diff --git a/Annex/Content.hs b/Annex/Content.hs --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -611,7 +611,10 @@ ( return True , do s <- calcRepo $ gitAnnexLocation key - liftIO $ copyFileExternal CopyTimeStamps s file + liftIO $ ifM (doesFileExist s) + ( copyFileExternal CopyTimeStamps s file + , return False + ) ) {- Blocks writing to an annexed file, and modifies file permissions to diff --git a/Annex/Init.hs b/Annex/Init.hs --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -128,6 +128,7 @@ createAnnexDirectory tmp liftIO $ writeFile f "" uncrippled <- liftIO $ probe f + void $ liftIO $ tryIO $ allowWrite f liftIO $ removeFile f return $ not uncrippled where @@ -137,8 +138,9 @@ createSymbolicLink f f2 nukeFile f2 preventWrite f - allowWrite f - return True + -- Should be unable to write to the file, but some crippled + -- filesystems ignore write bit removals. + not <$> catchBoolIO (writeFile f "2" >> return True) #endif checkCrippledFileSystem :: Annex () diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs --- a/Assistant/WebApp/Configurators/Ssh.hs +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -212,23 +212,26 @@ - to prompt for its password. -} enableSshRemote :: (RemoteConfig -> Maybe SshData) -> (SshInput -> RemoteName -> Handler Html) -> (SshData -> UUID -> Handler Html) -> UUID -> Handler Html -enableSshRemote getsshinput rsyncnetsetup genericsetup u = do +enableSshRemote getsshdata rsyncnetsetup genericsetup u = do m <- fromMaybe M.empty . M.lookup u <$> liftAnnex readRemoteLog - case (mkSshInput . unmangle <$> getsshinput m, M.lookup "name" m) of - (Just sshinput, Just reponame) -> sshConfigurator $ do - ((result, form), enctype) <- liftH $ - runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ sshInputAForm textField sshinput - case result of - FormSuccess sshinput' - | isRsyncNet (inputHostname sshinput') -> - void $ liftH $ rsyncnetsetup sshinput' reponame - | otherwise -> do - s <- liftAssistant $ testServer sshinput' - case s of - Left status -> showform form enctype status - Right (sshdata, _u) -> void $ liftH $ genericsetup - ( sshdata { sshRepoName = reponame } ) u - _ -> showform form enctype UntestedServer + case (unmangle <$> getsshdata m, M.lookup "name" m) of + (Just sshdata, Just reponame) + | isGitLab sshdata -> enableGitLab sshdata + | otherwise -> sshConfigurator $ do + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + sshInputAForm textField $ mkSshInput sshdata + case result of + FormSuccess sshinput + | isRsyncNet (inputHostname sshinput) -> + void $ liftH $ rsyncnetsetup sshinput reponame + | otherwise -> do + s <- liftAssistant $ testServer sshinput + case s of + Left status -> showform form enctype status + Right (sshdata', _u) -> void $ liftH $ genericsetup + ( sshdata' { sshRepoName = reponame } ) u + _ -> showform form enctype UntestedServer _ -> redirect AddSshR where unmangle sshdata = sshdata @@ -672,7 +675,7 @@ isRsyncNet Nothing = False isRsyncNet (Just host) = ".rsync.net" `T.isSuffixOf` T.toLower host -data GitLabUrl = GitLabUrl Text +data GitLabUrl = GitLabUrl { unGitLabUrl :: Text } badGitLabUrl :: Text badGitLabUrl = "Bad SSH clone url. Expected something like: git@gitlab.com:yourlogin/annex.git" @@ -698,6 +701,18 @@ , sshRepoUrl = Just (T.unpack t) } +isGitLab :: SshData -> Bool +isGitLab d = T.pack "gitlab.com" `T.isSuffixOf` (T.toLower (sshHostName d)) + +toGitLabUrl :: SshData -> GitLabUrl +toGitLabUrl d = GitLabUrl $ T.concat + [ fromMaybe (T.pack "git") (sshUserName d) + , T.pack "@" + , sshHostName d + , T.pack ":" + , sshDirectory d + ] + {- Try to ssh into the gitlab server, verify we can access the repository, - and get the uuid of the repository, if it already has one. - @@ -735,8 +750,8 @@ , Param (fromRef Annex.Branch.name) ] -gitLabUrlAForm :: AForm Handler GitLabUrl -gitLabUrlAForm = GitLabUrl <$> areq check_input (bfs "SSH clone url") Nothing +gitLabUrlAForm :: Maybe GitLabUrl -> AForm Handler GitLabUrl +gitLabUrlAForm defval = GitLabUrl <$> areq check_input (bfs "SSH clone url") (unGitLabUrl <$> defval) where check_input = checkBool (isJust . parseGitLabUrl . GitLabUrl) badGitLabUrl textField @@ -744,9 +759,13 @@ getAddGitLabR :: Handler Html getAddGitLabR = postAddGitLabR postAddGitLabR :: Handler Html -postAddGitLabR = sshConfigurator $ do +postAddGitLabR = promptGitLab Nothing + +promptGitLab :: Maybe GitLabUrl -> Handler Html +promptGitLab defval = sshConfigurator $ do ((result, form), enctype) <- liftH $ - runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout gitLabUrlAForm + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + gitLabUrlAForm defval case result of FormSuccess gitlaburl -> do (status, msshdata, u) <- liftAnnex $ testGitLabUrl gitlaburl @@ -757,3 +776,6 @@ _ -> showform form enctype UntestedServer where showform form enctype status = $(widgetFile "configurators/gitlab.com/add") + +enableGitLab :: SshData -> Handler Html +enableGitLab = promptGitLab . Just . toGitLabUrl diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,19 @@ +git-annex (5.20150731) unstable; urgency=medium + + * webapp: Support enabling known gitlab.com remotes. + * Fix rsync special remote to work when -Jn is used for concurrent + uploads. + * The last release accidentially removed a number of options from the + copy command. (-J, file matching options, etc). These have been added + back. + * init: Detect when the filesystem is crippled such that it ignores + attempts to remove the write bit from a file, and enable direct mode. + Seen with eg, NTFS fuse on linux. + * Fix man page installation by cabal install; all the new man pages are + now installed. + + -- Joey Hess Fri, 31 Jul 2015 11:34:36 -0400 + git-annex (5.20150727) unstable; urgency=medium * Fix bug that prevented uploads to remotes using new-style chunking diff --git a/Command/Copy.hs b/Command/Copy.hs --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -15,9 +15,10 @@ import Annex.NumCopies cmd :: Command -cmd = command "copy" SectionCommon - "copy content of files to/from another repository" - paramPaths (seek <--< optParser) +cmd = withGlobalOptions (jobsOption : annexedMatchingOptions) $ + command "copy" SectionCommon + "copy content of files to/from another repository" + paramPaths (seek <--< optParser) data CopyOptions = CopyOptions { moveOptions :: Command.Move.MoveOptions diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -39,6 +39,7 @@ import Types.Creds import Types.Key (isChunkKey) import Annex.DirHashes +import Utility.Tmp import qualified Data.Map as M @@ -252,16 +253,8 @@ - up trees for rsync. -} withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a withRsyncScratchDir a = do - p <- liftIO getPID t <- fromRepo gitAnnexTmpObjectDir - createAnnexDirectory t - let tmp = t "rsynctmp" show p - nuke tmp - liftIO $ createDirectoryIfMissing True tmp - nuke tmp `after` a tmp - where - nuke d = liftIO $ whenM (doesDirectoryExist d) $ - removeDirectoryRecursive d + withTmpDirIn t "rsynctmp" a rsyncRetrieve :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool rsyncRetrieve o k dest meterupdate = diff --git a/Setup.hs b/Setup.hs --- a/Setup.hs +++ b/Setup.hs @@ -13,6 +13,7 @@ import Control.Applicative import Control.Monad import System.Directory +import Data.List import qualified Build.DesktopFile as DesktopFile import qualified Build.Configure as Configure @@ -22,17 +23,17 @@ { preConf = \_ _ -> do Configure.run Configure.tests return (Nothing, []) - , postInst = myPostInst + , postCopy = myPostCopy } -myPostInst :: Args -> InstallFlags -> PackageDescription -> LocalBuildInfo -> IO () -myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do +myPostCopy :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO () +myPostCopy _ (CopyFlags { copyVerbosity }) pkg lbi = do installGitAnnexShell dest verbosity pkg lbi installManpages dest verbosity pkg lbi installDesktopFile dest verbosity pkg lbi where dest = NoCopyDest - verbosity = fromFlag installVerbosity + verbosity = fromFlag copyVerbosity installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () installGitAnnexShell copyDest verbosity pkg lbi = @@ -51,10 +52,14 @@ installOrdinaryFiles verbosity dstManDir =<< srcManpages where dstManDir = mandir (absoluteInstallDirs pkg lbi copyDest) "man1" - srcManpages = zip (repeat srcManDir) - <$> filterM doesFileExist manpages - srcManDir = "" - manpages = ["git-annex.1", "git-annex-shell.1"] + srcManpages = do + havemans <- doesDirectoryExist srcManDir + if havemans + then zip (repeat srcManDir) + . filter (".1" `isSuffixOf`) + <$> getDirectoryContents srcManDir + else return [] + srcManDir = "man" installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () installDesktopFile copyDest _verbosity pkg lbi = diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +git-annex (5.20150731) unstable; urgency=medium + + * webapp: Support enabling known gitlab.com remotes. + * Fix rsync special remote to work when -Jn is used for concurrent + uploads. + * The last release accidentially removed a number of options from the + copy command. (-J, file matching options, etc). These have been added + back. + * init: Detect when the filesystem is crippled such that it ignores + attempts to remove the write bit from a file, and enable direct mode. + Seen with eg, NTFS fuse on linux. + * Fix man page installation by cabal install; all the new man pages are + now installed. + + -- Joey Hess Fri, 31 Jul 2015 11:34:36 -0400 + git-annex (5.20150727) unstable; urgency=medium * Fix bug that prevented uploads to remotes using new-style chunking diff --git a/doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn b/doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn --- a/doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn +++ b/doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn @@ -4,3 +4,5 @@ This is a SMOP; it needs to detect that the repo is on gitlab and use a custom enabling process and no the generic one, which doesn't work. --[[Joey]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn b/doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn --- a/doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn +++ b/doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn @@ -10,3 +10,13 @@ This does not happen when I try the same setup on a self-hosted repo. Unsure what is causing git-annex-shell to behave this way on gitlab. --[[Joey]] + +> Here's a minimal way to reproduce this problem: +> +> 1. Make a new, empty repository on gitlab.com. +> 2. Run command: ssh git@gitlab.com git-annex-shell 'gcryptsetup' '/~/username/reponame.git' ':id:dummy' +> +> Leads to the failure as described above. But, the repo is new and empty +> and has not had any opportunity to get a git-annex uuid set.. +> +> I wonder what version of git-annex gitlab.com is running? --[[Joey]] diff --git a/doc/bugs/rsync_remote_with_-J2_fails.mdwn b/doc/bugs/rsync_remote_with_-J2_fails.mdwn --- a/doc/bugs/rsync_remote_with_-J2_fails.mdwn +++ b/doc/bugs/rsync_remote_with_-J2_fails.mdwn @@ -53,3 +53,5 @@ annex-uuid = f05581cc-7236-41ed-9db8-49424f863307 """]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn b/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn @@ -0,0 +1,90 @@ +### Please describe the problem. + +subject + what was described in https://git-annex.branchable.com/forum/some_symlinks_left_in_direct_mode/ some time ago I guess + +### What steps will reproduce the problem? + +clone git annex repo, switch immediately to direct mode and observe files being left as dangling symlinks, get some content -- they get into normal files, drop them -- broken symlinks again. I thought (and I believe that it was like that) that in direct mode -- no symlinks to .git/annex regardless of the state (present/absent). + +full example /transcript is below + +### What version of git-annex are you using? On what operating system? + +Debian sid 5.20150710-2 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$> git clone git://github.com/datalad/testrepo--basic--r1 +Cloning into 'testrepo--basic--r1'... +remote: Counting objects: 23, done. +remote: Total 23 (delta 0), reused 0 (delta 0), pack-reused 23 +Receiving objects: 100% (23/23), done. +Resolving deltas: 100% (3/3), done. +Checking connectivity... done. +2 22606.....................................:Tue 28 Jul 2015 04:11:23 PM EDT:. +hopa:/tmp +$> cd testrepo--basic--r1 +INFO.txt test-annex.dat@ test.dat +2 22607.....................................:Tue 28 Jul 2015 04:11:30 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> ls -l +total 12 +-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt +lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat +-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat +2 22608.....................................:Tue 28 Jul 2015 04:11:31 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> git annex direct +(merging origin/git-annex into git-annex...) +(recording state in git...) +commit +(recording state in git...) +On branch master +Your branch is up-to-date with 'origin/master'. +nothing to commit, working directory clean +ok +direct ok +2 22609.....................................:Tue 28 Jul 2015 04:11:36 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> ls -l +total 12 +-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt +lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat +-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat +2 22610.....................................:Tue 28 Jul 2015 04:11:38 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> git annex get test-annex.dat +get test-annex.dat (from web...) +/tmp/testrepo--basic--r1/.git/annex/tmp/SHA256E-s4--1 100%[===========================================================================================================================>] 4 --.-KB/s in 0s +ok +(recording state in git...) +2 22611.....................................:Tue 28 Jul 2015 04:11:45 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> ls -lta +total 12 +drwxrwxrwt 126 root root 4020 Jul 28 16:11 ../ +drwx------ 3 yoh yoh 120 Jul 28 16:11 ./ +-rw------- 1 yoh yoh 4 Jul 28 16:11 test-annex.dat +drwx------ 9 yoh yoh 300 Jul 28 16:11 .git/ +-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt +-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat +2 22612.....................................:Tue 28 Jul 2015 04:11:49 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> git annex drop test-annex.dat +drop test-annex.dat (checking https://raw.githubusercontent.com/datalad/testrepo--basic--r1/master/test.dat...) ok +(recording state in git...) +2 22613.....................................:Tue 28 Jul 2015 04:11:57 PM EDT:. +(git)hopa:/tmp/testrepo--basic--r1[master] +$> ls -l +total 12 +-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt +lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat +-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat + + +# End of transcript or log. +"""]] diff --git a/doc/devblog/day_306__release_day.mdwn b/doc/devblog/day_306__release_day.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_306__release_day.mdwn @@ -0,0 +1,8 @@ +Made a release today, with recent work, including the optparse-applicative +transition and initial gitlab.com support in the webapp. + +I had time before the release to work out most of the wrinkles in the +gitlab.com support, but was not able to get gcrypt encrypted repos to work +with gitlab, for reasons that remain murky. Their git-annex-shell seems to +be misbehaving somehow. Will need to get some debugging assistance from the +gitlab.com developers to figure that out. diff --git a/doc/news/version_5.20150528.mdwn b/doc/news/version_5.20150528.mdwn deleted file mode 100644 --- a/doc/news/version_5.20150528.mdwn +++ /dev/null @@ -1,15 +0,0 @@ -git-annex 5.20150528 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * fromkey, registerurl: Allow urls to be specified instead of keys, - and generate URL keys. - * Linux standalone, OSX app: Improve runshell script to always quote - shell vars, so that it will work when eg, untarred into a directory - path with spaces in its name. - * Revert removal dependency on obsolete hamlet package, since the - autobuilders are not ready for this change yet and it prevented them - from building the webapp. Reopens: #786659 - * fsck: When checksumming a file fails due to a hardware fault, - the file is now moved to the bad directory, and the fsck proceeds. - Before, the fsck immediately failed. - * Linux standalone: The webapp was not built in the previous release, - this release fixes that oversight."""]] diff --git a/doc/news/version_5.20150731.mdwn b/doc/news/version_5.20150731.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20150731.mdwn @@ -0,0 +1,13 @@ +git-annex 5.20150731 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * webapp: Support enabling known gitlab.com remotes. + * Fix rsync special remote to work when -Jn is used for concurrent + uploads. + * The last release accidentially removed a number of options from the + copy command. (-J, file matching options, etc). These have been added + back. + * init: Detect when the filesystem is crippled such that it ignores + attempts to remove the write bit from a file, and enable direct mode. + Seen with eg, NTFS fuse on linux. + * Fix man page installation by cabal install; all the new man pages are + now installed."""]] diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn b/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn --- a/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn +++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn @@ -5,7 +5,7 @@ you can store your large files on GitLab, quite easily. Note that as I'm writing this, GitLab is providing this service for free, -and I don't know how much data they're willing to host for free. +and will store up to 10 gb per project. ## create the repository diff --git a/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment b/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment new file mode 100644 --- /dev/null +++ b/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="torarnv@6179ecd599a0e00709a67306f015e46307a66eb6" + nickname="torarnv" + subject="Git 2.5 allows smudge filters to not read all of stdin" + date="2015-07-29T10:35:07Z" + content=""" +It seems git 2.5 allows smudge filters to not read all of stdin: + +https://github.com/git/git/blob/master/Documentation/RelNotes/2.5.0.txt + +\" + * Filter scripts were run with SIGPIPE disabled on the Git side, + expecting that they may not read what Git feeds them to filter. + We however treated a filter that does not read its input fully + before exiting as an error. We no longer do and ignore EPIPE + when writing to feed the filter scripts. + + This changes semantics, but arguably in a good way. If a filter + can produce its output without fully consuming its input using + whatever magic, we now let it do so, instead of diagnosing it + as a programming error.\" +"""]] diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20150727 +Version: 5.20150731 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess diff --git a/templates/configurators/gitlab.com/add.hamlet b/templates/configurators/gitlab.com/add.hamlet --- a/templates/configurators/gitlab.com/add.hamlet +++ b/templates/configurators/gitlab.com/add.hamlet @@ -6,7 +6,10 @@ GitLab.com # provides free public and private git repositories, and supports # - git-annex. + git-annex. While the amount of data that can be stored there is limited # + (to 10 gb currently), # + it's enough for smaller repositories, # + or as a transfer point between larger repositories.

$case status $of UnusableServer msg