git-annex 5.20150727 → 5.20150731
raw patch · 19 files changed
+255/−62 lines, 19 filesdep ~awssetup-changed
Dependency ranges changed: aws
Files
- Annex/Content.hs +4/−1
- Annex/Init.hs +4/−2
- Assistant/WebApp/Configurators/Ssh.hs +43/−21
- CHANGELOG +16/−0
- Command/Copy.hs +4/−3
- Remote/Rsync.hs +2/−9
- Setup.hs +13/−8
- debian/changelog +16/−0
- doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn +2/−0
- doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn +10/−0
- doc/bugs/rsync_remote_with_-J2_fails.mdwn +2/−0
- doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn +90/−0
- doc/devblog/day_306__release_day.mdwn +8/−0
- doc/news/version_5.20150528.mdwn +0/−15
- doc/news/version_5.20150731.mdwn +13/−0
- doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn +1/−1
- doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment +22/−0
- git-annex.cabal +1/−1
- templates/configurators/gitlab.com/add.hamlet +4/−1
Annex/Content.hs view
@@ -611,7 +611,10 @@ ( return True , do s <- calcRepo $ gitAnnexLocation key- liftIO $ copyFileExternal CopyTimeStamps s file+ liftIO $ ifM (doesFileExist s)+ ( copyFileExternal CopyTimeStamps s file+ , return False+ ) ) {- Blocks writing to an annexed file, and modifies file permissions to
Annex/Init.hs view
@@ -128,6 +128,7 @@ createAnnexDirectory tmp liftIO $ writeFile f "" uncrippled <- liftIO $ probe f+ void $ liftIO $ tryIO $ allowWrite f liftIO $ removeFile f return $ not uncrippled where@@ -137,8 +138,9 @@ createSymbolicLink f f2 nukeFile f2 preventWrite f- allowWrite f- return True+ -- Should be unable to write to the file, but some crippled+ -- filesystems ignore write bit removals.+ not <$> catchBoolIO (writeFile f "2" >> return True) #endif checkCrippledFileSystem :: Annex ()
Assistant/WebApp/Configurators/Ssh.hs view
@@ -212,23 +212,26 @@ - to prompt for its password. -} enableSshRemote :: (RemoteConfig -> Maybe SshData) -> (SshInput -> RemoteName -> Handler Html) -> (SshData -> UUID -> Handler Html) -> UUID -> Handler Html-enableSshRemote getsshinput rsyncnetsetup genericsetup u = do+enableSshRemote getsshdata rsyncnetsetup genericsetup u = do m <- fromMaybe M.empty . M.lookup u <$> liftAnnex readRemoteLog- case (mkSshInput . unmangle <$> getsshinput m, M.lookup "name" m) of- (Just sshinput, Just reponame) -> sshConfigurator $ do- ((result, form), enctype) <- liftH $- runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ sshInputAForm textField sshinput- case result of- FormSuccess sshinput'- | isRsyncNet (inputHostname sshinput') ->- void $ liftH $ rsyncnetsetup sshinput' reponame- | otherwise -> do- s <- liftAssistant $ testServer sshinput'- case s of- Left status -> showform form enctype status- Right (sshdata, _u) -> void $ liftH $ genericsetup- ( sshdata { sshRepoName = reponame } ) u- _ -> showform form enctype UntestedServer+ case (unmangle <$> getsshdata m, M.lookup "name" m) of+ (Just sshdata, Just reponame)+ | isGitLab sshdata -> enableGitLab sshdata+ | otherwise -> sshConfigurator $ do+ ((result, form), enctype) <- liftH $+ runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $+ sshInputAForm textField $ mkSshInput sshdata+ case result of+ FormSuccess sshinput+ | isRsyncNet (inputHostname sshinput) ->+ void $ liftH $ rsyncnetsetup sshinput reponame+ | otherwise -> do+ s <- liftAssistant $ testServer sshinput+ case s of+ Left status -> showform form enctype status+ Right (sshdata', _u) -> void $ liftH $ genericsetup+ ( sshdata' { sshRepoName = reponame } ) u+ _ -> showform form enctype UntestedServer _ -> redirect AddSshR where unmangle sshdata = sshdata@@ -672,7 +675,7 @@ isRsyncNet Nothing = False isRsyncNet (Just host) = ".rsync.net" `T.isSuffixOf` T.toLower host -data GitLabUrl = GitLabUrl Text+data GitLabUrl = GitLabUrl { unGitLabUrl :: Text } badGitLabUrl :: Text badGitLabUrl = "Bad SSH clone url. Expected something like: git@gitlab.com:yourlogin/annex.git"@@ -698,6 +701,18 @@ , sshRepoUrl = Just (T.unpack t) } +isGitLab :: SshData -> Bool+isGitLab d = T.pack "gitlab.com" `T.isSuffixOf` (T.toLower (sshHostName d))++toGitLabUrl :: SshData -> GitLabUrl+toGitLabUrl d = GitLabUrl $ T.concat+ [ fromMaybe (T.pack "git") (sshUserName d)+ , T.pack "@"+ , sshHostName d+ , T.pack ":"+ , sshDirectory d+ ]+ {- Try to ssh into the gitlab server, verify we can access the repository, - and get the uuid of the repository, if it already has one. -@@ -735,8 +750,8 @@ , Param (fromRef Annex.Branch.name) ] -gitLabUrlAForm :: AForm Handler GitLabUrl-gitLabUrlAForm = GitLabUrl <$> areq check_input (bfs "SSH clone url") Nothing+gitLabUrlAForm :: Maybe GitLabUrl -> AForm Handler GitLabUrl+gitLabUrlAForm defval = GitLabUrl <$> areq check_input (bfs "SSH clone url") (unGitLabUrl <$> defval) where check_input = checkBool (isJust . parseGitLabUrl . GitLabUrl) badGitLabUrl textField@@ -744,9 +759,13 @@ getAddGitLabR :: Handler Html getAddGitLabR = postAddGitLabR postAddGitLabR :: Handler Html-postAddGitLabR = sshConfigurator $ do+postAddGitLabR = promptGitLab Nothing++promptGitLab :: Maybe GitLabUrl -> Handler Html+promptGitLab defval = sshConfigurator $ do ((result, form), enctype) <- liftH $- runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout gitLabUrlAForm+ runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $+ gitLabUrlAForm defval case result of FormSuccess gitlaburl -> do (status, msshdata, u) <- liftAnnex $ testGitLabUrl gitlaburl@@ -757,3 +776,6 @@ _ -> showform form enctype UntestedServer where showform form enctype status = $(widgetFile "configurators/gitlab.com/add")++enableGitLab :: SshData -> Handler Html+enableGitLab = promptGitLab . Just . toGitLabUrl
CHANGELOG view
@@ -1,3 +1,19 @@+git-annex (5.20150731) unstable; urgency=medium++ * webapp: Support enabling known gitlab.com remotes.+ * Fix rsync special remote to work when -Jn is used for concurrent+ uploads.+ * The last release accidentially removed a number of options from the+ copy command. (-J, file matching options, etc). These have been added+ back.+ * init: Detect when the filesystem is crippled such that it ignores+ attempts to remove the write bit from a file, and enable direct mode.+ Seen with eg, NTFS fuse on linux.+ * Fix man page installation by cabal install; all the new man pages are+ now installed.++ -- Joey Hess <id@joeyh.name> Fri, 31 Jul 2015 11:34:36 -0400+ git-annex (5.20150727) unstable; urgency=medium * Fix bug that prevented uploads to remotes using new-style chunking
Command/Copy.hs view
@@ -15,9 +15,10 @@ import Annex.NumCopies cmd :: Command-cmd = command "copy" SectionCommon- "copy content of files to/from another repository"- paramPaths (seek <--< optParser)+cmd = withGlobalOptions (jobsOption : annexedMatchingOptions) $+ command "copy" SectionCommon+ "copy content of files to/from another repository"+ paramPaths (seek <--< optParser) data CopyOptions = CopyOptions { moveOptions :: Command.Move.MoveOptions
Remote/Rsync.hs view
@@ -39,6 +39,7 @@ import Types.Creds import Types.Key (isChunkKey) import Annex.DirHashes+import Utility.Tmp import qualified Data.Map as M @@ -252,16 +253,8 @@ - up trees for rsync. -} withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a withRsyncScratchDir a = do- p <- liftIO getPID t <- fromRepo gitAnnexTmpObjectDir- createAnnexDirectory t- let tmp = t </> "rsynctmp" </> show p- nuke tmp- liftIO $ createDirectoryIfMissing True tmp- nuke tmp `after` a tmp- where- nuke d = liftIO $ whenM (doesDirectoryExist d) $- removeDirectoryRecursive d+ withTmpDirIn t "rsynctmp" a rsyncRetrieve :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool rsyncRetrieve o k dest meterupdate =
Setup.hs view
@@ -13,6 +13,7 @@ import Control.Applicative import Control.Monad import System.Directory+import Data.List import qualified Build.DesktopFile as DesktopFile import qualified Build.Configure as Configure@@ -22,17 +23,17 @@ { preConf = \_ _ -> do Configure.run Configure.tests return (Nothing, []) - , postInst = myPostInst+ , postCopy = myPostCopy } -myPostInst :: Args -> InstallFlags -> PackageDescription -> LocalBuildInfo -> IO ()-myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do+myPostCopy :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO ()+myPostCopy _ (CopyFlags { copyVerbosity }) pkg lbi = do installGitAnnexShell dest verbosity pkg lbi installManpages dest verbosity pkg lbi installDesktopFile dest verbosity pkg lbi where dest = NoCopyDest- verbosity = fromFlag installVerbosity+ verbosity = fromFlag copyVerbosity installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () installGitAnnexShell copyDest verbosity pkg lbi =@@ -51,10 +52,14 @@ installOrdinaryFiles verbosity dstManDir =<< srcManpages where dstManDir = mandir (absoluteInstallDirs pkg lbi copyDest) </> "man1"- srcManpages = zip (repeat srcManDir)- <$> filterM doesFileExist manpages- srcManDir = ""- manpages = ["git-annex.1", "git-annex-shell.1"]+ srcManpages = do+ havemans <- doesDirectoryExist srcManDir+ if havemans+ then zip (repeat srcManDir)+ . filter (".1" `isSuffixOf`)+ <$> getDirectoryContents srcManDir+ else return []+ srcManDir = "man" installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () installDesktopFile copyDest _verbosity pkg lbi =
debian/changelog view
@@ -1,3 +1,19 @@+git-annex (5.20150731) unstable; urgency=medium++ * webapp: Support enabling known gitlab.com remotes.+ * Fix rsync special remote to work when -Jn is used for concurrent+ uploads.+ * The last release accidentially removed a number of options from the+ copy command. (-J, file matching options, etc). These have been added+ back.+ * init: Detect when the filesystem is crippled such that it ignores+ attempts to remove the write bit from a file, and enable direct mode.+ Seen with eg, NTFS fuse on linux.+ * Fix man page installation by cabal install; all the new man pages are+ now installed.++ -- Joey Hess <id@joeyh.name> Fri, 31 Jul 2015 11:34:36 -0400+ git-annex (5.20150727) unstable; urgency=medium * Fix bug that prevented uploads to remotes using new-style chunking
doc/bugs/enabling_existing_gitlab_repo_in_webapp_broken.mdwn view
@@ -4,3 +4,5 @@ This is a SMOP; it needs to detect that the repo is on gitlab and use a custom enabling process and no the generic one, which doesn't work. --[[Joey]]++> [[fixed|done]] --[[Joey]]
doc/bugs/gitlab_repos_cannot_use_gcrypt.mdwn view
@@ -10,3 +10,13 @@ This does not happen when I try the same setup on a self-hosted repo. Unsure what is causing git-annex-shell to behave this way on gitlab. --[[Joey]]++> Here's a minimal way to reproduce this problem:+>+> 1. Make a new, empty repository on gitlab.com.+> 2. Run command: ssh git@gitlab.com git-annex-shell 'gcryptsetup' '/~/username/reponame.git' ':id:dummy'+> +> Leads to the failure as described above. But, the repo is new and empty+> and has not had any opportunity to get a git-annex uuid set..+> +> I wonder what version of git-annex gitlab.com is running? --[[Joey]]
doc/bugs/rsync_remote_with_-J2_fails.mdwn view
@@ -53,3 +53,5 @@ annex-uuid = f05581cc-7236-41ed-9db8-49424f863307 """]]++> [[fixed|done]] --[[Joey]]
+ doc/bugs/symlinks_to_absent_files_remain_upon_switching_to_direct_mode2.mdwn view
@@ -0,0 +1,90 @@+### Please describe the problem.++subject + what was described in https://git-annex.branchable.com/forum/some_symlinks_left_in_direct_mode/ some time ago I guess++### What steps will reproduce the problem?++clone git annex repo, switch immediately to direct mode and observe files being left as dangling symlinks, get some content -- they get into normal files, drop them -- broken symlinks again. I thought (and I believe that it was like that) that in direct mode -- no symlinks to .git/annex regardless of the state (present/absent).++full example /transcript is below++### What version of git-annex are you using? On what operating system?++Debian sid 5.20150710-2++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++$> git clone git://github.com/datalad/testrepo--basic--r1+Cloning into 'testrepo--basic--r1'...+remote: Counting objects: 23, done.+remote: Total 23 (delta 0), reused 0 (delta 0), pack-reused 23+Receiving objects: 100% (23/23), done.+Resolving deltas: 100% (3/3), done.+Checking connectivity... done.+2 22606.....................................:Tue 28 Jul 2015 04:11:23 PM EDT:.+hopa:/tmp+$> cd testrepo--basic--r1+INFO.txt test-annex.dat@ test.dat+2 22607.....................................:Tue 28 Jul 2015 04:11:30 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> ls -l+total 12+-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt+lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat+-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat+2 22608.....................................:Tue 28 Jul 2015 04:11:31 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> git annex direct+(merging origin/git-annex into git-annex...)+(recording state in git...)+commit +(recording state in git...)+On branch master+Your branch is up-to-date with 'origin/master'.+nothing to commit, working directory clean+ok+direct ok+2 22609.....................................:Tue 28 Jul 2015 04:11:36 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> ls -l +total 12+-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt+lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat+-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat+2 22610.....................................:Tue 28 Jul 2015 04:11:38 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> git annex get test-annex.dat+get test-annex.dat (from web...) +/tmp/testrepo--basic--r1/.git/annex/tmp/SHA256E-s4--1 100%[===========================================================================================================================>] 4 --.-KB/s in 0s +ok+(recording state in git...)+2 22611.....................................:Tue 28 Jul 2015 04:11:45 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> ls -lta+total 12+drwxrwxrwt 126 root root 4020 Jul 28 16:11 ../+drwx------ 3 yoh yoh 120 Jul 28 16:11 ./+-rw------- 1 yoh yoh 4 Jul 28 16:11 test-annex.dat+drwx------ 9 yoh yoh 300 Jul 28 16:11 .git/+-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt+-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat+2 22612.....................................:Tue 28 Jul 2015 04:11:49 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> git annex drop test-annex.dat+drop test-annex.dat (checking https://raw.githubusercontent.com/datalad/testrepo--basic--r1/master/test.dat...) ok+(recording state in git...)+2 22613.....................................:Tue 28 Jul 2015 04:11:57 PM EDT:.+(git)hopa:/tmp/testrepo--basic--r1[master]+$> ls -l +total 12+-rw------- 1 yoh yoh 53 Jul 28 16:11 INFO.txt+lrwxrwxrwx 1 yoh yoh 186 Jul 28 16:11 test-annex.dat -> .git/annex/objects/zk/71/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat+-rw------- 1 yoh yoh 4 Jul 28 16:11 test.dat+++# End of transcript or log.+"""]]
+ doc/devblog/day_306__release_day.mdwn view
@@ -0,0 +1,8 @@+Made a release today, with recent work, including the optparse-applicative+transition and initial gitlab.com support in the webapp.++I had time before the release to work out most of the wrinkles in the+gitlab.com support, but was not able to get gcrypt encrypted repos to work+with gitlab, for reasons that remain murky. Their git-annex-shell seems to+be misbehaving somehow. Will need to get some debugging assistance from the+gitlab.com developers to figure that out.
− doc/news/version_5.20150528.mdwn
@@ -1,15 +0,0 @@-git-annex 5.20150528 released with [[!toggle text="these changes"]]-[[!toggleable text="""- * fromkey, registerurl: Allow urls to be specified instead of keys,- and generate URL keys.- * Linux standalone, OSX app: Improve runshell script to always quote- shell vars, so that it will work when eg, untarred into a directory- path with spaces in its name.- * Revert removal dependency on obsolete hamlet package, since the- autobuilders are not ready for this change yet and it prevented them- from building the webapp. Reopens: #786659- * fsck: When checksumming a file fails due to a hardware fault,- the file is now moved to the bad directory, and the fsck proceeds.- Before, the fsck immediately failed.- * Linux standalone: The webapp was not built in the previous release,- this release fixes that oversight."""]]
+ doc/news/version_5.20150731.mdwn view
@@ -0,0 +1,13 @@+git-annex 5.20150731 released with [[!toggle text="these changes"]]+[[!toggleable text="""+ * webapp: Support enabling known gitlab.com remotes.+ * Fix rsync special remote to work when -Jn is used for concurrent+ uploads.+ * The last release accidentially removed a number of options from the+ copy command. (-J, file matching options, etc). These have been added+ back.+ * init: Detect when the filesystem is crippled such that it ignores+ attempts to remove the write bit from a file, and enable direct mode.+ Seen with eg, NTFS fuse on linux.+ * Fix man page installation by cabal install; all the new man pages are+ now installed."""]]
doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn view
@@ -5,7 +5,7 @@ you can store your large files on GitLab, quite easily. Note that as I'm writing this, GitLab is providing this service for free,-and I don't know how much data they're willing to host for free.+and will store up to 10 gb per project. ## create the repository
+ doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment view
@@ -0,0 +1,22 @@+[[!comment format=mdwn+ username="torarnv@6179ecd599a0e00709a67306f015e46307a66eb6"+ nickname="torarnv"+ subject="Git 2.5 allows smudge filters to not read all of stdin"+ date="2015-07-29T10:35:07Z"+ content="""+It seems git 2.5 allows smudge filters to not read all of stdin:++https://github.com/git/git/blob/master/Documentation/RelNotes/2.5.0.txt++\"+ * Filter scripts were run with SIGPIPE disabled on the Git side,+ expecting that they may not read what Git feeds them to filter.+ We however treated a filter that does not read its input fully+ before exiting as an error. We no longer do and ignore EPIPE+ when writing to feed the filter scripts.++ This changes semantics, but arguably in a good way. If a filter+ can produce its output without fully consuming its input using+ whatever magic, we now let it do so, instead of diagnosing it+ as a programming error.\"+"""]]
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20150727+Version: 5.20150731 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name>
templates/configurators/gitlab.com/add.hamlet view
@@ -6,7 +6,10 @@ <a href="http://gitlab.com/"> GitLab.com # provides free public and private git repositories, and supports #- git-annex.+ git-annex. While the amount of data that can be stored there is limited #+ (<a href="https://about.gitlab.com/gitlab-com/">to 10 gb currently</a>), #+ it's enough for smaller repositories, #+ or as a transfer point between larger repositories. <p> $case status $of UnusableServer msg