diff --git a/Annex/Content.hs b/Annex/Content.hs --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -56,6 +56,7 @@ import Types.Key import Utility.DataUnits import Utility.CopyFile +import Utility.Metered import Config import Git.SharedRepository import Annex.Perms @@ -658,8 +659,9 @@ Annex.Branch.commit "update" {- Downloads content from any of a list of urls. -} -downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool -downloadUrl urls file = go =<< annexWebDownloadCommand <$> Annex.getGitConfig +downloadUrl :: Key -> MeterUpdate -> [Url.URLString] -> FilePath -> Annex Bool +downloadUrl k p urls file = concurrentMeteredFile file (Just p) k $ + go =<< annexWebDownloadCommand <$> Annex.getGitConfig where go Nothing = do a <- ifM commandProgressDisabled @@ -697,18 +699,21 @@ ) ) -{- Blocks writing to an annexed file, and modifies file permissions to - - allow reading it, per core.sharedRepository setting. -} +{- Normally, blocks writing to an annexed file, and modifies file + - permissions to allow reading it. + - + - When core.sharedRepository is set, the write bits are not removed from + - the file, but instead the appropriate group write bits are set. This is + - necessary to let other users in the group lock the file. + -} freezeContent :: FilePath -> Annex () freezeContent file = unlessM crippledFileSystem $ withShared go where go GroupShared = liftIO $ modifyFileMode file $ - removeModes writeModes . - addModes [ownerReadMode, groupReadMode] + addModes [ownerReadMode, groupReadMode, ownerWriteMode, groupWriteMode] go AllShared = liftIO $ modifyFileMode file $ - removeModes writeModes . - addModes readModes + addModes (readModes ++ writeModes) go _ = liftIO $ modifyFileMode file $ removeModes writeModes . addModes [ownerReadMode] diff --git a/Annex/ReplaceFile.hs b/Annex/ReplaceFile.hs --- a/Annex/ReplaceFile.hs +++ b/Annex/ReplaceFile.hs @@ -27,9 +27,10 @@ replaceFile file action = do misctmpdir <- fromRepo gitAnnexTmpMiscDir void $ createAnnexDirectory misctmpdir - let basetmp = takeFileName file + filemax <- liftIO $ fileNameLengthLimit misctmpdir + let basetmp = take (filemax `div` 2) (takeFileName file) withTmpDirIn misctmpdir basetmp $ \tmpdir -> do - let tmpfile = tmpdir <> basetmp + let tmpfile = tmpdir basetmp action tmpfile liftIO $ replaceFileFrom tmpfile file diff --git a/Assistant/Threads/XMPPClient.hs b/Assistant/Threads/XMPPClient.hs --- a/Assistant/Threads/XMPPClient.hs +++ b/Assistant/Threads/XMPPClient.hs @@ -7,7 +7,7 @@ module Assistant.Threads.XMPPClient where -import Assistant.Common +import Assistant.Common hiding (ProtocolError) import Assistant.XMPP import Assistant.XMPP.Client import Assistant.NetMessager diff --git a/Assistant/WebApp/Common.hs b/Assistant/WebApp/Common.hs --- a/Assistant/WebApp/Common.hs +++ b/Assistant/WebApp/Common.hs @@ -13,5 +13,5 @@ import Assistant.WebApp.Form as X import Assistant.WebApp.Types as X import Assistant.WebApp.RepoId as X -import Utility.Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option) +import Utility.Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option, PermissionDenied) import Data.Text as X (Text) diff --git a/Assistant/WebApp/Configurators/Preferences.hs b/Assistant/WebApp/Configurators/Preferences.hs --- a/Assistant/WebApp/Configurators/Preferences.hs +++ b/Assistant/WebApp/Configurators/Preferences.hs @@ -118,5 +118,5 @@ inAutoStartFile :: Annex Bool inAutoStartFile = do - here <- fromRepo Git.repoPath + here <- liftIO . absPath =<< fromRepo Git.repoPath any (`equalFilePath` here) <$> liftIO readAutoStartFile diff --git a/Backend/Hash.hs b/Backend/Hash.hs --- a/Backend/Hash.hs +++ b/Backend/Hash.hs @@ -108,17 +108,16 @@ {- A key's checksum is checked during fsck. -} checkKeyChecksum :: Hash -> Key -> FilePath -> Annex Bool -checkKeyChecksum hash key file = go `catchHardwareFault` hwfault +checkKeyChecksum hash key file = catchIOErrorType HardwareFault hwfault $ do + fast <- Annex.getState Annex.fast + mstat <- liftIO $ catchMaybeIO $ getFileStatus file + case (mstat, fast) of + (Just stat, False) -> do + filesize <- liftIO $ getFileSize' file stat + showAction "checksum" + check <$> hashFile hash file filesize + _ -> return True where - go = do - fast <- Annex.getState Annex.fast - mstat <- liftIO $ catchMaybeIO $ getFileStatus file - case (mstat, fast) of - (Just stat, False) -> do - filesize <- liftIO $ getFileSize' file stat - showAction "checksum" - check <$> hashFile hash file filesize - _ -> return True expected = keyHash key check s | s == expected = True diff --git a/BuildFlags.hs b/BuildFlags.hs --- a/BuildFlags.hs +++ b/BuildFlags.hs @@ -67,6 +67,8 @@ #endif #ifdef WITH_CONCURRENTOUTPUT , "ConcurrentOutput" +#else +#warning Building without ConcurrentOutput #endif #ifdef WITH_DNS , "DNS" diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,30 @@ +git-annex (5.20151208) unstable; urgency=medium + + * Build with -j1 again to get reproducible build. + * Display progress meter in -J mode when copying from a local git repo, + to a local git repo, and from a remote git repo. + * Display progress meter in -J mode when downloading from the web. + * map: Improve display of git remotes with non-ssh urls, including http + and gcrypt. + * When core.sharedRepository is set, annex object files are not made mode + 444, since that prevents a user other than the file owner from locking + them. Instead, a mode such as 664 is used in this case. + * tahoe: Include tahoe capabilities in whereis display. + * import: Changed to honor annex.largefiles settings. + * addurl, importfeed: Changed to honor annex.largefiles settings, + when the content of the url is downloaded. (Not when using --fast or + --relaxed.) + * webapp: Fix bugs that could result in a relative path such as "." + being written to ~/.config/git-annex/autostart, and ignore any such + relative paths in the file. + This was a reversion caused by the relative path changes in 5.20150113. + * dropunused: Make more robust when trying to drop an object that has + already been dropped. + * Fix reversion in handling of long filenames, particularly when using + addurl/importfeed, which was introduced in the previous release. + + -- Joey Hess Tue, 08 Dec 2015 11:14:03 -0400 + git-annex (5.20151116) unstable; urgency=medium * Use concurrent-output library when configured with -fConcurrentOutput. diff --git a/Command/Add.hs b/Command/Add.hs --- a/Command/Add.hs +++ b/Command/Add.hs @@ -73,14 +73,18 @@ startSmall :: FilePath -> CommandStart startSmall file = do showStart "add" file + next $ next $ addSmall file + +addSmall :: FilePath -> Annex Bool +addSmall file = do showNote "non-large file; adding content to git repository" - next $ performAdd file + addFile file -performAdd :: FilePath -> CommandPerform -performAdd file = do +addFile :: FilePath -> Annex Bool +addFile file = do ps <- forceParams Annex.Queue.addCommand "add" (ps++[Param "--"]) [file] - next $ return True + return True {- The add subcommand annexes a file, generating a key for it using a - backend, and then moving it into the annex directory and setting up @@ -97,7 +101,7 @@ | otherwise -> do showStart "add" file next $ if isSymbolicLink s - then performAdd file + then next $ addFile file else perform file addpresent key = ifM isDirect ( do diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -29,6 +29,7 @@ import Types.UrlContents import Config import Annex.Content.Direct +import Annex.FileMatcher import Logs.Location import Utility.Metered import qualified Annex.Transfer as Transfer @@ -252,9 +253,9 @@ tmp <- fromRepo $ gitAnnexTmpObjectLocation key showOutput ok <- Transfer.notifyTransfer Transfer.Download (Just file) $ - Transfer.download webUUID key (Just file) Transfer.forwardRetry Transfer.noObserver $ const $ do + Transfer.download webUUID key (Just file) Transfer.forwardRetry Transfer.noObserver $ \p -> do liftIO $ createDirectoryIfMissing True (parentDir tmp) - downloadUrl [videourl] tmp + downloadUrl key p [videourl] tmp if ok then do cleanup webUUID quviurl file key (Just tmp) @@ -294,9 +295,9 @@ downloadWeb :: URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) downloadWeb url urlinfo file = do let dummykey = addSizeUrlKey urlinfo $ Backend.URL.fromUrl url Nothing - let downloader f _ = do + let downloader f p = do showOutput - downloadUrl [url] f + downloadUrl dummykey p [url] f showAction $ "downloading " ++ url ++ " " downloadWith downloader dummykey webUUID url file @@ -335,17 +336,28 @@ addSizeUrlKey urlinfo key = key { keySize = Url.urlSize urlinfo } cleanup :: UUID -> URLString -> FilePath -> Key -> Maybe FilePath -> Annex () -cleanup u url file key mtmp = do - when (isJust mtmp) $ - logStatus key InfoPresent - setUrlPresent u key url - Command.Add.addLink file key Nothing - whenM isDirect $ do - void $ addAssociatedFile key file - {- For moveAnnex to work in direct mode, the symlink - - must already exist, so flush the queue. -} - Annex.Queue.flush - maybe noop (moveAnnex key) mtmp +cleanup u url file key mtmp = case mtmp of + Nothing -> go + Just tmp -> do + largematcher <- largeFilesMatcher + ifM (checkFileMatcher largematcher file) + ( go + , do + liftIO $ renameFile tmp file + void $ Command.Add.addSmall file + ) + where + go = do + when (isJust mtmp) $ + logStatus key InfoPresent + setUrlPresent u key url + Command.Add.addLink file key Nothing + whenM isDirect $ do + void $ addAssociatedFile key file + {- For moveAnnex to work in direct mode, the symlink + - must already exist, so flush the queue. -} + Annex.Queue.flush + maybe noop (moveAnnex key) mtmp nodownload :: URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) nodownload url urlinfo file diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -14,6 +14,7 @@ import qualified Git import Command.Unused (withUnusedMaps, UnusedMaps(..), startUnused) import Annex.NumCopies +import Annex.Content cmd :: Command cmd = command "dropunused" SectionMaintenance @@ -37,14 +38,20 @@ withUnusedMaps (start from numcopies) (rangesToDrop o) start :: Maybe Remote -> NumCopies -> UnusedMaps -> Int -> CommandStart -start from numcopies = startUnused "dropunused" (perform from numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpObjectLocation) +start from numcopies = startUnused "dropunused" + (perform from numcopies) + (performOther gitAnnexBadLocation) + (performOther gitAnnexTmpObjectLocation) perform :: Maybe Remote -> NumCopies -> Key -> CommandPerform perform from numcopies key = case from of Just r -> do showAction $ "from " ++ Remote.name r Command.Drop.performRemote key Nothing numcopies r - Nothing -> Command.Drop.performLocal key Nothing numcopies [] + Nothing -> ifM (inAnnex key) + ( Command.Drop.performLocal key Nothing numcopies [] + , next (return True) + ) performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform performOther filespec key = do diff --git a/Command/Import.hs b/Command/Import.hs --- a/Command/Import.hs +++ b/Command/Import.hs @@ -19,6 +19,8 @@ import Types.Key import Annex.CheckIgnore import Annex.NumCopies +import Types.FileMatcher +import Annex.FileMatcher cmd :: Command cmd = withGlobalOptions (jobsOption : fileMatchingOptions) $ notBareRepo $ @@ -64,10 +66,11 @@ inrepops <- liftIO $ filter (dirContains repopath) <$> mapM absPath (importFiles o) unless (null inrepops) $ do error $ "cannot import files from inside the working tree (use git annex add instead): " ++ unwords inrepops - withPathContents (start (duplicateMode o)) (importFiles o) + largematcher <- largeFilesMatcher + withPathContents (start largematcher (duplicateMode o)) (importFiles o) -start :: DuplicateMode -> (FilePath, FilePath) -> CommandStart -start mode (srcfile, destfile) = +start :: FileMatcher Annex -> DuplicateMode -> (FilePath, FilePath) -> CommandStart +start largematcher mode (srcfile, destfile) = ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) ( do ma <- pickaction @@ -124,7 +127,10 @@ liftIO $ if mode == Duplicate || mode == SkipDuplicates then void $ copyFileExternal CopyAllMetaData srcfile destfile else moveFile srcfile destfile - Command.Add.perform destfile + ifM (checkFileMatcher largematcher destfile) + ( Command.Add.perform destfile + , next $ Command.Add.addSmall destfile + ) notoverwriting why = do warning $ "not overwriting existing " ++ destfile ++ " " ++ why stop diff --git a/Command/Map.hs b/Command/Map.hs --- a/Command/Map.hs +++ b/Command/Map.hs @@ -15,6 +15,7 @@ import qualified Git.Url import qualified Git.Config import qualified Git.Construct +import qualified Remote import qualified Annex import Annex.UUID import Logs.UUID @@ -46,7 +47,9 @@ liftIO $ writeFile file (drawMap rs umap trusted) next $ next $ ifM (Annex.getState Annex.fast) - ( return True + ( do + showLongNote $ "left map in " ++ file + return True , do showLongNote $ "running: dot -Tx11 " ++ file showOutput @@ -164,9 +167,8 @@ {- Checks if two repos are the same. -} same :: Git.Repo -> Git.Repo -> Bool same a b - | both Git.repoIsSsh = matching Git.Url.authority && matching Git.repoPath - | both Git.repoIsUrl && neither Git.repoIsSsh = matching show - | neither Git.repoIsSsh = matching Git.repoPath + | both Git.repoIsUrl = matching Git.Url.scheme && matching Git.Url.authority && matching Git.repoPath + | neither Git.repoIsUrl = matching Git.repoPath | otherwise = False where matching t = t a == t b @@ -192,7 +194,13 @@ tryScan :: Git.Repo -> Annex (Maybe Git.Repo) tryScan r | Git.repoIsSsh r = sshscan - | Git.repoIsUrl r = return Nothing + | Git.repoIsUrl r = case Git.remoteName r of + -- Can't scan a non-ssh url, so use any cached uuid for it. + Just n -> Just <$> (either + (const (pure r)) + (liftIO . setUUID r . Remote.uuid) + =<< Remote.byName' n) + Nothing -> return $ Just r | otherwise = liftIO $ safely $ Git.Config.read r where pipedconfig pcmd params = liftIO $ safely $ diff --git a/Config/Files.hs b/Config/Files.hs --- a/Config/Files.hs +++ b/Config/Files.hs @@ -26,8 +26,11 @@ readAutoStartFile :: IO [FilePath] readAutoStartFile = do f <- autoStartFile - nub . map dropTrailingPathSeparator . lines + filter valid . nub . map dropTrailingPathSeparator . lines <$> catchDefaultIO "" (readFile f) + where + -- Ignore any relative paths; some old buggy versions added eg "." + valid = isAbsolute modifyAutoStartFile :: ([FilePath] -> [FilePath]) -> IO () modifyAutoStartFile func = do @@ -42,12 +45,16 @@ - present, it's moved to the top, so it will be used as the default - when opening the webapp. -} addAutoStartFile :: FilePath -> IO () -addAutoStartFile path = modifyAutoStartFile $ (:) path +addAutoStartFile path = do + path' <- absPath path + modifyAutoStartFile $ (:) path' {- Removes a directory from the autostart file. -} removeAutoStartFile :: FilePath -> IO () -removeAutoStartFile path = modifyAutoStartFile $ - filter (not . equalFilePath path) +removeAutoStartFile path = do + path' <- absPath path + modifyAutoStartFile $ + filter (not . equalFilePath path') {- The path to git-annex is written here; which is useful when cabal - has installed it to some awful non-PATH location. -} diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -18,8 +18,9 @@ if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi $(CABAL) configure --ghc-options="$(shell Build/collect-ghc-options.sh)" +# -j1 is used for reproducible build git-annex: Build/SysConfig.hs - $(CABAL) build + $(CABAL) build -j1 ln -sf dist/build/git-annex/git-annex git-annex man/%.1: doc/%.mdwn @@ -242,8 +243,7 @@ $(MAKE) -C standalone/android # We bypass cabal, and only run the main ghc --make command for a -# fast development built. Note: Does not rebuild C libraries, or link -# executable. +# fast development built. fast: dist/caballog @$$(grep 'ghc --make' dist/caballog | head -n 1 | sed -e 's/-package-id [^ ]*//g' -e 's/-hide-all-packages//') -O0 -j -dynamic @ln -sf dist/build/git-annex/git-annex git-annex diff --git a/Messages/Progress.hs b/Messages/Progress.hs --- a/Messages/Progress.hs +++ b/Messages/Progress.hs @@ -29,8 +29,8 @@ {- Shows a progress meter while performing a transfer of a key. - The action is passed a callback to use to update the meter. -} -metered :: Maybe MeterUpdate -> Key -> AssociatedFile -> (MeterUpdate -> Annex a) -> Annex a -metered combinemeterupdate key _af a = case keySize key of +metered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a +metered combinemeterupdate key a = case keySize key of Nothing -> nometer Just size -> withOutputType (go $ fromInteger size) where @@ -66,11 +66,19 @@ {- Use when the progress meter is only desired for concurrent - output; as when a command's own progress output is preferred. -} -concurrentMetered :: Maybe MeterUpdate -> Key -> AssociatedFile -> (MeterUpdate -> Annex a) -> Annex a -concurrentMetered combinemeterupdate key af a = withOutputType go +concurrentMetered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a +concurrentMetered combinemeterupdate key a = withOutputType go where - go (ConcurrentOutput _) = metered combinemeterupdate key af a - go _ = a (fromMaybe (const noop) combinemeterupdate) + go (ConcurrentOutput _) = metered combinemeterupdate key a + go _ = a (fromMaybe nullMeterUpdate combinemeterupdate) + +{- Poll file size to display meter, but only for concurrent output. -} +concurrentMeteredFile :: FilePath -> Maybe MeterUpdate -> Key -> Annex a -> Annex a +concurrentMeteredFile file combinemeterupdate key a = withOutputType go + where + go (ConcurrentOutput _) = metered combinemeterupdate key $ \p -> + watchFileSize file p a + go _ = a {- Progress dots. -} showProgressDots :: Annex () diff --git a/Remote/External.hs b/Remote/External.hs --- a/Remote/External.hs +++ b/Remote/External.hs @@ -369,12 +369,8 @@ startExternal externaltype = do errrelayer <- mkStderrRelayer liftIO $ do - (Just hin, Just hout, Just herr, pid) <- createProcess $ - (proc cmd []) - { std_in = CreatePipe - , std_out = CreatePipe - , std_err = CreatePipe - } + (Just hin, Just hout, Just herr, pid) <- + createProcess p `catchIO` runerr fileEncoding hin fileEncoding hout fileEncoding herr @@ -390,7 +386,14 @@ } where cmd = externalRemoteProgram externaltype + p = (proc cmd []) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + runerr _ = error ("Cannot run " ++ cmd ++ " -- Make sure it's in your PATH and is executable.") + checkearlytermination Nothing = noop checkearlytermination (Just exitcode) = ifM (inPath cmd) ( error $ unwords [ "failed to run", cmd, "(" ++ show exitcode ++ ")" ] @@ -503,9 +506,9 @@ mkmulti (u, s, f) = (u, s, mkSafeFilePath f) retrieveUrl :: Retriever -retrieveUrl = fileRetriever $ \f k _p -> do +retrieveUrl = fileRetriever $ \f k p -> do us <- getWebUrls k - unlessM (downloadUrl us f) $ + unlessM (downloadUrl k p us f) $ error "failed to download content" checkKeyUrl :: Git.Repo -> CheckPresent diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -421,7 +421,7 @@ {- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) -copyFromRemote r key file dest p = concurrentMetered (Just p) key file $ +copyFromRemote r key file dest p = concurrentMetered (Just p) key $ copyFromRemote' r key file dest copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) @@ -440,12 +440,13 @@ copier <- mkCopier hardlink params runTransfer (Transfer Download u key) file noRetry noObserver - (\p -> copier object dest p checksuccess) - | Git.repoIsSsh (repo r) = unVerified $ feedprogressback $ \feeder -> do + (\p -> copier object dest (combineMeterUpdate p meterupdate) checksuccess) + | Git.repoIsSsh (repo r) = unVerified $ feedprogressback $ \p -> do direct <- isDirect - Ssh.rsyncHelper (Just feeder) + Ssh.rsyncHelper (Just (combineMeterUpdate meterupdate p)) =<< Ssh.rsyncParamsRemote direct r Download key dest file - | Git.repoIsHttp (repo r) = unVerified $ Annex.Content.downloadUrl (keyUrls r key) dest + | Git.repoIsHttp (repo r) = unVerified $ + Annex.Content.downloadUrl key meterupdate (keyUrls r key) dest | otherwise = error "copying from non-ssh, non-http remote not supported" where {- Feed local rsync's progress info back to the remote, @@ -522,7 +523,7 @@ ) | Git.repoIsSsh (repo r) = ifM (Annex.Content.preseedTmp key file) - ( fst <$> concurrentMetered Nothing key af + ( fst <$> concurrentMetered Nothing key (copyFromRemote' r key af file) , return False ) @@ -533,17 +534,19 @@ {- Tries to copy a key's content to a remote's annex. -} copyToRemote :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool -copyToRemote r key file p = concurrentMetered (Just p) key file $ copyToRemote' r key file +copyToRemote r key file meterupdate = + concurrentMetered (Just meterupdate) key $ + copyToRemote' r key file copyToRemote' :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool -copyToRemote' r key file p +copyToRemote' r key file meterupdate | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) (return False) $ commitOnCleanup r $ copylocal =<< Annex.Content.prepSendAnnex key | Git.repoIsSsh (repo r) = commitOnCleanup r $ Annex.Content.sendAnnex key noop $ \object -> do direct <- isDirect - Ssh.rsyncHelper (Just p) + Ssh.rsyncHelper (Just meterupdate) =<< Ssh.rsyncParamsRemote direct r Upload key object file | otherwise = error "copying to non-ssh repo not supported" where @@ -563,10 +566,11 @@ ensureInitialized copier <- mkCopier hardlink params let verify = Annex.Content.RemoteVerify r - runTransfer (Transfer Download u key) file noRetry noObserver $ const $ - Annex.Content.saveState True `after` + runTransfer (Transfer Download u key) file noRetry noObserver $ \p -> + let p' = combineMeterUpdate meterupdate p + in Annex.Content.saveState True `after` Annex.Content.getViaTmp verify key - (\dest -> copier object dest p (liftIO checksuccessio)) + (\dest -> copier object dest p' (liftIO checksuccessio)) ) fsckOnRemote :: Git.Repo -> [CommandParam] -> Annex (IO Bool) @@ -638,19 +642,8 @@ where sameDeviceIds a b = (==) <$> getDeviceId a <*> getDeviceId b getDeviceId f = deviceID <$> liftIO (getFileStatus $ parentDir f) - docopy = liftIO $ bracket - (forkIO $ watchfilesize zeroBytesProcessed) - (void . tryIO . killThread) - (const $ copyFileExternal CopyTimeStamps src dest) - watchfilesize oldsz = do - threadDelay 500000 -- 0.5 seconds - v <- catchMaybeIO $ toBytesProcessed <$> getFileSize dest - case v of - Just sz - | sz /= oldsz -> do - p sz - watchfilesize sz - _ -> watchfilesize oldsz + docopy = liftIO $ watchFileSize dest p $ + copyFileExternal CopyTimeStamps src dest #endif dorsync = Ssh.rsyncHelper (Just p) $ rsyncparams ++ [File src, File dest] diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -155,8 +155,8 @@ specialRemote' cfg c preparestorer prepareretriever prepareremover preparecheckpresent baser = encr where encr = baser - { storeKey = \k f p -> cip >>= storeKeyGen k f p - , retrieveKeyFile = \k f d p -> cip >>= unVerified . retrieveKeyFileGen k f d p + { storeKey = \k _f p -> cip >>= storeKeyGen k p + , retrieveKeyFile = \k _f d p -> cip >>= unVerified . retrieveKeyFileGen k d p , retrieveKeyFileCheap = \k f d -> cip >>= maybe (retrieveKeyFileCheap baser k f d) -- retrieval of encrypted keys is never cheap @@ -183,12 +183,12 @@ safely a = catchNonAsync a (\e -> warning (show e) >> return False) -- chunk, then encrypt, then feed to the storer - storeKeyGen k f p enc = safely $ preparestorer k $ safely . go + storeKeyGen k p enc = safely $ preparestorer k $ safely . go where go (Just storer) = preparecheckpresent k $ safely . go' storer go Nothing = return False go' storer (Just checker) = sendAnnex k rollback $ \src -> - displayprogress p k f $ \p' -> + displayprogress p k $ \p' -> storeChunks (uuid baser) chunkconfig k src p' (storechunk enc storer) checker @@ -204,10 +204,10 @@ storer (enck k) (ByteContent encb) p -- call retriever to get chunks; decrypt them; stream to dest file - retrieveKeyFileGen k f dest p enc = + retrieveKeyFileGen k dest p enc = safely $ prepareretriever k $ safely . go where - go (Just retriever) = displayprogress p k f $ \p' -> + go (Just retriever) = displayprogress p k $ \p' -> retrieveChunks retriever (uuid baser) chunkconfig enck k dest p' (sink dest enc) go Nothing = return False @@ -227,8 +227,8 @@ chunkconfig = chunkConfig cfg - displayprogress p k f a - | displayProgress cfg = metered (Just p) k f a + displayprogress p k a + | displayProgress cfg = metered (Just p) k a | otherwise = a p {- Sink callback for retrieveChunks. Stores the file content into the diff --git a/Remote/S3.hs b/Remote/S3.hs --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -249,8 +249,8 @@ Nothing -> \_ _ _ -> do warnMissingCredPairFor "S3" (AWS.creds $ uuid r) return False - Just geturl -> fileRetriever $ \f k _p -> - unlessM (downloadUrl [geturl k] f) $ + Just geturl -> fileRetriever $ \f k p -> + unlessM (downloadUrl k p [geturl k] f) $ error "failed to download content" retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs --- a/Remote/Tahoe.hs +++ b/Remote/Tahoe.hs @@ -75,7 +75,7 @@ , lockContent = Nothing , checkPresent = checkKey u hdl , checkPresentCheap = False - , whereisKey = Nothing + , whereisKey = Just (getWhereisKey u) , remoteFsck = Nothing , repairRepo = Nothing , config = c @@ -230,6 +230,12 @@ getCapability :: UUID -> Key -> Annex (Maybe Capability) getCapability u k = getRemoteState u k + +getWhereisKey :: UUID -> Key -> Annex [String] +getWhereisKey u k = disp <$> getCapability u k + where + disp Nothing = [] + disp (Just c) = [c] {- tahoe put outputs a single line, containing the capability. -} parsePut :: String -> Maybe Capability diff --git a/Remote/Web.hs b/Remote/Web.hs --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -72,7 +72,7 @@ } downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) -downloadKey key _file dest _p = unVerified $ get =<< getWebUrls key +downloadKey key _af dest p = unVerified $ get =<< getWebUrls key where get [] = do warning "no known url" @@ -84,13 +84,13 @@ case downloader of QuviDownloader -> do #ifdef WITH_QUVI - flip downloadUrl dest + flip (downloadUrl key p) dest =<< withQuviOptions Quvi.queryLinks [Quvi.httponly, Quvi.quiet] u' #else warning "quvi support needed for this url" return False #endif - _ -> downloadUrl [u'] dest + _ -> downloadUrl key p [u'] dest downloadKeyCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool downloadKeyCheap _ _ _ = return False diff --git a/Test.hs b/Test.hs --- a/Test.hs +++ b/Test.hs @@ -83,6 +83,7 @@ import qualified Utility.Exception import qualified Utility.Hash import qualified Utility.Scheduled +import qualified Utility.Scheduled.QuickCheck import qualified Utility.HumanTime import qualified Utility.ThreadScheduler import qualified Utility.Base64 @@ -157,7 +158,7 @@ , testProperty "prop_parse_show_TrustLog" Logs.Trust.prop_parse_show_TrustLog , testProperty "prop_hashes_stable" Utility.Hash.prop_hashes_stable , testProperty "prop_mac_stable" Utility.Hash.prop_mac_stable - , testProperty "prop_schedule_roundtrips" Utility.Scheduled.prop_schedule_roundtrips + , testProperty "prop_schedule_roundtrips" Utility.Scheduled.QuickCheck.prop_schedule_roundtrips , testProperty "prop_past_sane" Utility.Scheduled.prop_past_sane , testProperty "prop_duration_roundtrips" Utility.HumanTime.prop_duration_roundtrips , testProperty "prop_metadata_sane" Types.MetaData.prop_metadata_sane diff --git a/Utility/Exception.hs b/Utility/Exception.hs --- a/Utility/Exception.hs +++ b/Utility/Exception.hs @@ -20,7 +20,8 @@ catchNonAsync, tryNonAsync, tryWhenExists, - catchHardwareFault, + catchIOErrorType, + IOErrorType(..) ) where import Control.Monad.Catch as X hiding (Handler) @@ -88,11 +89,11 @@ v <- tryJust (guard . isDoesNotExistError) a return (eitherToMaybe v) -{- Catches only exceptions caused by hardware faults. - - Ie, disk IO error. -} -catchHardwareFault :: MonadCatch m => m a -> (IOException -> m a) -> m a -catchHardwareFault a onhardwareerr = catchIO a onlyhw +{- Catches only IO exceptions of a particular type. + - Ie, use HardwareFault to catch disk IO errors. -} +catchIOErrorType :: MonadCatch m => IOErrorType -> (IOException -> m a) -> m a -> m a +catchIOErrorType errtype onmatchingerr a = catchIO a onlymatching where - onlyhw e - | ioeGetErrorType e == HardwareFault = onhardwareerr e + onlymatching e + | ioeGetErrorType e == errtype = onmatchingerr e | otherwise = throwM e diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs --- a/Utility/FileSystemEncoding.hs +++ b/Utility/FileSystemEncoding.hs @@ -29,6 +29,7 @@ import qualified Data.Hash.MD5 as MD5 import Data.Word import Data.Bits.Utils +import Data.List import Data.List.Utils import qualified Data.ByteString.Lazy as L #ifdef mingw32_HOST_OS @@ -125,12 +126,12 @@ {- Like encodeW8 and decodeW8, but NULs are passed through unchanged. -} encodeW8NUL :: [Word8] -> FilePath -encodeW8NUL = join nul . map encodeW8 . split (s2w8 nul) +encodeW8NUL = intercalate nul . map encodeW8 . split (s2w8 nul) where nul = ['\NUL'] decodeW8NUL :: FilePath -> [Word8] -decodeW8NUL = join (s2w8 nul) . map decodeW8 . split nul +decodeW8NUL = intercalate (s2w8 nul) . map decodeW8 . split nul where nul = ['\NUL'] diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs --- a/Utility/FreeDesktop.hs +++ b/Utility/FreeDesktop.hs @@ -59,7 +59,7 @@ | null l = "" | otherwise = (intercalate ";" $ map (escapesemi . toString) l) ++ ";" where - escapesemi = join "\\;" . split ";" + escapesemi = intercalate "\\;" . split ";" genDesktopEntry :: String -> String -> Bool -> FilePath -> Maybe String -> [String] -> DesktopEntry genDesktopEntry name comment terminal program icon categories = catMaybes diff --git a/Utility/Metered.hs b/Utility/Metered.hs --- a/Utility/Metered.hs +++ b/Utility/Metered.hs @@ -18,7 +18,9 @@ import System.Posix.Types import Data.Int import Data.Bits.Utils +import Control.Concurrent import Control.Concurrent.Async +import Control.Monad.IO.Class (MonadIO) {- An action that can be run repeatedly, updating it on the bytes processed. - @@ -29,6 +31,9 @@ nullMeterUpdate :: MeterUpdate nullMeterUpdate _ = return () +combineMeterUpdate :: MeterUpdate -> MeterUpdate -> MeterUpdate +combineMeterUpdate a b = \n -> a n >> b n + {- Total number of bytes processed so far. -} newtype BytesProcessed = BytesProcessed Integer deriving (Eq, Ord, Show) @@ -145,6 +150,23 @@ where k = 1024 chunkOverhead = 2 * sizeOf (1 :: Int) -- GHC specific + +{- Runs an action, watching a file as it grows and updating the meter. -} +watchFileSize :: (MonadIO m, MonadMask m) => FilePath -> MeterUpdate -> m a -> m a +watchFileSize f p a = bracket + (liftIO $ forkIO $ watcher zeroBytesProcessed) + (liftIO . void . tryIO . killThread) + (const a) + where + watcher oldsz = do + v <- catchMaybeIO $ toBytesProcessed <$> getFileSize f + newsz <- case v of + Just sz | sz /= oldsz -> do + p sz + return sz + _ -> return oldsz + threadDelay 500000 -- 0.5 seconds + watcher newsz data OutputHandler = OutputHandler { quietMode :: Bool diff --git a/Utility/Path.hs b/Utility/Path.hs --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -89,7 +89,7 @@ upFrom :: FilePath -> Maybe FilePath upFrom dir | length dirs < 2 = Nothing - | otherwise = Just $ joinDrive drive (join s $ init dirs) + | otherwise = Just $ joinDrive drive (intercalate s $ init dirs) where -- on Unix, the drive will be "/" when the dir is absolute, otherwise "" (drive, path) = splitDrive dir @@ -149,7 +149,7 @@ relPathDirToFileAbs :: FilePath -> FilePath -> FilePath relPathDirToFileAbs from to | takeDrive from /= takeDrive to = to - | otherwise = join s $ dotdots ++ uncommon + | otherwise = intercalate s $ dotdots ++ uncommon where s = [pathSeparator] pfrom = split s from @@ -288,7 +288,6 @@ if l <= 0 then return 255 else return $ minimum [l, 255] - where #endif {- Given a string that we'd like to use as the basis for FilePath, but that diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -14,6 +14,7 @@ import Data.String.Utils import System.FilePath import Data.Char +import Data.List import Control.Applicative import Prelude @@ -85,7 +86,7 @@ shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' - escaped = join "'\"'\"'" $ split "'" f + escaped = intercalate "'\"'\"'" $ split "'" f -- | Unescapes a set of shellEscaped words or filenames. shellUnEscape :: String -> [String] diff --git a/Utility/Scheduled.hs b/Utility/Scheduled.hs --- a/Utility/Scheduled.hs +++ b/Utility/Scheduled.hs @@ -23,12 +23,10 @@ toRecurrance, toSchedule, parseSchedule, - prop_schedule_roundtrips, prop_past_sane, ) where import Utility.Data -import Utility.QuickCheck import Utility.PartialPrelude import Utility.Misc @@ -38,6 +36,7 @@ import Data.Time.Calendar import Data.Time.Calendar.WeekDate import Data.Time.Calendar.OrdinalDate +import Data.Time.Format () import Data.Tuple.Utils import Data.Char import Control.Applicative @@ -336,41 +335,6 @@ (rws, tws) = separate (== "at") (words s) recurrance = unwords rws scheduledtime = unwords tws - -instance Arbitrary Schedule where - arbitrary = Schedule <$> arbitrary <*> arbitrary - -instance Arbitrary ScheduledTime where - arbitrary = oneof - [ pure AnyTime - , SpecificTime - <$> choose (0, 23) - <*> choose (1, 59) - ] - -instance Arbitrary Recurrance where - arbitrary = oneof - [ pure Daily - , Weekly <$> arbday - , Monthly <$> arbday - , Yearly <$> arbday - , Divisible - <$> positive arbitrary - <*> oneof -- no nested Divisibles - [ pure Daily - , Weekly <$> arbday - , Monthly <$> arbday - , Yearly <$> arbday - ] - ] - where - arbday = oneof - [ Just <$> nonNegative arbitrary - , pure Nothing - ] - -prop_schedule_roundtrips :: Schedule -> Bool -prop_schedule_roundtrips s = toSchedule (fromSchedule s) == Just s prop_past_sane :: Bool prop_past_sane = and diff --git a/Utility/Scheduled/QuickCheck.hs b/Utility/Scheduled/QuickCheck.hs new file mode 100644 --- /dev/null +++ b/Utility/Scheduled/QuickCheck.hs @@ -0,0 +1,51 @@ +{- quickcheck for scheduled activities + - + - Copyright 2013-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Utility.Scheduled.QuickCheck where + +import Utility.Scheduled +import Utility.QuickCheck + +import Control.Applicative +import Prelude + +instance Arbitrary Schedule where + arbitrary = Schedule <$> arbitrary <*> arbitrary + +instance Arbitrary ScheduledTime where + arbitrary = oneof + [ pure AnyTime + , SpecificTime + <$> choose (0, 23) + <*> choose (1, 59) + ] + +instance Arbitrary Recurrance where + arbitrary = oneof + [ pure Daily + , Weekly <$> arbday + , Monthly <$> arbday + , Yearly <$> arbday + , Divisible + <$> positive arbitrary + <*> oneof -- no nested Divisibles + [ pure Daily + , Weekly <$> arbday + , Monthly <$> arbday + , Yearly <$> arbday + ] + ] + where + arbday = oneof + [ Just <$> nonNegative arbitrary + , pure Nothing + ] + +prop_schedule_roundtrips :: Schedule -> Bool +prop_schedule_roundtrips s = toSchedule (fromSchedule s) == Just s diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs --- a/Utility/Tmp.hs +++ b/Utility/Tmp.hs @@ -88,8 +88,9 @@ makenewdir (tmpdir template) (0 :: Int) makenewdir t n = do let dir = t ++ "." ++ show n - either (const $ makenewdir t $ n + 1) (const $ return dir) - =<< tryIO (createDirectory dir) + catchIOErrorType AlreadyExists (const $ makenewdir t $ n + 1) $ do + createDirectory dir + return dir {- It's not safe to use a FilePath of an existing file as the template - for openTempFile, because if the FilePath is really long, the tmpfile diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,30 @@ +git-annex (5.20151208) unstable; urgency=medium + + * Build with -j1 again to get reproducible build. + * Display progress meter in -J mode when copying from a local git repo, + to a local git repo, and from a remote git repo. + * Display progress meter in -J mode when downloading from the web. + * map: Improve display of git remotes with non-ssh urls, including http + and gcrypt. + * When core.sharedRepository is set, annex object files are not made mode + 444, since that prevents a user other than the file owner from locking + them. Instead, a mode such as 664 is used in this case. + * tahoe: Include tahoe capabilities in whereis display. + * import: Changed to honor annex.largefiles settings. + * addurl, importfeed: Changed to honor annex.largefiles settings, + when the content of the url is downloaded. (Not when using --fast or + --relaxed.) + * webapp: Fix bugs that could result in a relative path such as "." + being written to ~/.config/git-annex/autostart, and ignore any such + relative paths in the file. + This was a reversion caused by the relative path changes in 5.20150113. + * dropunused: Make more robust when trying to drop an object that has + already been dropped. + * Fix reversion in handling of long filenames, particularly when using + addurl/importfeed, which was introduced in the previous release. + + -- Joey Hess Tue, 08 Dec 2015 11:14:03 -0400 + git-annex (5.20151116) unstable; urgency=medium * Use concurrent-output library when configured with -fConcurrentOutput. diff --git a/debian/rules b/debian/rules --- a/debian/rules +++ b/debian/rules @@ -25,4 +25,10 @@ override_dh_fixperms: dh_fixperms -Xld-linux +# Do not add "ldconfig" trigger since all libraries in the standalone build +# are private copies and the trigger activate-noawait trigger may not work +# on elderly distributions (e.g. squeeze) +override_dh_makeshlibs: + dh_makeshlibs --noscripts + endif diff --git a/doc/bugs/20151116_tests_fail_on_OS_X.mdwn b/doc/bugs/20151116_tests_fail_on_OS_X.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/20151116_tests_fail_on_OS_X.mdwn @@ -0,0 +1,40 @@ +Test output on OS X. Btw, the previous version works: + +``` +OK (4.87s) + map: OK (0.33s) + uninit: Deleted branch git-annex (was 9a28b38). +OK (0.38s) + uninit (in git-annex branch): Switched to branch 'git-annex' +OK (0.26s) + upgrade: OK (0.21s) + whereis: OK (0.61s) + hook remote: OK (0.91s) + directory remote: OK (0.85s) + rsync remote: OK (1.40s) + bup remote: Reinitialized existing Git repository in /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/.bup/ +Initialized empty Git repository in /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/tmprepo0/dir/ + content cannot be completely removed from bup remote +OK (2.62s) + crypto: OK (4.16s) + preferred content: OK (1.97s) + add subdirs: Merge made by the 'recursive' strategy. + conflictor.variant-cc12 | 1 + + conflictor/subfile | 1 + + 2 files changed, 2 insertions(+) + create mode 120000 conflictor.variant-cc12 + create mode 120000 conflictor/subfile +To /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/repo + e4d3d14..60148c4 git-annex -> synced/git-annex + efda76d..f76baae master -> synced/master +OK (0.55s) + addurl: git-annex: .git/annex/objects/2Q/J8/SHA256E-s3--2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/SHA256E-s3--2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae: createSymbolicLink: invalid argument (File name too long) +FAIL (0.29s) + addurl failed on file:///private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/tmprepo0/myurl + +2 out of 150 tests failed (126.13s) + (This could be due to a bug in git-annex, or an incompatability + with utilities, such as git, installed on this system.) +``` + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/A_weird___34__.__34___repository_shows_up_when_switching.mdwn b/doc/bugs/A_weird___34__.__34___repository_shows_up_when_switching.mdwn --- a/doc/bugs/A_weird___34__.__34___repository_shows_up_when_switching.mdwn +++ b/doc/bugs/A_weird___34__.__34___repository_shows_up_when_switching.mdwn @@ -23,3 +23,4 @@ ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn b/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. +git-annex ignores changing of remote location in .git/config + +### What steps will reproduce the problem? +1. Change Hostname of previously working remote so that the existing remote will no longer work and produce "ssh: connect to host port 22: Network is unreachable" errors. +2. Stop all running git-annex processes. +3. Edit DNS name in corresponding remote .git/config. +4. Restart git-annex. +5. Turns out: git-annex still uses the old remote's DNS name. + +### What version of git-annex are you using? On what operating system? +newest one available. +5.20151116-gbe86081 + +### Please provide any additional information below. +See following entry in log, occurring plenty often: +[[!format sh """ +ssh: connect to host some.unreachable.dns.net port 22: Network is unreachable +rsync: connection unexpec + rsync failed -- run git annex again to resume file transfer +tedly closed (0 bytes received so far) [Receiver] +rsync error: unexplained error (code 255) at io.c(226) [Receiver=3.1.1] +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Trust me I love git-annex. Had dreams of something like git-annex for almost 10 years. I instantly got stuck on git-annex when I have read the first few sentences about it. Since then, things in my life have changed in a quite serious manner. + +THANKS for your help! diff --git a/doc/bugs/Unable_to_take_transfer_lock.mdwn b/doc/bugs/Unable_to_take_transfer_lock.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +`git annex get --from=myremote' show "transfer already in progress, or unable to take transfer lock". + +**Likely Problem discovered!** I copied the whole remote repo (was residing on a Linux machine accessed via SMB) and pasted it into my Mac. The problem was resolved. It is likely something to do with the fact that Mac OS filesystem is **case-insensitive**. I discovered this issue when I tried to copy that remote repo with ``cp`` rather than ``tar``ing it first. + + +### What steps will reproduce the problem? + +Remote is on a Linux machine. +Do `git annex get' on a Mac OS. +Do the same on another Linux machine. + +The git annex client on the Linux machine works. +Mac OS doesn't. + +rsync version for all machines: 3.1.1 + +### What version of git-annex are you using? On what operating system? + +git-annex 5.20151019 + +Lubuntu 15.10. +Mac OS X 10.11.1 (El Capitan) + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes. + +It's wonderful. This should be a problem with Mac OS. diff --git a/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn @@ -0,0 +1,9 @@ +### Please describe the problem. + +Due to the very unstable kernel on my laptop atm I need to reboot quite frequently... and this issue with annex assistant immediately starting synchronizing (although I did disable all synchronizations in webapp, so it would be another issue I guess), I wanted just to disable autostart of git-annex altogether. I have removed $HOME/.config/autostart/git-annex.desktop but to my surprise upon gnome crashing/relogin I had the same ssh key passphrase dialog and that file reappeared. + +### What version of git-annex are you using? On what operating system? + +5.20151116+gitg5416a1a-1~ndall+1 + +[[!meta author=yoh]] diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +git-annex's WebDAV support does not like (aka it does not work) the WebDAV server of the freenet cloud. + +### What steps will reproduce the problem? + +My first attempt was: + + WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none + initremote webdav (testing WebDAV server...) + git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 401, statusMessage = "Unauthorized"}) [("Date","Fri, 04 Dec 2015 12:20:57 GMT"),("Server","Apache/2.2.16 (Debian)"),("WWW-Authenticate","Basic realm=\"MD-Cloud\""), + ("Vary","Accept-Encoding"),("Content-Encoding","gzip"),("Content-Length","20"),("Connection","close"),("Content-Type","text/html; charset=iso-8859-15"),("X-Response-Body-Start",""),("X-Request-URL","MKCOL https://webmail.freenet.de:443/webdav/tmp")] + (CJ {expose = []}): user error failed + git-annex: initremote: 1 failed + +Ok this fails (what is the error?). However, it does create a folder "tmp" in the "cloud". A second attempt yields another error: + + WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none + initremote webdav (testing WebDAV server...) + git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 501, statusMessage = "Not Implemented"}) [("Date","Fri, 04 Dec 2015 12:21:22 GMT"),("Server","Apache/2.2.16 (Debian)"),("Content-Length","349"),("Connection","close"), + ("Content-Type","application/xml; charset=utf-8"),("X-Response-Body-Start","\n\n Sabre\\DAV\\Exception\\NotImplemented\n + This server is not compatible with OS/X finder. Consider using a different WebDAV client or webserver.\n 1.8.6\n\n"),("X-Request-URL","PUT https://webmail.freenet.de:443/webdav + /tmp/git-annex-test")] (CJ {expose = []}): user error failed + git-annex: initremote: 1 failed + +which is I guess the same. The WebDAV server does support writing and locking files. I tried writing using davfs2 and locking-unlocking using cadaver. I guess in the end that it's the server's fault, but it would be great to know what exactly fails at this point :). + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20151116-gbe86081 + +Gentoo Linux + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +First time trying WebDAV support. diff --git a/doc/bugs/git-annex_creates_many_zombies.mdwn b/doc/bugs/git-annex_creates_many_zombies.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git-annex_creates_many_zombies.mdwn @@ -0,0 +1,33 @@ +### Please describe the problem. + +In a debian unstable docker container, I run `git-annex assistant --auto` on a few repositories. + +After a few days, I have many (> 10000 right now) zombie processes: + + % ps x | grep "git-annex.*defunct" | wc -l + 10325 + +### What steps will reproduce the problem? + +Keep `git-annex assistant` running. + +### What version of git-annex are you using? On what operating system? + +Binary version 5.20151116-gbe86081 on Debian Linux unstable (up-to-date from a few days ago when my docker container started) + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I couldn't live without git-annex to synchronize my documents over multiple hosts and archives :-) + +> [[done]]; this is a docker bug if anything, and I really don't see any +> way git-annex can solve it. --[[Joey]] diff --git a/doc/bugs/regression_with_S3_creds_not_used.mdwn b/doc/bugs/regression_with_S3_creds_not_used.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/regression_with_S3_creds_not_used.mdwn @@ -0,0 +1,59 @@ +### Please describe the problem. + +I am having a problem on Windows in which the AWS access credentials are stored but not used. When I have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` variables are set, everything works correctly. However if I open a new window that doesn't have envs set, git annex sync --content fails with a Forbidden message. Looking in the creds directory, I see plaintext creds stored. + +### What steps will reproduce the problem? + +``` +# set AWS_ACCESS_KEY_ID=thing1 +# set AWS_SECRET_ACCESS_KEY=thing2 +# git init test +# cd test +# git annex init +# git annex initremote s3cloud type=S3 encryption=pubkey keyid=key embedcreds=yes + ... everything works, annex-uuid is created, etc ... + +# now open a new window, navigate to proper repo + +# git annex sync --content + +... forbidden messages + +# Reset the env vars +... sync works as normal ... +``` + +Note that this fails *silently* when using the webapp. The app shows percentage of upload, and the uploads "complete" but the log shows: + +``` + S3Error {s3StatusCode = Status {statusCode = 403, statusMessage = "Forbidden"}, s3ErrorCode = "InvalidAccessKeyId", s3ErrorMessage = "The AWS Access Key Id you provided does not exist in our records.", s3ErrorResource = Nothing, s3ErrorHostId = Just "/MfOhrvmoi1uTsuJDaV2c+3jxqt+/cAC8KBvGkbE+os1BAPAw/bLeNHiUF8kbSJ+2cMuNO+0mzQ=", s3ErrorAccessKeyId = Just "redacted", s3ErrorStringToSign = Nothing, s3ErrorBucket = Nothing, s3ErrorEndpointRaw = Nothing, s3ErrorEndpoint = Nothing} +``` + +### What version of git-annex are you using? On what operating system? + +I'm on Windows: + +``` +git-annex version: 5.20151019-gcc50c00 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feed +s Quvi TDFA TorrentParser Database +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 +SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEI +N256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glac +ier ddar hook external +``` + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I have successfully used git annex on linux and mac, and it is great! diff --git a/doc/bugs/remote_repo_marked_as___34__here__34__.mdwn b/doc/bugs/remote_repo_marked_as___34__here__34__.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/remote_repo_marked_as___34__here__34__.mdwn @@ -0,0 +1,38 @@ +### Please describe the problem. + +``git annex info`` shows a local repo as "not here" (withou the ``[here]`` tag), and shows a remote repo with the ``[here]`` tag. + +I had to manually edit `.git/config` to set my local repo's UUID to the correct one. + +> Well, git-annex can only display information that you've told it. +> In this case you seem to have done something wrong with UUIDs, and so it +> displayed [here] next to the UUID of the local repo, as it's supposed to. +> There's no bug here I think. [[done]] --[[Joey]] + +### What steps will reproduce the problem? + +I just ``git annex dead`` an old repo, and ``git annex semitrust`` a new repo by the same name as the old repo. + + +### What version of git-annex are you using? On what operating system? + +5.20150731-1build1 + +Lubuntu 15.10 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, lots of luck (need more skill). + +git-annex may need more error catches to prevent a wrong order of commands from corrupting a repo. But still, git annex works great. diff --git a/doc/coding_style.mdwn b/doc/coding_style.mdwn --- a/doc/coding_style.mdwn +++ b/doc/coding_style.mdwn @@ -98,3 +98,16 @@ or vertical whitespace around your code, making it hard to fit enough of it on the screen, consider finding a better abstraction, so the code that does fit on the screen is easily understandable. ;) + +----- + +Note for emacs users: You can put the following snippet into a file called +`.dir-locals.el` at root of git-annex's source tree to use tabs for indentation: + + ((nil . ((indent-tabs-mode . t) + (tab-width . 8) + (fill-column . 80))) + ;; Warn about spaces used for indentation: + (haskell-mode . ((eval . (highlight-regexp "^ *"))))) + +Also consider [haskell-tab-indent-mode](https://spwhitton.name/tech/code/haskell-tab-indent/). The standard indentation modes that come with haskell-mode do not work well with tabs for indentation. This mode works well for hacking on git-annex. diff --git a/doc/devblog/day_338__week_in_review.mdwn b/doc/devblog/day_338__week_in_review.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_338__week_in_review.mdwn @@ -0,0 +1,16 @@ +Monday: Some finishing touches on the pid locking support, and released +5.20151116. After the release I noticed that concurrent downloads didn't +always include a progress meter, and made the necessary changes to fix +that. + +Wednesday: This was a day of minor bug fixing and responding to questions +etc. Message backlog got down below 90, not bad. + +Thursday: I've been distracted from coding today with an idea of making +some new stickers. Hexagonal this time, and even better, composable... +So they can show git-annex getting as big as you want. ;) + +[[stickers/hex.svg]] + +The design is done, see [[stickers]], and seems to work well, and even +better is easy to modify. May find time to get these printed at some point. diff --git a/doc/devblog/day_339_smudging_out_direct_mode.mdwn b/doc/devblog/day_339_smudging_out_direct_mode.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_339_smudging_out_direct_mode.mdwn @@ -0,0 +1,61 @@ +I'm considering ways to get rid of direct mode, replacing it with something +better implemented using [[todo/smudge]] filters. + +## git-lfs + +I started by trying out git-lfs, to see what I can learn from it. My +feeling is that git-lfs brings an admirable simplicity to using git with +large files. For example, it uses a push-hook to automatically +upload file contents before pushing a branch. + +But its simplicity comes at the cost of being centralized. You can't make a +git-lfs repository locally and clone it onto other drive and have the local +repositories interoperate to pass file contents around. Everything has to +go back through a centralized server. I'm willing to pay complexity costs +for decentralization. + +Its simplicity also means that the user doesn't have much control over what +files are present in their checkout of a repository. git-lfs downloads +all the files in the work tree. It doesn't have facilities for dropping +files to free up space, or for configuring a repository to only want to get +a subset of files in the first place. Some of this could be added to it +I suppose. + +I also noticed that git-lfs uses twice the disk space, at least when +initially adding files. It keep a copy of the file in .git/lfs/objects/, +in addition to the copy in the working tree. That copy seems to be +necessary due to the way git smudge filters work, to avoid data loss. Of +course, git-annex manages to avoid that duplication when using symlinks, +and its direct mode also avoids that duplication (at the cost of some +robustness). I'd like to keep git-annex's single local copy feature +if possible. + +## replacing direct mode + +Anyway, as smudge/clean filters stand now, they can't be used to set up +git-annex symlinks; their interface doesn't allow it. But, I was able to +think up a design that uses smudge/clean filters to cover the same use +cases that direct mode covers now. + +Thanks to the clean filter, adding a file with `git add` would check in a +small file that points to the git-annex object. + +In the same repository, you could also use `git annex add` to check +in a git-annex symlink, which would protect the object from modification, +in the good old indirect mode way. `git annex lock` and `git annex unlock` +could switch a file between those two modes. + +So this allows mixing directly writable annexed files and locked down +annexed files in the same repository. All regular git commands and all +git-annex commands can be used on both sorts of files. Workflows could +develop where a file starts out unlocked, but once it's done, is locked to +prevent accidental edits and archived away or published. + +That's much more flexible than the current direct mode, and I think it will +be able to be implemented in a simpler, more scalable, and robust way too. +I can lose the direct mode merge code, and remove hundreds of lines of +other special cases for direct mode. + +The downside, perhaps, is that for a repository to be usable on a crippled +filesystem, all the files in it will need to be unlocked. A file can't +easily be unlocked in one checkout and locked in another checkout. diff --git a/doc/devblog/day_340__post_turkey_catchup.mdwn b/doc/devblog/day_340__post_turkey_catchup.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_340__post_turkey_catchup.mdwn @@ -0,0 +1,9 @@ +Spent a couple of days catching up on backlog, and my backlog is down to 80 +messages now. Lowest in recent memory. + +Made the annex.largefiles config be honored by `git annex import`, `git +annex addurl`, and even `git annex importfeed`. + +Planning to dive into smudge filters soon. The design seems ready to go, +although there is some complication in needing to keep track of mappings +between worktree files and annex keys. diff --git a/doc/devblog/day_341__starting_smudge.mdwn b/doc/devblog/day_341__starting_smudge.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_341__starting_smudge.mdwn @@ -0,0 +1,24 @@ +I've gotten git-annex working as a smudge/clean filter today in the +`smudge` branch. It works ok in a local git repository. `git add` lets +git-annex decide if it wants to annex a file's content, and checking out +branches and other git commands involving those files works pretty well. + +It can sometimes be slow; git's smudge interface necessarily needs to +copy the content of files around, particularly when checking out files, +and so it's never going to be as fast as the good old git-annex symlink +approach. Most of the slow parts are things that can't be done in direct +mode repos though, like switching branches, so that isn't a regression. + +No git-annex commands to manage the annexed content work yet. That +will need a key to worktree file mapping to be maintained, and implementing +that mapping and ensuring its always consistent is probably going to be +the harder part of this. + +Also there's the question of how to handle upgrades from direct mode +repositories. This will be an upgrade from annex.version 5 to 6, and you +won't want to do it until all computers that have clones of a repository +have upgraded to git-annex 6.x, since older versions won't be able to work +with the upgraded repository. So, the repository upgrade will need to be +run manually initially, and it seems I'll need to keep supporting direct +mode for v5 repos in a transition period, which will probably be measured +in years. diff --git a/doc/devblog/day_342__continuing_smudge.mdwn b/doc/devblog/day_342__continuing_smudge.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_342__continuing_smudge.mdwn @@ -0,0 +1,30 @@ +Made a lot of progress today. Implemented the database mapping a key to its +associated files. As expected this database, when updated by the +smudge/clean filters, is not always consistent with the current git work tree. +In particular, commands like `git mv` don't update the database with the +new filename. So queries of the database will need to do some additional +work first to get it updated with any staged changes. But the database is +good enough for a proof of concept, I hope. + +Then I got git-annex commands treating smudged files as annexed files. +So this works: + + joey@darkstar:~/tmp/new>git annex init + init ok + (recording state in git...) + joey@darkstar:~/tmp/new>cp ~/some.mp3 . + joey@darkstar:~/tmp/new>git add some.mp3 + joey@darkstar:~/tmp/new>git diff --cached + diff --git a/some.mp3 b/some.mp3 + new file mode 100644 + index 0000000..2df8868 + --- /dev/null + +++ b/some.mp3 + @@ -0,0 +1 @@ + +/annex/objects/SHA256E-s191213--e4b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.mp3 + joey@darkstar:~/tmp/new>git annex whereis some.mp3 + whereis some.mp3 (1 copy) + 7de17427-329a-46ec-afd0-0a088f0d0b1b -- joey@darkstar:~/tmp/new [here] + ok + +get/drop don't yet update the smudged files, and that's the next step. diff --git a/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment b/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment new file mode 100644 --- /dev/null +++ b/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="jckuester@11e9fceb442172b6db63320bb545797a47a4c103" + nickname="jckuester" + subject="error during "git clone git://git.kitenet.net/git-annex"" + date="2015-11-22T15:38:57Z" + content=""" +error: unable to create file doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn (File name too long) +fatal: unable to checkout working tree +"""]] diff --git a/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment b/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment new file mode 100644 --- /dev/null +++ b/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-11-30T19:13:34Z" + content=""" +I guess you guys with the badly limited filename lenghs are using encfs or +something like that. encfs is [really shoddy](https://defuse.ca/audits/encfs.htm), +and I feel kind of like someone is complaining that the git repo can't be +checked out on MS-DOS. +8.3 filenames are so 1985, and 128 filenames are so.. 1995? +"""]] diff --git a/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn @@ -0,0 +1,9 @@ +I am confused on how to invoke the web app. I am learning how to use this tool, however I don't feel quite confident yet. I wanted to test out the web app, to play around with configurations and auto syncing, however I stumbled over this: + + ➜ ~ git annex help webapp // works fine + ➜ ~ git annex webapp + Invalid argument `webapp' // uhh what. + +git-annex-webapp didn't work either. + +I installed with brew. El Capitan, latest update as of right now. diff --git a/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn b/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn @@ -0,0 +1,8 @@ +Hi, + +I am currently trying to move nearly all of my files into git-annex, and I was wondering if there is a (plumbing-level?) way to easily figure out if a file I find on a random hard disk is already contained in my annex repository? + +I.e., is there an easy way to compute the key for a file (which is not in an annex) and look up if the file is already contained in a given annex repository? I've looked a bit at the plumbing-level commands but none seem to exactly fit the bill :-/ + +Cheers, +Alex diff --git a/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn @@ -0,0 +1,21 @@ +I use the command line almost every hour, I'm familiar with it. But it's a pita on the phone. + +I'd like to create an "graphical" Android application that lists/browses the repo's content and allows the user to manipulate the files/directories. + +git-annex for Android (5.0) is installed and working fine. I was messing around (on my rooted phone) with the rwx permissions to run the git-annex binary from my app as it's a different user and has no access to those binaries by default. But no success yet: + +``` +shell@jfltexx:/storage/sdcard1/repo $ /data/data/ga.androidterm/lib/lib.git-annex.so list + +lib.git-annex.so: git: createProcess: runInteractiveProcess: exec: permission denied (Permission denied) +``` + +Question 1: How should I set up the permissions to be able to successfully run any git-annex command from my (non-commercial, private) Android app? + +Question 2: What if I include all the git-annex binaries in my app as a resource and write them to disk after install and execute them? (Permissions should work.) + + +Use case: family videos are stored in a repo at home. This repo is cloned to my phone's sdcard. I want to browse the repo on my phone and mark some directories to be transferred from home to phone. My app would be a simple frontend for git-annex where you can browse and do some basic stuff: get, drop, manipulate metadata (and of course start playing a video). + +Thank you for any ideas, +Bence diff --git a/doc/forum/assistant_and_archive_folder.mdwn b/doc/forum/assistant_and_archive_folder.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/assistant_and_archive_folder.mdwn @@ -0,0 +1,13 @@ +I have a setup with a windows desktop and a linux laptop with both an annex repository, interconnected via a backup repository in the middle (connected via ssh). +Both computers are setup as clients and all 3 machines are running the assistant. + +Today i tried creating an archive folder and a putting some files in it. To my understanding those files should have been dropped in the 2 machines but kept on the server.. instead I have them copied on each machine. Listing where the files are on both computers shows that both know that themselves and the server have a copy, but they don't know anything about each other. + +AFAIK it looks like the bug reported in https://git-annex.branchable.com/forum/question_about_assistant_and___47__archive__47__/ + +laptop: git-annex version: 5.20151116+gitg5416a1a-1~ndall+1 + +server: git-annex version: 5.20151109-g08bb3b1 + +windows: the latest build on the website + diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -789,16 +789,26 @@ * `annex.largefiles` - Allows configuring which files `git annex add` and the assistant consider - to be large enough to need to be added to the annex. By default, - all files are added to the annex. + Allows configuring which files are considered to be large enough to + need to be added to the annex. By default, all specified files are added + to the annex, but configuring this can cause the small files to be + checked into git, without using the annex. The value is a preferred content expression. See [[git-annex-preferred-content]](1) for details. Example: - annex.largefiles = largerthan=100kb and not (include=*.c or include=*.h) + annex.largefiles = largerthan=100kb and not (include=*.c or include=*.h) + + This setting is checked by `git annex add`, `git annex import` and the assistant. + It's also used by `git annex addurl` and `git annex importfeed` when + downloading files. + + It can be useful to temporarily override it via -c at the command line. + For example: + + git annex add -c annex.largefiles='include=*' 99kbfile * `annex.numcopies` diff --git a/doc/install/Windows/comment_2_ef329ea100b36a1180e232a1f97369d6._comment b/doc/install/Windows/comment_2_ef329ea100b36a1180e232a1f97369d6._comment new file mode 100644 --- /dev/null +++ b/doc/install/Windows/comment_2_ef329ea100b36a1180e232a1f97369d6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="eren.m.dursun@73116dce47e54243c7ee1637f1289afb02f07f4b" + nickname="eren.m.dursun" + subject="Re: Compatibility with recent version of Git for Windows" + date="2015-11-29T19:24:03Z" + content=""" +To install git-annex next to 64bit git on Windows: + +Start cmd.exe with administrator privileges and run +mklink /J \"C:\Program Files (x86)\Git\" \"C:\Program Files\Git\" +to add a sym-link. Install git-annex. +"""]] diff --git a/doc/install/Windows/comment_3_10fb90bc434b4da189ed7235818ef30f._comment b/doc/install/Windows/comment_3_10fb90bc434b4da189ed7235818ef30f._comment new file mode 100644 --- /dev/null +++ b/doc/install/Windows/comment_3_10fb90bc434b4da189ed7235818ef30f._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-11-30T19:05:36Z" + content=""" +@eren, some of the programs included with git-annex need to use libraries +from Git for Windows. In particular, I think you'll find that rsync doesn't +work if you use the 64 bit Git for Windows. +"""]] diff --git a/doc/logo.mdwn b/doc/logo.mdwn --- a/doc/logo.mdwn +++ b/doc/logo.mdwn @@ -11,3 +11,11 @@ [[logo-old.svg]] [[logo-old-bw.svg]] + +---- + + Copyright: 2007 Henrik Nyh + 2010 Joey Hess + 2013 John Lawrence + License: other + Free to modify and redistribute with due credit, and obviously free to use. diff --git a/doc/news/version_5.20150930.mdwn b/doc/news/version_5.20150930.mdwn deleted file mode 100644 --- a/doc/news/version_5.20150930.mdwn +++ /dev/null @@ -1,19 +0,0 @@ -git-annex 5.20150930 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Added new linux standalone "ancient" build to support kernels - like 2.6.32. - * info: Don't allow use in a non-git-annex repository, since it - uses the git-annex branch and would create it if it were missing. - * assistant: When updating ~/.ssh/config, preserve any symlinks. - * webapp: Remove the "disable remote" feature from the UI. - * S3: When built with aws-0.13.0, supports using more storage classes. - In particular, storageclass=STANDARD\_IA to use Amazon's - new Infrequently Accessed storage, and storageclass=NEARLINE - to use Google's NearLine storage. - * Improve ~/.ssh/config modification code to not add trailing spaces - to lines it cannot parse. - * Fix a crash at direct mode merge time when .git/index doesn't exist - yet. Triggered by eg, git-annex sync --no-commit in a fresh clone of - a repository. - * status: Show added but not yet committed files. - * Added stack.yaml to support easy builds from source with stack."""]] diff --git a/doc/news/version_5.20151208.mdwn b/doc/news/version_5.20151208.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20151208.mdwn @@ -0,0 +1,24 @@ +git-annex 5.20151208 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Build with -j1 again to get reproducible build. + * Display progress meter in -J mode when copying from a local git repo, + to a local git repo, and from a remote git repo. + * Display progress meter in -J mode when downloading from the web. + * map: Improve display of git remotes with non-ssh urls, including http + and gcrypt. + * When core.sharedRepository is set, annex object files are not made mode + 444, since that prevents a user other than the file owner from locking + them. Instead, a mode such as 664 is used in this case. + * tahoe: Include tahoe capabilities in whereis display. + * import: Changed to honor annex.largefiles settings. + * addurl, importfeed: Changed to honor annex.largefiles settings, + when the content of the url is downloaded. (Not when using --fast or + --relaxed.) + * webapp: Fix bugs that could result in a relative path such as "." + being written to ~/.config/git-annex/autostart, and ignore any such + relative paths in the file. + This was a reversion caused by the relative path changes in 5.20150113. + * dropunused: Make more robust when trying to drop an object that has + already been dropped. + * Fix reversion in handling of long filenames, particularly when using + addurl/importfeed, which was introduced in the previous release."""]] diff --git a/doc/not.mdwn b/doc/not.mdwn --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -52,7 +52,7 @@ * Similarly, git-annex is not [git-lfs](https://github.com/github/git-lfs), which also uses git smudge filters, and appears to lack git-annex's - widely distributed storage. + widely distributed storage and partial checkouts. * git-annex is also not [boar](http://code.google.com/p/boar/), although it shares many of its goals and characteristics. Boar implements diff --git a/doc/special_remotes/S3/comment_20_ce4ce50d51675ee4746a2528d4bea750._comment b/doc/special_remotes/S3/comment_20_ce4ce50d51675ee4746a2528d4bea750._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/S3/comment_20_ce4ce50d51675ee4746a2528d4bea750._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="cantora@432fae6be728a32ac472387df86a8922f059d4a6" + nickname="cantora" + subject="How to view configuration of special remotes?" + date="2015-12-08T08:29:12Z" + content=""" +I don't remember which gpg key my s3 remote is using, but I can't seem to get git annex to tell me about the configuration of my s3 remote, which has the gpg key ID that I need to find (I need to restore it from many backed up keys, but need to know which one). Is there a way to view the remote metadata? I was hoping to see a command like `git annex remoteinfo NAME`. (git annex version: `5.20140408-gb37d538`). +"""]] diff --git a/doc/special_remotes/bup/comment_15_91e34a0087eaec6ad3c393179621a471._comment b/doc/special_remotes/bup/comment_15_91e34a0087eaec6ad3c393179621a471._comment deleted file mode 100644 --- a/doc/special_remotes/bup/comment_15_91e34a0087eaec6ad3c393179621a471._comment +++ /dev/null @@ -1,7 +0,0 @@ -[[!comment format=mdwn - username="https://woid.cryptobitch.de/foobar" - subject="obnam" - date="2015-11-12T11:50:52Z" - content=""" -Will obnam work as well with git-annex for backups? -"""]] diff --git a/doc/special_remotes/comment_29_8393a14f084d022986c8245ee01f4198._comment b/doc/special_remotes/comment_29_8393a14f084d022986c8245ee01f4198._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/comment_29_8393a14f084d022986c8245ee01f4198._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="encryptio@2557a3f5b4ef0cddf8f011d7dbb3e76d46ed9c79" + nickname="encryptio" + subject="git-annex-remote-b2" + date="2015-12-04T03:19:29Z" + content=""" +I wrote an external remote for [Backblaze's B2](https://www.backblaze.com/b2/cloud-storage.html) object storage service: [git-annex-remote-b2](https://github.com/encryptio/git-annex-remote-b2). I can't be arsed to write a full page on it, but the README covers everything an experienced git-annex user needs to know to use it. +"""]] diff --git a/doc/special_remotes/ipfs/comment_5_ba7d08b2bcb6614102c48b4cb3de5b1a._comment b/doc/special_remotes/ipfs/comment_5_ba7d08b2bcb6614102c48b4cb3de5b1a._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/ipfs/comment_5_ba7d08b2bcb6614102c48b4cb3de5b1a._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="rob.syme@92895c98b16fd7a88bed5f10913c522ebfd76c31" + nickname="rob.syme" + subject="Finding IPFS hash" + date="2015-11-18T14:55:33Z" + content=""" +The use case is only very silly - I just wanted to look directly at the data stored at that IPFS hash - it was not important and certainly not worth changing the way git-annex stores its keys. + +On a new machine with a new version of git-annex, I can no longer initialize a new IFPS remote: + + ⟫ ipfs version + ipfs version 0.3.10-dev + + ⟫ git annex version + git-annex version: 5.20151116-gbe86081 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput DNS Feeds Quvi TDFA TorrentParser Database + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + + ⟫ git annex initremote ipfs type=external externaltype=ipfs encryption=none + initremote ipfs + git-annex: git-annex-remote-ipfs: createProcess: runInteractiveProcess: exec: does not exist (No such file or directory) + failed + git-annex: initremote: 1 failed + +It looks like I'm missing a binary somewhere, but I'm not sure which one is missing. +"""]] diff --git a/doc/special_remotes/ipfs/comment_6_8621bbd87982f73cf401cfa09d95d1e4._comment b/doc/special_remotes/ipfs/comment_6_8621bbd87982f73cf401cfa09d95d1e4._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/ipfs/comment_6_8621bbd87982f73cf401cfa09d95d1e4._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-11-18T16:22:51Z" + content=""" +@rob.syme I think that error message is pretty clear: git-annex-remote-ipfs +is not present in your PATH or perhaps is not executable. + +(Still, the error message could be improved a bit, doing that now.) +"""]] diff --git a/doc/special_remotes/ipfs/comment_7_062f266f26b5788b418de6955bff442d._comment b/doc/special_remotes/ipfs/comment_7_062f266f26b5788b418de6955bff442d._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/ipfs/comment_7_062f266f26b5788b418de6955bff442d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="rob.syme@92895c98b16fd7a88bed5f10913c522ebfd76c31" + nickname="rob.syme" + subject="git-annex-remote-ipfs" + date="2015-11-19T14:46:03Z" + content=""" +Silly me - I messed up my bashrc. Fixed now, thanks! +"""]] diff --git a/doc/stickers.mdwn b/doc/stickers.mdwn new file mode 100644 --- /dev/null +++ b/doc/stickers.mdwn @@ -0,0 +1,21 @@ +Hexagonal sticker design using the [[logo]] with add-on tiles to expand +the annex. + +[[stickers/hex.svg]] +[download hex.svg](hex.svg) + +[[stickers/hexdemos.svg]] +[download hexdemos.svg](hexdemos.svg) + +This complies with +and so is intended to combine with other hexagonal stickers. + +Some of the add-on tiles contain enough white-space to add other art work +as desired. + +It includes some smaller tiles that can bridge between hexagonal and +square/regtangular stickers, as illustrated in +. + +License: Same as git-annex [[logo]]; Joey would appreciate spare +git-annex stickers mailed his way, but does not require it. ;) diff --git a/doc/stickers/comment_1_96f9b35048793c1cc0902cced82b65c5._comment b/doc/stickers/comment_1_96f9b35048793c1cc0902cced82b65c5._comment new file mode 100644 --- /dev/null +++ b/doc/stickers/comment_1_96f9b35048793c1cc0902cced82b65c5._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://bret.io/" + subject="Woo! " + date="2015-11-19T23:48:01Z" + content=""" +Rad! Cant wait to add it to: + + + +I like the idea of adapters too :) + +https://www.stickermule.com/ and https://hexi.pics/help_en both print high quality spec compliant stickers. +"""]] diff --git a/doc/stickers/comment_2_39cfee29eae5e69d88d84b2c65eb83fe._comment b/doc/stickers/comment_2_39cfee29eae5e69d88d84b2c65eb83fe._comment new file mode 100644 --- /dev/null +++ b/doc/stickers/comment_2_39cfee29eae5e69d88d84b2c65eb83fe._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 2" + date="2015-11-20T08:37:39Z" + content=""" +I don't usually like any decals on my stuff.. but I've never come across this before and it looks interesting. I'm really tempted.. + +However, I wouldn't be able to do it without having some nice way of dealing with tolerance stacking. The overlapping stickers (bottom-right) on the photo above would drive me mad! :) + +I went ahead and [raised an issue](https://github.com/terinjokes/StickerConstructorSpec/issues/18) in case they want to deal with this in the specification by having a(some?) guide specification(s) too. +"""]] diff --git a/doc/stickers/hex.svg b/doc/stickers/hex.svg new file mode 100644 --- /dev/null +++ b/doc/stickers/hex.svg @@ -0,0 +1,1017 @@ + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + workshop + <- cutafterprinting -> + + + (remove yellow borders before printing) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + git-annex tiling stickerscompliant withhttps://github.com/terinjokes/StickerConstructorSpec/ + + diff --git a/doc/stickers/hexdemos.svg b/doc/stickers/hexdemos.svg new file mode 100644 --- /dev/null +++ b/doc/stickers/hexdemos.svg @@ -0,0 +1,628 @@ + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn b/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn --- a/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn +++ b/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn @@ -2,3 +2,5 @@ N.B. I do understand that use-case might be somewhat vague, let me know if I should expand reasoning [[!meta author=yoh]] + +> [[done]] --[[Joey]] diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn --- a/doc/todo/smudge.mdwn +++ b/doc/todo/smudge.mdwn @@ -15,6 +15,10 @@ > git to handle this sort of case in an efficient way.. just needs someone > to do the work. --[[Joey]] +>> Update 2015: git status only calls the clean filter for files +>> that the index says are modified, so this is no longer a problem. +>> --[[Joey]] + ---- The clean filter is run when files are staged for commit. So a user could copy @@ -36,35 +40,26 @@ and have it use git-annex when .gitattributes says to. Also, annexed files can be directly modified without having to `git annex unlock`. -### design +### configuration In .gitattributes, the user would put something like "* filter=git-annex". This way they could control which files are annexed vs added normally. -(git-annex could have further controls to allow eg, passing small files -through to regular processing. At least .gitattributes is a special case, -it should never be annexed...) - -For files not configured this way, git-annex could continue to use -its symlink method -- this would preserve backwards compatability, -and even allow mixing the two methods in a repo as desired. - -To find files in the repository that are annexed, git-annex would do -`ls-files` as now, but would check if found files have the appropriate -filter, rather than the current symlink checks. To determine the key -of a file, rather than reading its symlink, git-annex would need to -look up the git blob associated with the file -- this can be done -efficiently using the existing code in `Branch.catFile`. - -The clean filter would inject the file's content into the annex, and hard -link from the annex to the file. Avoiding duplication of data. +It would also be good to allow using this without having to specify +the files in .gitattributes. Just use "* filter=git-annex" there, and then +let git-annex decide which files to annex and which to pass through the +smudge and clean filters as-is. The smudge filter can just read a little of +its input to see if it's a pointer to an annexed file. The clean filter +could apply annex.largefiles to decide whether to annex a file's content or +not. -The smudge filter can't do that, so to avoid duplication of data, it -might always create an empty file. To get the content, `git annex get` -could be used (which would hard link it). A `post-checkout` hook might -be used to set up hard links for all currently available content. +For files not configured this way in .gitattributes, git-annex could +continue to use its symlink method -- this would preserve backwards +compatability, and even allow mixing the two methods in a repo as desired. +(But not switching an existing repo between indirect and direct modes; +the user decides which mode to use when adding files to the repo.) -#### clean +### clean The trick is doing it efficiently. Since git a2b665d, v1.7.4.1, something like this works to provide a filename to the clean script: @@ -100,7 +95,7 @@ > but it seems to avoid this problem. > --[[Joey]] -#### smudge +### smudge The smudge script can also be provided a filename with %f, but it cannot directly write to the file or git gets unhappy. @@ -108,18 +103,220 @@ > Still the case in 2015. Means an unnecesary read and pipe of the file > even if the content is already locally available on disk. --[[Joey]] -### dealing with partial content availability +### partial checkouts -The smudge filter cannot be allowed to fail, that leaves the tree and -index in a weird state. So if a file's content is requested by calling -the smudge filter, the trick is to instead provide dummy content, -indicating it is not available (and perhaps saying to run "git-annex get"). +.. Are very important, otherwise a repo can't scale past the size of the +smallest client's disk! -Then, in the clean filter, it has to detect that it's cleaning a file -with that dummy content, and make sure to provide the same identifier as -it would if the file content was there. +It would be nice if the smudge filter could hard link or symlink a work +tree file to the annex object. +But currently, the smudge filter can't modify the work tree file on its own +-- git always modifies the file after getting the output of the smudge +filter, and will stumble over any modifications that the smudge filter +makes. And, it's important that the smudge filter never fail as that will +leave the repo in a bad state. + +Seems the best that can be done is for the smudge filter to copy from the +annex object when the object is present. When it's not present, the smudge +filter should provide a pointer to its content. + +The clean filter should detect when it's operating on that pointer file. + I've a demo implementation of this technique in the scripts below. + +### deduplication + +.. Is nice; needing 2 copies of every annexed file is annoying. + +Unfortunately, when using smudge/clean, `git merge` does not preserve a +smudged file in the work tree when renaming it. It instead deletes the old +file and asks the smudge filter to smudge the new filename. + +So, copies need to be maintained in .git/annex/objects, though it's ok +to use hard links to the work tree files. + +Even if hard links are used, smudge needs to output the content of an +annexed file, which will result in duplication when merging in renames of +files. + +### design + +Goal: Get rid of current direct mode, using smudge/clean filters instead to +cover the same use cases, more flexibly and robustly. + +Use case 1: + +A user wants to be able to edit files, and git-add, git commit, +without needing to worry about using git-annex to unlock files, add files, +etc. + +Use case 2: + +Using git-annex on a crippled filesystem that does not support symlinks. + +Data: + +* An annex pointer file has as its first line the git-annex key + that it's standing in for. Subsequent lines of the file might + be a message saying that the file's content is not currently available. + An annex pointer file is checked into the git repository the same way + that an annex symlink is checked in. +* A file map is maintained by git-annex, to keep track of the keys + that are used by files in the working tree. + +Configuration: + +* .gitattributes tells git which files to use git-annex's smudge/clean + filters with. Typically, all files except for dotfiles: + + * filter=annex + .* !filter + +* annex.largefiles tells git-annex which files should in fact be put in + the annex. Other files are passed through the smudge/clean as-is and + have their contents stored in git. + +* annex.direct is repurposed to configure how the assistant adds files. + When set to true, they're added unlocked. + +git-annex clean: + +* Run by `git add` (and diff and status, etc), and passed the + filename, as well as fed the file content on stdin. + + Look at configuration to decide if this file's content belongs in the + annex. If not, output the file content to stdout. + + Generate annex key from filename and content from stdin. + + Hard link .git/annex/objects to the file, if it doesn't already exist. + (On platforms not supporting hardlinks, copy the file to + .git/annex/objects.) + + This is done to prevent losing the only copy of a file when eg + doing a git checkout of a different branch, or merging a commit that + renames or deletes a file. But, no attempt is made to + protect the object from being modified. If a user wants to + protect object contents from modification, they should use + `git annex add`, not `git add`, or they can `git annex lock` after adding,. + + There could be a configuration knob to cause a copy to be made to + .git/annex/objects -- useful for those crippled filesystems. It might + also drop that copy once the object gets uploaded to another repo ... + But that gets complicated quickly. + + Update file map. + + Output the pointer file content to stdout. + +git-annex smudge: + +* Run by eg `git checkout` + and passed the filename, as well as fed the pointer file content on stdin. + + Update file map. + + When an object is present in the annex, outputs its content to stdout. + Otherwise, outputs the file pointer content. + +git annex direct/indirect: + + Previously these commands switched in and out of direct mode. + Now they become no-ops. + +git annex lock/unlock: + + Makes sense for these to change to switch files between using + git-annex symlinks and pointers. So, this provides both a way to + transition repositories to using pointers, and a cleaner unlock/lock + for repos using symlinks. + + unlock will stage a pointer file, and will copy the content of the object + out of .git/annex/objects to the work tree file. (Might want a --hardlink + switch.) + + lock will replace the current work tree file with the symlink, and stage it. + Note that multiple work tree files could point to the same object. + So, if the link count is > 1, replace the annex object with a copy of + itself to break such a hard link. Always finish by locking down the + permissions of the annex object. + +#### file map + +The file map needs to map from `Key -> [File]`. `File -> Key` +seems useful to have, but in practice is not worthwhile. + +Drop and get operations need to know what files in the work tree use a +given key in order to update the work tree. + +git-annex commands that look at annex symlinks to get keys to act on will +need fall back to either consulting the file map, or looking at the staged +file to see if it's a pointer to a key. So a `File -> Key` map is a possible +optimisation. + +Question: If the smudge/clean filters update the file map incrementally +based on the pointer files they generate/see, will the result +always be consistent with the content of the working tree? + +This depends on when git calls the smudge/clean filters and on what. +In particular: + +* Does the clean filter always get called when adding a relevant + file to git? Yes. +* Is the clean filter called at any other time? Yes, for example + git diff will clean relevant modified files to generate the diff. + So, the clean filter may see file versions that have not yet been staged + in git. +* Is the clean filter ever passed content not in the work tree? + I don't think so, but not 100% sure. +* Is the smudge filter always called when git updates a relevant file + in the work tree? Yes. +* Is the smudge filter called at any other time? Seems unlikely but then + there could be situations with a detached work tree or such. +* Does git call any useful hooks when removing a file from the work tree, + or converting it to not be annexed? + No! + +From this analysis, any file map generated by the smudge/clean filters +is necessary potentially innaccurate. It may list deleted files. +It may or may not reflect current unstaged changes from the work tree. + +Follows that any use of the file map needs to verify the info from it, +and throw out bad cached info (updating the map to match reality). + +When downloading a key, check if the files listed in the file map are +still pointer files in the work tree, and only replace them with the +content if so. + +When dropping a key, check if the files listed for it in the file map are +unmodified in the work tree, and are staged as pointers to the key, +and only reset them to the pointers if so. Note that this means that +a modified work tree file that has not yet been staged, but that +corresponds to a key, won't be reset when the key is dropped. +This is probably not a big deal; the user will either add the +file, which will add the key back, or reset the file. + +Does the `File -> Key` map have any benefits given this innaccuracy? +Answer seems to be no; any answer that map gives may be innaccurate and +needs to be verified by looking at actual repo content, so might as well +just look at the repo content in the first place.. + +#### Upgrading + +annex.version changes to 6 + +Upgrade should be handled automatically. + +On upgrade, update .gitattributes with a stock configuration, unless +it already mentions "filter=annex". + +Upgrading a direct mode repo needs to switch it out of bare mode, and +needs to run `git annex unlock` on all files (or reach the same result). +So will need to stage changes to all annexed files. + +When a repo has some clones indirect and some direct, the upgraded repo +will have all files unlocked, necessarily in all clones. ---- diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20151116 +Version: 5.20151208 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess diff --git a/man/git-annex-clean.1 b/man/git-annex-clean.1 new file mode 100644 --- /dev/null +++ b/man/git-annex-clean.1 @@ -0,0 +1,33 @@ +.TH git-annex-clean 1 +.SH NAME +git-annex-clean \- git filter driver for git-annex +.PP +.SH SYNOPSIS +git annex clean +.PP +.SH DESCRIPTION +When git-annex is used as a git filter driver, this command is run +by git commands such as \fBgit add\fP. It generates a file that +is added to the git repository and points to the git-annex object +containing the content of a large file. +.PP +To configure git to use git-annex as a git filter driver, place the +following in the .gitattributes file: +.PP + * filter=annex +\&.* !filter +.PP +The annex.largefiles config is consulted to decide if a given file should +be added to git as\-is, or if its content are large enough to need to use +git-annex. +.PP +.SH SEE ALSO +git-annex(1) +.PP +git-annex\-smudge(1) +.PP +.SH AUTHOR +Joey Hess +.PP +.PP + diff --git a/man/git-annex-smudge.1 b/man/git-annex-smudge.1 new file mode 100644 --- /dev/null +++ b/man/git-annex-smudge.1 @@ -0,0 +1,44 @@ +.TH git-annex-smudge 1 +.SH NAME +git-annex-smudge \- git filter driver for git-annex +.PP +.SH SYNOPSIS +git annex smudge [\-\-clean] file +.PP +.SH DESCRIPTION +This command lets git-annex be used as a git filter driver which lets +annexed files in the git repository to be unlocked at all times, instead +of being symlinks. +.PP +When adding a file with \fBgit add\fP, the annex.largefiles config is +consulted to decide if a given file should be added to git as\-is, +or if its content are large enough to need to use git-annex. To force a +file that would normally be added to the annex to be added to git as\-is, +this can be temporarily overridden. For example: +.PP + git \-c annex.largefiles='exclude=*' add myfile +.PP +The git configuration to use this command as a filter driver is as follows. +This is normally set up for you by git-annex init, so you should +not need to configure it manually. +.PP + [filter "annex"] + smudge = git-annex smudge %f + clean = git-annex smudge \-\-clean %f +.PP +To make git use that filter driver, it needs to be configured in +the .gitattributes file or in \fB.git/config/attributes\fP. The latter +is normally configured when a repository is initialized, with the following +contents: +.PP + * filter=annex +\&.* !filter +.PP +.SH SEE ALSO +git-annex(1) +.PP +.SH AUTHOR +Joey Hess +.PP +.PP + diff --git a/man/git-annex.1 b/man/git-annex.1 --- a/man/git-annex.1 +++ b/man/git-annex.1 @@ -674,9 +674,10 @@ The default reserve is 1 megabyte. .IP .IP "\fBannex.largefiles\fP" -Allows configuring which files \fBgit annex add\fP and the assistant consider -to be large enough to need to be added to the annex. By default, -all files are added to the annex. +Allows configuring which files are considered to be large enough to +need to be added to the annex. By default, all specified files are added +to the annex, but configuring this can cause the small files to be +checked into git, without using the annex. .IP The value is a preferred content expression. See git-annex\-preferred\-content(1) for details. @@ -684,6 +685,15 @@ Example: .IP annex.largefiles = largerthan=100kb and not (include=*.c or include=*.h) +.IP +This setting is checked by \fBgit annex add\fP, \fBgit annex import\fP and the assistant. +It's also used by \fBgit annex addurl\fP and \fBgit annex importfeed\fP when +downloading files. +.IP +It can be useful to temporarily override it via \-c at the command line. +For example: +.IP + git annex add \-c annex.largefiles='include=*' 99kbfile .IP .IP "\fBannex.numcopies\fP" This is a deprecated setting. You should instead use the