diff --git a/Annex/Branch.hs b/Annex/Branch.hs --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -29,6 +29,7 @@ import qualified Data.Set as S import qualified Data.Map as M import Data.Bits.Utils +import Control.Concurrent (threadDelay) import Common.Annex import Annex.BranchState @@ -232,28 +233,32 @@ {- Commits the staged changes in the index to the branch. - - - Ensures that the branch's index file is first updated to the state + - Ensures that the branch's index file is first updated to merge the state - of the branch at branchref, before running the commit action. This - is needed because the branch may have had changes pushed to it, that - are not yet reflected in the index. - - - - Also safely handles a race that can occur if a change is being pushed - - into the branch at the same time. When the race happens, the commit will - - be made on top of the newly pushed change, but without the index file - - being updated to include it. The result is that the newly pushed - - change is reverted. This race is detected and another commit made - - to fix it. - - The branchref value can have been obtained using getBranch at any - previous point, though getting it a long time ago makes the race - more likely to occur. + - + - Note that changes may be pushed to the branch at any point in time! + - So, there's a race. If the commit is made using the newly pushed tip of + - the branch as its parent, and that ref has not yet been merged into the + - index, then the result is that the commit will revert the pushed + - changes, since they have not been merged into the index. This race + - is detected and another commit made to fix it. + - + - (It's also possible for the branch to be overwritten, + - losing the commit made here. But that's ok; the data is still in the + - index and will get committed again later.) -} commitIndex :: JournalLocked -> Git.Ref -> String -> [Git.Ref] -> Annex () commitIndex jl branchref message parents = do showStoringStateAction - commitIndex' jl branchref message parents -commitIndex' :: JournalLocked -> Git.Ref -> String -> [Git.Ref] -> Annex () -commitIndex' jl branchref message parents = do + commitIndex' jl branchref message message 0 parents +commitIndex' :: JournalLocked -> Git.Ref -> String -> String -> Integer -> [Git.Ref] -> Annex () +commitIndex' jl branchref message basemessage retrynum parents = do updateIndex jl branchref committedref <- inRepo $ Git.Branch.commitAlways Git.Branch.AutomaticCommit message fullname parents setIndexSha committedref @@ -276,12 +281,16 @@ | otherwise = True -- race! {- To recover from the race, union merge the lost refs - - into the index, and recommit on top of the bad commit. -} + - into the index. -} fixrace committedref lostrefs = do + showSideAction "recovering from race" + let retrynum' = retrynum+1 + -- small sleep to let any activity that caused + -- the race settle down + liftIO $ threadDelay (100000 + fromInteger retrynum') mergeIndex jl lostrefs - commitIndex jl committedref racemessage [committedref] - - racemessage = message ++ " (recovery from race)" + let racemessage = basemessage ++ " (recovery from race #" ++ show retrynum' ++ "; expected commit parent " ++ show branchref ++ " but found " ++ show lostrefs ++ " )" + commitIndex' jl committedref racemessage basemessage retrynum' [committedref] {- Lists all files on the branch. There may be duplicates in the list. -} files :: Annex [FilePath] diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -42,7 +42,7 @@ matchfile <- getTopFilePath <$> inRepo (toTopFilePath file) return $ MatchingFile FileInfo { matchFile = matchfile - , relFile = file + , currFile = file } matchAll :: FileMatcher Annex diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs --- a/Annex/Ssh.hs +++ b/Annex/Ssh.hs @@ -1,6 +1,6 @@ {- git-annex ssh interface, with connection caching - - - Copyright 2012-2014 Joey Hess + - Copyright 2012-2015 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -8,14 +8,14 @@ {-# LANGUAGE CPP #-} module Annex.Ssh ( - sshCachingOptions, + sshOptions, sshCacheDir, sshReadPort, forceSshCleanup, - sshCachingEnv, - sshCachingTo, - inRepoWithSshCachingTo, - runSshCaching, + sshOptionsEnv, + sshOptionsTo, + inRepoWithSshOptionsTo, + runSshOptions, sshAskPassEnv, runSshAskPass ) where @@ -41,20 +41,26 @@ #endif {- Generates parameters to ssh to a given host (or user@host) on a given - - port, with connection caching. -} -sshCachingOptions :: (String, Maybe Integer) -> [CommandParam] -> Annex [CommandParam] -sshCachingOptions (host, port) opts = go =<< sshInfo (host, port) + - port. This includes connection caching parameters, and any ssh-options. -} +sshOptions :: (String, Maybe Integer) -> RemoteGitConfig -> [CommandParam] -> Annex [CommandParam] +sshOptions (host, port) gc opts = go =<< sshCachingInfo (host, port) where go (Nothing, params) = ret params go (Just socketfile, params) = do prepSocket socketfile ret params - ret ps = return $ ps ++ opts ++ portParams port ++ [Param "-T"] + ret ps = return $ concat + [ ps + , map Param (remoteAnnexSshOptions gc) + , opts + , portParams port + , [Param "-T"] + ] {- Returns a filename to use for a ssh connection caching socket, and - parameters to enable ssh connection caching. -} -sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam]) -sshInfo (host, port) = go =<< sshCacheDir +sshCachingInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam]) +sshCachingInfo (host, port) = go =<< sshCacheDir where go Nothing = return (Nothing, []) go (Just dir) = do @@ -227,50 +233,61 @@ | otherwise = aux (p,q:ps) rest readPort p = fmap fst $ listToMaybe $ reads p -{- When this env var is set, git-annex runs ssh with parameters - - to use the socket file that the env var contains. +{- When this env var is set, git-annex runs ssh with the specified + - options. (The options are separated by newlines.) - - This is a workaround for GIT_SSH not being able to contain - additional parameters to pass to ssh. -} -sshCachingEnv :: String -sshCachingEnv = "GIT_ANNEX_SSHCACHING" +sshOptionsEnv :: String +sshOptionsEnv = "GIT_ANNEX_SSHOPTION" +toSshOptionsEnv :: [CommandParam] -> String +toSshOptionsEnv = unlines . toCommand + +fromSshOptionsEnv :: String -> [CommandParam] +fromSshOptionsEnv = map Param . lines + {- Enables ssh caching for git push/pull to a particular - remote git repo. (Can safely be used on non-ssh remotes.) - + - Also propigates any configured ssh-options. + - - Like inRepo, the action is run with the local git repo. - But here it's a modified version, with gitEnv to set GIT_SSH=git-annex, - - and sshCachingEnv set so that git-annex will know what socket + - and sshOptionsEnv set so that git-annex will know what socket - file to use. -} -inRepoWithSshCachingTo :: Git.Repo -> (Git.Repo -> IO a) -> Annex a -inRepoWithSshCachingTo remote a = - liftIO . a =<< sshCachingTo remote =<< gitRepo +inRepoWithSshOptionsTo :: Git.Repo -> RemoteGitConfig -> (Git.Repo -> IO a) -> Annex a +inRepoWithSshOptionsTo remote gc a = + liftIO . a =<< sshOptionsTo remote gc =<< gitRepo -{- To make any git commands be run with ssh caching enabled, - - alters the local Git.Repo's gitEnv to set GIT_SSH=git-annex, - - and set sshCachingEnv so that git-annex will know what socket - - file to use. -} -sshCachingTo :: Git.Repo -> Git.Repo -> Annex Git.Repo -sshCachingTo remote g +{- To make any git commands be run with ssh caching enabled, + - and configured ssh-options alters the local Git.Repo's gitEnv + - to set GIT_SSH=git-annex, and sets sshOptionsEnv. -} +sshOptionsTo :: Git.Repo -> RemoteGitConfig -> Git.Repo -> Annex Git.Repo +sshOptionsTo remote gc g | not (Git.repoIsUrl remote) || Git.repoIsHttp remote = uncached | otherwise = case Git.Url.hostuser remote of Nothing -> uncached Just host -> do - (msockfile, _) <- sshInfo (host, Git.Url.port remote) + (msockfile, _) <- sshCachingInfo (host, Git.Url.port remote) case msockfile of Nothing -> return g Just sockfile -> do command <- liftIO readProgramFile prepSocket sockfile + let val = toSshOptionsEnv $ concat + [ sshConnectionCachingParams sockfile + , map Param (remoteAnnexSshOptions gc) + ] liftIO $ do - g' <- addGitEnv g sshCachingEnv sockfile + g' <- addGitEnv g sshOptionsEnv val addGitEnv g' "GIT_SSH" command where uncached = return g -runSshCaching :: [String] -> FilePath -> IO () -runSshCaching args sockfile = do - let args' = toCommand (sshConnectionCachingParams sockfile) ++ args +runSshOptions :: [String] -> String -> IO () +runSshOptions args s = do + let args' = toCommand (fromSshOptionsEnv s) ++ args let p = proc "ssh" args' exitWith =<< waitForProcess . processHandle =<< createProcess p diff --git a/Assistant.hs b/Assistant.hs --- a/Assistant.hs +++ b/Assistant.hs @@ -73,6 +73,7 @@ - stdout and stderr descriptors. -} startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe String -> Maybe HostName -> Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex () startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = do + Annex.changeState $ \s -> s { Annex.daemon = True } pidfile <- fromRepo gitAnnexPidFile logfile <- fromRepo gitAnnexLogFile diff --git a/Assistant/Pairing.hs b/Assistant/Pairing.hs --- a/Assistant/Pairing.hs +++ b/Assistant/Pairing.hs @@ -58,6 +58,15 @@ } deriving (Eq, Read, Show) +checkSane :: PairData -> Bool +checkSane p = all (not . any isControl) + [ fromMaybe "" (remoteHostName p) + , remoteUserName p + , remoteDirectory p + , remoteSshPubKey p + , fromUUID (pairUUID p) + ] + type UserName = String {- A pairing that is in progress has a secret, a thread that is diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -50,6 +50,7 @@ delayadd <- liftAnnex $ maybe delayaddDefault (return . Just . Seconds) =<< annexDelayAdd <$> Annex.getGitConfig + msg <- liftAnnex Command.Sync.commitMsg waitChangeTime $ \(changes, time) -> do readychanges <- handleAdds havelsof delayadd changes if shouldCommit False time (length readychanges) readychanges @@ -60,7 +61,7 @@ , "changes" ] void $ alertWhile commitAlert $ - liftAnnex commitStaged + liftAnnex $ commitStaged msg recordCommit let numchanges = length readychanges mapM_ checkChangeContent readychanges @@ -212,15 +213,15 @@ recentchanges = filter thissecond changes timeDelta c = now `diffUTCTime` changeTime c -commitStaged :: Annex Bool -commitStaged = do +commitStaged :: String -> Annex Bool +commitStaged msg = do {- This could fail if there's another commit being made by - something else. -} v <- tryNonAsync Annex.Queue.flush case v of Left _ -> return False Right _ -> do - ok <- Command.Sync.commitStaged Git.Branch.AutomaticCommit "" + ok <- Command.Sync.commitStaged Git.Branch.AutomaticCommit msg when ok $ Command.Sync.updateSyncBranch =<< inRepo Git.Branch.current return ok diff --git a/Assistant/Threads/PairListener.hs b/Assistant/Threads/PairListener.hs --- a/Assistant/Threads/PairListener.hs +++ b/Assistant/Threads/PairListener.hs @@ -16,13 +16,11 @@ import Assistant.Alert import Assistant.DaemonStatus import Utility.ThreadScheduler -import Utility.Format import Git import Network.Multicast import Network.Socket import qualified Data.Text as T -import Data.Char pairListenerThread :: UrlRenderer -> NamedThread pairListenerThread urlrenderer = namedThread "PairListener" $ do @@ -39,16 +37,18 @@ Nothing -> go reqs cache sock Just m -> do debug ["received", show msg] - sane <- checkSane msg (pip, verified) <- verificationCheck m =<< (pairingInProgress <$> getDaemonStatus) let wrongstage = maybe False (\p -> pairMsgStage m <= inProgressPairStage p) pip let fromus = maybe False (\p -> remoteSshPubKey (pairMsgData m) == remoteSshPubKey (inProgressPairData p)) pip - case (wrongstage, fromus, sane, pairMsgStage m) of + case (wrongstage, fromus, checkSane (pairMsgData m), pairMsgStage m) of (_, True, _, _) -> do debug ["ignoring message that looped back"] go reqs cache sock - (_, _, False, _) -> go reqs cache sock + (_, _, False, _) -> do + liftAnnex $ warning + "illegal control characters in pairing message; ignoring" + go reqs cache sock -- PairReq starts a pairing process, so a -- new one is always heeded, even if -- some other pairing is in process. @@ -83,19 +83,10 @@ "detected possible pairing brute force attempt; disabled pairing" stopSending pip return (Nothing, False) - |otherwise = return (Just pip, verified && sameuuid) + | otherwise = return (Just pip, verified && sameuuid) where verified = verifiedPairMsg m pip sameuuid = pairUUID (inProgressPairData pip) == pairUUID (pairMsgData m) - - checkSane msg - {- Control characters could be used in a - - console poisoning attack. -} - | any isControl (filter (/= '\n') (decode_c msg)) = do - liftAnnex $ warning - "illegal control characters in pairing message; ignoring" - return False - | otherwise = return True {- PairReqs invalidate the cache of recently finished pairings. - This is so that, if a new pairing is started with the diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -115,7 +115,7 @@ - blocking the response to the browser on it. -} openFileBrowser :: Handler Bool openFileBrowser = do - path <- liftAnnex $ fromRepo Git.repoPath + path <- liftIO . absPath =<< liftAnnex (fromRepo Git.repoPath) #ifdef darwin_HOST_OS let cmd = "open" let p = proc cmd [path] diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs --- a/Build/LinuxMkLibs.hs +++ b/Build/LinuxMkLibs.hs @@ -44,31 +44,40 @@ -- Various files used by runshell to set up env vars used by the -- linker shims. writeFile (top "libdirs") (unlines libdirs) - writeFile (top "linker") - (Prelude.head $ filter ("ld-linux" `isInfixOf`) libs') writeFile (top "gconvdir") (parentDir $ Prelude.head $ filter ("/gconv/" `isInfixOf`) glibclibs) - mapM_ (installLinkerShim top) exes + let linker = Prelude.head $ filter ("ld-linux" `isInfixOf`) libs' + mapM_ (installLinkerShim top linker) exes {- Installs a linker shim script around a binary. - - Note that each binary is put into its own separate directory, - to avoid eg git looking for binaries in its directory rather - - than in PATH.-} -installLinkerShim :: FilePath -> FilePath -> IO () -installLinkerShim top exe = do - createDirectoryIfMissing True shimdir + - than in PATH. + - + - The linker is symlinked to a file with the same basename as the binary, + - since that looks better in ps than "ld-linux.so". + -} +installLinkerShim :: FilePath -> FilePath -> FilePath -> IO () +installLinkerShim top linker exe = do + createDirectoryIfMissing True (top shimdir) + createDirectoryIfMissing True (top exedir) renameFile exe exedest + link <- relPathDirToFile (top exedir) (top ++ linker) + unlessM (doesFileExist (top exelink)) $ + createSymbolicLink link (top exelink) writeFile exe $ unlines [ "#!/bin/sh" - , "exec \"$GIT_ANNEX_LINKER\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_SHIMMED/" ++ base ++ "/" ++ base ++ "\" \"$@\"" + , "exec \"$GIT_ANNEX_DIR/" ++ exelink ++ "\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_DIR/shimmed/" ++ base ++ "/" ++ base ++ "\" \"$@\"" ] modifyFileMode exe $ addModes executeModes where base = takeFileName exe - shimdir = top "shimmed" base - exedest = shimdir base + shimdir = "shimmed" base + exedir = "exe" + exedest = top shimdir base + exelink = exedir base {- Converting symlinks to hard links simplifies the binary shimming - process. -} diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,40 @@ +git-annex (5.20150219) unstable; urgency=medium + + * glacier: Detect when the glacier command in PATH is the wrong one, + from boto, rather than from glacier-cli, and refuse to use it, + since the boto program fails to fail when passed + parameters it does not understand. + * groupwanted: New command to set the groupwanted preferred content + expression. + * import: Support file matching options such as --exclude, --include, + --smallerthan, --largerthan + * The file matching options are now only accepted by commands that + can actually use them, instead of by all commands. + * import: Avoid checksumming file twice when run in the default + or --duplicate mode. + * Windows: Fix bug in dropping an annexed file, which + caused a symlink to be staged that contained backslashes. + * webapp: Fix reversion in opening webapp when starting it manually + inside a repository. + * assistant: Improve sanity check for control characters when pairing. + * Improve race recovery code when committing to git-annex branch. + * addurl: Avoid crash if quvi is not installed, when git-annex was + built with process-1.2 + * bittorrent: Fix mojibake introduced in parsing arai2c progress output. + * fsck --from: If a download from a remote fails, propagate the failure. + * metadata: When setting metadata, do not recurse into directories by + default, since that can be surprising behavior and difficult to recover + from. The old behavior is available by using --force. + * sync, assistant: Include repository name in head branch commit message. + * The ssh-options git config is now used by gcrypt, rsync, and ddar + special remotes that use ssh as a transport. + * sync, assistant: Use the ssh-options git config when doing git pull + and push. + * remotedaemon: Use the ssh-options git config. + * Linux standalone: Improved process names of linker shimmed programs. + + -- Joey Hess Thu, 19 Feb 2015 14:16:03 -0400 + git-annex (5.20150205) unstable; urgency=medium * info: Can now display info about a given uuid. diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs --- a/CmdLine/GitAnnex.hs +++ b/CmdLine/GitAnnex.hs @@ -68,6 +68,7 @@ import qualified Command.Dead import qualified Command.Group import qualified Command.Wanted +import qualified Command.GroupWanted import qualified Command.Schedule import qualified Command.Ungroup import qualified Command.Vicfg @@ -142,6 +143,7 @@ , Command.Dead.cmd , Command.Group.cmd , Command.Wanted.cmd + , Command.GroupWanted.cmd , Command.Schedule.cmd , Command.Ungroup.cmd , Command.Vicfg.cmd @@ -216,6 +218,6 @@ go [] = dispatch True args cmds gitAnnexOptions [] header Git.CurrentRepo.get go ((v, a):rest) = maybe (go rest) a =<< getEnv v envmodes = - [ (sshCachingEnv, runSshCaching args) + [ (sshOptionsEnv, runSshOptions args) , (sshAskPassEnv, runSshAskPass) ] diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -1,6 +1,6 @@ {- git-annex options - - - Copyright 2010, 2013 Joey Hess + - Copyright 2010-2015 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -22,6 +22,8 @@ import CmdLine.Option import CmdLine.Usage +-- Options that are accepted by all git-annex sub-commands, +-- although not always used. gitAnnexOptions :: [Option] gitAnnexOptions = commonOptions ++ [ Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber) @@ -34,39 +36,11 @@ "override trust setting to untrusted" , Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE") "override git configuration setting" - , Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob) - "skip files matching the glob pattern" - , Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob) - "limit to files matching the glob pattern" - , Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote) - "match files present in a remote" - , Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber) - "skip files with fewer copies" - , Option [] ["lackingcopies"] (ReqArg (Limit.addLackingCopies False) paramNumber) - "match files that need more copies" - , Option [] ["approxlackingcopies"] (ReqArg (Limit.addLackingCopies True) paramNumber) - "match files that need more copies (faster)" - , Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName) - "match files using a key-value backend" - , Option [] ["inallgroup"] (ReqArg Limit.addInAllGroup paramGroup) - "match files present in all remotes in a group" - , Option [] ["largerthan"] (ReqArg Limit.addLargerThan paramSize) - "match files larger than a size" - , Option [] ["smallerthan"] (ReqArg Limit.addSmallerThan paramSize) - "match files smaller than a size" - , Option [] ["metadata"] (ReqArg Limit.addMetaData "FIELD=VALUE") - "match files with attached metadata" - , Option [] ["want-get"] (NoArg Limit.Wanted.addWantGet) - "match files the repository wants to get" - , Option [] ["want-drop"] (NoArg Limit.Wanted.addWantDrop) - "match files the repository wants to drop" - , Option ['T'] ["time-limit"] (ReqArg Limit.addTimeLimit paramTime) - "stop after the specified amount of time" , Option [] ["user-agent"] (ReqArg setuseragent paramName) "override default User-Agent" , Option [] ["trust-glacier"] (NoArg (Annex.setFlag "trustglacier")) "Trust Amazon Glacier inventory" - ] ++ matcherOptions + ] where trustArg t = ReqArg (Remote.forceTrust t) paramRemote setnumcopies v = maybe noop @@ -77,6 +51,7 @@ >>= pure . (\r -> r { gitGlobalOpts = gitGlobalOpts r ++ [Param "-c", Param v] }) >>= Annex.changeGitRepo +-- Options for matching on annexed keys, rather than work tree files. keyOptions :: [Option] keyOptions = [ Option ['A'] ["all"] (NoArg (Annex.setFlag "all")) @@ -87,6 +62,69 @@ "operate on specified key" ] +-- Options to match properties of annexed files. +annexedMatchingOptions :: [Option] +annexedMatchingOptions = concat + [ nonWorkTreeMatchingOptions' + , fileMatchingOptions' + , combiningOptions + , [timeLimitOption] + ] + +-- Matching options that don't need to examine work tree files. +nonWorkTreeMatchingOptions :: [Option] +nonWorkTreeMatchingOptions = nonWorkTreeMatchingOptions' ++ combiningOptions + +nonWorkTreeMatchingOptions' :: [Option] +nonWorkTreeMatchingOptions' = + [ Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote) + "match files present in a remote" + , Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber) + "skip files with fewer copies" + , Option [] ["lackingcopies"] (ReqArg (Limit.addLackingCopies False) paramNumber) + "match files that need more copies" + , Option [] ["approxlackingcopies"] (ReqArg (Limit.addLackingCopies True) paramNumber) + "match files that need more copies (faster)" + , Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName) + "match files using a key-value backend" + , Option [] ["inallgroup"] (ReqArg Limit.addInAllGroup paramGroup) + "match files present in all remotes in a group" + , Option [] ["metadata"] (ReqArg Limit.addMetaData "FIELD=VALUE") + "match files with attached metadata" + , Option [] ["want-get"] (NoArg Limit.Wanted.addWantGet) + "match files the repository wants to get" + , Option [] ["want-drop"] (NoArg Limit.Wanted.addWantDrop) + "match files the repository wants to drop" + ] + +-- Options to match files which may not yet be annexed. +fileMatchingOptions :: [Option] +fileMatchingOptions = fileMatchingOptions' ++ combiningOptions + +fileMatchingOptions' :: [Option] +fileMatchingOptions' = + [ Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob) + "skip files matching the glob pattern" + , Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob) + "limit to files matching the glob pattern" + , Option [] ["largerthan"] (ReqArg Limit.addLargerThan paramSize) + "match files larger than a size" + , Option [] ["smallerthan"] (ReqArg Limit.addSmallerThan paramSize) + "match files smaller than a size" + ] + +combiningOptions :: [Option] +combiningOptions = + [ longopt "not" "negate next option" + , longopt "and" "both previous and next option must match" + , longopt "or" "either previous or next option must match" + , shortopt "(" "open group of options" + , shortopt ")" "close group of options" + ] + where + longopt o = Option [] [o] $ NoArg $ Limit.addToken o + shortopt o = Option o [] $ NoArg $ Limit.addToken o + fromOption :: Option fromOption = fieldOption ['f'] "from" paramRemote "source remote" @@ -99,3 +137,8 @@ jsonOption :: Option jsonOption = Option ['j'] ["json"] (NoArg (Annex.setOutput JSONOutput)) "enable JSON output" + +timeLimitOption :: Option +timeLimitOption = Option ['T'] ["time-limit"] + (ReqArg Limit.addTimeLimit paramTime) + "stop after the specified amount of time" diff --git a/CmdLine/Option.hs b/CmdLine/Option.hs --- a/CmdLine/Option.hs +++ b/CmdLine/Option.hs @@ -7,10 +7,10 @@ module CmdLine.Option ( commonOptions, - matcherOptions, flagOption, fieldOption, optionName, + optionParam, ArgDescr(..), OptDescr(..), ) where @@ -21,9 +21,9 @@ import qualified Annex import Types.Messages import Types.DesktopNotify -import Limit import CmdLine.Usage +-- Options accepted by both git-annex and git-annex-shell sub-commands. commonOptions :: [Option] commonOptions = [ Option [] ["force"] (NoArg (setforce True)) @@ -56,18 +56,6 @@ unsetdebug = Annex.changeGitConfig $ \c -> c { annexDebug = False } setdesktopnotify v = Annex.changeState $ \s -> s { Annex.desktopnotify = Annex.desktopnotify s <> v } -matcherOptions :: [Option] -matcherOptions = - [ longopt "not" "negate next option" - , longopt "and" "both previous and next option must match" - , longopt "or" "either previous or next option must match" - , shortopt "(" "open group of options" - , shortopt ")" "close group of options" - ] - where - longopt o = Option [] [o] $ NoArg $ addToken o - shortopt o = Option o [] $ NoArg $ addToken o - {- An option that sets a flag. -} flagOption :: String -> String -> String -> Option flagOption short opt description = @@ -81,3 +69,6 @@ {- The flag or field name used for an option. -} optionName :: Option -> String optionName (Option _ o _ _) = Prelude.head o + +optionParam :: Option -> String +optionParam o = "--" ++ optionName o diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -32,6 +32,27 @@ withFilesInGit a params = seekActions $ prepFiltered a $ seekHelper LsFiles.inRepo params +withFilesInGitNonRecursive :: (FilePath -> CommandStart) -> CommandSeek +withFilesInGitNonRecursive a params = ifM (Annex.getState Annex.force) + ( withFilesInGit a params + , if null params + then needforce + else seekActions $ prepFiltered a (getfiles [] params) + ) + where + getfiles c [] = return (reverse c) + getfiles c (p:ps) = do + (fs, cleanup) <- inRepo $ LsFiles.inRepo [p] + case fs of + [f] -> do + void $ liftIO $ cleanup + getfiles (f:c) ps + [] -> do + void $ liftIO $ cleanup + getfiles c ps + _ -> needforce + needforce = error "Not recursively setting metadata. Use --force to do that." + withFilesNotInGit :: Bool -> (FilePath -> CommandStart) -> CommandSeek withFilesNotInGit skipdotfiles a params | skipdotfiles = do @@ -66,14 +87,20 @@ void $ commandAction $ a f k withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek -withPathContents a params = seekActions $ - map a . concat <$> liftIO (mapM get params) +withPathContents a params = do + matcher <- Limit.getMatcher + seekActions $ map a <$> (filterM (checkmatch matcher) =<< ps) where + ps = concat <$> liftIO (mapM get params) get p = ifM (isDirectory <$> getFileStatus p) ( map (\f -> (f, makeRelative (parentDir p) f)) <$> dirContentsRecursiveSkipping (".git" `isSuffixOf`) True p , return [(p, takeFileName p)] ) + checkmatch matcher (f, relf) = matcher $ MatchingFile $ FileInfo + { currFile = f + , matchFile = relf + } withWords :: ([String] -> CommandStart) -> CommandSeek withWords a params = seekActions $ return [a params] diff --git a/Command.hs b/Command.hs --- a/Command.hs +++ b/Command.hs @@ -55,7 +55,7 @@ {- Adds options to a command. -} withOptions :: [Option] -> Command -> Command -withOptions o c = c { cmdoptions = o } +withOptions o c = c { cmdoptions = cmdoptions c ++ o } {- For start and perform stages to indicate what step to run next. -} next :: a -> Annex (Maybe a) diff --git a/Command/Add.hs b/Command/Add.hs --- a/Command/Add.hs +++ b/Command/Add.hs @@ -35,9 +35,11 @@ import Control.Exception (IOException) cmd :: [Command] -cmd = [notBareRepo $ withOptions [includeDotFilesOption] $ - command "add" paramPaths seek SectionCommon - "add files to annex"] +cmd = [notBareRepo $ withOptions addOptions $ + command "add" paramPaths seek SectionCommon "add files to annex"] + +addOptions :: [Option] +addOptions = includeDotFilesOption : fileMatchingOptions includeDotFilesOption :: Option includeDotFilesOption = flagOption [] "include-dotfiles" "don't skip dotfiles" diff --git a/Command/Drop.hs b/Command/Drop.hs --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -23,8 +23,11 @@ import qualified Data.Set as S cmd :: [Command] -cmd = [withOptions [dropFromOption] $ command "drop" paramPaths seek +cmd = [withOptions (dropOptions) $ command "drop" paramPaths seek SectionCommon "indicate content of files not currently wanted"] + +dropOptions :: [Option] +dropOptions = dropFromOption : annexedMatchingOptions dropFromOption :: Option dropFromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote" diff --git a/Command/Find.hs b/Command/Find.hs --- a/Command/Find.hs +++ b/Command/Find.hs @@ -20,7 +20,8 @@ import Types.Key cmd :: [Command] -cmd = [mkCommand $ command "find" paramPaths seek SectionQuery "lists available files"] +cmd = [withOptions annexedMatchingOptions $ mkCommand $ + command "find" paramPaths seek SectionQuery "lists available files"] mkCommand :: Command -> Command mkCommand = noCommit . noMessages . withOptions [formatOption, print0Option, jsonOption] diff --git a/Command/FindRef.hs b/Command/FindRef.hs --- a/Command/FindRef.hs +++ b/Command/FindRef.hs @@ -11,8 +11,9 @@ import qualified Command.Find as Find cmd :: [Command] -cmd = [Find.mkCommand $ command "findref" paramRef seek SectionPlumbing - "lists files in a git ref"] +cmd = [withOptions nonWorkTreeMatchingOptions $ Find.mkCommand $ + command "findref" paramRef seek SectionPlumbing + "lists files in a git ref"] seek :: CommandSeek seek refs = do diff --git a/Command/Fix.hs b/Command/Fix.hs --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -19,8 +19,9 @@ #endif cmd :: [Command] -cmd = [notDirect $ noCommit $ command "fix" paramPaths seek - SectionMaintenance "fix up symlinks to point to annexed content"] +cmd = [notDirect $ noCommit $ withOptions annexedMatchingOptions $ + command "fix" paramPaths seek + SectionMaintenance "fix up symlinks to point to annexed content"] seek :: CommandSeek seek = withFilesInGit $ whenAnnexed start diff --git a/Command/Fsck.hs b/Command/Fsck.hs --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -62,7 +62,7 @@ , startIncrementalOption , moreIncrementalOption , incrementalScheduleOption - ] ++ keyOptions + ] ++ keyOptions ++ annexedMatchingOptions seek :: CommandSeek seek ps = do @@ -141,7 +141,10 @@ dispatch (Right True) = withtmp $ \tmpfile -> ifM (getfile tmpfile) ( go True (Just tmpfile) - , go True Nothing + , do + warning "failed to download file from remote" + void $ go True Nothing + return False ) dispatch (Right False) = go False Nothing go present localcopy = check diff --git a/Command/Get.hs b/Command/Get.hs --- a/Command/Get.hs +++ b/Command/Get.hs @@ -21,7 +21,7 @@ SectionCommon "make content of annexed files available"] getOptions :: [Option] -getOptions = fromOption : keyOptions +getOptions = fromOption : annexedMatchingOptions ++ keyOptions seek :: CommandSeek seek ps = do diff --git a/Command/GroupWanted.hs b/Command/GroupWanted.hs new file mode 100644 --- /dev/null +++ b/Command/GroupWanted.hs @@ -0,0 +1,45 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.GroupWanted where + +import Common.Annex +import qualified Annex +import Command +import Logs.PreferredContent +import Types.Messages +import Types.Group + +import qualified Data.Map as M + +cmd :: [Command] +cmd = [command "groupwanted" (paramPair paramGroup (paramOptional paramExpression)) seek + SectionSetup "get or set groupwanted expression"] + +seek :: CommandSeek +seek = withWords start + +start :: [String] -> CommandStart +start (g:[]) = next $ performGet g +start (g:expr:[]) = do + showStart "groupwanted" g + next $ performSet g expr +start _ = error "Specify a group." + +performGet :: Group -> CommandPerform +performGet g = do + Annex.setOutput QuietOutput + m <- groupPreferredContentMapRaw + liftIO $ putStrLn $ fromMaybe "" $ M.lookup g m + next $ return True + +performSet :: Group -> String -> CommandPerform +performSet g expr = case checkPreferredContentExpression expr of + Just e -> error $ "Parse error: " ++ e + Nothing -> do + groupPreferredContentSet g expr + next $ return True diff --git a/Command/Import.hs b/Command/Import.hs --- a/Command/Import.hs +++ b/Command/Import.hs @@ -21,42 +21,38 @@ SectionCommon "move and add files from outside git working copy"] opts :: [Option] -opts = - [ duplicateOption - , deduplicateOption - , cleanDuplicatesOption - , skipDuplicatesOption - ] - -duplicateOption :: Option -duplicateOption = flagOption [] "duplicate" "do not delete source files" - -deduplicateOption :: Option -deduplicateOption = flagOption [] "deduplicate" "delete source files whose content was imported before" +opts = duplicateModeOptions ++ fileMatchingOptions -cleanDuplicatesOption :: Option -cleanDuplicatesOption = flagOption [] "clean-duplicates" "delete duplicate source files (import nothing)" +data DuplicateMode = Default | Duplicate | DeDuplicate | CleanDuplicates | SkipDuplicates + deriving (Eq, Enum, Bounded) -skipDuplicatesOption :: Option -skipDuplicatesOption = flagOption [] "skip-duplicates" "import only new files" +associatedOption :: DuplicateMode -> Maybe Option +associatedOption Default = Nothing +associatedOption Duplicate = Just $ + flagOption [] "duplicate" "do not delete source files" +associatedOption DeDuplicate = Just $ + flagOption [] "deduplicate" "delete source files whose content was imported before" +associatedOption CleanDuplicates = Just $ + flagOption [] "clean-duplicates" "delete duplicate source files (import nothing)" +associatedOption SkipDuplicates = Just $ + flagOption [] "skip-duplicates" "import only new files" -data DuplicateMode = Default | Duplicate | DeDuplicate | CleanDuplicates | SkipDuplicates - deriving (Eq) +duplicateModeOptions :: [Option] +duplicateModeOptions = mapMaybe associatedOption [minBound..maxBound] getDuplicateMode :: Annex DuplicateMode -getDuplicateMode = gen - <$> getflag duplicateOption - <*> getflag deduplicateOption - <*> getflag cleanDuplicatesOption - <*> getflag skipDuplicatesOption +getDuplicateMode = go . catMaybes <$> mapM getflag [minBound..maxBound] where - getflag = Annex.getFlag . optionName - gen False False False False = Default - gen True False False False = Duplicate - gen False True False False = DeDuplicate - gen False False True False = CleanDuplicates - gen False False False True = SkipDuplicates - gen _ _ _ _ = error "bad combination of --duplicate, --deduplicate, --clean-duplicates, --skip-duplicates" + getflag m = case associatedOption m of + Nothing -> return Nothing + Just o -> ifM (Annex.getFlag (optionName o)) + ( return (Just m) + , return Nothing + ) + go [] = Default + go [m] = m + go ms = error $ "cannot combine " ++ + unwords (map (optionParam . fromJust . associatedOption) ms) seek :: CommandSeek seek ps = do @@ -67,14 +63,8 @@ start mode (srcfile, destfile) = ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) ( do - isdup <- do - backend <- chooseBackend destfile - let ks = KeySource srcfile srcfile Nothing - v <- genKey ks backend - case v of - Just (k, _) -> not . null <$> keyLocations k - _ -> return False - case pickaction isdup of + ma <- pickaction + case ma of Nothing -> stop Just a -> do showStart "import" destfile @@ -101,15 +91,16 @@ , notoverwriting "(use --force to override)" ) notoverwriting why = error $ "not overwriting existing " ++ destfile ++ " " ++ why - pickaction isdup = case mode of - DeDuplicate - | isdup -> Just deletedup - | otherwise -> Just importfile - CleanDuplicates - | isdup -> Just deletedup - | otherwise -> Nothing - SkipDuplicates - | isdup -> Nothing - | otherwise -> Just importfile - _ -> Just importfile - + checkdup dupa notdupa = do + backend <- chooseBackend destfile + let ks = KeySource srcfile srcfile Nothing + v <- genKey ks backend + isdup <- case v of + Just (k, _) -> not . null <$> keyLocations k + _ -> return False + return $ if isdup then dupa else notdupa + pickaction = case mode of + DeDuplicate -> checkdup (Just deletedup) (Just importfile) + CleanDuplicates -> checkdup (Just deletedup) Nothing + SkipDuplicates -> checkdup Nothing (Just importfile) + _ -> return (Just importfile) diff --git a/Command/Info.hs b/Command/Info.hs --- a/Command/Info.hs +++ b/Command/Info.hs @@ -77,7 +77,7 @@ type StatState = StateT StatInfo Annex cmd :: [Command] -cmd = [noCommit $ dontCheck repoExists $ withOptions [jsonOption] $ +cmd = [noCommit $ dontCheck repoExists $ withOptions (jsonOption : annexedMatchingOptions) $ command "info" (paramOptional $ paramRepeating paramItem) seek SectionQuery "shows information about the specified item or the repository as a whole"] diff --git a/Command/List.hs b/Command/List.hs --- a/Command/List.hs +++ b/Command/List.hs @@ -24,8 +24,9 @@ import Git.Types (RemoteName) cmd :: [Command] -cmd = [noCommit $ withOptions [allrepos] $ command "list" paramPaths seek - SectionQuery "show which remotes contain files"] +cmd = [noCommit $ withOptions (allrepos : annexedMatchingOptions) $ + command "list" paramPaths seek + SectionQuery "show which remotes contain files"] allrepos :: Option allrepos = flagOption [] "allrepos" "show all repositories, not only remotes" diff --git a/Command/Lock.hs b/Command/Lock.hs --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -13,7 +13,8 @@ import qualified Annex cmd :: [Command] -cmd = [notDirect $ command "lock" paramPaths seek SectionCommon +cmd = [notDirect $ withOptions annexedMatchingOptions $ + command "lock" paramPaths seek SectionCommon "undo unlock command"] seek :: CommandSeek diff --git a/Command/Log.hs b/Command/Log.hs --- a/Command/Log.hs +++ b/Command/Log.hs @@ -39,7 +39,7 @@ command "log" paramPaths seek SectionQuery "shows location log"] options :: [Option] -options = passthruOptions ++ [gourceOption] +options = passthruOptions ++ [gourceOption] ++ annexedMatchingOptions passthruOptions :: [Option] passthruOptions = map odate ["since", "after", "until", "before"] ++ diff --git a/Command/MetaData.hs b/Command/MetaData.hs --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -28,7 +28,7 @@ , untagOption , getOption , jsonOption - ] ++ keyOptions + ] ++ keyOptions ++ annexedMatchingOptions storeModMeta :: ModMeta -> Annex () storeModMeta modmeta = Annex.changeState $ @@ -58,9 +58,12 @@ getfield <- getOptionField getOption $ \ms -> return $ either error id . mkMetaField <$> ms now <- liftIO getPOSIXTime + let seeker = if null modmeta + then withFilesInGit + else withFilesInGitNonRecursive withKeyOptions (startKeys now getfield modmeta) - (withFilesInGit (whenAnnexed $ start now getfield modmeta)) + (seeker $ whenAnnexed $ start now getfield modmeta) ps start :: POSIXTime -> Maybe MetaField -> [ModMeta] -> FilePath -> Key -> CommandStart diff --git a/Command/Migrate.hs b/Command/Migrate.hs --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -18,7 +18,7 @@ import qualified Command.Fsck cmd :: [Command] -cmd = [notDirect $ +cmd = [notDirect $ withOptions annexedMatchingOptions $ command "migrate" paramPaths seek SectionUtility "switch data to different backend"] diff --git a/Command/Mirror.hs b/Command/Mirror.hs --- a/Command/Mirror.hs +++ b/Command/Mirror.hs @@ -18,7 +18,7 @@ import Config.NumCopies cmd :: [Command] -cmd = [withOptions (fromToOptions ++ keyOptions) $ +cmd = [withOptions (fromToOptions ++ annexedMatchingOptions ++ keyOptions) $ command "mirror" paramPaths seek SectionCommon "mirror content of files to/from another repository"] diff --git a/Command/Move.hs b/Command/Move.hs --- a/Command/Move.hs +++ b/Command/Move.hs @@ -22,7 +22,7 @@ SectionCommon "move content of files to/from another repository"] moveOptions :: [Option] -moveOptions = fromToOptions ++ keyOptions +moveOptions = fromToOptions ++ keyOptions ++ annexedMatchingOptions seek :: CommandSeek seek ps = do diff --git a/Command/Sync.hs b/Command/Sync.hs --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -12,6 +12,7 @@ mergeLocal, mergeRemote, commitStaged, + commitMsg, pushBranch, updateBranch, syncBranch, @@ -41,10 +42,12 @@ import Logs.Location import Annex.Drop import Annex.UUID +import Logs.UUID import Annex.AutoMerge import Annex.Ssh import Control.Concurrent.MVar +import qualified Data.Map as M cmd :: [Command] cmd = [withOptions syncOptions $ @@ -145,8 +148,8 @@ commit :: CommandStart commit = next $ next $ do - commitmessage <- fromMaybe "git-annex automatic sync" - <$> Annex.getField (optionName messageOption) + commitmessage <- maybe commitMsg return + =<< Annex.getField (optionName messageOption) showStart "commit" "" Annex.Branch.commit "update" ifM isDirect @@ -163,6 +166,12 @@ return True ) +commitMsg :: Annex String +commitMsg = do + u <- getUUID + m <- uuidMap + return $ "git-annex in " ++ fromMaybe "unknown" (M.lookup u m) + commitStaged :: Git.Branch.CommitMode -> String -> Annex Bool commitStaged commitmode commitmessage = go =<< inRepo Git.Branch.currentUnsafe where @@ -225,8 +234,9 @@ stopUnless fetch $ next $ mergeRemote remote branch where - fetch = inRepoWithSshCachingTo (Remote.repo remote) $ Git.Command.runBool - [Param "fetch", Param $ Remote.name remote] + fetch = inRepoWithSshOptionsTo (Remote.repo remote) (Remote.gitconfig remote) $ + Git.Command.runBool + [Param "fetch", Param $ Remote.name remote] {- The remote probably has both a master and a synced/master branch. - Which to merge from? Well, the master has whatever latest changes @@ -261,7 +271,7 @@ showStart "push" (Remote.name remote) next $ next $ do showOutput - ok <- inRepoWithSshCachingTo (Remote.repo remote) $ + ok <- inRepoWithSshOptionsTo (Remote.repo remote) (Remote.gitconfig remote) $ pushBranch remote branch unless ok $ do warning $ unwords [ "Pushing to " ++ Remote.name remote ++ " failed." ] diff --git a/Command/Unannex.hs b/Command/Unannex.hs --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -23,7 +23,8 @@ import Command.PreCommit (lockPreCommitHook) cmd :: [Command] -cmd = [command "unannex" paramPaths seek SectionUtility +cmd = [withOptions annexedMatchingOptions $ + command "unannex" paramPaths seek SectionUtility "undo accidential add command"] seek :: CommandSeek diff --git a/Command/Unlock.hs b/Command/Unlock.hs --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -19,7 +19,8 @@ , c "edit" "same as unlock" ] where - c n = notDirect . command n paramPaths seek SectionCommon + c n = notDirect . withOptions annexedMatchingOptions + . command n paramPaths seek SectionCommon seek :: CommandSeek seek = withFilesInGit $ whenAnnexed start diff --git a/Command/WebApp.hs b/Command/WebApp.hs --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -195,10 +195,15 @@ #endif openBrowser :: Maybe FilePath -> FilePath -> String -> Maybe Handle -> Maybe Handle -> IO () +openBrowser mcmd htmlshim realurl outh errh = do + htmlshim' <- absPath htmlshim + openBrowser' mcmd htmlshim' realurl outh errh + +openBrowser' :: Maybe FilePath -> FilePath -> String -> Maybe Handle -> Maybe Handle -> IO () #ifndef __ANDROID__ -openBrowser mcmd htmlshim _realurl outh errh = runbrowser +openBrowser' mcmd htmlshim _realurl outh errh = runbrowser #else -openBrowser mcmd htmlshim realurl outh errh = do +openBrowser' mcmd htmlshim realurl outh errh = do recordUrl url {- Android's `am` command does not work reliably across the - wide range of Android devices. Intead, FIFO should be set to diff --git a/Command/Whereis.hs b/Command/Whereis.hs --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -16,7 +16,7 @@ import Logs.Web cmd :: [Command] -cmd = [noCommit $ withOptions (jsonOption : keyOptions) $ +cmd = [noCommit $ withOptions (jsonOption : annexedMatchingOptions ++ keyOptions) $ command "whereis" paramPaths seek SectionQuery "lists repositories that have file content"] diff --git a/Git/Config.hs b/Git/Config.hs --- a/Git/Config.hs +++ b/Git/Config.hs @@ -66,10 +66,9 @@ home <- myHomeDir ifM (doesFileExist $ home ".gitconfig") ( do - repo <- Git.Construct.fromUnknown - repo' <- withHandle StdoutHandle createProcessSuccess p $ - hRead repo - return $ Just repo' + repo <- withHandle StdoutHandle createProcessSuccess p $ + hRead (Git.Construct.fromUnknown) + return $ Just repo , return Nothing ) where diff --git a/Git/Construct.hs b/Git/Construct.hs --- a/Git/Construct.hs +++ b/Git/Construct.hs @@ -19,8 +19,8 @@ fromRemotes, fromRemoteLocation, repoAbsPath, - newFrom, checkForRepo, + newFrom, ) where #ifndef mingw32_HOST_OS @@ -48,7 +48,7 @@ Nothing -> case upFrom dir of Nothing -> return Nothing Just d -> seekUp d - Just loc -> Just <$> newFrom loc + Just loc -> pure $ Just $ newFrom loc {- Local Repo constructor, accepts a relative or absolute path. -} fromPath :: FilePath -> IO Repo @@ -62,7 +62,7 @@ | otherwise = error $ "internal error, " ++ dir ++ " is not absolute" where - ret = newFrom . LocalUnknown + ret = pure . newFrom . LocalUnknown {- Git always looks for "dir.git" in preference to - to "dir", even if dir ends in a "/". -} canondir = dropTrailingPathSeparator dir @@ -90,13 +90,13 @@ fromUrlStrict :: String -> IO Repo fromUrlStrict url | startswith "file://" url = fromAbsPath $ unEscapeString $ uriPath u - | otherwise = newFrom $ Url u + | otherwise = pure $ newFrom $ Url u where u = fromMaybe bad $ parseURI url bad = error $ "bad url " ++ url {- Creates a repo that has an unknown location. -} -fromUnknown :: IO Repo +fromUnknown :: Repo fromUnknown = newFrom Unknown {- Converts a local Repo into a remote repo, using the reference repo @@ -223,8 +223,8 @@ gitdirprefix = "gitdir: " gitSignature file = doesFileExist $ dir file -newFrom :: RepoLocation -> IO Repo -newFrom l = return Repo +newFrom :: RepoLocation -> Repo +newFrom l = Repo { location = l , config = M.empty , fullconfig = M.empty @@ -233,5 +233,4 @@ , gitEnv = Nothing , gitGlobalOpts = [] } - diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs --- a/Git/CurrentRepo.hs +++ b/Git/CurrentRepo.hs @@ -50,8 +50,8 @@ configure (Just d) _ = do absd <- absPath d curr <- getCurrentDirectory - r <- newFrom $ Local { gitdir = absd, worktree = Just curr } - Git.Config.read r + Git.Config.read $ newFrom $ + Local { gitdir = absd, worktree = Just curr } configure Nothing Nothing = error "Not in a git repository." addworktree w r = changelocation r $ diff --git a/Limit.hs b/Limit.hs --- a/Limit.hs +++ b/Limit.hs @@ -239,7 +239,7 @@ checkkey sz key = return $ keySize key `vs` Just sz check _ sz (Just key) = checkkey sz key check fi sz Nothing = do - filesize <- liftIO $ catchMaybeIO $ getFileSize (relFile fi) + filesize <- liftIO $ catchMaybeIO $ getFileSize (currFile fi) return $ filesize `vs` Just sz addMetaData :: String -> Annex () @@ -271,7 +271,7 @@ else return True lookupFileKey :: FileInfo -> Annex (Maybe Key) -lookupFileKey = Backend.lookupFile . relFile +lookupFileKey = Backend.lookupFile . currFile checkKey :: (Key -> Annex Bool) -> MatchInfo -> Annex Bool checkKey a (MatchingFile fi) = lookupFileKey fi >>= maybe (return False) a diff --git a/Locations.hs b/Locations.hs --- a/Locations.hs +++ b/Locations.hs @@ -75,6 +75,7 @@ import Types.UUID import Types.Difference import qualified Git +import Git.FilePath import Annex.DirHashes {- Conventions: @@ -154,7 +155,7 @@ currdir <- getCurrentDirectory let absfile = fromMaybe whoops $ absNormPathUnix currdir file loc <- gitAnnexLocation' key r config False - relPathDirToFile (parentDir absfile) loc + toInternalGitPath <$> relPathDirToFile (parentDir absfile) loc where whoops = error $ "unable to normalize " ++ file diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -44,7 +44,7 @@ -- There is only one bittorrent remote, and it always exists. list :: Annex [Git.Repo] list = do - r <- liftIO $ Git.Construct.remoteNamed "bittorrent" Git.Construct.fromUnknown + r <- liftIO $ Git.Construct.remoteNamed "bittorrent" (pure Git.Construct.fromUnknown) return [r] gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) diff --git a/Remote/Ddar.hs b/Remote/Ddar.hs --- a/Remote/Ddar.hs +++ b/Remote/Ddar.hs @@ -23,7 +23,10 @@ import Annex.Ssh import Annex.UUID -type DdarRepo = String +data DdarRepo = DdarRepo + { ddarRepoConfig :: RemoteGitConfig + , ddarRepoLocation :: String + } remote :: RemoteType remote = RemoteType { @@ -62,18 +65,18 @@ , config = c , repo = r , gitconfig = gc - , localpath = if ddarLocal ddarrepo && not (null ddarrepo) - then Just ddarrepo + , localpath = if ddarLocal ddarrepo && not (null $ ddarRepoLocation ddarrepo) + then Just $ ddarRepoLocation ddarrepo else Nothing , remotetype = remote , availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable , readonly = False , mkUnavailable = return Nothing - , getInfo = return [("repo", ddarrepo)] + , getInfo = return [("repo", ddarRepoLocation ddarrepo)] , claimUrl = Nothing , checkUrl = Nothing } - ddarrepo = fromMaybe (error "missing ddarrepo") $ remoteAnnexDdarRepo gc + ddarrepo = maybe (error "missing ddarrepo") (DdarRepo gc) (remoteAnnexDdarRepo gc) specialcfg = (specialRemoteCfg c) -- chunking would not improve ddar { chunkConfig = NoChunks @@ -100,7 +103,7 @@ [ Param "c" , Param "-N" , Param $ key2file k - , Param ddarrepo + , Param $ ddarRepoLocation ddarrepo , File src ] liftIO $ boolSystem "ddar" params @@ -110,25 +113,23 @@ splitRemoteDdarRepo ddarrepo = (host, ddarrepo') where - (host, remainder) = span (/= ':') ddarrepo + (host, remainder) = span (/= ':') (ddarRepoLocation ddarrepo) ddarrepo' = drop 1 remainder {- Return the command and parameters to use for a ddar call that may need to be - made on a remote repository. This will call ssh if needed. -} - ddarRemoteCall :: DdarRepo -> Char -> [CommandParam] -> Annex (String, [CommandParam]) ddarRemoteCall ddarrepo cmd params | ddarLocal ddarrepo = return ("ddar", localParams) | otherwise = do - remoteCachingParams <- sshCachingOptions (host, Nothing) [] - return ("ssh", remoteCachingParams ++ remoteParams) + os <- sshOptions (host, Nothing) (ddarRepoConfig ddarrepo) remoteParams + return ("ssh", os) where (host, ddarrepo') = splitRemoteDdarRepo ddarrepo - localParams = Param [cmd] : Param ddarrepo : params + localParams = Param [cmd] : Param (ddarRepoLocation ddarrepo) : params remoteParams = Param host : Param "ddar" : Param [cmd] : Param ddarrepo' : params {- Specialized ddarRemoteCall that includes extraction command and flags -} - ddarExtractRemoteCall :: DdarRepo -> Key -> Annex (String, [CommandParam]) ddarExtractRemoteCall ddarrepo k = ddarRemoteCall ddarrepo 'x' [Param "--force-stdout", Param $ key2file k] @@ -152,13 +153,13 @@ ddarDirectoryExists :: DdarRepo -> Annex (Either String Bool) ddarDirectoryExists ddarrepo | ddarLocal ddarrepo = do - maybeStatus <- liftIO $ tryJust (guard . isDoesNotExistError) $ getFileStatus ddarrepo + maybeStatus <- liftIO $ tryJust (guard . isDoesNotExistError) $ getFileStatus $ ddarRepoLocation ddarrepo return $ case maybeStatus of Left _ -> Right False Right status -> Right $ isDirectory status | otherwise = do - sshCachingParams <- sshCachingOptions (host, Nothing) [] - exitCode <- liftIO $ safeSystem "ssh" $ sshCachingParams ++ params + ps <- sshOptions (host, Nothing) (ddarRepoConfig ddarrepo) params + exitCode <- liftIO $ safeSystem "ssh" ps case exitCode of ExitSuccess -> return $ Right True ExitFailure 1 -> return $ Right False @@ -195,4 +196,4 @@ Right False -> return False ddarLocal :: DdarRepo -> Bool -ddarLocal = notElem ':' +ddarLocal = notElem ':' . ddarRepoLocation diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -70,7 +70,7 @@ gen baser u c gc = do -- doublecheck that cache matches underlying repo's gcrypt-id -- (which might not be set), only for local repos - (mgcryptid, r) <- getGCryptId True baser + (mgcryptid, r) <- getGCryptId True baser gc g <- gitRepo case (mgcryptid, Git.GCrypt.remoteRepoId g (Git.remoteName baser)) of (Just gcryptid, Just cachedgcryptid) @@ -99,7 +99,7 @@ gen' r u c gc = do cst <- remoteCost gc $ if repoCheap r then nearlyCheapRemoteCost else expensiveRemoteCost - (rsynctransport, rsyncurl) <- rsyncTransportToObjects r + (rsynctransport, rsyncurl) <- rsyncTransportToObjects r gc let rsyncopts = Remote.Rsync.genRsyncOpts c gc rsynctransport rsyncurl let this = Remote { uuid = u @@ -139,13 +139,13 @@ { displayProgress = False } | otherwise = specialRemoteCfg c -rsyncTransportToObjects :: Git.Repo -> Annex ([CommandParam], String) -rsyncTransportToObjects r = do - (rsynctransport, rsyncurl, _) <- rsyncTransport r +rsyncTransportToObjects :: Git.Repo -> RemoteGitConfig -> Annex ([CommandParam], String) +rsyncTransportToObjects r gc = do + (rsynctransport, rsyncurl, _) <- rsyncTransport r gc return (rsynctransport, rsyncurl ++ "/annex/objects") -rsyncTransport :: Git.Repo -> Annex ([CommandParam], String, AccessMethod) -rsyncTransport r +rsyncTransport :: Git.Repo -> RemoteGitConfig -> Annex ([CommandParam], String, AccessMethod) +rsyncTransport r gc | "ssh://" `isPrefixOf` loc = sshtransport $ break (== '/') $ drop (length "ssh://") loc | "//:" `isInfixOf` loc = othertransport | ":" `isInfixOf` loc = sshtransport $ separate (== ':') loc @@ -156,7 +156,7 @@ let rsyncpath = if "/~/" `isPrefixOf` path then drop 3 path else path - opts <- sshCachingOptions (host, Nothing) [] + opts <- sshOptions (host, Nothing) gc [] return (rsyncShell $ Param "ssh" : opts, host ++ ":" ++ rsyncpath, AccessShell) othertransport = return ([], loc, AccessDirect) @@ -218,7 +218,7 @@ setupRepo :: Git.GCrypt.GCryptId -> Git.Repo -> Annex AccessMethod setupRepo gcryptid r | Git.repoIsUrl r = do - (_, _, accessmethod) <- rsyncTransport r + (_, _, accessmethod) <- rsyncTransport r def case accessmethod of AccessDirect -> rsyncsetup AccessShell -> ifM gitannexshellsetup @@ -240,7 +240,7 @@ -} rsyncsetup = Remote.Rsync.withRsyncScratchDir $ \tmp -> do liftIO $ createDirectoryIfMissing True $ tmp objectDir - (rsynctransport, rsyncurl, _) <- rsyncTransport r + (rsynctransport, rsyncurl, _) <- rsyncTransport r def let tmpconfig = tmp "config" void $ liftIO $ rsync $ rsynctransport ++ [ Param $ rsyncurl ++ "/config" @@ -376,7 +376,7 @@ getGCryptUUID :: Bool -> Git.Repo -> Annex (Maybe UUID) getGCryptUUID fast r = (genUUIDInNameSpace gCryptNameSpace <$>) . fst - <$> getGCryptId fast r + <$> getGCryptId fast r def coreGCryptId :: String coreGCryptId = "core.gcrypt-id" @@ -389,22 +389,22 @@ - tries git-annex-shell and direct rsync of the git config file. - - (Also returns a version of input repo with its config read.) -} -getGCryptId :: Bool -> Git.Repo -> Annex (Maybe Git.GCrypt.GCryptId, Git.Repo) -getGCryptId fast r +getGCryptId :: Bool -> Git.Repo -> RemoteGitConfig -> Annex (Maybe Git.GCrypt.GCryptId, Git.Repo) +getGCryptId fast r gc | Git.repoIsLocal r || Git.repoIsLocalUnknown r = extract <$> liftIO (catchMaybeIO $ Git.Config.read r) | not fast = extract . liftM fst <$> getM (eitherToMaybe <$>) [ Ssh.onRemote r (Git.Config.fromPipe r, return (Left undefined)) "configlist" [] [] - , getConfigViaRsync r + , getConfigViaRsync r gc ] | otherwise = return (Nothing, r) where extract Nothing = (Nothing, r) extract (Just r') = (Git.Config.getMaybe coreGCryptId r', r') -getConfigViaRsync :: Git.Repo -> Annex (Either SomeException (Git.Repo, String)) -getConfigViaRsync r = do - (rsynctransport, rsyncurl, _) <- rsyncTransport r +getConfigViaRsync :: Git.Repo -> RemoteGitConfig -> Annex (Either SomeException (Git.Repo, String)) +getConfigViaRsync r gc = do + (rsynctransport, rsyncurl, _) <- rsyncTransport r gc liftIO $ do withTmpFile "tmpconfig" $ \tmpconfig _ -> do void $ rsync $ rsynctransport ++ diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -506,7 +506,7 @@ [ ("GIT_WORK_TREE", Git.repoPath r') , ("GIT_DIR", Git.localGitDir r') ] environ - batchCommandEnv program (Param "fsck" : params) $ Just environ' + batchCommandEnv program (Param "fsck" : params) (Just environ') {- The passed repair action is run in the Annex monad of the remote. -} repairRemote :: Git.Repo -> Annex Bool -> Annex (IO Bool) diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Glacier (remote, jobList) where +module Remote.Glacier (remote, jobList, checkSaneGlacierCommand) where import qualified Data.Map as M import qualified Data.Text as T @@ -223,7 +223,9 @@ (M.lookup "datacenter" c) glacierEnv :: RemoteConfig -> UUID -> Annex (Maybe [(String, String)]) -glacierEnv c u = go =<< getRemoteCredPairFor "glacier" c creds +glacierEnv c u = do + liftIO checkSaneGlacierCommand + go =<< getRemoteCredPairFor "glacier" c creds where go Nothing = return Nothing go (Just (user, pass)) = do @@ -301,3 +303,14 @@ | otherwise -> parse c rest parse c (_:rest) = parse c rest + +-- boto's version of glacier exits 0 when given a parameter it doesn't +-- understand. See https://github.com/boto/boto/issues/2942 +checkSaneGlacierCommand :: IO () +checkSaneGlacierCommand = + whenM ((Nothing /=) <$> catchMaybeIO shouldfail) $ + error wrongcmd + where + test = proc "glacier" ["--compatibility-test-git-annex"] + shouldfail = withQuietOutput createProcessSuccess test + wrongcmd = "The glacier program in PATH seems to be from boto, not glacier-cli. Cannot use this program." diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -59,7 +59,7 @@ liftIO $ mapM construct $ remotepairs m where remotepairs = M.toList . M.filterWithKey match - construct (k,_) = Git.Construct.remoteNamedFromKey k Git.Construct.fromUnknown + construct (k,_) = Git.Construct.remoteNamedFromKey k (pure Git.Construct.fromUnknown) match k _ = startswith "remote." k && endswith (".annex-"++s) k {- Sets up configuration for a special remote in .git/config. -} diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -30,7 +30,7 @@ toRepo r gc sshcmd = do let opts = map Param $ remoteAnnexSshOptions gc let host = fromMaybe (error "bad ssh url") $ Git.Url.hostuser r - params <- sshCachingOptions (host, Git.Url.port r) opts + params <- sshOptions (host, Git.Url.port r) gc opts return $ params ++ Param host : sshcmd {- Generates parameters to run a git-annex-shell command on a remote diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -121,8 +121,8 @@ let (port, sshopts') = sshReadPort sshopts userhost = takeWhile (/=':') url -- Connection caching - (Param "ssh":) <$> sshCachingOptions - (userhost, port) + (Param "ssh":) <$> sshOptions + (userhost, port) gc (map Param $ loginopt ++ sshopts') "rsh":rshopts -> return $ map Param $ "rsh" : loginopt ++ rshopts diff --git a/Remote/Web.hs b/Remote/Web.hs --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -38,7 +38,7 @@ -- a new release to the survivors by carrier pigeon.) list :: Annex [Git.Repo] list = do - r <- liftIO $ Git.Construct.remoteNamed "web" Git.Construct.fromUnknown + r <- liftIO $ Git.Construct.remoteNamed "web" (pure Git.Construct.fromUnknown) return [r] gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs --- a/RemoteDaemon/Core.hs +++ b/RemoteDaemon/Core.hs @@ -117,14 +117,16 @@ gen r = case Git.location r of Git.Url u -> case M.lookup (uriScheme u) remoteTransports of Just transport - | remoteAnnexSync (extractRemoteGitConfig r (Git.repoDescribe r)) -> do + | remoteAnnexSync gc -> do ichan <- newTChanIO :: IO (TChan Consumed) return $ Just ( r - , (transport r (RemoteURI u) h ichan ochan, ichan) + , (transport (RemoteRepo r gc) (RemoteURI u) h ichan ochan, ichan) ) _ -> return Nothing _ -> return Nothing + where + gc = extractRemoteGitConfig r (Git.repoDescribe r) genTransportHandle :: IO TransportHandle genTransportHandle = do diff --git a/RemoteDaemon/Transport/Ssh.hs b/RemoteDaemon/Transport/Ssh.hs --- a/RemoteDaemon/Transport/Ssh.hs +++ b/RemoteDaemon/Transport/Ssh.hs @@ -22,13 +22,13 @@ import Control.Concurrent.Async transport :: Transport -transport r url h@(TransportHandle g s) ichan ochan = do +transport rr@(RemoteRepo r gc) url h@(TransportHandle g s) ichan ochan = do -- enable ssh connection caching wherever inLocalRepo is called - g' <- liftAnnex h $ sshCachingTo r g - transport' r url (TransportHandle g' s) ichan ochan + g' <- liftAnnex h $ sshOptionsTo r gc g + transport' rr url (TransportHandle g' s) ichan ochan transport' :: Transport -transport' r url transporthandle ichan ochan = do +transport' (RemoteRepo r _) url transporthandle ichan ochan = do v <- liftAnnex transporthandle $ git_annex_shell r "notifychanges" [] [] case v of diff --git a/RemoteDaemon/Types.hs b/RemoteDaemon/Types.hs --- a/RemoteDaemon/Types.hs +++ b/RemoteDaemon/Types.hs @@ -14,6 +14,7 @@ import qualified Annex import qualified Git.Types as Git import qualified Utility.SimpleProtocol as Proto +import Types.GitConfig import Network.URI import Control.Concurrent @@ -27,7 +28,7 @@ -- from a Chan, and emits others to another Chan. type Transport = RemoteRepo -> RemoteURI -> TransportHandle -> TChan Consumed -> TChan Emitted -> IO () -type RemoteRepo = Git.Repo +data RemoteRepo = RemoteRepo Git.Repo RemoteGitConfig type LocalRepo = Git.Repo -- All Transports share a single AnnexState MVar diff --git a/Test.hs b/Test.hs --- a/Test.hs +++ b/Test.hs @@ -191,6 +191,7 @@ , testCase "edit (pre-commit)" test_edit_precommit , testCase "partial commit" test_partial_commit , testCase "fix" test_fix + , testCase "direct" test_direct , testCase "trust" test_trust , testCase "fsck (basics)" test_fsck_basic , testCase "fsck (bare)" test_fsck_bare @@ -226,6 +227,7 @@ , testCase "crypto" test_crypto , testCase "preferred content" test_preferred_content , testCase "add subdirs" test_add_subdirs + , testCase "addurl" test_addurl ] -- this test case create the main repo @@ -329,6 +331,9 @@ git_annex "numcopies" ["1"] @? "numcopies config failed" git_annex "drop" [annexedfile] @? "drop failed though origin has copy" annexed_notpresent annexedfile + -- make sure that the correct symlink is staged for the file + -- after drop + git_annex_expectoutput "status" [] [] inmainrepo $ annexed_present annexedfile test_drop_untrustedremote :: Assertion @@ -530,6 +535,14 @@ subdir = "s" newfile = subdir ++ "/" ++ annexedfile +test_direct :: Assertion +test_direct = intmpclonerepoInDirect $ do + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "direct" [] @? "switch to direct mode failed" + annexed_present annexedfile + git_annex "indirect" [] @? "switch to indirect mode failed" + test_trust :: Assertion test_trust = intmpclonerepo $ do git_annex "trust" [repo] @? "trust failed" @@ -1343,6 +1356,17 @@ writeFile ("dir2" "foo") $ content annexedfile setCurrentDirectory "dir" git_annex "add" [".." "dir2"] @? "add of ../subdir failed" + +test_addurl :: Assertion +test_addurl = intmpclonerepo $ do + -- file:// only; this test suite should not hit the network + f <- absPath "myurl" + let url = replace "\\" "/" ("file:///" ++ dropDrive f) + writeFile f "foo" + git_annex "addurl" [url] @? ("addurl failed on " ++ url) + let dest = "addurlurldest" + git_annex "addurl" ["--file", dest, url] @? ("addurl failed on " ++ url ++ " with --file") + doesFileExist dest @? (dest ++ " missing after addurl --file") -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. diff --git a/Types/FileMatcher.hs b/Types/FileMatcher.hs --- a/Types/FileMatcher.hs +++ b/Types/FileMatcher.hs @@ -19,8 +19,10 @@ | MatchingKey Key data FileInfo = FileInfo - { relFile :: FilePath -- may be relative to cwd - , matchFile :: FilePath -- filepath to match on; may be relative to top + { currFile :: FilePath + -- ^ current path to the file, for operations that examine it + , matchFile :: FilePath + -- ^ filepath to match on; may be relative to top of repo or cwd } type FileMatcherMap a = M.Map UUID (Utility.Matcher.Matcher (S.Set UUID -> MatchInfo -> a Bool)) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -15,6 +15,7 @@ import Common import qualified Git import qualified Git.Config +import qualified Git.Construct import Utility.DataUnits import Config.Cost import Types.Distribution @@ -193,3 +194,5 @@ notempty (Just "") = Nothing notempty (Just s) = Just s +instance Default RemoteGitConfig where + def = extractRemoteGitConfig Git.Construct.fromUnknown "dummy" diff --git a/Utility/Metered.hs b/Utility/Metered.hs --- a/Utility/Metered.hs +++ b/Utility/Metered.hs @@ -1,6 +1,6 @@ {- Metered IO - - - Copyright 2012, 2013 Joey Hess + - Copyright 2012-2105 Joey Hess - - License: BSD-2-clause -} @@ -17,6 +17,7 @@ import Foreign.Storable (Storable(sizeOf)) import System.Posix.Types import Data.Int +import Data.Bits.Utils {- An action that can be run repeatedly, updating it on the bytes processed. - @@ -163,12 +164,13 @@ p = proc cmd (toCommand params) feedprogress prev buf h = do - s <- hGetSomeString h 80 - if null s + b <- S.hGetSome h 80 + if S.null b then return True else do - putStr s + S.hPut stdout b hFlush stdout + let s = w82s (S.unpack b) let (mbytes, buf') = progressparser (buf++s) case mbytes of Nothing -> feedprogress prev buf' h diff --git a/Utility/Path.hs b/Utility/Path.hs --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -66,7 +66,7 @@ absPathFrom dir path = simplifyPath (combine dir path) {- On Windows, this converts the paths to unix-style, in order to run - - MissingH's absNormPath on them. Resulting path will use / separators. -} + - MissingH's absNormPath on them. -} absNormPathUnix :: FilePath -> FilePath -> Maybe FilePath #ifndef mingw32_HOST_OS absNormPathUnix dir path = MissingH.absNormPath dir path diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs --- a/Utility/Quvi.hs +++ b/Utility/Quvi.hs @@ -22,6 +22,7 @@ = Quvi04 | Quvi09 | NoQuvi + deriving (Show) data Page = Page { pageTitle :: String @@ -61,7 +62,8 @@ m = M.fromList $ map (separate (== '=')) $ lines s probeVersion :: IO QuviVersion -probeVersion = examine <$> processTranscript "quvi" ["--version"] Nothing +probeVersion = catchDefaultIO NoQuvi $ + examine <$> processTranscript "quvi" ["--version"] Nothing where examine (s, True) | "quvi v0.4" `isInfixOf` s = Quvi04 diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,40 @@ +git-annex (5.20150219) unstable; urgency=medium + + * glacier: Detect when the glacier command in PATH is the wrong one, + from boto, rather than from glacier-cli, and refuse to use it, + since the boto program fails to fail when passed + parameters it does not understand. + * groupwanted: New command to set the groupwanted preferred content + expression. + * import: Support file matching options such as --exclude, --include, + --smallerthan, --largerthan + * The file matching options are now only accepted by commands that + can actually use them, instead of by all commands. + * import: Avoid checksumming file twice when run in the default + or --duplicate mode. + * Windows: Fix bug in dropping an annexed file, which + caused a symlink to be staged that contained backslashes. + * webapp: Fix reversion in opening webapp when starting it manually + inside a repository. + * assistant: Improve sanity check for control characters when pairing. + * Improve race recovery code when committing to git-annex branch. + * addurl: Avoid crash if quvi is not installed, when git-annex was + built with process-1.2 + * bittorrent: Fix mojibake introduced in parsing arai2c progress output. + * fsck --from: If a download from a remote fails, propagate the failure. + * metadata: When setting metadata, do not recurse into directories by + default, since that can be surprising behavior and difficult to recover + from. The old behavior is available by using --force. + * sync, assistant: Include repository name in head branch commit message. + * The ssh-options git config is now used by gcrypt, rsync, and ddar + special remotes that use ssh as a transport. + * sync, assistant: Use the ssh-options git config when doing git pull + and push. + * remotedaemon: Use the ssh-options git config. + * Linux standalone: Improved process names of linker shimmed programs. + + -- Joey Hess Thu, 19 Feb 2015 14:16:03 -0400 + git-annex (5.20150205) unstable; urgency=medium * info: Can now display info about a given uuid. diff --git a/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment b/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment new file mode 100644 --- /dev/null +++ b/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="THANK YOU JOEY" + date="2015-02-09T14:04:27Z" + content=""" +for the MD5/MD5E (and now I have found \"email replies to me\" - I will become a power user of branchable ;) ) +"""]] diff --git a/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment b/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment new file mode 100644 --- /dev/null +++ b/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/814e4910-8e9b-4fe5-83ef-ff863c1a7314" + nickname="BehemothTheCat" + subject="push fails" + date="2015-02-14T00:11:40Z" + content=""" +These instructions don't work for me, unfortunately. + +This step: + + git push origin master git-annex + +results in: + + To ssh://my.server.com/home/itz/git/annex.git + ! [rejected] git-annex -> git-annex (non-fast-forward) + error: failed to push some refs to 'ssh://my.server.com/home/itz/git/annex.git' + hint: Updates were rejected because a pushed branch tip is behind its remote + hint: counterpart. Check out this branch and integrate the remote changes + hint: (e.g. 'git pull ...') before pushing again. + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + +Versions: git 1:1.9.1-1~bpo70+2 , git-annex 5.20141024~bpo70+1 (both packaged by Debian, same on local and remote) + +And yes, I did a pull on the master branch first. Afraid to do anything +with the git-annex branch without explicit instruction. + +"""]] diff --git a/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment b/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment new file mode 100644 --- /dev/null +++ b/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-02-17T21:54:33Z" + content=""" +Since the two repos git-annex branches have diverged, you need to run `git +annex merge` to merge them before you can push that branch. + +Of course, `git annex sync` handles all that for you. It can be used +against a bare repository as well as a non-bare. +"""]] diff --git a/doc/bugs/5.20140517_fails_to_talk_to_other_5.x_git-annex_remotes.mdwn b/doc/bugs/5.20140517_fails_to_talk_to_other_5.x_git-annex_remotes.mdwn --- a/doc/bugs/5.20140517_fails_to_talk_to_other_5.x_git-annex_remotes.mdwn +++ b/doc/bugs/5.20140517_fails_to_talk_to_other_5.x_git-annex_remotes.mdwn @@ -34,4 +34,4 @@ If this is intended behavior, it seems to me the major version of git annex should be bumped, at the very least... -- [[anarcat]] -[[!tag moreinfo]] +> [[done]], seems operator error --[[Joey]] diff --git a/doc/bugs/How_to_use_a_DRA_bucket_in_Google_cloud_storage__63__.mdwn b/doc/bugs/How_to_use_a_DRA_bucket_in_Google_cloud_storage__63__.mdwn --- a/doc/bugs/How_to_use_a_DRA_bucket_in_Google_cloud_storage__63__.mdwn +++ b/doc/bugs/How_to_use_a_DRA_bucket_in_Google_cloud_storage__63__.mdwn @@ -21,3 +21,7 @@ ### Please provide any additional information below. There didn't seem to be any extra logs and `--debug` didn't seem to add anything useful. + +> Closing, as it seems others have gotten this to work with a more recent +> version of git-annex. Please followup if it doesn't work. [[done]] +> --[[Joey]] diff --git a/doc/bugs/aria2c_display_broken_in_git-annex.mdwn b/doc/bugs/aria2c_display_broken_in_git-annex.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/aria2c_display_broken_in_git-annex.mdwn @@ -0,0 +1,147 @@ +[[!meta title="aria2c output very verbose (non-console mode)"]] +[[!tag confirmed]] + +### Please describe the problem. + +Instead of displaying what are probably carriage returns, git-annex strips those out and outputs a *lot* of lines when downloading stuff through aria2c. On small downloads, it's not a big deal, but on large downloads it can flood the screen pretty badly. + +### What steps will reproduce the problem? + +Just download a torrent with git-annex. + +### What version of git-annex are you using? On what operating system? + +`5.20150205-gbf9058a` on current debian jessie. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +anarcat@marcos:iso(master +)$ /srv/downloads.kitenet.net/git-annex/linux/current/git-annex.linux/git-annex addurl http://images.kali.org/kali-linux-1.1.0-amd64.torrent +(downloading torrent file...) +--2015-02-09 22:12:51-- http://images.kali.org/kali-linux-1.1.0-amd64.torrent +Résolution de images.kali.org (images.kali.org)… 50.7.37.130 +Connexion à images.kali.org (images.kali.org)|50.7.37.130|:80… connecté. +requête HTTP transmise, en attente de la réponse… 200 OK +Taille : 233152 (228K) [application/octet-stream] +Sauvegarde en : « ../.git/annex/misctmp/torrent32659 » + +100%[=====================================================================================================================================================================================================>] 233 152 610KB/s ds 0,4s + +2015-02-09 22:12:52 (610 KB/s) — « ../.git/annex/misctmp/torrent32659 » sauvegardé [233152/233152] + +addurl images.kali.org_kali_linux_1.1.0_amd64.torrent/kali_linux_1.1.0_amd64.iso (from bittorrent) + + +02/09 22:12:52 [NOTICE] IPv4 DHT: listening on UDP port 6940 + +02/09 22:12:52 [ERROR] Erreur d'intégrité détectée. fichier=../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64.torrent/kali-linux-1.1.0-amd64 + +02/09 22:12:52 [NOTICE] IPv4 BitTorrent: listening on TCP port 6923 + +02/09 22:12:52 [NOTICE] IPv6 BitTorrent: listening on TCP port 6923 +[#066a28 0B/2.8GiB(0%) CN:0 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:1 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:2 DL:0B] +[#066a28 16KiB/2.8GiB(0%) CN:44 SD:2 DL:11KiB ETA:72h44m54s] +[#066a28 32KiB/2.8GiB(0%) CN:44 SD:4 DL:11KiB ETA:72h55m47s] +[#066a28 48KiB/2.8GiB(0%) CN:44 SD:5 DL:10KiB ETA:79h7m17s] +[#066a28 64KiB/2.8GiB(0%) CN:49 SD:6 DL:10KiB ETA:77h27m] +[#066a28 96KiB/2.8GiB(0%) CN:47 SD:6 DL:13KiB ETA:60h20m41s] +[#066a28 160KiB/2.8GiB(0%) CN:45 SD:6 DL:20KiB ETA:41h20m43s] +[#066a28 304KiB/2.8GiB(0%) CN:45 SD:6 DL:33KiB ETA:24h28m29s] +[#066a28 384KiB/2.8GiB(0%) CN:45 SD:6 DL:38KiB ETA:21h34m21s] +[#066a28 416KiB/2.8GiB(0%) CN:44 SD:6 DL:37KiB ETA:21h54m17s] +[#066a28 512KiB/2.8GiB(0%) CN:44 SD:6 DL:42KiB ETA:19h26m19s] +[#066a28 544KiB/2.8GiB(0%) CN:44 SD:6 DL:41KiB ETA:19h48m32s] +[#066a28 624KiB/2.8GiB(0%) CN:44 SD:6 DL:44KiB ETA:18h38m8s] +[#066a28 720KiB/2.8GiB(0%) CN:44 SD:6 DL:52KiB ETA:15h42m33s] +[#066a28 736KiB/2.8GiB(0%) CN:44 SD:6 DL:50KiB ETA:16h30m42s] +[#066a28 768KiB/2.8GiB(0%) CN:44 SD:7 DL:52KiB ETA:15h42m44s] +[#066a28 896KiB/2.8GiB(0%) CN:44 SD:7 DL:57KiB ETA:14h20m4s] +[#066a28 1.0MiB/2.8GiB(0%) CN:44 SD:7 DL:69KiB ETA:11h58m54s] +[#066a28 1.1MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB ETA:10h21m22s] +[#066a28 1.2MiB/2.8GiB(0%) CN:44 SD:7 DL:87KiB ETA:9h24m15s] +[#066a28 1.4MiB/2.8GiB(0%) CN:44 SD:6 DL:90KiB ETA:9h8m26s] +[#066a28 1.4MiB/2.8GiB(0%) CN:44 SD:6 DL:88KiB ETA:9h23m40s] +[#066a28 1.5MiB/2.8GiB(0%) CN:44 SD:6 DL:85KiB ETA:9h41m21s] +[#066a28 1.6MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB ETA:9h12m17s] +[#066a28 1.6MiB/2.8GiB(0%) CN:44 SD:6 DL:86KiB ETA:9h32m9s] +[#066a28 1.7MiB/2.8GiB(0%) CN:44 SD:6 DL:85KiB UL:451KiB(288KiB) ETA:9h37m58s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB UL:168KiB(288KiB) ETA:9h16m53s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:84KiB UL:105KiB(288KiB) ETA:9h47m6s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB UL:76KiB(288KiB) ETA:9h16m44s] +[#066a28 1.9MiB/2.8GiB(0%) CN:44 SD:7 DL:85KiB UL:59KiB(288KiB) ETA:9h36m53s] +[#066a28 1.9MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB UL:49KiB(288KiB) ETA:10h24m56s] +[#066a28 2.1MiB/2.8GiB(0%) CN:44 SD:7 DL:80KiB UL:42KiB(288KiB) ETA:10h15m19s] +[#066a28 2.2MiB/2.8GiB(0%) CN:44 SD:7 DL:75KiB UL:68KiB(544KiB) ETA:10h56m] +[#066a28 2.3MiB/2.8GiB(0%) CN:44 SD:7 DL:75KiB UL:60KiB(544KiB) ETA:10h54m12s] +[#066a28 2.4MiB/2.8GiB(0%) CN:44 SD:7 DL:70KiB UL:54KiB(544KiB) ETA:11h39m51s] +[#066a28 2.4MiB/2.8GiB(0%) CN:44 SD:7 DL:71KiB UL:49KiB(544KiB) ETA:11h29m45s] +[#066a28 2.6MiB/2.8GiB(0%) CN:44 SD:7 DL:74KiB UL:44KiB(544KiB) ETA:11h1m53s] +[#066a28 2.6MiB/2.8GiB(0%) CN:44 SD:7 DL:73KiB UL:41KiB(544KiB) ETA:11h14m27s] +[#066a28 2.7MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:38KiB(544KiB) ETA:10h52m4s] +[#066a28 2.7MiB/2.8GiB(0%) CN:44 SD:7 DL:73KiB UL:19KiB(544KiB) ETA:11h18m54s] +[#066a28 2.8MiB/2.8GiB(0%) CN:44 SD:7 DL:78KiB UL:27KiB(544KiB) ETA:10h31m16s] +[#066a28 2.9MiB/2.8GiB(0%) CN:44 SD:7 DL:74KiB UL:25KiB(544KiB) ETA:11h7m40s] +[#066a28 2.9MiB/2.8GiB(0%) CN:44 SD:7 DL:72KiB UL:22KiB(544KiB) ETA:11h22m19s] +[#066a28 3.0MiB/2.8GiB(0%) CN:44 SD:7 DL:77KiB UL:41KiB(800KiB) ETA:10h42m53s] +[#066a28 3.0MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB UL:38KiB(800KiB) ETA:10h24m56s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:82KiB UL:35KiB(800KiB) ETA:10h4m14s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:72KiB(800KiB) ETA:10h44m14s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:69KiB UL:113KiB(1.0MiB) ETA:11h48m20s] +[#066a28 3.3MiB/2.8GiB(0%) CN:44 SD:8 DL:69KiB UL:92KiB(1.0MiB) ETA:11h54m37s] +[#066a28 3.4MiB/2.8GiB(0%) CN:44 SD:8 DL:73KiB UL:78KiB(1.0MiB) ETA:11h17m38s] +[#066a28 3.6MiB/2.8GiB(0%) CN:44 SD:8 DL:78KiB UL:67KiB(1.0MiB) ETA:10h27m51s] +[#066a28 3.7MiB/2.8GiB(0%) CN:44 SD:9 DL:76KiB UL:59KiB(1.0MiB) ETA:10h46m10s] +[#066a28 3.7MiB/2.8GiB(0%) CN:44 SD:9 DL:72KiB UL:53KiB(1.0MiB) ETA:11h21m24s] +[#066a28 3.8MiB/2.8GiB(0%) CN:44 SD:10 DL:75KiB UL:72KiB(1.2MiB) ETA:10h55m4s] +[#066a28 3.9MiB/2.8GiB(0%) CN:44 SD:10 DL:74KiB UL:66KiB(1.2MiB) ETA:11h9m6s] +[#066a28 4.0MiB/2.8GiB(0%) CN:44 SD:10 DL:80KiB UL:62KiB(1.3MiB) ETA:10h12m8s] +[#066a28 4.1MiB/2.8GiB(0%) CN:44 SD:10 DL:84KiB UL:58KiB(1.3MiB) ETA:9h45m9s] +[#066a28 4.2MiB/2.8GiB(0%) CN:44 SD:9 DL:88KiB UL:54KiB(1.3MiB) ETA:9h17m20s] +[#066a28 4.2MiB/2.8GiB(0%) CN:44 SD:9 DL:83KiB UL:46KiB(1.3MiB) ETA:9h53m47s] +[#066a28 4.3MiB/2.8GiB(0%) CN:44 SD:8 DL:75KiB UL:42KiB(1.3MiB) ETA:10h55m25s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:8 DL:82KiB UL:39KiB(1.3MiB) ETA:10h3m52s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:80KiB UL:36KiB(1.3MiB) ETA:10h18m43s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:53KiB(1.5MiB) ETA:10h49m36s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:66KiB UL:48KiB(1.5MiB) ETA:12h22m22s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:7 DL:57KiB UL:43KiB(1.5MiB) ETA:14h27m44s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:7 DL:55KiB UL:39KiB(1.5MiB) ETA:14h52m9s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:8 DL:52KiB UL:37KiB(1.5MiB) ETA:15h48m10s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:8 DL:50KiB UL:21KiB(1.5MiB) ETA:16h13m54s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:9 DL:42KiB UL:30KiB(1.5MiB) ETA:19h24m47s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:9 DL:41KiB UL:53KiB(1.8MiB) ETA:19h44m57s] +[#066a28 4.7MiB/2.8GiB(0%) CN:44 SD:9 DL:32KiB UL:48KiB(1.8MiB) ETA:25h20m56s] +[#066a28 4.7MiB/2.8GiB(0%) CN:44 SD:10 DL:38KiB UL:44KiB(1.8MiB) ETA:21h24m33s] +[#066a28 4.8MiB/2.8GiB(0%) CN:44 SD:10 DL:34KiB UL:40KiB(1.8MiB) ETA:23h45m37s] +[#066a28 4.8MiB/2.8GiB(0%) CN:48 SD:10 DL:31KiB UL:56KiB(2.0MiB) ETA:26h11m56s] +[#066a28 4.8MiB/2.8GiB(0%) CN:46 SD:10 DL:30KiB UL:52KiB(2.0MiB) ETA:27h15m40s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:32KiB UL:75KiB(2.0MiB) ETA:25h13m45s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:36KiB UL:66KiB(2.0MiB) ETA:22h32m9s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:35KiB UL:61KiB(2.0MiB) ETA:23h9m10s] +[#066a28 5.0MiB/2.8GiB(0%) CN:44 SD:9 DL:36KiB UL:80KiB(2.3MiB) ETA:22h43m56s] +[#066a28 5.0MiB/2.8GiB(0%) CN:44 SD:9 DL:35KiB UL:69KiB(2.3MiB) ETA:23h6m29s] +[#066a28 5.1MiB/2.8GiB(0%) CN:44 SD:10 DL:37KiB UL:62KiB(2.3MiB) ETA:21h58m37s] +[#066a28 5.1MiB/2.8GiB(0%) CN:44 SD:10 DL:38KiB UL:58KiB(2.3MiB) ETA:21h19m1s] +[#066a28 5.1MiB/2.8GiB(0%) CN:49 SD:12 DL:35KiB UL:55KiB(2.3MiB) ETA:23h3s] +[#066a28 5.2MiB/2.8GiB(0%) CN:48 SD:11 DL:38KiB UL:48KiB(2.3MiB) ETA:21h41m12s] +[#066a28 5.2MiB/2.8GiB(0%) CN:45 SD:11 DL:32KiB UL:64KiB(2.6MiB) ETA:25h8m51s] +[#066a28 5.2MiB/2.8GiB(0%) CN:44 SD:12 DL:32KiB UL:59KiB(2.6MiB) ETA:25h17m48s] +^C +anarcat@marcos:iso(master +%)$ aria2c kali-linux-1.1.0-amd64.torrent + +02/09 22:17:16 [NOTICE] IPv4 DHT: listening on UDP port 6963 + +02/09 22:17:16 [NOTICE] IPv4 BitTorrent: listening on TCP port 6950 + +02/09 22:17:16 [NOTICE] IPv6 BitTorrent: listening on TCP port 6950 +[#51d0f6 5.2MiB/2.8GiB(0%) CN:44 SD:12 DL:241KiB ETA:3h25m7s] + +# End of transcript or log. +"""]] + +Could be caused by [[bittorrent_special_url_double-encoding]]? --[[anarcat]] diff --git a/doc/bugs/assistant_windows_does_not_start.mdwn b/doc/bugs/assistant_windows_does_not_start.mdwn --- a/doc/bugs/assistant_windows_does_not_start.mdwn +++ b/doc/bugs/assistant_windows_does_not_start.mdwn @@ -33,3 +33,5 @@ # End of transcript or log. """]] + +> [[done]]; never heard back, so I assume my comment is right. --[[Joey]] diff --git a/doc/bugs/bencode.mdwn b/doc/bugs/bencode.mdwn --- a/doc/bugs/bencode.mdwn +++ b/doc/bugs/bencode.mdwn @@ -26,3 +26,7 @@ # End of transcript or log. """]] + +> Closing, since it's been over a month with no followup to my question, +> and I cannot find any license problems at all with bencode. --[[Joey]] +> [[done]] diff --git a/doc/bugs/bittorrent_special_url_double-encoding.mdwn b/doc/bugs/bittorrent_special_url_double-encoding.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/bittorrent_special_url_double-encoding.mdwn @@ -0,0 +1,177 @@ +[[!meta title="mojibake in aria2c output"]] + +### Please describe the problem. + +French messages in the `aria2c` client called from `git-annex` are garbled, probably double utf-8 encoded. + +### What steps will reproduce the problem? + +download a bittorrent url with git-torrent using a UTF8 locale. + +### What version of git-annex are you using? On what operating system? + +`5.20150205-gbf9058a` on current debian jessie. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +anarcat@marcos:iso(master)$ /srv/downloads.kitenet.net/git-annex/linux/current/git-annex.linux/git-annex addurl http://images.kali.org/kali-linux-1.1.0-amd64-mini.torrent +(downloading torrent file...) +--2015-02-09 22:08:58-- http://images.kali.org/kali-linux-1.1.0-amd64-mini.torrent +Résolution de images.kali.org (images.kali.org)… 50.7.37.130 +Connexion à images.kali.org (images.kali.org)|50.7.37.130|:80… connecté. +requête HTTP transmise, en attente de la réponse… 200 OK +Taille : 2668 (2,6K) [application/octet-stream] +Sauvegarde en : « ../.git/annex/misctmp/torrent31107 » + +100%[=====================================================================================================================================================================================================>] 2 668 --.-K/s ds 0,002s + +2015-02-09 22:08:59 (1,56 MB/s) — « ../.git/annex/misctmp/torrent31107 » sauvegardé [2668/2668] + +addurl images.kali.org_kali_linux_1.1.0_amd64_mini.torrent/kali_linux_1.1.0_amd64_mini.iso (from bittorrent) + + +02/09 22:08:59 [ERROR] Exception caught while loading DHT routing table from /home/anarcat/.aria2/dht.dat +Exception: [DHTRoutingTableDeserializer.cc:83] errorCode=1 Failed to load DHT routing table from /home/anarcat/.aria2/dht.dat + +02/09 22:08:59 [NOTICE] IPv4 DHT: listening on UDP port 6912 + +02/09 22:08:59 [ERROR] Erreur d'intégrité détectée. fichier=../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini + +02/09 22:08:59 [NOTICE] IPv4 BitTorrent: listening on TCP port 6910 + +02/09 22:08:59 [NOTICE] IPv6 BitTorrent: listening on TCP port 6910 +[#9acd0a 0B/27MiB(0%) CN:0 SD:0 DL:0B] +[#9acd0a 0B/27MiB(0%) CN:29 SD:0 DL:0B] +[#9acd0a 0B/27MiB(0%) CN:29 SD:1 DL:0B] +[#9acd0a 0B/27MiB(0%) CN:27 SD:5 DL:0B] +[#9acd0a 464KiB/27MiB(1%) CN:27 SD:6 DL:612KiB ETA:44s] +[#9acd0a 1.0MiB/27MiB(3%) CN:27 SD:6 DL:604KiB ETA:44s] +[#9acd0a 1.5MiB/27MiB(5%) CN:27 SD:7 DL:583KiB ETA:44s] +[#9acd0a 2.2MiB/27MiB(8%) CN:26 SD:8 DL:600KiB ETA:42s] +[#9acd0a 2.7MiB/27MiB(10%) CN:26 SD:10 DL:596KiB ETA:41s] +[#9acd0a 3.2MiB/27MiB(12%) CN:26 SD:11 DL:579KiB ETA:41s] +[#9acd0a 3.8MiB/27MiB(14%) CN:27 SD:11 DL:584KiB ETA:40s] +[#9acd0a 4.4MiB/27MiB(16%) CN:27 SD:11 DL:584KiB ETA:39s] +[#9acd0a 5.0MiB/27MiB(18%) CN:26 SD:11 DL:589KiB ETA:38s] +[#9acd0a 5.5MiB/27MiB(20%) CN:26 SD:11 DL:590KiB ETA:37s] +[#9acd0a 6.2MiB/27MiB(23%) CN:26 SD:11 DL:595KiB ETA:35s] +[#9acd0a 6.8MiB/27MiB(25%) CN:26 SD:11 DL:596KiB ETA:34s] +[#9acd0a 7.3MiB/27MiB(27%) CN:24 SD:9 DL:594KiB ETA:33s] +[#9acd0a 7.9MiB/27MiB(29%) CN:24 SD:9 DL:594KiB ETA:32s] +[#9acd0a 8.4MiB/27MiB(31%) CN:24 SD:10 DL:591KiB ETA:32s] +[#9acd0a 9.1MiB/27MiB(33%) CN:24 SD:10 DL:595KiB ETA:30s] +[#9acd0a 9.7MiB/27MiB(35%) CN:24 SD:10 DL:596KiB ETA:29s] +[#9acd0a 10MiB/27MiB(37%) CN:22 SD:9 DL:589KiB ETA:29s] +[#9acd0a 10MiB/27MiB(39%) CN:10 SD:8 DL:588KiB ETA:28s] +[#9acd0a 11MiB/27MiB(41%) CN:9 SD:8 DL:583KiB ETA:27s] +[#9acd0a 11MiB/27MiB(43%) CN:9 SD:8 DL:587KiB ETA:26s] +[#9acd0a 12MiB/27MiB(45%) CN:8 SD:7 DL:586KiB ETA:25s] +[#9acd0a 12MiB/27MiB(48%) CN:8 SD:7 DL:584KiB ETA:24s] +[#9acd0a 13MiB/27MiB(50%) CN:8 SD:8 DL:585KiB ETA:23s] +[#9acd0a 14MiB/27MiB(52%) CN:8 SD:8 DL:574KiB ETA:23s] +[#9acd0a 14MiB/27MiB(53%) CN:8 SD:8 DL:567KiB ETA:22s] +[#9acd0a 14MiB/27MiB(55%) CN:8 SD:8 DL:556KiB ETA:22s] +[#9acd0a 15MiB/27MiB(57%) CN:8 SD:8 DL:555KiB ETA:21s] +[#9acd0a 15MiB/27MiB(59%) CN:8 SD:8 DL:550KiB ETA:20s] +[#9acd0a 16MiB/27MiB(60%) CN:8 SD:8 DL:543KiB ETA:19s] +[#9acd0a 16MiB/27MiB(62%) CN:8 SD:8 DL:540KiB ETA:18s] +[#9acd0a 17MiB/27MiB(64%) CN:8 SD:8 DL:537KiB ETA:18s] +[#9acd0a 18MiB/27MiB(66%) CN:8 SD:8 DL:535KiB ETA:17s] +[#9acd0a 18MiB/27MiB(68%) CN:8 SD:8 DL:525KiB ETA:16s] +[#9acd0a 18MiB/27MiB(70%) CN:8 SD:8 DL:522KiB ETA:15s] +[#9acd0a 19MiB/27MiB(72%) CN:8 SD:8 DL:522KiB ETA:14s] +[#9acd0a 19MiB/27MiB(74%) CN:8 SD:8 DL:523KiB ETA:13s] +[#9acd0a 20MiB/27MiB(76%) CN:8 SD:8 DL:517KiB ETA:12s] +[#9acd0a 21MiB/27MiB(78%) CN:8 SD:8 DL:522KiB ETA:11s] +[#9acd0a 21MiB/27MiB(80%) CN:13 SD:8 DL:532KiB ETA:10s] +[#9acd0a 22MiB/27MiB(82%) CN:13 SD:8 DL:541KiB ETA:8s] +[#9acd0a 22MiB/27MiB(84%) CN:13 SD:8 DL:544KiB ETA:7s] +[#9acd0a 23MiB/27MiB(86%) CN:13 SD:8 DL:552KiB ETA:6s] +[#9acd0a 23MiB/27MiB(88%) CN:13 SD:8 DL:549KiB ETA:5s] +[#9acd0a 24MiB/27MiB(91%) CN:13 SD:8 DL:558KiB ETA:4s] +[#9acd0a 25MiB/27MiB(93%) CN:13 SD:8 DL:570KiB ETA:3s] +[#9acd0a 25MiB/27MiB(95%) CN:13 SD:8 DL:576KiB ETA:2s] +[#9acd0a 26MiB/27MiB(97%) CN:13 SD:8 DL:587KiB ETA:1s] +[#9acd0a 26MiB/27MiB(99%) CN:13 SD:8 DL:592KiB] + +02/09 22:09:53 [NOTICE] Le téléchargement des fichiers sélectionnés est terminé. + +02/09 22:09:53 [NOTICE] Le partage (seeding) est terminé +[#9acd0a SEED(0.0) CN:9 SD:4] +[#9acd0a SEED(0.0) CN:0 SD:0] + +02/09 22:09:55 [NOTICE] Téléchargement terminé: ../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini + +02/09 22:09:55 [NOTICE] Votre ratio de partage était de 0.0, envoyé/téléchargé=0B/27MiB + + +Résultats du téléchargement: +gid |stat|avg speed |path/URI +======+====+===========+======================================================= +9acd0a|OK | 518KiB/s|../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini/kali-linux-1.1.0-amd64-mini.iso + +Légende du statut: +(OK): téléchargement terminé. +ok +addurl images.kali.org_kali_linux_1.1.0_amd64_mini.torrent/kali_linux_1.1.0_amd64_mini.txt.sha1sum (from bittorrent) + + +02/09 22:09:56 [NOTICE] IPv4 DHT: listening on UDP port 6990 + +02/09 22:09:56 [NOTICE] Le fichier de contrôle défectueux ../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini.aria2 a été supprimé car le fichier téléchargé ../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini n'existe pas. + +02/09 22:09:56 [ERROR] Erreur d'intégrité détectée. fichier=../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini + +02/09 22:09:56 [NOTICE] IPv4 BitTorrent: listening on TCP port 6899 + +02/09 22:09:56 [NOTICE] IPv6 BitTorrent: listening on TCP port 6899 +[#5f49f7 0B/75B(0%) CN:0 SD:0 DL:0B] +[#5f49f7 0B/75B(0%) CN:28 SD:0 DL:0B] +[#5f49f7 0B/75B(0%) CN:28 SD:1 DL:0B] + +02/09 22:10:00 [NOTICE] Le téléchargement des fichiers sélectionnés est terminé. + +02/09 22:10:00 [NOTICE] Le partage (seeding) est terminé +[#5f49f7 SEED(0.0) CN:22 SD:3] +[#5f49f7 SEED(0.0) CN:0 SD:0] + +02/09 22:10:02 [NOTICE] Téléchargement terminé: ../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini + +02/09 22:10:02 [NOTICE] Votre ratio de partage était de 0.0, envoyé/téléchargé=0B/75B + + +Résultats du téléchargement: +gid |stat|avg speed |path/URI +======+====+===========+======================================================= +5f49f7|OK | 19B/s|../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64-mini.torrent/kali-linux-1.1.0-amd64-mini/kali-linux-1.1.0-amd64-mini.txt.sha1sum + +Légende du statut: +(OK): téléchargement terminé. +ok +(recording state in git...) +anarcat@marcos:iso(master +)$ locale +LANG=fr_CA.UTF-8 +LANGUAGE= +LC_CTYPE="fr_CA.UTF-8" +LC_NUMERIC="fr_CA.UTF-8" +LC_TIME="fr_CA.UTF-8" +LC_COLLATE="fr_CA.UTF-8" +LC_MONETARY="fr_CA.UTF-8" +LC_MESSAGES="fr_CA.UTF-8" +LC_PAPER="fr_CA.UTF-8" +LC_NAME="fr_CA.UTF-8" +LC_ADDRESS="fr_CA.UTF-8" +LC_TELEPHONE="fr_CA.UTF-8" +LC_MEASUREMENT="fr_CA.UTF-8" +LC_IDENTIFICATION="fr_CA.UTF-8" +LC_ALL= +# End of transcript or log. +"""]] + +Previous similar UTF-8 bug: [[forget_corrupts_non-ascii_chars]]. Looks similar. --[[anarcat]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/box.com.mdwn b/doc/bugs/box.com.mdwn --- a/doc/bugs/box.com.mdwn +++ b/doc/bugs/box.com.mdwn @@ -31,3 +31,5 @@ # End of transcript or log. """]] + +> [[done]]; fixed in [[!commit d84eab8a8af0c8cfa85de5baabfd9a2cf306f968]] diff --git a/doc/bugs/fsck_of_special_remotes_incorrectly_reports_ok.mdwn b/doc/bugs/fsck_of_special_remotes_incorrectly_reports_ok.mdwn --- a/doc/bugs/fsck_of_special_remotes_incorrectly_reports_ok.mdwn +++ b/doc/bugs/fsck_of_special_remotes_incorrectly_reports_ok.mdwn @@ -33,3 +33,4 @@ built using Homebrew (i.e. in a cabal sandbox) on OS X Yosemite 10.10.1 +> [[fixed|done]] as described in comment --[[Joey]] diff --git a/doc/bugs/git-annex_on_NAS_eats_all_memory.mdwn b/doc/bugs/git-annex_on_NAS_eats_all_memory.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git-annex_on_NAS_eats_all_memory.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. + +"Git annex sync" stopped working when syncing my NAS to my Laptop, it just eats up all the memory on the NAS without any result. When running the debug mode I found out that it stops working when it tries to run "git-annex-shell 'configlist' 'mygitannexfolder'". +That's the first command to be run on the NAS over ssh. + +I traced the problem down and the following command also won't work without any output just eating up all memory: +ssh myuser@mynas "git-annex-shell 'configlist' 'mygitannexfolder'" + +But when I log in to the NAS and run the git-annex-shell script inside the .ssh folder with exactly the same command, it works perfectly and returns the uuid and so on. + +### What steps will reproduce the problem? +Execute on PC (doesn't work, just uses all memory and crashes): ssh myuser@mynas "git-annex-shell 'configlist' 'mygitannexfolder'" +Execute on NAS (should work): ./.ssh/git-annex-shell "git-annex-shell 'configlist' 'mygitannexfolder'" + + +### What version of git-annex are you using? On what operating system? +git-annex version: 5.20140412ubuntu1 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 + +and the current bundle on the NAS + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn --- a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn +++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn @@ -25,3 +25,5 @@ WebApp crashed: : hPutStr: illegal operation (handle is closed """]] + +[[!meta title="assistant does not support nonstandard --git-dir"]] diff --git a/doc/bugs/glacier_fails_to_copy.mdwn b/doc/bugs/glacier_fails_to_copy.mdwn --- a/doc/bugs/glacier_fails_to_copy.mdwn +++ b/doc/bugs/glacier_fails_to_copy.mdwn @@ -80,3 +80,6 @@ # End of transcript or log. """]] + +> git-annex will now detect this misconfiguration and refuse to use it, so +> [[done]] --[[Joey]] diff --git a/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path.mdwn b/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path.mdwn --- a/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path.mdwn +++ b/doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path.mdwn @@ -14,3 +14,5 @@ git-annex version: 5.20141016-g26b38fd on Arch Linux git-annex version: 5.20140717 on Ubuntu 14.10 + +> [[done]]; see comment diff --git a/doc/bugs/main_repo_not_available_on_downloads.kitenet.net.mdwn b/doc/bugs/main_repo_not_available_on_downloads.kitenet.net.mdwn --- a/doc/bugs/main_repo_not_available_on_downloads.kitenet.net.mdwn +++ b/doc/bugs/main_repo_not_available_on_downloads.kitenet.net.mdwn @@ -22,4 +22,13 @@ Thanks! -- [[anarcat]] -> [[done]]; apparently some pebak. --[[Joey]] +> done; apparently some pebak. --[[Joey]] + +> > it's baaaack! :) -- [[anarcat]] + +>>> Please don't reuse old bug reports for unrelated issues. It muddies the +>>> waters. +>>> +>>> I've fixed git-update-server-info hook on the repository, fixing this +>>> second problem. +>>> [[done]] --[[Joey]] diff --git a/doc/bugs/rsync_remote_is_not_working.mdwn b/doc/bugs/rsync_remote_is_not_working.mdwn --- a/doc/bugs/rsync_remote_is_not_working.mdwn +++ b/doc/bugs/rsync_remote_is_not_working.mdwn @@ -24,3 +24,5 @@ * Kill the webapp process, re-run webapp, add remote again, it worked very quickly * But future interaction with the remote still requires password, both commandline & webapp +> Closing this bug report since it seems it was largely due to a +> misunderstanding of what "small archive" does. [[done]] --[[Joey]] diff --git a/doc/bugs/ssh-options_seems_to_be_ignored.mdwn b/doc/bugs/ssh-options_seems_to_be_ignored.mdwn --- a/doc/bugs/ssh-options_seems_to_be_ignored.mdwn +++ b/doc/bugs/ssh-options_seems_to_be_ignored.mdwn @@ -43,3 +43,6 @@ # End of transcript or log. """]] + +> [[fixed|done]], ssh-options is now propigated everywhere that ssh +> connection caching goes --[[Joey]] diff --git a/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. +`git-annex sync gcryptremote` makes too many ssh connections one after another for each sub-task involved, potentially triggering firewall rate-limits on the SSH server. + +### What steps will reproduce the problem? +sync with gcrypt remote while watching sshd logs on the server (I was getting >=5 connections per single sync in quick succession) + +### What version of git-annex are you using? On what operating system? +Fedora 19, installed through cabal (without s3 and webapp support as the deps were failing to build) + + $ git annex version + git-annex version: 5.20150205 + build flags: Assistant Pairing WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/webapp_does_not_start_on_android.mdwn b/doc/bugs/webapp_does_not_start_on_android.mdwn --- a/doc/bugs/webapp_does_not_start_on_android.mdwn +++ b/doc/bugs/webapp_does_not_start_on_android.mdwn @@ -18,3 +18,5 @@ Using 4.3/4.4 daily build (nov 1st 2014) apk. CyanogemMod 11 (M10) + +> Seems to be a dup of [[Android_Default_startup_command]]; [[done]] --[[Joey]] diff --git a/doc/bugs/weird_entry_in_process_list.mdwn b/doc/bugs/weird_entry_in_process_list.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/weird_entry_in_process_list.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +The standalone linux binaries do not show up as `git-annex` in the process list, but as `ld-linux-x86-64` - it's pretty confusing! + +### What steps will reproduce the problem? + +Install the standalone binaries from downloads.kitenet.net, run git-annex. + +### What version of git-annex are you using? On what operating system? + +Today's snapshot from downloads.k.n. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +root@koumbit-mp-test:/var/isuma/media/video# top -b -n 1 | head -10 +top - 14:00:09 up 15 days, 23:25, 4 users, load average: 1.18, 1.26, 1.34 +Tasks: 216 total, 1 running, 213 sleeping, 0 stopped, 2 zombie +Cpu(s): 0.4%us, 0.1%sy, 0.0%ni, 99.3%id, 0.2%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 6122044k total, 5469364k used, 652680k free, 321080k buffers +Swap: 2928632k total, 0k used, 2928632k free, 4009592k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +28261 root 20 0 4528 652 528 D 79 0.0 0:01.28 ld-linux-x86-64 + 1381 root 20 0 126m 13m 4060 S 2 0.2 190:25.64 Xorg + 1 root 20 0 8356 812 684 S 0 0.0 0:05.50 init +root@koumbit-mp-test:/var/isuma/media/video# ps axf | grep annex + 9861 pts/2 S+ 0:00 | \_ git annex add hd high high~ ipod ipod~ large low mp4_sd raw small wc xlarge + 9862 pts/2 Sl+ 3:50 | \_ /opt/git-annex.linux//lib64/ld-linux-x86-64.so.2 --library-path /opt/git-annex.linux//etc/ld.so.conf.d:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex.linux//usr/lib:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu:/opt/git-annex.linux//lib64:/opt/git-annex.linux//lib/x86_64-linux-gnu: /opt/git-annex.linux/shimmed/git-annex/git-annex add hd high high~ ipod ipod~ large low mp4_sd raw small wc xlarge + 9878 pts/2 S+ 0:00 | \_ /opt/git-annex.linux//lib64/ld-linux-x86-64.so.2 --library-path /opt/git-annex.linux//etc/ld.so.conf.d:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex.linux//usr/lib:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu:/opt/git-annex.linux//lib64:/opt/git-annex.linux//lib/x86_64-linux-gnu: /opt/git-annex.linux/shimmed/git/git --git-dir=.git --work-tree=. check-attr -z --stdin annex.backend annex.numcopies -- + 9881 pts/2 S+ 0:01 | \_ /opt/git-annex.linux//lib64/ld-linux-x86-64.so.2 --library-path /opt/git-annex.linux//etc/ld.so.conf.d:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex.linux//usr/lib:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu:/opt/git-annex.linux//lib64:/opt/git-annex.linux//lib/x86_64-linux-gnu: /opt/git-annex.linux/shimmed/git/git --git-dir=.git --work-tree=. cat-file --batch + 9882 pts/2 S+ 0:00 | \_ /opt/git-annex.linux//lib64/ld-linux-x86-64.so.2 --library-path /opt/git-annex.linux//etc/ld.so.conf.d:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex.linux//usr/lib:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu:/opt/git-annex.linux//lib64:/opt/git-annex.linux//lib/x86_64-linux-gnu: /opt/git-annex.linux/shimmed/git/git --git-dir=.git --work-tree=. cat-file --batch +28293 pts/2 R+ 0:00 | \_ /opt/git-annex.linux//lib64/ld-linux-x86-64.so.2 --library-path /opt/git-annex.linux//etc/ld.so.conf.d:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex.linux//usr/lib:/opt/git-annex.linux//usr/lib/x86_64-linux-gnu:/opt/git-annex.linux//lib64:/opt/git-annex.linux//lib/x86_64-linux-gnu: /opt/git-annex.linux/shimmed/sha256sum/sha256sum .git/annex/misctmp/videonew9862 +# End of transcript or log. +"""]] + +couldn't it alter its process name to make this a little more intuitive? This is especially problematic because i am trying to hook git-annex into Puppet and Facter, which require me to guess where the various git-annex repos are on the server. The way i was doing that so far was with `lsof -c 'git-annex' -F0tn`, which is obviously failing under those circumstances.... Unless there's a better way to find those repos across the system? I assume there's a git-annex assistant running here... --[[anarcat]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 10 "Box.com (done)" 74 "My phone (or MP3 player)" 25 "Tahoe-LAFS" 13 "OpenStack SWIFT" 35 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 10 "Box.com (done)" 74 "My phone (or MP3 player)" 25 "Tahoe-LAFS" 14 "OpenStack SWIFT" 36 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/design/caching_database.mdwn b/doc/design/caching_database.mdwn --- a/doc/design/caching_database.mdwn +++ b/doc/design/caching_database.mdwn @@ -25,11 +25,11 @@ ## implementation plan -1. Implement for metadata, on a branch, with sqlite. +1. Store incremental fsck info in db, on a branch, with sqlite. 2. Make sure that builds on all platforms. -3. Add associated file mappings support. This is needed to fully +3. Implement for metadata, on a branch, with sqlite. +4. Add associated file mappings support. This is needed to fully use the caching database to construct views. -4. Store incremental fsck info in db. 5. Replace .map files with 3. for direct mode. ## sqlite or not? @@ -39,20 +39,21 @@ to involve some technical debt (eg, database migrations). It would be great if there were some haskell thing like acid-state -that I could use instead. But, acid-sate needs to load the whole +that I could use instead. But, acid-state needs to load the whole DB into memory. In the comments of [[bugs/incremental_fsck_should_not_use_sticky_bit]] I examined several other haskell database-like things, and found them all wanting, except for -possibly TCache. +possibly TCache. (And TCache is backed by persistent/sqlite anyway.) -TODO: This seems promising; investigate it: - -It uses LMDB, which is a C library, and its PVar is a variable named by a -bytestring, so it's essentially a key/value store where the values can be -arbitrary Haskell data types. Since git-annex already has Keys, and most -of the need for the database is to look up some cached value for a Key, -this seems like a pretty good fit! +## one db or multiple? +Using a single database will use less space. Eg, each Key will only need to +appear in it once, with proper normalization. + +OTOH, it's more complicated, and harder to recover from problems. + +Currently leaning toward one database per purpose. + ## case study: persistent with sqllite Here's a non-normalized database schema in persistent's syntax. @@ -128,15 +129,19 @@ actually doing a join at the SQL level, so this could be sped up using eg, esquelito. -Update2: Using esquelito to do a join got this down to 0.250s. +Update2: Using esquelito to do a join got this down to 0.109s. +See `database` branch for code. -Code: +Update3: Converting to a single un-normalized table for AssociatedFiles +avoids the join, and increased lookup speed to 0.087s. Of course, when +a key has multiple associated files, this will use more disk space, due +to not normalizing the key. Compare the above with 1000 calls to `associatedFiles`, which is approximately as fast as just opening and reading 1000 files, so will take well under 0.05s with a **cold** cache. -So, we're looking at nearly an order of magnitude slowdown using sqlite and +So, we're looking at maybe 50% slowdown using sqlite and persistent for associated files. OTOH, the normalized schema should perform better when adding an associated file to a key that already has many. diff --git a/doc/design/roadmap.mdwn b/doc/design/roadmap.mdwn --- a/doc/design/roadmap.mdwn +++ b/doc/design/roadmap.mdwn @@ -1,23 +1,38 @@ -## roadmap +## ahead -Just finished the -[sustaining git-annex development](https://campaign.joeyh.name/) year -(starting September 2013). +* [[assistant/deltas]] +* [[assistant/gpgkeys]] +* [[assistant/telehash]] +* [[design/requests_routing]] +* [[design/v6]] -* Month 1 [[!traillink assistant/encrypted_git_remotes]] -* Month 2 [[!traillink assistant/disaster_recovery]] -* Month 3 [[!traillink direct_mode]] guard [[!traillink assistant/upgrading]] -* Month 4 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, external special remotes -* Month 5 user-driven features and polishing -* Month 6 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]] -* Month 7 user-driven features and polishing -* Month 8 [[!traillink git-remote-daemon]] -* Month 9 Brazil!, [[!traillink assistant/sshpassword]] -* Month 10 polish [[assistant/Windows]] port -* Month 11 [[!traillink assistant/chunks]] -* Month 12 user-driven features and polishing +## now -Deferred until later: +* Feb 2015 user-driven features and polishing, [[design/caching_database]] -* Month XX [[!traillink assistant/deltas]], [[!traillink assistant/gpgkeys]] -* Month XX [[!traillink assistant/telehash]] +## the rearview + +* Jan 2015 Android 5, relative paths, workload [[tuning]] +* Dec 2014 [[todo/extensible_addurl]], bittorrent special remote +* Nov 2014 direct mode proxy, undo command, diffdriver +* Oct 2014 user-driven features and polishing, S3 multipart +* Sep 2014 vacation + +2013-2014 [crowdfunded](https://campaign.joeyh.name/) year + +* Aug 2014 user-driven features and polishing +* Jul 2014 [[!traillink assistant/chunks]] +* Jun 2014 polish [[assistant/Windows]] port +* May 2014 Brazil!, [[!traillink assistant/sshpassword]] +* Apr 2014 [[!traillink git-remote-daemon]] +* Mar 2014 user-driven features and polishing +* Feb 2014 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]] +* Jan 2014 user-driven features and polishing +* Dec 2013 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, external special remotes +* Nov 2013 [[!traillink direct_mode]] guard [[!traillink assistant/upgrading]] +* Oct 2013 [[!traillink assistant/disaster_recovery]] +* Sep 2013 [[!traillink assistant/encrypted_git_remotes]] + +2012-2013 kickstarted year developing [[git-annex assistant|assistant]] + +2010-2011 initial git-annex development diff --git a/doc/devblog.mdwn b/doc/devblog.mdwn --- a/doc/devblog.mdwn +++ b/doc/devblog.mdwn @@ -1,5 +1,4 @@ -Work on git-annex is [crowdfunded](https://campaign.joeyh.name/). -Joey blogs about his progress here on a semi-daily basis. +Joey blogs about his work here on a semi-daily basis. [[!sidebar content=""" [[!calendar type="month" pages="page(devblog/*)"]] diff --git a/doc/devblog/day_249_onward.mdwn b/doc/devblog/day_249_onward.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_249_onward.mdwn @@ -0,0 +1,28 @@ +Made a release yesterday, and caught up on most recent messages +earlier this week. Backlog stands at 128 messages. + +Had to deal with an ugly problem with /usr/bin/glacier today. Seems that +there are multiple programs all using that name, some of them shipping in +some linux distributions, and the one from boto fails to fail when passed +parameters it doesn't understand. Yugh! I had to make git-annex probe to +make sure the right glacier program is installed. + +I'm planning to deprecate the glacier special remote at some point. +Instead, I'd like to make the S3 special remote support the S3-glacier +lifecycle, so objects can be uploaded to S3, set to transition to +glacier, and then if necessary pulled back from glacier to S3. That should +be much simpler and less prone to break. + +But not yet; [haskell-aws needs glacier support added](https://github.com/aristidb/aws/issues/81). +Or I could use the new amazonka library, but I'd rather stick with +haskell-aws. + +Some other minor improvements today included adding `git annex +groupwanted`, which makes for easier examples than using vicfg, and +making `git annex import` support options like --include and --exclude. + +Also I moved a many file matching options to only be accepted by +the commands that actually use them. Of the remaining common +options, most of them make sense for every command to accept (eg, --force +and --debug). It would make sense to move --backend, --notify-start/finish, +and perhaps --user-agent. Eventually. diff --git a/doc/devblog/day_250__backog_bugfixing.mdwn b/doc/devblog/day_250__backog_bugfixing.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_250__backog_bugfixing.mdwn @@ -0,0 +1,10 @@ +Plowing through the backlog today, and fixing quite a few bugs! Got the +backlog down to 87 messages from ~140. And some of the things I got to were +old and/or hard. + +About a third of the day was spent revisiting +[[bugs/git-annex_branch_shows_commit_with_looong_commitlog]]. +I still don't understand how that behavior can happen, but I have a +donated repository where it did happen. Made several changes to try to make +the problem less likely to occur, and not as annoying when it does occur, +and maybe get me more info if it does happen to someone again. diff --git a/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn b/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn @@ -0,0 +1,12 @@ +Many more little improvements made yesterday and part of today. While +it's only been a week since the last release, it feels almost time +to make another one, after so many recent bug fixes and small improvements. + +I've updated the [[design/roadmap]]. I have been operating without a +roadmap for half a year, and it would be nice to have some plans. +Keeping up with bug reports and requests as they come in is a fine mode +of work, but it can feel a little aimless. It's good to have a planned out +course, or at least some longer term goals. + +After the next release, I've penciled in the second half of this month to +work on the [[design/caching_database]]. diff --git a/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn b/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn @@ -0,0 +1,58 @@ +[[!meta title="day 254 sqlite for incremental fsck"]] + +Yesterday I did a little more investigation of key/value stores. +I'd love a pure haskell key/value store that didn't buffer everything in +memory, and that allowed concurrent readers, and was ACID, and production +quality. But so far, I have not found anything that meets all those +criteria. It seems that sqlite is the best choice for now. + +Started working on the `database` branch today. The plan is to use +sqlite for incremental fsck first, and if that works well, do the rest +of what's planned in [[design/caching_database]]. + +At least for now, I'm going to use a dedicated database file for each +different thing. (This may not be as space-efficient due to lacking +normalization, but it keeps things simple.) + +So, .git/annex/fsck.db will be used by incremental fsck, and it has +a super simple Persistent database schema: + +[[!format haskell """ +Fscked + key SKey + UniqueKey key +"""]] + +It was pretty easy to implement this and make incremental fsck use it. The +hard part is making it both fast and robust. + +At first, I was doing everything inside a single `runSqlite` action. +Including creating the table. But, it turns out that runs as a single +transaction, and if it was interrupted, this left the database in a +state where it exists, but has no tables. Hard to recover from. + +So, I separated out creating the database, made that be done in a separate +transation and fully atomically. Now `fsck --incremental` could be crtl-c'd +and resumed with `fsck --more`, but it would lose the transaction and so +not remember anything had been checked. + +To fix that, I tried making a separate transation per file fscked. That +worked, and it resumes nicely where it left off, but all those transactions +made it much slower. + +To fix the speed, I made it commit just one transaction per minute. This +seems like an ok balance. Having fsck re-do one minute's work when restarting +an interrupted incremental fsck is perfectly reasonable, and now the speed, +using the sqlite database, is nearly as fast as the old sticky bit hack was. +(Specifically, 6m7s old vs 6m27s new, fscking 37000 files from cold cache +in --fast mode.) + +There is still a problem with multiple concurrent `fsck --more` +failing. Probably a concurrent writer problem? And, some porting will be +required to get sqlite and persistent working on Windows and Android. +So the branch isn't ready to merge yet, but it seems promising. + +In retrospect, while incremental fsck has the simplest database schema, it +might be one of the harder things listed in [[design/caching_database]], +just because it involves so many writes to the database. The other use +cases are more read heavy. diff --git a/doc/devblog/day_253__ssh-options.mdwn b/doc/devblog/day_253__ssh-options.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_253__ssh-options.mdwn @@ -0,0 +1,5 @@ +Spent a couple hours to make the ssh-options git config setting be used +in more places. Now it's used everywhere that git-annex supports ssh +caching, including the `git pull` and `git push` done by `sync` and by the +assistant. Also the `remotedaemon` and the gcrypt, rsync, and ddar +special remotes. diff --git a/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn b/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn @@ -0,0 +1,34 @@ +Worked today on making incremental fsck's use of sqlite be safe with +multiple concurrent fsck processes. + +The first problem was that having `fsck --incremental` running and starting a +new `fsck --incremental` caused it to crash. And with good reason, since +starting a new incremental fsck deletes the old database, the old process +was left writing to a database that had been deleted and recreated out from +underneath it. Fixed with some locking. + +Next problem is harder. Sqlite doesn't support multiple concurrent writers +at all. One of them will fail to write. It's not even possible to have two +processes building up separate transactions at the same time. Before using +sqlite, incremental fsck could work perfectly well with multiple fsck +processes running concurrently. I'd like to keep that working. + +My partial solution, so far, is to make git-annex buffer writes, and every +so often send them all to sqlite at once, in a transaction. So most of the +time, nothing is writing to the database. (And if it gets unlucky and +a write fails due to a collision with another writer, it can just wait and +retry the write later.) This lets multiple processes write to the database +successfully. + +But, for the purposes of concurrent, incremental fsck, it's not ideal. +Each process doesn't immediately learn of files that another process has +checked. So they'll tend to do redundant work. Only way I can see to +improve this is to use some other mechanism for short-term IPC between the +fsck processes. + +---- + +Also, I made `git annex fsck --from remote --incremental` use a different +database per remote. This is a real improvement over the sticky bits; +multiple incremental fscks can be in progress at once, +checking different remotes. diff --git a/doc/devblog/day_256__sqlite_concurrency_argh.mdwn b/doc/devblog/day_256__sqlite_concurrency_argh.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_256__sqlite_concurrency_argh.mdwn @@ -0,0 +1,28 @@ +Breaking news: gitlab.com repositories now support git-annex! + +* [GitLab Annex solves the problem of versioning large binaries with git](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/) +* [freely licensed source code](https://gitlab.com/gitlab-org/gitlab-shell) + +A very nice surprise! More git hosters should do this.. + +---- + +Back to sqlite concurrency, I thought I had it dealt with, but more testing +today has turned up a lot more problems with sqlite and concurrent writers +(and readers). + +First, I noticed that a process can be happily writing changes to the +database, but if a second process starts reading from the database, this +will make the writier start failing with BUSY, and keep failing until the +second process goes idle. It turns out the solution to this is to use WAL +mode, which prevents readers from blocking writers. + +After several hours (persistent doesn't make it easy to enable WAL mode), +it seemed pretty robust with concurrent fsck. + +But then I saw SELECT fail with BUSY. I don't understand why a reader would +fail in WAL mode; that's counter to the documentation. My best guess is +that this happens when a checkpoint is being made. + +This seems to be a real bug in sqlite. It may only affect the older +versions bundled with persistent. diff --git a/doc/direct_mode.mdwn b/doc/direct_mode.mdwn --- a/doc/direct_mode.mdwn +++ b/doc/direct_mode.mdwn @@ -104,6 +104,17 @@ tree to reflect any changes staged or committed by the git command, with appropriate handling of the direct mode files. +## undoing changes in direct mode + +There is also the `undo` command to do the equivalent of the above revert +in a simpler way. Say you made a change in direct mode, the assistant +dutifully committed it and you realise your mistake, you can try: + + git annex undo file + +to revert the last change to `file`. Note that you can use the `--depth` +flag to revert earlier versions of the file. + ## forcing git to use the work tree in direct mode This is for experts only. You can lose data doing this, or check enormous diff --git a/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment b/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment new file mode 100644 --- /dev/null +++ b/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://id.koumbit.net/anarcat" + subject="comment 16" + date="2015-02-17T05:22:00Z" + content=""" +i believe this is [answered here](https://git-annex.branchable.com/todo/windows_support/#comment-e72601243c643d7821e68d3a04489fcb). TLDR; basically NTFS + symlink works in Linux, but not in Windows/Cygwin, which git-annex seems to be using. YMMV. +"""]] diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn @@ -0,0 +1,4 @@ + +I'm trying to find a way of syncing data across multiple platforms, including an android device. The latter I want to sync the data so that it's on the devices SD filesystem and independent of network. As far as I can tell git-annex's client is the only one that can do this, all other self hosted storage clients I've found are only network viewers (owncloud, seafile), which is no good to me. I want to sync my ABC sheet music collection onto it, viewable independent of network access, updating changes automatically. Other parts of the data set (i.e. large collection of cannon cr2 raws), I want synced between my laptop and desktop. + +I've set up git annex with a central git repo and another client directory which syncs fine to the server (set as a 'full backup'). Now I want to download this whole archive onto the android device. I've set up the client on the device and it works and will upload to the server, however I can't see any way of getting it to download. Surely git-annex has the capability to pull data? diff --git a/doc/forum/canceling_wrong_repository_merge.mdwn b/doc/forum/canceling_wrong_repository_merge.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/canceling_wrong_repository_merge.mdwn @@ -0,0 +1,3 @@ +so i mistakenly merged two unrelated repos together. i have "canceled" the merge as in reverted it by removing the created files, but then those zillion of small files will stick around the git repository forever. + +is there a way to use something like `git annex forget` for this? i know about things like [git rebase --onto](https://sethrobertson.github.io/GitFixUm/fixup.html#remove_deep) but that won't propagate across all repositories... can git-annex give me a hand here? --[[anarcat]] diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn @@ -0,0 +1,25 @@ +I am using git-annex to store the output of a build process that produces large binary files (~1GB). These files are built on a number of workers which then commit the files to a central server (git annex add && copy to origin) + +I am trying to delete some of the older builds but have been unable to see any disk space be freed on the server (Disk usage is identical before and after the delete). + + df -h + mcchicken-srv://repo 284G 114G 157G 43% /srv/repo + +To delete the files I have performed the following: + +* git clone +* git annex drop <1gb_file> +* git rm <1gb_file> +* git commit +* git annex sync +* [log into server] +* git annex sync +* git annex unused +* git annex dropunused + + +I have also tried other variations of the above technique that I have found online, but to no avail. + +In all cases the files appear to not be in the repo when a clone is performed, However the disk usage on the server never decreases. + +How would I delete the older builds in a manner that will free up disk space? diff --git a/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn @@ -0,0 +1,9 @@ +I have following usecase: + +I want to use a central repos for xbmc/kodi where I can play and delete files from kodi somethimes also with a random file manager. But I normaly if a file gets deleted and synced with git annex sync it deletes the file "head" in every repository. + +I would like if its only a git annex drop of the content, if thats was the last copy ok then its ok for me that its gone. But if its not the last copy it shhould just delete it from this repo but not delete the heads of the other repositories. + +I know that the actual file is still in the other repositories but the entry is gone, I would love if it would be more like a git annex drop instead of a git rm. + +Can I do that with setting this repos readonly or is my usecase not supported/doable with git annex? diff --git a/doc/forum/optimising_lookupkey.mdwn b/doc/forum/optimising_lookupkey.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/optimising_lookupkey.mdwn @@ -0,0 +1,13 @@ +to work around [[forum/original_filename_on_s3/]], i need to get the key from a file, and i'm not within the git-annex process. i know there's `git annex lookupkey $FILE`, but that incurs significant overhead because the whole git annex runtime needs to fire up. in my tests, this takes around 25ms on average. + +could i optimise this by simply doing a `readlink` call on the git checkout? it sure looks like `readlink | basename` is all I really need, and that can probably be done below 10ms (4ms in my tests). how reliable are those links anyways, and is that what lookupkey does? + +similarly, i wonder if it's safe to bypass git-annex and talk straight with git to extract location tracking? i can jump from 90ms to below 10ms for such requests if I turn `git annex find ` into the convoluted: + +
+git annex lookupkey $file
+printf $key | md5sum
+git cat-file -p refs/heads/git-annex:$hash/${key}.log
+
+ +thanks. --[[anarcat]] diff --git a/doc/forum/original_filename_on_s3.mdwn b/doc/forum/original_filename_on_s3.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/original_filename_on_s3.mdwn @@ -0,0 +1,5 @@ +I understand that [[special_remotes/S3/]] is seen as a "backend" storage mechanism, but since S3 files are available directly on the web, it would be nice to have the real filenames up there. + +Is there a way to set that up? I know about [[tips/publishing_your_files_to_the_public/]], but it assumes you have a local git repo with all the data in the first place, something which may not be available... + +my use case is that we have ~1TB of files already stored in S3 under specific filenames, and those filenames are how the files are accessed on the main website. changing all those filenames would be a significant burden... i'm not even sure this can be done cheaply on S3 in the first place. --[[anarcat]] diff --git a/doc/forum/root_assistant__63__.mdwn b/doc/forum/root_assistant__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/root_assistant__63__.mdwn @@ -0,0 +1,3 @@ +How safe (or not) is it to run the assistant as root? + +If not safe, what would be a good way to sync directories like /usr/local ? diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -272,7 +272,12 @@ (Note that using `--deduplicate` or `--clean-duplicates` with the WORM backend does not look at file content, but filename and mtime.) + + To control which files are imported, many of the MATCHING OPTIONS can + be used. + git annex import /dir --include='*.png' + * `importfeed [url ...]` Imports the contents of podcast feeds. Only downloads files whose @@ -504,6 +509,21 @@ Without an expression, displays the current preferred content setting of the repository. +* `groupwanted groupname [expression]` + + Sets or displays the groupwanted expression. This will be used by + repositories that are in the group, and that have their preferred + content expression set to "groupwanted". + + For example, to configure a group named redundantarchive, and + make repositories in the group want to contain 3 copies of every file: + + git annex groupwanted redundantarchive "not (copies=redundantarchive:3)" + for repo in foo bar baz; do + git annex group $repo redundantarchive + git annex wanted $repo groupwanted + done + * `schedule repository [expression]` When run with an expression, configures scheduled jobs to run at a @@ -709,7 +729,7 @@ When no item is specified, displays statistics and information for the repository as a whole. - When a directory is specified, the file matching options can be used + When a directory is specified, the MATCHING OPTIONS can be used to select the files in the directory that are included in the statistics. To only show the data that can be gathered quickly, use `--fast`. @@ -972,9 +992,6 @@ This is similar to the find command, but instead of finding files in the current work tree, it finds files in the specified git ref. - Most MATCHING OPTIONS can be used with findref, to limit the files it - finds. However, the --include and --exclude options will not work. - * `proxy -- git cmd [options]` Only useful in a direct mode repository, this runs the specified git @@ -1375,7 +1392,9 @@ When a repository is in one of the standard predefined groups, like "backup" and "client", setting its preferred content to "standard" will use a -built-in preferred content expression developed for that group. +built-in preferred content expression developed for that group. Or, +setting its preferred content to "groupwanted" will make it use whatever +groupwanted expression you set for the group. # SCHEDULED JOBS diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn --- a/doc/install/Windows.mdwn +++ b/doc/install/Windows.mdwn @@ -2,6 +2,7 @@ * First, [install git](http://git-scm.com/downloads) (msysgit 1.9 or newer is needed) _Be sure to tell the msysgit installer to add git to the PATH._ + That is, select "Use Git from the Windows Command Prompt" * Then, [install git-annex](https://downloads.kitenet.net/git-annex/windows/current/) This port is now in reasonably good shape for command-line use of diff --git a/doc/internals/hashing/comment_5_b0cb207a85cda5a0ff2ea71caca22c0d._comment b/doc/internals/hashing/comment_5_b0cb207a85cda5a0ff2ea71caca22c0d._comment new file mode 100644 --- /dev/null +++ b/doc/internals/hashing/comment_5_b0cb207a85cda5a0ff2ea71caca22c0d._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://id.koumbit.net/anarcat" + subject="why md5sum?" + date="2015-02-13T15:59:46Z" + content=""" +why the extra processing to generate the hashing directories? + +we already have a hash here, for example, `SHA256E-s8242375--5f82490990812ad3feabb02355750710a9d94283ab256d1c691c3bf8d7d9fbe3.ogg` has a loon `5f82490990812ad3feabb02355750710a9d94283ab256d1c691c3bf8d7d9fbe3` hash. Why not use the first characters of that? This is will not change for a give file, and has a higher chance of generating collisions (which is a good thing here, because we can reuse directories). + +In other words, why aren't the hashes of `SHA256E-s8242375--5f82490990812ad3feabb02355750710a9d94283ab256d1c691c3bf8d7d9fbe3.ogg` simply `5f8/249`? --[[anarcat]] +"""]] diff --git a/doc/internals/hashing/comment_6_edb5c3388b5ac3481403c7accf9bb3f2._comment b/doc/internals/hashing/comment_6_edb5c3388b5ac3481403c7accf9bb3f2._comment new file mode 100644 --- /dev/null +++ b/doc/internals/hashing/comment_6_edb5c3388b5ac3481403c7accf9bb3f2._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""re: why md5sum?""" + date="2015-02-17T21:51:59Z" + content=""" +Not all types of keys contain hashes. +"""]] diff --git a/doc/news/version_5.20141219.mdwn b/doc/news/version_5.20141219.mdwn deleted file mode 100644 --- a/doc/news/version_5.20141219.mdwn +++ /dev/null @@ -1,20 +0,0 @@ -git-annex 5.20141219 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Webapp: When adding a new box.com remote, use the new style chunking. - Thanks, Jon Ander Peñalba. - * External special remote protocol now includes commands for setting - and getting the urls associated with a key. - * Urls can now be claimed by remotes. This will allow creating, - for example, a external special remote that handles magnet: and - *.torrent urls. - * Use wget -q --show-progress for less verbose wget output, - when built with wget 1.16. - * Added bittorrent special remote. - * addurl behavior change: When downloading an url ending in .torrent, - it will download files from bittorrent, instead of the old behavior - of adding the torrent file to the repository. - * Added Recommends on aria2. - * When possible, build with the haskell torrent library for parsing - torrent files. As a fallback, can instead use btshowmetainfo from - bittornado | bittorrent. - * Fix build with -f-S3."""]] diff --git a/doc/news/version_5.20150219.mdwn b/doc/news/version_5.20150219.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20150219.mdwn @@ -0,0 +1,34 @@ +git-annex 5.20150219 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * glacier: Detect when the glacier command in PATH is the wrong one, + from boto, rather than from glacier-cli, and refuse to use it, + since the boto program fails to fail when passed + parameters it does not understand. + * groupwanted: New command to set the groupwanted preferred content + expression. + * import: Support file matching options such as --exclude, --include, + --smallerthan, --largerthan + * The file matching options are now only accepted by commands that + can actually use them, instead of by all commands. + * import: Avoid checksumming file twice when run in the default + or --duplicate mode. + * Windows: Fix bug in dropping an annexed file, which + caused a symlink to be staged that contained backslashes. + * webapp: Fix reversion in opening webapp when starting it manually + inside a repository. + * assistant: Improve sanity check for control characters when pairing. + * Improve race recovery code when committing to git-annex branch. + * addurl: Avoid crash if quvi is not installed, when git-annex was + built with process-1.2 + * bittorrent: Fix mojibake introduced in parsing arai2c progress output. + * fsck --from: If a download from a remote fails, propagate the failure. + * metadata: When setting metadata, do not recurse into directories by + default, since that can be surprising behavior and difficult to recover + from. The old behavior is available by using --force. + * sync, assistant: Include repository name in head branch commit message. + * The ssh-options git config is now used by gcrypt, rsync, and ddar + special remotes that use ssh as a transport. + * sync, assistant: Use the ssh-options git config when doing git pull + and push. + * remotedaemon: Use the ssh-options git config. + * Linux standalone: Improved process names of linker shimmed programs."""]] diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -125,8 +125,8 @@ The "groupwanted" keyword can be used to refer to a preferred content expression that is associated with a group. This is like the "standard" -keyword, but you can set up groupwanted preferred content expressions -using `git annex vicfg`. +keyword, but you can configure the preferred content expressions +using `git annex groupwanted`. Note that when writing a groupwanted preferred content expression, you can use all of the keywords listed above, including "standard". @@ -134,7 +134,8 @@ For example, to make a variant of the standard client preferred content expression that does not want files in the "out" directory, you -could set `groupwanted client = standard and exclude=out/*`. +could run: `git annex groupwanted client "standard and exclude=out/*"` + Then repositories that are in the client group and have their preferred content expression set to "groupwanted" will use that, while other client repositories that have their preferred content expression @@ -143,6 +144,17 @@ Or, you could make a new group, with your own custom preferred content expression tuned for your needs, and every repository you put in this group and make its preferred content be "groupwanted" will use it. + +For example, the archive group only wants to archive 1 copy of each file, +spread amoung every repository in the group. +Here's how to configure a group named redundantarchive, that instead +wants to contain 3 copies of each file: + + git annex groupwanted redundantarchive "not (copies=redundantarchive:3)" + for repo in foo bar baz; do + git annex group $repo redundantarchive + git annex wanted $repo groupwanted + done ### difference: metadata matching diff --git a/doc/preferred_content/standard_groups.mdwn b/doc/preferred_content/standard_groups.mdwn --- a/doc/preferred_content/standard_groups.mdwn +++ b/doc/preferred_content/standard_groups.mdwn @@ -72,7 +72,7 @@ `(not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1` Note that if you want to archive multiple copies (not a bad idea!), -you can set `groupwanted archive` to a version of +you can set `git-annex groupwanted archive` to a version of the above preferred content expression with a larger number of copies than 1. Then make the archive repositories have a preferred content expression of "groupwanted" in order to use your modified diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -3,12 +3,10 @@ * The [[git-annex assistant|assistant]] is included in git-annex, and extends its use cases into new territory. -* [git-annex-watcher](https://github.com/rubiojr/git-annex-watcher) - is a status icon for your desktop. -* [[forum/gadu_-_git-annex_disk_usage]] is a du like utility that - is git-annex aware. -* [sizes](http://hackage.haskell.org/package/sizes) is another du-like - utility, with a `-A` switch that enables git-annex support. +* [gitlab-shell](https://gitlab.com/gitlab-org/gitlab-shell) supports + git-annex. So git-annex can be used with repositries served by Gitlab, + including gitlab.com, or deploy your own. + [See Gitlab's announcment](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/) * Emacs Org mode can auto-commit attached files to git-annex. * [git annex darktable integration](https://github.com/xxv/darktable-git-annex) * [Magit](http://github.com/magit/magit), an Emacs mode for Git, has @@ -19,3 +17,9 @@ built by the Brazilian [Mocambos network](http://www.mocambos.net/) is [using git-annex to connect isolated communities](http://www.modspil.dk/itpolitik/baob_xia.html). Repositories sync over satellite internet and/or sneakernet. +* [[forum/gadu_-_git-annex_disk_usage]] is a du like utility that + is git-annex aware. +* [sizes](http://hackage.haskell.org/package/sizes) is another du-like + utility, with a `-A` switch that enables git-annex support. +* [git-annex-watcher](https://github.com/rubiojr/git-annex-watcher) + is a status icon for your desktop. diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn --- a/doc/special_remotes/S3.mdwn +++ b/doc/special_remotes/S3.mdwn @@ -68,4 +68,4 @@ then use the same bucket. * `x-amz-meta-*` are passed through as http headers when storing keys - in S3. + in S3. see [the Internet Archive S3 interface documentation](https://archive.org/help/abouts3.txt) for example headers. diff --git a/doc/special_remotes/bittorrent.mdwn b/doc/special_remotes/bittorrent.mdwn --- a/doc/special_remotes/bittorrent.mdwn +++ b/doc/special_remotes/bittorrent.mdwn @@ -27,3 +27,5 @@ treat this special remote as one of the required [[copies]]. It's probably a good idea to configure git-annex to fully distrust this remote, by running `git annex untrust bittorrent` + +This feature is available only from version `5.20141219`. diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -6,7 +6,7 @@ the drive's mountpoint as a directory remote. Note that directory remotes have a special directory structure -(by design, the same as the \[[rsync|rsync]] remote). +(by design, the same as the [[rsync|rsync]] remote). If you just want two copies of your repository with the files "visible" in the tree in both, the directory special remote is not what you want. Instead, you should use a regular `git clone` of your git-annex repository. diff --git a/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl b/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl --- a/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl +++ b/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl @@ -158,7 +158,7 @@ Manually specify the path to B<.xbmc/userdata/Database>. This overrides B<--home>. -Note that this doesn't point directly to the datbase itself, because +Note that this doesn't point directly to the database itself, because there are usually many database files and we want to automatically find the latest. This may be a stupid limitation. diff --git a/doc/todo/direct_mode_undo.mdwn b/doc/todo/direct_mode_undo.mdwn --- a/doc/todo/direct_mode_undo.mdwn +++ b/doc/todo/direct_mode_undo.mdwn @@ -84,3 +84,5 @@ Also, --depth could make undo look for an older commit than the most recent one to affect the specified file. + +See [[direct_mode]] for documentation about this feature. diff --git a/doc/todo/do_not_commit_with_empty_messages.mdwn b/doc/todo/do_not_commit_with_empty_messages.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/do_not_commit_with_empty_messages.mdwn @@ -0,0 +1,15 @@ +it seems that `git-annex` sometimes does commits with empty commit messages. this makes rebasing git-annex branches much much harder than they need to, because rebase freaks out on those weird commits: + +
+anarcat@marcos:video$ git rebase --continue
+Waiting for Emacs...
+Aborting commit due to empty commit message.
+Could not commit staged changes.
+
+ +This was trying to fix [[a broken merge|forum/canceling_wrong_repository_merge/]]... --[[anarcat]] + +> While I think it's silly to use empty dummy commit messages when there +> is nothing of value to say about the commit, I guess I can add value +> by putting in the name of the repository where the commit was made. So, +> [[done]] --[[Joey]] diff --git a/doc/todo/server-level_daemon__63__.mdwn b/doc/todo/server-level_daemon__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/server-level_daemon__63__.mdwn @@ -0,0 +1,3 @@ +coming from [[bugs/weird_entry_in_process_list]] - are there plans to make an init.d / systemd .service file for git-annex? + +my use case is that i have dedicated machines that will sync a common directory. they will run only one assistant - would patches to make a `git-annex` user, and the associated startup scripts, in the debian package be welcome? --[[anarcat]] diff --git a/git-annex.1 b/git-annex.1 --- a/git-annex.1 +++ b/git-annex.1 @@ -254,6 +254,11 @@ (Note that using \fB\-\-deduplicate\fP or \fB\-\-clean\-duplicates\fP with the WORM backend does not look at file content, but filename and mtime.) .IP +To control which files are imported, many of the MATCHING OPTIONS can +be used. +.IP + git annex import /dir \-\-include='*.png' +.IP .IP "\fBimportfeed [url ...]\fP" Imports the contents of podcast feeds. Only downloads files whose urls have not already been added to the repository before, so you can @@ -468,6 +473,20 @@ Without an expression, displays the current preferred content setting of the repository. .IP +.IP "\fBgroupwanted groupname [expression]\fP" +Sets or displays the groupwanted expression. This will be used by +repositories that are in the group, and that have their preferred +content expression set to "groupwanted". +.IP +For example, to configure a group named redundantarchive, and +make repositories in the group want to contain 3 copies of every file: +.IP + git annex groupwanted redundantarchive "not (copies=redundantarchive:3)" + for repo in foo bar baz; do + git annex group $repo redundantarchive + git annex wanted $repo groupwanted + done +.IP .IP "\fBschedule repository [expression]\fP" When run with an expression, configures scheduled jobs to run at a particular time. This can be used to make the assistant periodically run @@ -656,7 +675,7 @@ When no item is specified, displays statistics and information for the repository as a whole. .IP -When a directory is specified, the file matching options can be used +When a directory is specified, the MATCHING OPTIONS can be used to select the files in the directory that are included in the statistics. .IP To only show the data that can be gathered quickly, use \fB\-\-fast\fP. @@ -896,9 +915,6 @@ This is similar to the find command, but instead of finding files in the current work tree, it finds files in the specified git ref. .IP -Most MATCHING OPTIONS can be used with findref, to limit the files it -finds. However, the \-\-include and \-\-exclude options will not work. -.IP .IP "\fBproxy \-\- git cmd [options]\fP" Only useful in a direct mode repository, this runs the specified git command with a temporary work tree, and updates the working tree to @@ -1246,7 +1262,9 @@ .PP When a repository is in one of the standard predefined groups, like "backup" and "client", setting its preferred content to "standard" will use a -built\-in preferred content expression developed for that group. +built\-in preferred content expression developed for that group. Or, +setting its preferred content to "groupwanted" will make it use whatever +groupwanted expression you set for the group. .PP .SH SCHEDULED JOBS The git\-annex assistant daemon can be configured to run scheduled jobs. diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20150205 +Version: 5.20150219 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell --- a/standalone/linux/skel/runshell +++ b/standalone/linux/skel/runshell @@ -66,10 +66,8 @@ GIT_ANNEX_LD_LIBRARY_PATH="$base/$lib:$GIT_ANNEX_LD_LIBRARY_PATH" done export GIT_ANNEX_LD_LIBRARY_PATH -GIT_ANNEX_LINKER="$base/$(cat $base/linker)" -export GIT_ANNEX_LINKER -GIT_ANNEX_SHIMMED="$base/shimmed" -export GIT_ANNEX_SHIMMED +GIT_ANNEX_DIR="$base" +export GIT_ANNEX_DIR ORIG_GCONV_PATH="$GCONV_PATH" export ORIG_GCONV_PATH