diff --git a/Annex.hs b/Annex.hs --- a/Annex.hs +++ b/Annex.hs @@ -142,7 +142,7 @@ , backends = [] , remotes = [] , remoteannexstate = M.empty - , output = defaultMessageState + , output = def , force = False , fast = False , daemon = False diff --git a/Annex/Content.hs b/Annex/Content.hs --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -57,6 +57,7 @@ import Annex.Content.Direct import Annex.ReplaceFile import Utility.LockFile +import Messages.Progress {- Checks if a given key's content is currently present. -} inAnnex :: Key -> Annex Bool @@ -555,12 +556,17 @@ downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool downloadUrl urls file = go =<< annexWebDownloadCommand <$> Annex.getGitConfig where - go Nothing = Url.withUrlOptions $ \uo -> - anyM (\u -> Url.download u file uo) urls - go (Just basecmd) = liftIO $ anyM (downloadcmd basecmd) urls + go Nothing = do + a <- ifM commandProgressDisabled + ( return Url.downloadQuiet + , return Url.download + ) + Url.withUrlOptions $ \uo -> + anyM (\u -> a u file uo) urls + go (Just basecmd) = anyM (downloadcmd basecmd) urls downloadcmd basecmd url = - boolSystem "sh" [Param "-c", Param $ gencmd url basecmd] - <&&> doesFileExist file + progressCommand "sh" [Param "-c", Param $ gencmd url basecmd] + <&&> liftIO (doesFileExist file) gencmd url = massReplace [ ("%file", shellEscape file) , ("%url", shellEscape url) diff --git a/Annex/Fixup.hs b/Annex/Fixup.hs --- a/Annex/Fixup.hs +++ b/Annex/Fixup.hs @@ -11,6 +11,7 @@ import Git.Config import Types.GitConfig import qualified Git.Construct as Construct +import qualified Git.BuildVersion import Utility.Path import Utility.SafeCommand import Utility.Directory @@ -27,10 +28,19 @@ fixupRepo :: Repo -> GitConfig -> IO Repo fixupRepo r c = do - r' <- fixupSubmodule r c + let r' = disableWildcardExpansion r + r'' <- fixupSubmodule r' c if annexDirect c - then fixupDirect r' - else return r' + then fixupDirect r'' + else return r'' + +{- Disable git's built-in wildcard expansion, which is not wanted + - when using it as plumbing by git-annex. -} +disableWildcardExpansion :: Repo -> Repo +disableWildcardExpansion r + | Git.BuildVersion.older "1.8.1" = r + | otherwise = r + { gitGlobalOpts = gitGlobalOpts r ++ [Param "--literal-pathspecs"] } {- Direct mode repos have core.bare=true, but are not really bare. - Fix up the Repo to be a non-bare repo, and arrange for git commands diff --git a/Annex/Init.hs b/Annex/Init.hs --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -162,13 +162,17 @@ #else tmp <- fromRepo gitAnnexTmpMiscDir let f = tmp "gaprobe" + let f2 = tmp "gaprobe2" createAnnexDirectory tmp liftIO $ do nukeFile f + nukeFile f2 ms <- tryIO $ do createNamedPipe f ownerReadMode + createLink f f2 getFileStatus f nukeFile f + nukeFile f2 return $ either (const False) isNamedPipe ms #endif diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -290,7 +290,7 @@ -- files. The ls-files is run on a batch of files. findnew [] = return ([], noop) findnew pending@(exemplar:_) = do - let segments = segmentXargs $ map changeFile pending + let segments = segmentXargsUnordered $ map changeFile pending rs <- liftAnnex $ forM segments $ \fs -> inRepo (Git.LsFiles.notInRepo False fs) let (newfiles, cleanup) = foldl' @@ -457,7 +457,7 @@ -} findopenfiles keysources = ifM crippledFileSystem ( liftIO $ do - let segments = segmentXargs $ map keyFilename keysources + let segments = segmentXargsUnordered $ map keyFilename keysources concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) , do tmpdir <- fromRepo gitAnnexTmpMiscDir diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs --- a/Build/LinuxMkLibs.hs +++ b/Build/LinuxMkLibs.hs @@ -69,6 +69,8 @@ createSymbolicLink link (top exelink) writeFile exe $ unlines [ "#!/bin/sh" + , "GIT_ANNEX_PROGRAMPATH=\"$0\"" + , "export GIT_ANNEX_PROGRAMPATH" , "exec \"$GIT_ANNEX_DIR/" ++ exelink ++ "\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_DIR/shimmed/" ++ base ++ "/" ++ base ++ "\" \"$@\"" ] modifyFileMode exe $ addModes executeModes diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,39 @@ +git-annex (5.20150406) unstable; urgency=medium + + * Prevent git-ls-files from double-expanding wildcards when an + unexpanded wildcard is passed to a git-annex command like add or find. + * Fix make build target. Thanks, Justin Geibel. + * Fix GETURLS in external special remote protocol to strip + downloader prefix from logged url info before checking for the + specified prefix. + * importfeed: Avoid downloading a redundant item from a feed whose + guid has been downloaded before, even when the url has changed. + * importfeed: Always store itemid in metadata; before this was only + done when annex.genmetadata was set. + * Relax debian package dependencies to git >= 1:1.8.1 rather + than needing >= 1:2.0. + * test: Fix --list-tests + * addurl --file: When used with a special remote that claims + urls and checks their contents, don't override the user's provided + filename with filenames that the special remote suggests. Also, + don't allow adding the url if the special remote says it contains + multiple files. + * import: --deduplicate and --cleanduplicates now output the keys + corresponding to duplicated files they process. + * expire: New command, for expiring inactive repositories. + * fsck: Record fsck activity for use by expire command. + * Fix truncation of parameters that could occur when using xargs git-annex. + * Significantly sped up processing of large numbers of directories + passed to a single git-annex command. + * version: Add --raw + * init: Improve fifo test to detect NFS systems that support fifos + but not well enough for sshcaching. + * --quiet now suppresses progress displays from eg, rsync. + (The option already suppressed git-annex's own built-in progress + displays.) + + -- Joey Hess Mon, 06 Apr 2015 12:48:48 -0400 + git-annex (5.20150327) unstable; urgency=medium * readpresentkey: New plumbing command for checking location log. diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs --- a/CmdLine/GitAnnex.hs +++ b/CmdLine/GitAnnex.hs @@ -45,6 +45,7 @@ import qualified Command.InitRemote import qualified Command.EnableRemote import qualified Command.Fsck +import qualified Command.Expire import qualified Command.Repair import qualified Command.Unused import qualified Command.DropUnused @@ -169,6 +170,7 @@ , Command.VCycle.cmd , Command.Fix.cmd , Command.Fsck.cmd + , Command.Expire.cmd , Command.Repair.cmd , Command.Unused.cmd , Command.DropUnused.cmd diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -215,8 +215,8 @@ seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [FilePath] -> Annex [FilePath] seekHelper a params = do - ll <- inRepo $ \g -> - runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g) params + ll <- inRepo $ \g -> concat <$> forM (segmentXargsOrdered params) + (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) forM_ (map fst $ filter (null . snd) $ zip params ll) $ \p -> unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ error $ p ++ " not found" diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -64,25 +64,36 @@ r <- Remote.claimingUrl u if Remote.uuid r == webUUID || raw then void $ commandAction $ startWeb relaxed optfile pathdepth u - else do - pathmax <- liftIO $ fileNameLengthLimit "." - let deffile = fromMaybe (urlString2file u pathdepth pathmax) optfile - res <- tryNonAsync $ maybe - (error $ "unable to checkUrl of " ++ Remote.name r) - (flip id u) - (Remote.checkUrl r) - case res of - Left e -> void $ commandAction $ do - showStart "addurl" u - warning (show e) - next $ next $ return False - Right (UrlContents sz mf) -> do - void $ commandAction $ - startRemote r relaxed (maybe deffile fromSafeFilePath mf) u sz - Right (UrlMulti l) -> - forM_ l $ \(u', sz, f) -> - void $ commandAction $ - startRemote r relaxed (deffile fromSafeFilePath f) u' sz + else checkUrl r u optfile relaxed pathdepth + +checkUrl :: Remote -> URLString -> Maybe FilePath -> Bool -> Maybe Int -> Annex () +checkUrl r u optfile relaxed pathdepth = do + pathmax <- liftIO $ fileNameLengthLimit "." + let deffile = fromMaybe (urlString2file u pathdepth pathmax) optfile + go deffile =<< maybe + (error $ "unable to checkUrl of " ++ Remote.name r) + (tryNonAsync . flip id u) + (Remote.checkUrl r) + where + + go _ (Left e) = void $ commandAction $ do + showStart "addurl" u + warning (show e) + next $ next $ return False + go deffile (Right (UrlContents sz mf)) = do + let f = fromMaybe (maybe deffile fromSafeFilePath mf) optfile + void $ commandAction $ + startRemote r relaxed f u sz + go deffile (Right (UrlMulti l)) + | isNothing optfile = + forM_ l $ \(u', sz, f) -> + void $ commandAction $ + startRemote r relaxed (deffile fromSafeFilePath f) u' sz + | otherwise = error $ unwords + [ "That url contains multiple files according to the" + , Remote.name r + , " remote; cannot add it to a single file." + ] startRemote :: Remote -> Bool -> FilePath -> URLString -> Maybe Integer -> CommandStart startRemote r relaxed file uri sz = do diff --git a/Command/Expire.hs b/Command/Expire.hs new file mode 100644 --- /dev/null +++ b/Command/Expire.hs @@ -0,0 +1,106 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Expire where + +import Common.Annex +import Command +import Logs.Activity +import Logs.UUID +import Logs.MapLog +import Logs.Trust +import Annex.UUID +import qualified Remote +import Utility.HumanTime + +import Data.Time.Clock.POSIX +import qualified Data.Map as M + +cmd :: [Command] +cmd = [withOptions [activityOption, noActOption] $ command "expire" paramExpire seek + SectionMaintenance "expire inactive repositories"] + +paramExpire :: String +paramExpire = (paramRepeating $ paramOptional paramRemote ++ ":" ++ paramTime) + +activityOption :: Option +activityOption = fieldOption [] "activity" "Name" "specify activity" + +noActOption :: Option +noActOption = flagOption [] "no-act" "don't really do anything" + +seek :: CommandSeek +seek ps = do + expire <- parseExpire ps + wantact <- getOptionField activityOption (pure . parseActivity) + noact <- getOptionFlag noActOption + actlog <- lastActivities wantact + u <- getUUID + us <- filter (/= u) . M.keys <$> uuidMap + descs <- uuidMap + seekActions $ pure $ map (start expire noact actlog descs) us + +start :: Expire -> Bool -> Log Activity -> M.Map UUID String -> UUID -> CommandStart +start (Expire expire) noact actlog descs u = + case lastact of + Just ent | notexpired ent -> checktrust (== DeadTrusted) $ do + showStart "unexpire" desc + showNote =<< whenactive + unless noact $ + trustSet u SemiTrusted + _ -> checktrust (/= DeadTrusted) $ do + showStart "expire" desc + showNote =<< whenactive + unless noact $ + trustSet u DeadTrusted + where + lastact = changed <$> M.lookup u actlog + whenactive = case lastact of + Just (Date t) -> do + d <- liftIO $ durationSince $ posixSecondsToUTCTime t + return $ "last active: " ++ fromDuration d ++ " ago" + _ -> return "no activity" + desc = fromUUID u ++ " " ++ fromMaybe "" (M.lookup u descs) + notexpired ent = case ent of + Unknown -> False + Date t -> case lookupexpire of + Just (Just expiretime) -> t >= expiretime + _ -> True + lookupexpire = headMaybe $ catMaybes $ + map (`M.lookup` expire) [Just u, Nothing] + checktrust want a = ifM (want <$> lookupTrust u) + ( do + void a + next $ next $ return True + , stop + ) + +data Expire = Expire (M.Map (Maybe UUID) (Maybe POSIXTime)) + +parseExpire :: [String] -> Annex Expire +parseExpire [] = error "Specify an expire time." +parseExpire ps = do + now <- liftIO getPOSIXTime + Expire . M.fromList <$> mapM (parse now) ps + where + parse now s = case separate (== ':') s of + (t, []) -> return (Nothing, parsetime now t) + (n, t) -> do + r <- Remote.nameToUUID n + return (Just r, parsetime now t) + parsetime _ "never" = Nothing + parsetime now s = case parseDuration s of + Nothing -> error $ "bad expire time: " ++ s + Just d -> Just (now - durationToPOSIXTime d) + +parseActivity :: Maybe String -> Maybe Activity +parseActivity Nothing = Nothing +parseActivity (Just s) = case readish s of + Nothing -> error $ "Unknown activity. Choose from: " ++ + unwords (map show [minBound..maxBound :: Activity]) + Just v -> Just v + diff --git a/Command/Fsck.hs b/Command/Fsck.hs --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -23,6 +23,7 @@ import Annex.Link import Logs.Location import Logs.Trust +import Logs.Activity import Config.NumCopies import Annex.UUID import Utility.DataUnits @@ -74,41 +75,7 @@ (withFilesInGit $ whenAnnexed $ start from i) ps withFsckDb i FsckDb.closeDb - -getIncremental :: UUID -> Annex Incremental -getIncremental u = do - i <- maybe (return False) (checkschedule . parseDuration) - =<< Annex.getField (optionName incrementalScheduleOption) - starti <- Annex.getFlag (optionName startIncrementalOption) - morei <- Annex.getFlag (optionName moreIncrementalOption) - case (i, starti, morei) of - (False, False, False) -> return NonIncremental - (False, True, False) -> startIncremental - (False ,False, True) -> contIncremental - (True, False, False) -> - maybe startIncremental (const contIncremental) - =<< getStartTime u - _ -> error "Specify only one of --incremental, --more, or --incremental-schedule" - where - startIncremental = do - recordStartTime u - ifM (FsckDb.newPass u) - ( StartIncremental <$> FsckDb.openDb u - , error "Cannot start a new --incremental fsck pass; another fsck process is already running." - ) - contIncremental = ContIncremental <$> FsckDb.openDb u - - checkschedule Nothing = error "bad --incremental-schedule value" - checkschedule (Just delta) = do - Annex.addCleanup FsckCleanup $ do - v <- getStartTime u - case v of - Nothing -> noop - Just started -> do - now <- liftIO getPOSIXTime - when (now - realToFrac started >= durationToPOSIXTime delta) $ - resetStartTime u - return True + recordActivity Fsck u start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart start from inc file key = do @@ -421,8 +388,6 @@ return $ (if ok then "dropped from " else "failed to drop from ") ++ Remote.name remote -data Incremental = StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle | NonIncremental - runFsck :: Incremental -> FilePath -> Key -> Annex Bool -> CommandStart runFsck inc file key a = ifM (needFsck inc key) ( do @@ -497,3 +462,40 @@ #else fromfile >= fromstatus #endif + +data Incremental = StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle | NonIncremental + +getIncremental :: UUID -> Annex Incremental +getIncremental u = do + i <- maybe (return False) (checkschedule . parseDuration) + =<< Annex.getField (optionName incrementalScheduleOption) + starti <- getOptionFlag startIncrementalOption + morei <- getOptionFlag moreIncrementalOption + case (i, starti, morei) of + (False, False, False) -> return NonIncremental + (False, True, False) -> startIncremental + (False ,False, True) -> contIncremental + (True, False, False) -> + maybe startIncremental (const contIncremental) + =<< getStartTime u + _ -> error "Specify only one of --incremental, --more, or --incremental-schedule" + where + startIncremental = do + recordStartTime u + ifM (FsckDb.newPass u) + ( StartIncremental <$> FsckDb.openDb u + , error "Cannot start a new --incremental fsck pass; another fsck process is already running." + ) + contIncremental = ContIncremental <$> FsckDb.openDb u + + checkschedule Nothing = error "bad --incremental-schedule value" + checkschedule (Just delta) = do + Annex.addCleanup FsckCleanup $ do + v <- getStartTime u + case v of + Nothing -> noop + Just started -> do + now <- liftIO getPOSIXTime + when (now - realToFrac started >= durationToPOSIXTime delta) $ + resetStartTime u + return True diff --git a/Command/Import.hs b/Command/Import.hs --- a/Command/Import.hs +++ b/Command/Import.hs @@ -15,6 +15,7 @@ import Backend import Remote import Types.KeySource +import Types.Key cmd :: [Command] cmd = [withOptions opts $ notBareRepo $ command "import" paramPaths seek @@ -72,8 +73,8 @@ , stop ) where - deletedup = do - showNote "duplicate" + deletedup k = do + showNote $ "duplicate of " ++ key2file k liftIO $ removeFile srcfile next $ return True importfile = do @@ -88,17 +89,19 @@ | isDirectory s = notoverwriting "(is a directory)" | otherwise = ifM (Annex.getState Annex.force) ( liftIO $ nukeFile destfile - , notoverwriting "(use --force to override)" + , notoverwriting "(use --force to override, or a duplication option such as --deduplicate to clean up)" ) notoverwriting why = error $ "not overwriting existing " ++ destfile ++ " " ++ why checkdup dupa notdupa = do backend <- chooseBackend destfile let ks = KeySource srcfile srcfile Nothing v <- genKey ks backend - isdup <- case v of - Just (k, _) -> not . null <$> keyLocations k - _ -> return False - return $ if isdup then dupa else notdupa + case v of + Just (k, _) -> ifM (not . null <$> keyLocations k) + ( return (maybe Nothing (\a -> Just (a k)) dupa) + , return notdupa + ) + _ -> return notdupa pickaction = case mode of DeDuplicate -> checkdup (Just deletedup) (Just importfile) CleanDuplicates -> checkdup (Just deletedup) Nothing diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -73,7 +73,7 @@ v <- findDownloads url case v of [] -> do - feedProblem url "bad feed content" + feedProblem url "bad feed content; no enclosures to download" next $ return True l -> do ok <- and <$> mapM (performDownload opts cache) l @@ -96,22 +96,33 @@ data DownloadLocation = Enclosure URLString | QuviLink URLString +type ItemId = String + data Cache = Cache { knownurls :: S.Set URLString + , knownitems :: S.Set ItemId , template :: Utility.Format.Format } getCache :: Maybe String -> Annex Cache getCache opttemplate = ifM (Annex.getState Annex.force) - ( ret S.empty + ( ret S.empty S.empty , do showSideAction "checking known urls" - ret =<< S.fromList <$> knownUrls + (is, us) <- unzip <$> (mapM knownItems =<< knownUrls) + ret (S.fromList us) (S.fromList (concat is)) ) where tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate - ret s = return $ Cache s tmpl + ret us is = return $ Cache us is tmpl +knownItems :: (Key, URLString) -> Annex ([ItemId], URLString) +knownItems (k, u) = do + itemids <- S.toList . S.filter (/= noneValue) . S.map fromMetaValue + . currentMetaDataValues itemIdField + <$> getCurrentMetaData k + return (itemids, u) + findDownloads :: URLString -> Annex [ToDownload] findDownloads u = go =<< downloadFeed u where @@ -191,12 +202,18 @@ where forced = Annex.getState Annex.force - {- Avoids downloading any urls that are already known to be + {- Avoids downloading any items that are already known to be - associated with a file in the annex, unless forced. -} checkknown url a - | S.member url (knownurls cache) = ifM forced (a, return True) + | knownitemid || S.member url (knownurls cache) + = ifM forced (a, return True) | otherwise = a + knownitemid = case getItemId (item todownload) of + -- only when it's a permalink + Just (True, itemid) -> S.member itemid (knownitems cache) + _ -> False + rundownload url extension getter = do dest <- makeunique url (1 :: Integer) $ feedFile (template cache) todownload extension @@ -211,8 +228,10 @@ checkFeedBroken (feedurl todownload) else do forM_ ks $ \key -> - whenM (annexGenMetaData <$> Annex.getGitConfig) $ - addMetaData key $ extractMetaData todownload + ifM (annexGenMetaData <$> Annex.getGitConfig) + ( addMetaData key $ extractMetaData todownload + , addMetaData key $ minimalMetaData todownload + ) showEndOk return True @@ -275,6 +294,12 @@ tometa (k, v) = (mkMetaFieldUnchecked k, S.singleton (toMetaValue v)) meta = MetaData $ M.fromList $ map tometa $ extractFields i +minimalMetaData :: ToDownload -> MetaData +minimalMetaData i = case getItemId (item i) of + (Nothing) -> emptyMetaData + (Just (_, itemid)) -> MetaData $ M.singleton itemIdField + (S.singleton $ toMetaValue itemid) + {- Extract fields from the feed and item, that are both used as metadata, - and to generate the filename. -} extractFields :: ToDownload -> [(String, String)] @@ -296,11 +321,17 @@ feedauthor = getFeedAuthor $ feed i itemauthor = getItemAuthor $ item i +itemIdField :: MetaField +itemIdField = mkMetaFieldUnchecked "itemid" + extractField :: String -> [Maybe String] -> (String, String) -extractField k [] = (k, "none") +extractField k [] = (k, noneValue) extractField k (Just v:_) | not (null v) = (k, v) extractField k (_:rest) = extractField k rest + +noneValue :: String +noneValue = "none" {- Called when there is a problem with a feed. - Throws an error if the feed is broken, otherwise shows a warning. -} diff --git a/Command/Info.hs b/Command/Info.hs --- a/Command/Info.hs +++ b/Command/Info.hs @@ -198,6 +198,7 @@ [ remote_name , remote_description , remote_uuid + , remote_trust , remote_cost , remote_type ] @@ -265,6 +266,10 @@ remote_uuid :: Remote -> Stat remote_uuid r = simpleStat "uuid" $ pure $ fromUUID $ Remote.uuid r + +remote_trust :: Remote -> Stat +remote_trust r = simpleStat "trust" $ lift $ + showTrustLevel <$> lookupTrust (Remote.uuid r) remote_cost :: Remote -> Stat remote_cost r = simpleStat "cost" $ pure $ diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs --- a/Command/TransferKeys.hs +++ b/Command/TransferKeys.hs @@ -16,7 +16,7 @@ import Annex.Transfer import qualified Remote import Types.Key -import Utility.SimpleProtocol (ioHandles) +import Utility.SimpleProtocol (dupIoHandles) import Git.Types (RemoteName) data TransferRequest = TransferRequest Direction Remote Key AssociatedFile @@ -30,7 +30,7 @@ start :: CommandStart start = do - (readh, writeh) <- liftIO ioHandles + (readh, writeh) <- liftIO dupIoHandles runRequests readh writeh runner stop where diff --git a/Command/Version.hs b/Command/Version.hs --- a/Command/Version.hs +++ b/Command/Version.hs @@ -18,16 +18,28 @@ import qualified Backend cmd :: [Command] -cmd = [noCommit $ noRepo startNoRepo $ dontCheck repoExists $ +cmd = [withOptions [rawOption] $ + noCommit $ noRepo startNoRepo $ dontCheck repoExists $ command "version" paramNothing seek SectionQuery "show version info"] +rawOption :: Option +rawOption = flagOption [] "raw" "output only program version" + seek :: CommandSeek -seek = withNothing start +seek = withNothing $ ifM (getOptionFlag rawOption) (startRaw, start) +startRaw :: CommandStart +startRaw = do + liftIO $ do + putStr SysConfig.packageversion + hFlush stdout + stop + start :: CommandStart start = do v <- getVersion liftIO $ do + showPackageVersion info "local repository version" $ fromMaybe "unknown" v info "supported repository version" supportedVersion diff --git a/Logs.hs b/Logs.hs --- a/Logs.hs +++ b/Logs.hs @@ -1,6 +1,6 @@ {- git-annex log file names - - - Copyright 2013-2014 Joey Hess + - Copyright 2013-2015 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -40,6 +40,7 @@ , preferredContentLog , requiredContentLog , scheduleLog + , activityLog , differenceLog ] @@ -84,8 +85,12 @@ scheduleLog :: FilePath scheduleLog = "schedule.log" +activityLog :: FilePath +activityLog = "activity.log" + differenceLog :: FilePath differenceLog = "difference.log" + {- The pathname of the location log file for a given key. -} locationLogFile :: GitConfig -> Key -> String diff --git a/Logs/Activity.hs b/Logs/Activity.hs new file mode 100644 --- /dev/null +++ b/Logs/Activity.hs @@ -0,0 +1,37 @@ +{- git-annex activity log + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Activity ( + Log, + Activity(..), + recordActivity, + lastActivities, +) where + +import Data.Time.Clock.POSIX + +import Common.Annex +import qualified Annex.Branch +import Logs +import Logs.UUIDBased + +data Activity = Fsck + deriving (Eq, Read, Show, Enum, Bounded) + +recordActivity :: Activity -> UUID -> Annex () +recordActivity act uuid = do + ts <- liftIO getPOSIXTime + Annex.Branch.change activityLog $ + showLog id . changeLog ts uuid (show act) . parseLog readish + +lastActivities :: Maybe Activity -> Annex (Log Activity) +lastActivities wantact = parseLog onlywanted <$> Annex.Branch.get activityLog + where + onlywanted s = case readish s of + Just a | wanted a -> Just a + _ -> Nothing + wanted a = maybe True (a ==) wantact diff --git a/Logs/Location.hs b/Logs/Location.hs --- a/Logs/Location.hs +++ b/Logs/Location.hs @@ -19,6 +19,7 @@ logChange, loggedLocations, loggedLocationsHistorical, + locationLog, loggedKeys, loggedKeysFor, ) where @@ -39,24 +40,32 @@ {- Log a change in the presence of a key's value in a repository. -} logChange :: Key -> UUID -> LogStatus -> Annex () -logChange key (UUID u) s = do +logChange = logChange' logNow + +logChange' :: (LogStatus -> String -> Annex LogLine) -> Key -> UUID -> LogStatus -> Annex () +logChange' mklog key (UUID u) s = do config <- Annex.getGitConfig - addLog (locationLogFile config key) =<< logNow s u -logChange _ NoUUID _ = noop + addLog (locationLogFile config key) =<< mklog s u +logChange' _ _ NoUUID _ = noop {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} loggedLocations :: Key -> Annex [UUID] -loggedLocations = getLoggedLocations currentLog +loggedLocations = getLoggedLocations currentLogInfo {- Gets the location log on a particular date. -} loggedLocationsHistorical :: RefDate -> Key -> Annex [UUID] -loggedLocationsHistorical = getLoggedLocations . historicalLog +loggedLocationsHistorical = getLoggedLocations . historicalLogInfo getLoggedLocations :: (FilePath -> Annex [String]) -> Key -> Annex [UUID] getLoggedLocations getter key = do config <- Annex.getGitConfig - map toUUID <$> (getter . locationLogFile config) key + map toUUID <$> getter (locationLogFile config key) + +locationLog :: Key -> Annex [LogLine] +locationLog key = do + config <- Annex.getGitConfig + currentLog (locationLogFile config key) {- Finds all keys that have location log information. - (There may be duplicate keys in the list.) -} diff --git a/Logs/Presence.hs b/Logs/Presence.hs --- a/Logs/Presence.hs +++ b/Logs/Presence.hs @@ -17,7 +17,8 @@ readLog, logNow, currentLog, - historicalLog + currentLogInfo, + historicalLogInfo, ) where import Data.Time.Clock.POSIX @@ -43,14 +44,17 @@ return $ LogLine now s i {- Reads a log and returns only the info that is still in effect. -} -currentLog :: FilePath -> Annex [String] -currentLog file = map info . filterPresent <$> readLog file +currentLogInfo :: FilePath -> Annex [String] +currentLogInfo file = map info <$> currentLog file +currentLog :: FilePath -> Annex [LogLine] +currentLog file = filterPresent <$> readLog file + {- Reads a historical version of a log and returns the info that was in - effect at that time. - - The date is formatted as shown in gitrevisions man page. -} -historicalLog :: RefDate -> FilePath -> Annex [String] -historicalLog refdate file = map info . filterPresent . parseLog +historicalLogInfo :: RefDate -> FilePath -> Annex [String] +historicalLogInfo refdate file = map info . filterPresent . parseLog <$> Annex.Branch.getHistorical refdate file diff --git a/Logs/Web.hs b/Logs/Web.hs --- a/Logs/Web.hs +++ b/Logs/Web.hs @@ -21,7 +21,6 @@ import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M -import Data.Tuple.Utils import Common.Annex import qualified Annex @@ -44,13 +43,15 @@ where go [] = return [] go (l:ls) = do - us <- currentLog l + us <- currentLogInfo l if null us then go ls else return us getUrlsWithPrefix :: Key -> String -> Annex [URLString] -getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) <$> getUrls key +getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) + . map (fst . getDownloader) + <$> getUrls key setUrlPresent :: UUID -> Key -> URLString -> Annex () setUrlPresent uuid key url = do @@ -68,7 +69,7 @@ logChange key uuid InfoMissing {- Finds all known urls. -} -knownUrls :: Annex [URLString] +knownUrls :: Annex [(Key, URLString)] knownUrls = do {- Ensure the git-annex branch's index file is up-to-date and - any journaled changes are reflected in it, since we're going @@ -78,10 +79,13 @@ Annex.Branch.withIndex $ do top <- fromRepo Git.repoPath (l, cleanup) <- inRepo $ Git.LsFiles.stagedDetails [top] - r <- mapM (geturls . snd3) $ filter (isUrlLog . fst3) l + r <- mapM getkeyurls l void $ liftIO cleanup return $ concat r where + getkeyurls (f, s, _) = case urlLogFileKey f of + Just k -> zip (repeat k) <$> geturls s + Nothing -> return [] geturls Nothing = return [] geturls (Just logsha) = getLog . L.unpack <$> catObject logsha diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ mans=$(shell find doc -maxdepth 1 -name git-annex*.mdwn | sed -e 's/^doc/man/' -e 's/\.mdwn/\.1/') -all=git-annex $(mans) docs +all=git-annex mans docs CABAL?=cabal # set to "./Setup" if you lack a cabal program GHC?=ghc @@ -103,7 +103,7 @@ Build/LinuxMkLibs: Build/LinuxMkLibs.hs $(GHC) --make $@ -Wall -sdist: clean $(mans) +sdist: clean mans ./Build/make-sdist.sh # Upload to hackage. diff --git a/Messages.hs b/Messages.hs --- a/Messages.hs +++ b/Messages.hs @@ -10,9 +10,6 @@ showStart', showNote, showAction, - showProgress, - metered, - meteredBytes, showSideAction, doSideAction, doQuietSideAction, @@ -33,28 +30,26 @@ showRaw, setupConsole, enableDebugOutput, - disableDebugOutput + disableDebugOutput, + commandProgressDisabled, ) where import Text.JSON -import Data.Progress.Meter -import Data.Progress.Tracker -import Data.Quantity import System.Log.Logger import System.Log.Formatter import System.Log.Handler (setFormatter) import System.Log.Handler.Simple -import Common hiding (handle) +import Common import Types import Types.Messages +import Messages.Internal import qualified Messages.JSON as JSON import Types.Key import qualified Annex -import Utility.Metered showStart :: String -> FilePath -> Annex () -showStart command file = handle (JSON.start command $ Just file) $ +showStart command file = handleMessage (JSON.start command $ Just file) $ flushed $ putStr $ command ++ " " ++ file ++ " " showStart' :: String -> Key -> Maybe FilePath -> Annex () @@ -62,42 +57,12 @@ fromMaybe (key2file key) afile showNote :: String -> Annex () -showNote s = handle (JSON.note s) $ +showNote s = handleMessage (JSON.note s) $ flushed $ putStr $ "(" ++ s ++ ") " showAction :: String -> Annex () showAction s = showNote $ s ++ "..." -{- Progress dots. -} -showProgress :: Annex () -showProgress = handle q $ - flushed $ putStr "." - -{- Shows a progress meter while performing a transfer of a key. - - The action is passed a callback to use to update the meter. -} -metered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a -metered combinemeterupdate key a = go (keySize key) - where - go (Just size) = meteredBytes combinemeterupdate size a - go _ = a (const noop) - -{- Shows a progress meter while performing an action on a given number - - of bytes. -} -meteredBytes :: Maybe MeterUpdate -> Integer -> (MeterUpdate -> Annex a) -> Annex a -meteredBytes combinemeterupdate size a = withOutputType go - where - go NormalOutput = do - progress <- liftIO $ newProgress "" size - meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1) - showOutput - r <- a $ \n -> liftIO $ do - setP progress $ fromBytesProcessed n - displayMeter stdout meter - maybe noop (\m -> m n) combinemeterupdate - liftIO $ clearMeter stdout meter - return r - go _ = a (const noop) - showSideAction :: String -> Annex () showSideAction m = Annex.getState Annex.output >>= go where @@ -108,7 +73,7 @@ Annex.changeState $ \s -> s { Annex.output = st' } | sideActionBlock st == InBlock = return () | otherwise = p - p = handle q $ putStrLn $ "(" ++ m ++ "...)" + p = handleMessage q $ putStrLn $ "(" ++ m ++ "...)" showStoringStateAction :: Annex () showStoringStateAction = showSideAction "recording state in git" @@ -130,12 +95,13 @@ where set o = Annex.changeState $ \s -> s { Annex.output = o } +{- Make way for subsequent output of a command. -} showOutput :: Annex () -showOutput = handle q $ - putStr "\n" +showOutput = unlessM commandProgressDisabled $ + handleMessage q $ putStr "\n" showLongNote :: String -> Annex () -showLongNote s = handle (JSON.note s) $ +showLongNote s = handleMessage (JSON.note s) $ putStrLn $ '\n' : indent s showEndOk :: Annex () @@ -145,7 +111,7 @@ showEndFail = showEndResult False showEndResult :: Bool -> Annex () -showEndResult ok = handle (JSON.end ok) $ putStrLn msg +showEndResult ok = handleMessage (JSON.end ok) $ putStrLn msg where msg | ok = "ok" @@ -159,7 +125,7 @@ warning' :: String -> Annex () warning' w = do - handle q $ putStr "\n" + handleMessage q $ putStr "\n" liftIO $ do hFlush stdout hPutStrLn stderr w @@ -175,7 +141,7 @@ {- Shows a JSON fragment only when in json mode. -} maybeShowJSON :: JSON a => [(String, a)] -> Annex () -maybeShowJSON v = handle (JSON.add v) q +maybeShowJSON v = handleMessage (JSON.add v) q {- Shows a complete JSON value, only when in json mode. -} showFullJSON :: JSON a => [(String, a)] -> Annex Bool @@ -190,16 +156,16 @@ - This is only needed when showStart and showEndOk is not used. -} showCustom :: String -> Annex Bool -> Annex () showCustom command a = do - handle (JSON.start command Nothing) q + handleMessage (JSON.start command Nothing) q r <- a - handle (JSON.end r) q + handleMessage (JSON.end r) q showHeader :: String -> Annex () -showHeader h = handle q $ +showHeader h = handleMessage q $ flushed $ putStr $ h ++ ": " showRaw :: String -> Annex () -showRaw s = handle q $ putStrLn s +showRaw s = handleMessage q $ putStrLn s setupConsole :: IO () setupConsole = do @@ -219,18 +185,11 @@ disableDebugOutput :: IO () disableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel NOTICE -handle :: IO () -> IO () -> Annex () -handle json normal = withOutputType go - where - go NormalOutput = liftIO normal - go QuietOutput = q - go JSONOutput = liftIO $ flushed json - -q :: Monad m => m () -q = noop - -flushed :: IO () -> IO () -flushed a = a >> hFlush stdout - -withOutputType :: (OutputType -> Annex a) -> Annex a -withOutputType a = outputType <$> Annex.getState Annex.output >>= a +{- Should commands that normally output progress messages have that + - output disabled? -} +commandProgressDisabled :: Annex Bool +commandProgressDisabled = withOutputType $ \t -> return $ case t of + QuietOutput -> True + ProgressOutput -> True + JSONOutput -> True + NormalOutput -> False diff --git a/Messages/Internal.hs b/Messages/Internal.hs new file mode 100644 --- /dev/null +++ b/Messages/Internal.hs @@ -0,0 +1,30 @@ +{- git-annex output messages + - + - Copyright 2010-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages.Internal where + +import Common +import Types +import Types.Messages +import qualified Annex + +handleMessage :: IO () -> IO () -> Annex () +handleMessage json normal = withOutputType go + where + go NormalOutput = liftIO normal + go QuietOutput = q + go ProgressOutput = q + go JSONOutput = liftIO $ flushed json + +q :: Monad m => m () +q = noop + +flushed :: IO () -> IO () +flushed a = a >> hFlush stdout + +withOutputType :: (OutputType -> Annex a) -> Annex a +withOutputType a = outputType <$> Annex.getState Annex.output >>= a diff --git a/Messages/Progress.hs b/Messages/Progress.hs new file mode 100644 --- /dev/null +++ b/Messages/Progress.hs @@ -0,0 +1,88 @@ +{- git-annex progress output + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages.Progress where + +import Common +import Messages +import Messages.Internal +import Utility.Metered +import Types +import Types.Messages +import Types.Key + +import Data.Progress.Meter +import Data.Progress.Tracker +import Data.Quantity + +{- Shows a progress meter while performing a transfer of a key. + - The action is passed a callback to use to update the meter. -} +metered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a +metered combinemeterupdate key a = go (keySize key) + where + go (Just size) = meteredBytes combinemeterupdate size a + go _ = a (const noop) + +{- Shows a progress meter while performing an action on a given number + - of bytes. -} +meteredBytes :: Maybe MeterUpdate -> Integer -> (MeterUpdate -> Annex a) -> Annex a +meteredBytes combinemeterupdate size a = withOutputType go + where + go NormalOutput = do + progress <- liftIO $ newProgress "" size + meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1) + showOutput + r <- a $ \n -> liftIO $ do + setP progress $ fromBytesProcessed n + displayMeter stdout meter + maybe noop (\m -> m n) combinemeterupdate + liftIO $ clearMeter stdout meter + return r + go _ = a (const noop) + +{- Progress dots. -} +showProgressDots :: Annex () +showProgressDots = handleMessage q $ + flushed $ putStr "." + +{- Runs a command, that may output progress to either stdout or + - stderr, as well as other messages. + - + - In quiet mode, the output is suppressed, except for error messages. + -} +progressCommand :: FilePath -> [CommandParam] -> Annex Bool +progressCommand cmd params = progressCommandEnv cmd params Nothing + +progressCommandEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> Annex Bool +progressCommandEnv cmd params environ = ifM commandProgressDisabled + ( do + oh <- mkOutputHandler + liftIO $ demeterCommandEnv oh cmd params environ + , liftIO $ boolSystemEnv cmd params environ + ) + +mkOutputHandler :: Annex OutputHandler +mkOutputHandler = OutputHandler + <$> commandProgressDisabled + <*> mkStderrEmitter + +mkStderrRelayer :: Annex (Handle -> IO ()) +mkStderrRelayer = do + quiet <- commandProgressDisabled + emitter <- mkStderrEmitter + return $ \h -> avoidProgress quiet h emitter + +{- Generates an IO action that can be used to emit stderr. + - + - When a progress meter is displayed, this takes care to avoid + - messing it up with interleaved stderr from a command. + -} +mkStderrEmitter :: Annex (String -> IO ()) +mkStderrEmitter = withOutputType go + where + go ProgressOutput = return $ \s -> hPutStrLn stderr ("E: " ++ s) + go _ = return (hPutStrLn stderr) diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -19,6 +19,7 @@ import Types.UrlContents import Types.CleanupActions import Types.Key +import Messages.Progress import Utility.Metered import Utility.Tmp import Backend.URL @@ -288,14 +289,15 @@ return (ps ++ opts) runAria :: [CommandParam] -> Annex Bool -runAria ps = liftIO . boolSystem "aria2c" =<< ariaParams ps +runAria ps = progressCommand "aria2c" =<< ariaParams ps -- Parse aria output to find "(n%)" and update the progress meter --- with it. The output is also output to stdout. +-- with it. ariaProgress :: Maybe Integer -> MeterUpdate -> [CommandParam] -> Annex Bool ariaProgress Nothing _ ps = runAria ps -ariaProgress (Just sz) meter ps = - liftIO . commandMeter (parseAriaProgress sz) meter "aria2c" +ariaProgress (Just sz) meter ps = do + oh <- mkOutputHandler + liftIO . commandMeter (parseAriaProgress sz) oh meter "aria2c" =<< ariaParams ps parseAriaProgress :: Integer -> ProgressParser diff --git a/Remote/Bup.hs b/Remote/Bup.hs --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -121,18 +121,22 @@ showOutput -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params -bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> Annex [CommandParam] -bupSplitParams r buprepo k src = do +bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> [CommandParam] +bupSplitParams r buprepo k src = let os = map Param $ remoteAnnexBupSplitOptions $ gitconfig r - showOutput -- make way for bup output - return $ bupParams "split" buprepo + in bupParams "split" buprepo (os ++ [Param "-q", Param "-n", Param (bupRef k)] ++ src) store :: Remote -> BupRepo -> Storer store r buprepo = byteStorer $ \k b p -> do - params <- bupSplitParams r buprepo k [] + let params = bupSplitParams r buprepo k [] + showOutput -- make way for bup output let cmd = proc "bup" (toCommand params) - liftIO $ withHandle StdinHandle createProcessSuccess cmd $ \h -> do + runner <- ifM commandProgressDisabled + ( return feedWithQuietOutput + , return (withHandle StdinHandle) + ) + liftIO $ runner createProcessSuccess cmd $ \h -> do meteredWrite p h b return True diff --git a/Remote/External.hs b/Remote/External.hs --- a/Remote/External.hs +++ b/Remote/External.hs @@ -17,6 +17,7 @@ import Config import Remote.Helper.Special import Utility.Metered +import Messages.Progress import Logs.Transfer import Logs.PreferredContent.Raw import Logs.RemoteState @@ -26,6 +27,7 @@ import Creds import Control.Concurrent.STM +import Control.Concurrent.Async import System.Log.Logger (debugM) import qualified Data.Map as M @@ -228,8 +230,7 @@ handleRemoteRequest (SETURIMISSING key uri) = withurl (SETURLMISSING key) uri handleRemoteRequest (GETURLS key prefix) = do - mapM_ (send . VALUE . fst . getDownloader) - =<< getUrlsWithPrefix key prefix + mapM_ (send . VALUE) =<< getUrlsWithPrefix key prefix send (VALUE "") -- end of list handleRemoteRequest (DEBUG msg) = liftIO $ debugM "external" msg handleRemoteRequest (VERSION _) = @@ -324,21 +325,28 @@ {- Starts an external remote process running, but does not handle checking - VERSION, etc. -} startExternal :: ExternalType -> Annex ExternalState -startExternal externaltype = liftIO $ do - (Just hin, Just hout, _, pid) <- createProcess $ (proc cmd []) - { std_in = CreatePipe - , std_out = CreatePipe - , std_err = Inherit - } - fileEncoding hin - fileEncoding hout - checkearlytermination =<< getProcessExitCode pid - return $ ExternalState - { externalSend = hin - , externalReceive = hout - , externalPid = pid - , externalPrepared = Unprepared - } +startExternal externaltype = do + errrelayer <- mkStderrRelayer + liftIO $ do + (Just hin, Just hout, Just herr, pid) <- createProcess $ + (proc cmd []) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + fileEncoding hin + fileEncoding hout + fileEncoding herr + stderrelay <- async $ errrelayer herr + checkearlytermination =<< getProcessExitCode pid + return $ ExternalState + { externalSend = hin + , externalReceive = hout + , externalShutdown = do + cancel stderrelay + void $ waitForProcess pid + , externalPrepared = Unprepared + } where cmd = externalRemoteProgram externaltype @@ -358,7 +366,7 @@ void $ atomically $ tryTakeTMVar v hClose $ externalSend st hClose $ externalReceive st - void $ waitForProcess $ externalPid st + externalShutdown st v = externalState external externalRemoteProgram :: ExternalType -> String diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs --- a/Remote/External/Types.hs +++ b/Remote/External/Types.hs @@ -70,7 +70,7 @@ data ExternalState = ExternalState { externalSend :: Handle , externalReceive :: Handle - , externalPid :: ProcessHandle + , externalShutdown :: IO () , externalPrepared :: PrepareStatus } diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -542,7 +542,8 @@ cache st = Annex.changeState $ \s -> s { Annex.remoteannexstate = M.insert (uuid r) st (Annex.remoteannexstate s) } go st a' = do - (ret, st') <- liftIO $ Annex.run st $ + curro <- Annex.getState Annex.output + (ret, st') <- liftIO $ Annex.run (st { Annex.output = curro }) $ catFileStop `after` a' cache st' return ret diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -42,6 +42,7 @@ import Remote.Helper.Encryptable as X import Remote.Helper.Messages import Annex.Content +import Messages.Progress import qualified Git import qualified Git.Command import qualified Git.Construct diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -17,6 +17,7 @@ import qualified CmdLine.GitAnnexShell.Fields as Fields import Types.Key import Remote.Helper.Messages +import Messages.Progress import Utility.Metered import Utility.Rsync import Types.Remote @@ -100,9 +101,14 @@ [] rsyncHelper :: Maybe MeterUpdate -> [CommandParam] -> Annex Bool -rsyncHelper callback params = do +rsyncHelper m params = do showOutput -- make way for progress bar - ifM (liftIO $ (maybe rsync rsyncProgress callback) params) + a <- case m of + Nothing -> return $ rsync params + Just meter -> do + oh <- mkOutputHandler + return $ rsyncProgress oh meter params + ifM (liftIO a) ( return True , do showLongNote "rsync failed -- run git annex again to resume file transfer" diff --git a/Remote/Hook.hs b/Remote/Hook.hs --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -17,6 +17,7 @@ import Annex.UUID import Remote.Helper.Special import Utility.Env +import Messages.Progress import qualified Data.Map as M @@ -113,7 +114,7 @@ where run command = do showOutput -- make way for hook output - ifM (liftIO $ boolSystemEnv "sh" [Param "-c", Param command] =<< hookEnv action k f) + ifM (progressCommandEnv "sh" [Param "-c", Param command] =<< liftIO (hookEnv action k f)) ( a , do warning $ hook ++ " hook exited nonzero!" diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -31,6 +31,7 @@ import Crypto import Utility.Rsync import Utility.CopyFile +import Messages.Progress import Utility.Metered import Utility.PID import Annex.Perms @@ -281,11 +282,15 @@ ) rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool -rsyncRemote direction o callback params = do +rsyncRemote direction o m params = do showOutput -- make way for progress bar - liftIO $ (maybe rsync rsyncProgress callback) $ - opts ++ [Params "--progress"] ++ params + case m of + Nothing -> liftIO $ rsync ps + Just meter -> do + oh <- mkOutputHandler + liftIO $ rsyncProgress oh meter ps where + ps = opts ++ [Params "--progress"] ++ params opts | direction == Download = rsyncDownloadOptions o | otherwise = rsyncUploadOptions o diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs --- a/RemoteDaemon/Core.hs +++ b/RemoteDaemon/Core.hs @@ -28,7 +28,7 @@ runForeground :: IO () runForeground = do - (readh, writeh) <- ioHandles + (readh, writeh) <- dupIoHandles ichan <- newTChanIO :: IO (TChan Consumed) ochan <- newTChanIO :: IO (TChan Emitted) diff --git a/Test.hs b/Test.hs --- a/Test.hs +++ b/Test.hs @@ -125,8 +125,8 @@ ingredients :: [Ingredient] ingredients = - [ rerunningTests [consoleTestReporter] - , listingTests + [ listingTests + , rerunningTests [consoleTestReporter] ] properties :: TestTree diff --git a/Types/Messages.hs b/Types/Messages.hs --- a/Types/Messages.hs +++ b/Types/Messages.hs @@ -7,8 +7,10 @@ module Types.Messages where -data OutputType = NormalOutput | QuietOutput | JSONOutput +import Data.Default +data OutputType = NormalOutput | QuietOutput | ProgressOutput | JSONOutput + data SideActionBlock = NoBlock | StartBlock | InBlock deriving (Eq) @@ -17,5 +19,6 @@ , sideActionBlock :: SideActionBlock } -defaultMessageState :: MessageState -defaultMessageState = MessageState NormalOutput NoBlock +instance Default MessageState + where + def = MessageState NormalOutput NoBlock diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs --- a/Upgrade/V2.hs +++ b/Upgrade/V2.hs @@ -16,6 +16,7 @@ import Annex.Content import Utility.Tmp import Logs +import Messages.Progress olddir :: Git.Repo -> FilePath olddir g @@ -44,7 +45,7 @@ old <- fromRepo olddir Annex.Branch.create - showProgress + showProgressDots e <- liftIO $ doesDirectoryExist old when e $ do @@ -53,12 +54,12 @@ mapM_ (\f -> inject f f) =<< logFiles old saveState False - showProgress + showProgressDots when e $ do inRepo $ Git.Command.run [Param "rm", Param "-r", Param "-f", Param "-q", File old] unless bare $ inRepo gitAttributesUnWrite - showProgress + showProgressDots unless bare push diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -87,7 +87,7 @@ pipeStrict :: [CommandParam] -> String -> IO String pipeStrict params input = do params' <- stdParams params - withBothHandles createProcessSuccess (proc gpgcmd params') $ \(to, from) -> do + withIOHandles createProcessSuccess (proc gpgcmd params') $ \(to, from) -> do hSetBinaryMode to True hSetBinaryMode from True hPutStr to input @@ -142,7 +142,7 @@ setup = liftIO . createProcess cleanup p (_, _, _, pid) = liftIO $ forceSuccessProcess p pid go p = do - let (to, from) = bothHandles p + let (to, from) = ioHandles p liftIO $ void $ forkIO $ do feeder to hClose to diff --git a/Utility/Metered.hs b/Utility/Metered.hs --- a/Utility/Metered.hs +++ b/Utility/Metered.hs @@ -1,4 +1,4 @@ -{- Metered IO +{- Metered IO and actions - - Copyright 2012-2105 Joey Hess - @@ -18,6 +18,7 @@ import System.Posix.Types import Data.Int import Data.Bits.Utils +import Control.Concurrent.Async {- An action that can be run repeatedly, updating it on the bytes processed. - @@ -145,8 +146,13 @@ k = 1024 chunkOverhead = 2 * sizeOf (undefined :: Int) -- GHC specific +data OutputHandler = OutputHandler + { quietMode :: Bool + , stderrHandler :: String -> IO () + } + {- Parses the String looking for a command's progress output, and returns - - Maybe the number of bytes rsynced so far, and any any remainder of the + - Maybe the number of bytes done so far, and any any remainder of the - string that could be an incomplete progress output. That remainder - should be prepended to future output, and fed back in. This interface - allows the command's output to be read in any desired size chunk, or @@ -155,11 +161,15 @@ type ProgressParser = String -> (Maybe BytesProcessed, String) {- Runs a command and runs a ProgressParser on its output, in order - - to update the meter. The command's output is also sent to stdout. -} -commandMeter :: ProgressParser -> MeterUpdate -> FilePath -> [CommandParam] -> IO Bool -commandMeter progressparser meterupdate cmd params = liftIO $ catchBoolIO $ - withHandle StdoutHandle createProcessSuccess p $ - feedprogress zeroBytesProcessed [] + - to update a meter. + -} +commandMeter :: ProgressParser -> OutputHandler -> MeterUpdate -> FilePath -> [CommandParam] -> IO Bool +commandMeter progressparser oh meterupdate cmd params = catchBoolIO $ + withOEHandles createProcessSuccess p $ \(outh, errh) -> do + ep <- async $ handlestderr errh + op <- async $ feedprogress zeroBytesProcessed [] outh + wait ep + wait op where p = proc cmd (toCommand params) @@ -168,8 +178,9 @@ if S.null b then return True else do - S.hPut stdout b - hFlush stdout + unless (quietMode oh) $ do + S.hPut stdout b + hFlush stdout let s = w82s (S.unpack b) let (mbytes, buf') = progressparser (buf++s) case mbytes of @@ -178,3 +189,40 @@ when (bytes /= prev) $ meterupdate bytes feedprogress bytes buf' h + + handlestderr h = unlessM (hIsEOF h) $ do + stderrHandler oh =<< hGetLine h + handlestderr h + +{- Runs a command, that may display one or more progress meters on + - either stdout or stderr, and prevents the meters from being displayed. + - + - The other command output is handled as configured by the OutputHandler. + -} +demeterCommand :: OutputHandler -> FilePath -> [CommandParam] -> IO Bool +demeterCommand oh cmd params = demeterCommandEnv oh cmd params Nothing + +demeterCommandEnv :: OutputHandler -> FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +demeterCommandEnv oh cmd params environ = catchBoolIO $ + withOEHandles createProcessSuccess p $ \(outh, errh) -> do + ep <- async $ avoidProgress True errh $ stderrHandler oh + op <- async $ avoidProgress True outh $ \l -> + unless (quietMode oh) $ + putStrLn l + wait ep + wait op + return True + where + p = (proc cmd (toCommand params)) + { env = environ } + +{- To suppress progress output, while displaying other messages, + - filter out lines that contain \r (typically used to reset to the + - beginning of the line when updating a progress display). + -} +avoidProgress :: Bool -> Handle -> (String -> IO ()) -> IO () +avoidProgress doavoid h emitter = unlessM (hIsEOF h) $ do + s <- hGetLine h + unless (doavoid && '\r' `elem` s) $ + emitter s + avoidProgress doavoid h emitter diff --git a/Utility/Path.hs b/Utility/Path.hs --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -170,17 +170,26 @@ == joinPath ["..", "..", "..", "..", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"] {- Given an original list of paths, and an expanded list derived from it, - - generates a list of lists, where each sublist corresponds to one of the - - original paths. When the original path is a directory, any items - - in the expanded list that are contained in that directory will appear in - - its segment. + - which may be arbitrarily reordered, generates a list of lists, where + - each sublist corresponds to one of the original paths. + - + - When the original path is a directory, any items in the expanded list + - that are contained in that directory will appear in its segment. + - + - The order of the original list of paths is attempted to be preserved in + - the order of the returned segments. However, doing so has a O^NM + - growth factor. So, if the original list has more than 100 paths on it, + - we stop preserving ordering at that point. Presumably a user passing + - that many paths in doesn't care too much about order of the later ones. -} segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]] segmentPaths [] new = [new] segmentPaths [_] new = [new] -- optimisation -segmentPaths (l:ls) new = [found] ++ segmentPaths ls rest +segmentPaths (l:ls) new = found : segmentPaths ls rest where - (found, rest)=partition (l `dirContains`) new + (found, rest) = if length ls < 100 + then partition (l `dirContains`) new + else break (\p -> not (l `dirContains` p)) new {- This assumes that it's cheaper to call segmentPaths on the result, - than it would be to run the action separately with each path. In diff --git a/Utility/Process.hs b/Utility/Process.hs --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -25,14 +25,16 @@ processTranscript, processTranscript', withHandle, - withBothHandles, + withIOHandles, + withOEHandles, withQuietOutput, + feedWithQuietOutput, createProcess, startInteractiveProcess, stdinHandle, stdoutHandle, stderrHandle, - bothHandles, + ioHandles, processHandle, devNull, ) where @@ -255,12 +257,12 @@ (stderrHandle, base { std_err = CreatePipe }) {- Like withHandle, but passes (stdin, stdout) handles to the action. -} -withBothHandles +withIOHandles :: CreateProcessRunner -> CreateProcess -> ((Handle, Handle) -> IO a) -> IO a -withBothHandles creator p a = creator p' $ a . bothHandles +withIOHandles creator p a = creator p' $ a . ioHandles where p' = p { std_in = CreatePipe @@ -268,6 +270,20 @@ , std_err = Inherit } +{- Like withHandle, but passes (stdout, stderr) handles to the action. -} +withOEHandles + :: CreateProcessRunner + -> CreateProcess + -> ((Handle, Handle) -> IO a) + -> IO a +withOEHandles creator p a = creator p' $ a . oeHandles + where + p' = p + { std_in = Inherit + , std_out = CreatePipe + , std_err = CreatePipe + } + {- Forces the CreateProcessRunner to run quietly; - both stdout and stderr are discarded. -} withQuietOutput @@ -281,6 +297,21 @@ } creator p' $ const $ return () +{- Stdout and stderr are discarded, while the process is fed stdin + - from the handle. -} +feedWithQuietOutput + :: CreateProcessRunner + -> CreateProcess + -> (Handle -> IO a) + -> IO a +feedWithQuietOutput creator p a = withFile devNull WriteMode $ \nullh -> do + let p' = p + { std_in = CreatePipe + , std_out = UseHandle nullh + , std_err = UseHandle nullh + } + creator p' $ a . stdinHandle + devNull :: FilePath #ifndef mingw32_HOST_OS devNull = "/dev/null" @@ -303,9 +334,12 @@ stderrHandle :: HandleExtractor stderrHandle (_, _, Just h, _) = h stderrHandle _ = error "expected stderrHandle" -bothHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) -bothHandles (Just hin, Just hout, _, _) = (hin, hout) -bothHandles _ = error "expected bothHandles" +ioHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +ioHandles (Just hin, Just hout, _, _) = (hin, hout) +ioHandles _ = error "expected ioHandles" +oeHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +oeHandles (_, Just hout, Just herr, _) = (hout, herr) +oeHandles _ = error "expected oeHandles" processHandle :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> ProcessHandle processHandle (_, _, _, pid) = pid diff --git a/Utility/Rsync.hs b/Utility/Rsync.hs --- a/Utility/Rsync.hs +++ b/Utility/Rsync.hs @@ -92,13 +92,13 @@ | rsyncUrlIsShell s = False | otherwise = ':' `notElem` s -{- Runs rsync, but intercepts its progress output and updates a meter. - - The progress output is also output to stdout. +{- Runs rsync, but intercepts its progress output and updates a progress + - meter. - - The params must enable rsync's --progress mode for this to work. -} -rsyncProgress :: MeterUpdate -> [CommandParam] -> IO Bool -rsyncProgress meterupdate = commandMeter parseRsyncProgress meterupdate "rsync" . rsyncParamsFixup +rsyncProgress :: OutputHandler -> MeterUpdate -> [CommandParam] -> IO Bool +rsyncProgress oh meter = commandMeter parseRsyncProgress oh meter "rsync" . rsyncParamsFixup {- Strategy: Look for chunks prefixed with \r (rsync writes a \r before - the first progress output, and each thereafter). The first number diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -101,14 +101,19 @@ prop_idempotent_shellEscape_multiword :: [String] -> Bool prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s -{- Segements a list of filenames into groups that are all below the manximum - - command-line length limit. Does not preserve order. -} -segmentXargs :: [FilePath] -> [[FilePath]] -segmentXargs l = go l [] 0 [] +{- Segments a list of filenames into groups that are all below the maximum + - command-line length limit. -} +segmentXargsOrdered :: [FilePath] -> [[FilePath]] +segmentXargsOrdered = reverse . map reverse . segmentXargsUnordered + +{- Not preserving data is a little faster, and streams better when + - there are a great many filesnames. -} +segmentXargsUnordered :: [FilePath] -> [[FilePath]] +segmentXargsUnordered l = go l [] 0 [] where - go [] c _ r = c:r + go [] c _ r = (c:r) go (f:fs) c accumlen r - | len < maxlen && newlen > maxlen = go (f:fs) [] 0 (c:r) + | newlen > maxlen && len < maxlen = go (f:fs) [] 0 (c:r) | otherwise = go fs (f:c) newlen r where len = length f diff --git a/Utility/SimpleProtocol.hs b/Utility/SimpleProtocol.hs --- a/Utility/SimpleProtocol.hs +++ b/Utility/SimpleProtocol.hs @@ -16,7 +16,7 @@ parse1, parse2, parse3, - ioHandles, + dupIoHandles, ) where import Data.Char @@ -80,8 +80,8 @@ - will mess up the protocol. To avoid that, close stdin, and - and duplicate stderr to stdout. Return two new handles - that are duplicates of the original (stdin, stdout). -} -ioHandles :: IO (Handle, Handle) -ioHandles = do +dupIoHandles :: IO (Handle, Handle) +dupIoHandles = do readh <- hDuplicate stdin writeh <- hDuplicate stdout nullh <- openFile devNull ReadMode diff --git a/Utility/Url.hs b/Utility/Url.hs --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -205,7 +205,7 @@ downloadQuiet = download' True download' :: Bool -> URLString -> FilePath -> UrlOptions -> IO Bool -download' quiet url file uo = +download' quiet url file uo = do case parseURIRelaxed url of Just u | uriScheme u == "file:" -> do @@ -224,7 +224,7 @@ -} #ifndef __ANDROID__ wgetparams = catMaybes - [ if Build.SysConfig.wgetquietprogress + [ if Build.SysConfig.wgetquietprogress && not quiet then Just $ Params "-q --show-progress" else Nothing , Just $ Params "--clobber -c -O" diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,39 @@ +git-annex (5.20150406) unstable; urgency=medium + + * Prevent git-ls-files from double-expanding wildcards when an + unexpanded wildcard is passed to a git-annex command like add or find. + * Fix make build target. Thanks, Justin Geibel. + * Fix GETURLS in external special remote protocol to strip + downloader prefix from logged url info before checking for the + specified prefix. + * importfeed: Avoid downloading a redundant item from a feed whose + guid has been downloaded before, even when the url has changed. + * importfeed: Always store itemid in metadata; before this was only + done when annex.genmetadata was set. + * Relax debian package dependencies to git >= 1:1.8.1 rather + than needing >= 1:2.0. + * test: Fix --list-tests + * addurl --file: When used with a special remote that claims + urls and checks their contents, don't override the user's provided + filename with filenames that the special remote suggests. Also, + don't allow adding the url if the special remote says it contains + multiple files. + * import: --deduplicate and --cleanduplicates now output the keys + corresponding to duplicated files they process. + * expire: New command, for expiring inactive repositories. + * fsck: Record fsck activity for use by expire command. + * Fix truncation of parameters that could occur when using xargs git-annex. + * Significantly sped up processing of large numbers of directories + passed to a single git-annex command. + * version: Add --raw + * init: Improve fifo test to detect NFS systems that support fifos + but not well enough for sshcaching. + * --quiet now suppresses progress displays from eg, rsync. + (The option already suppressed git-annex's own built-in progress + displays.) + + -- Joey Hess Mon, 06 Apr 2015 12:48:48 -0400 + git-annex (5.20150327) unstable; urgency=medium * readpresentkey: New plumbing command for checking location log. diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -75,7 +75,7 @@ lsof [!kfreebsd-i386 !kfreebsd-amd64 !hurd-any], ikiwiki, perlmagick, - git (>= 1:2.0), + git (>= 1:1.8.1), rsync, wget, curl, @@ -92,7 +92,7 @@ Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, - git (>= 1:2.0), + git (>= 1:1.8.1), rsync, wget, curl, diff --git a/doc/Android.mdwn b/doc/Android.mdwn --- a/doc/Android.mdwn +++ b/doc/Android.mdwn @@ -39,7 +39,7 @@ `rsync`, and `gpg`. To prevent the webapp from being automatically started -when a terminal window opens, go into the terminal preferences, to "Inital +when a terminal window opens, go into the terminal preferences, to "Initial Command", and clear out the default `git annex webapp` setting. Or, if you'd like to run the assistant automatically, but not open the diff --git a/doc/automatic_conflict_resolution.mdwn b/doc/automatic_conflict_resolution.mdwn --- a/doc/automatic_conflict_resolution.mdwn +++ b/doc/automatic_conflict_resolution.mdwn @@ -17,7 +17,7 @@ The "AAA" and "BBB" in the above example are essentially arbitrary (technically they are the MD5 checksum of the key). The automatic merge -conflict resoltuion is designed so that if two or more repositories both get +conflict resolution is designed so that if two or more repositories both get a merge conflict, and resolve it, the resolved repositories will not themselves conflict. This is why it doesn't use something nicer, like perhaps the name of the remote that the file came from. diff --git a/doc/bugs/--list-tests_runs_tests.mdwn b/doc/bugs/--list-tests_runs_tests.mdwn --- a/doc/bugs/--list-tests_runs_tests.mdwn +++ b/doc/bugs/--list-tests_runs_tests.mdwn @@ -23,3 +23,6 @@ [snip all the passing tests] All 140 tests passed (305.69s) """]] + +> [[fixed|done]] although I don't understand why tasty needs the +> `listingTests` ingredient to come first for it to work. --[[Joey]] diff --git a/doc/bugs/GETURLS_doesn__39__t_return_URLs_if_prefix_is_provided.mdwn b/doc/bugs/GETURLS_doesn__39__t_return_URLs_if_prefix_is_provided.mdwn --- a/doc/bugs/GETURLS_doesn__39__t_return_URLs_if_prefix_is_provided.mdwn +++ b/doc/bugs/GETURLS_doesn__39__t_return_URLs_if_prefix_is_provided.mdwn @@ -33,3 +33,5 @@ # End of transcript or log. """]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn b/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn @@ -0,0 +1,81 @@ +### Please describe the problem. + +Rebooted my laptop recently (after dunno how long of uptime) and annex assistant was shut off in previous uptime, but this time decided to leave it running. spotted that laptop is hot today to see that git-annex is busy. And I am not sure what it is really doing: top says + + 3087 yoh 30 10 520700 21084 14748 S 100.0 0.1 80:50.60 /usr/bin/git-annex remotedaemon + + +[[!format sh """ + +$> tail -20 /proc/3087/cwd/.git/annex/daemon.log + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +[2015-03-28 20:27:53 EDT] main: Syncing with debandy_Vault +hostname: Name or service not known +hostname: Name or service not known +To ssh://yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex/media/Vault/annex/ + 8c181af..a0b0bd6 git-annex -> synced/git-annex + 8352761..537d4a6 master -> synced/master + +"""]] + +in webapp I saw it struggling to connect to vagus which is offline, I turned it off, then reanabled syncing to above yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex . I don't know if annex was busier or not before, but it is 100% busy now, but nothing seems to be done -- no traffic, no changing fd's for that annex process + +strace shows busy reading from fd 24: + +[[!format sh """ +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +"""]] + + + $> ls -l /proc/3087/fd/24 + lr-x------ 1 yoh yoh 64 Mar 28 20:31 /proc/3087/fd/24 -> pipe:[794930] + + +so what could it be doing? + +[[!format sh """ + 2807 yoh 20 0 693M 110M 36948 S 1.4 0.7 11:15.32 ├─ /usr/bin/git-annex assistant --startdelay=5s + 3359 yoh 39 19 20996 3412 3096 S 0.0 0.0 0:00.01 │ ├─ git --git-dir=.git --work-tree=. check-attr -z --stdin annex.backend annex.numcopies -- + 3140 yoh 20 0 21128 3504 3188 S 0.0 0.0 0:00.01 │ ├─ git --git-dir=.git --work-tree=. check-ignore -z --stdin --verbose --non-matching + 3117 yoh 20 0 693M 110M 36948 S 0.0 0.7 0:00.00 │ ├─ /usr/bin/git-annex assistant --startdelay=5s + 3108 yoh 20 0 693M 110M 36948 S 0.0 0.7 0:00.15 │ ├─ /usr/bin/git-annex assistant --startdelay=5s + 3087 yoh 30 10 508M 21084 14748 S 97.9 0.1 1h29:16 │ ├─ /usr/bin/git-annex remotedaemon +24361 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.00 │ │ ├─ /usr/bin/git-annex remotedaemon +19799 yoh 30 10 0 0 0 Z 0.0 0.0 0:00.01 │ │ ├─ ssh +19440 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.00 │ │ ├─ /usr/bin/git-annex remotedaemon + 3222 yoh 30 10 27368 5796 3456 S 0.0 0.0 0:00.00 │ │ ├─ git --git-dir=.git --work-tree=. cat-file --batch + 3110 yoh 30 10 508M 21084 14748 R 97.5 0.1 1h29:10 │ │ ├─ /usr/bin/git-annex remotedaemon + 3090 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.98 │ │ ├─ /usr/bin/git-annex remotedaemon + 3089 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.03 │ │ └─ /usr/bin/git-annex remotedaemon +"""]] + +how could I figure out what is doing??? meanwhile I have just sent STOP signal to 3087 + +### What version of git-annex are you using? On what operating system? + +5.20150327+git27-g6af24b6-1 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/feeding_git_annex_with_xargs_can_fail.mdwn b/doc/bugs/feeding_git_annex_with_xargs_can_fail.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/feeding_git_annex_with_xargs_can_fail.mdwn @@ -0,0 +1,15 @@ +Feeding git-annex a long list off directories, eg with xargs can have +2 bad behaviors: + +* git-annex runs git ls-files and passes it the same long list. But the git + ls-files command is longer than the git-annex command often, so it gets + truncated and some files are not processed. + + > fixed --[[Joey]] + +* It can take a really long time for git-annex to chew through the + git-ls-files results. There is probably an exponential blowup in the time + relative to the number of parameters. Some of the stuff being done to + preserve original ordering etc is likely at fault. + + > [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/importfeed_fails_with_local_file_urls.mdwn b/doc/bugs/importfeed_fails_with_local_file_urls.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/importfeed_fails_with_local_file_urls.mdwn @@ -0,0 +1,28 @@ +Hi, + +I've a script which generates .rss files which reference local files with the file:// scheme. I can import the file:// urls with git annex addurl, but it fails with git annex importfeed: + + +`$ git annex importfeed --fast file:///path/to/local/rss/file.rss` + +`(checking known urls...)` + +`importfeed file:///path/to/local/rss/file.rss` + +`git-annex: /tmp/feed6757: openFile: resource busy (file is locked)` + +If I try to import it with `$ git annex importfeed --fast /path/to/local/rss/file.rss` I get + +`importfeed /path/to/local/rss/file.rss` + +` warning: bad feed content` + +`ok` + +But the directory stays empty. + + +Is it possible to use local files in rss format with items which reference local files using the file:// scheme as input for importfeed? + +Cheers, +Marco diff --git a/doc/bugs/package_build_fails_with_missing_man_directory.mdwn b/doc/bugs/package_build_fails_with_missing_man_directory.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/package_build_fails_with_missing_man_directory.mdwn @@ -0,0 +1,30 @@ +When building a debian package, the build fails due to a missing man/ directory. + + ./Build/mdwn2man man/git-annex-vpop.1 1 doc/git-annex-vpop.mdwn > man/git-annex-vpop.1 + /bin/sh: 1: cannot create man/git-annex-vpop.1: Directory nonexistent + +I was able to build the package with the following patch: + +[[!format patch """ +From: Justin Geibel +Date: Fri, 27 Mar 2015 16:21:13 -0400 +Subject: Fix build of man pages + +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index d6fb1a1..b36cbcd 100644 +--- a/Makefile ++++ b/Makefile +@@ -1,5 +1,5 @@ + mans=$(shell find doc -maxdepth 1 -name git-annex*.mdwn | sed -e 's/^doc/man/' -e 's/\.mdwn/\.1/') +-all=git-annex $(mans) docs ++all=git-annex mans docs + + CABAL?=cabal # set to "./Setup" if you lack a cabal program + GHC?=ghc +"""]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/unexpected_double_wildcard_expansion.mdwn b/doc/bugs/unexpected_double_wildcard_expansion.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/unexpected_double_wildcard_expansion.mdwn @@ -0,0 +1,15 @@ +From the forum, it seems that git-ls-files very unexpectedly expands +wildcards in filenames passed to it. (Not a documented or an expected +behavior.) + +This causes problems when eg, the user does `git annex add *.jpeg` and that +matches no files, but there are some jpegs in subdirectories. git-ls-files +re-expands the wildcard and finds those. + +Seems that the best fix is to make Git.LsFiles paper over this git +misfeature, by always escaping wildcards in paths passed +to git-ls-files. AFAIK, no callers of Git.LsFiles expect to provide it +wildcards, because I was completely surprised when I learned they were +expanded.. --[[Joey]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/chunking.mdwn b/doc/chunking.mdwn --- a/doc/chunking.mdwn +++ b/doc/chunking.mdwn @@ -13,7 +13,7 @@ initremote`, specifying the chunk size. Good chunk sizes will depend on the remote, but a good starting place -is probably `1MiB`. Very large chunks are problimatic, both because +is probably `1MiB`. Very large chunks are problematic, both because git-annex needs to buffer one chunk in memory when uploading, and because a larger chunk will make resuming interrupted transfers less efficient. On the other hand, when a file is split into a great many chunks, diff --git a/doc/design/assistant/xmpp_security.mdwn b/doc/design/assistant/xmpp_security.mdwn --- a/doc/design/assistant/xmpp_security.mdwn +++ b/doc/design/assistant/xmpp_security.mdwn @@ -18,11 +18,11 @@ use SPEKE (or similar methods like J-PAKE) to generate a shared key. Avoids active MITM attacks. Makes pairing harder, especially pairing between one's own devices, since the passphrase has to be entered on - all devices. Also problimatic when pairing more than 2 devices, + all devices. Also problematic when pairing more than 2 devices, especially when adding a device to the set later, since there would then be multiple different keys in use. * Rely on the user's gpg key, and do gpg key verification during XMPP - pairing. Problimatic because who wants to put their gpg key on their + pairing. Problematic because who wants to put their gpg key on their phone? Also, require the users be in the WOT and be gpg literate. Update: This seems unlikely to be worth doing. [[Telehash]] is better. diff --git a/doc/design/iabackup.mdwn b/doc/design/iabackup.mdwn --- a/doc/design/iabackup.mdwn +++ b/doc/design/iabackup.mdwn @@ -134,6 +134,8 @@ With that change, the server can check for files that not enough clients have verified they have recently, and distribute them to more clients. +(This is now implemented.) + Note that bad actors can lie about this verification; it's not a proof they still have the file. But, a bad actor could prove they have a file, and refuse to give it back if the IA needed to restore the backup, too. diff --git a/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment b/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment new file mode 100644 --- /dev/null +++ b/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlZF5AC-FSxwkiay5ZgEYZwUzN69Wa6PTE" + nickname="Sunke" + subject="filename from metadata?" + date="2015-04-06T18:00:25Z" + content=""" +Hi everbody, +is it possible to use a metadata field for the filename in a +metadata driven view? + +I am thinking of the following use case: + +git annex metadata --set artist=Led\ Zeppelin --set album=Led\ Zeppelin\ IV --set title=04\ Stairway\ to\ heaven some/weird/filename.mp3 +git annex view --filename-from title artist=* album=* + +result: +Led Zeppelin/Led Zeppelin IV/04 Stairway to heaven.mp3 +"""]] diff --git a/doc/devblog/day_268_stressed_out.mdwn b/doc/devblog/day_268_stressed_out.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_268_stressed_out.mdwn @@ -0,0 +1,18 @@ +While I plowed through a lot of backlog the past several days, +I still have some 120 messages piled deep. + +That work did result in a number of improvements, culminating in a rather +rushed release of version 5.20150327 today, to fix a regression affecting +`git annex sync` when using the standalone linux tarballs. Unfortunately, I +then had to update those tarballs a second time after the release +as the first fix was incomplete. + +And, I'm feeling super stressed out. At this point, I think I should step +away until the end of the month. Unfortunately, this will mean more backlog +later. Including lots of noise and hand-holding that I just don't seem to +have time for if I want to continue making forward progress. + +Maybe I'll think of a way to deal with it while I'm away. Currently, all I +have is that I may have to start ignoring irc and the forum, and +de-prioritizing bug reports that don't have either a working reproduction +recipe or multiple independent confirmations that it's a real bug. diff --git a/doc/devblog/day_269__wildcards_and_podcasts.mdwn b/doc/devblog/day_269__wildcards_and_podcasts.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_269__wildcards_and_podcasts.mdwn @@ -0,0 +1,26 @@ +Turns out that git has a feature I didn't know about; it will expand +wildcards and other stuff in filenames passed to many git commands. This is +on top of the shell's expansion. + +That led to some broken behavior by `git annex add 'foo.*'` +and, it could lead to other probably unwanted behavior, like `git annex +drop 'foo[barred]'` dropping a file named `food` in addition to +`foo[barred]` + +For now, I've disabled this git feature throughout git-annex. If you relied +on it for something, let me know, I might think about adding it back in +specific places where it makes sense. + +---- + +Improved `git annex importfeed` to check the itemid of the feed and avoid +re-downloading a file with the same itemid. Before, it would add duplicate +files if a feed kept the itemid the same, but changed the url. This was +easier than expected because annex.genmetadata already caused the itemid +to be stored in the git-annex metadata. I just had to make it check the +itemid metadata, and set itemid even when annex.genmetadata isn't set. + +---- + +Also got 4 other bug reports fixed, even though I feel I'm taking it easy +today. It's good to be relaxed again! diff --git a/doc/devblog/day_270__distributed_fsck.mdwn b/doc/devblog/day_270__distributed_fsck.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_270__distributed_fsck.mdwn @@ -0,0 +1,27 @@ +Added two options to `git annex fsck` that allow for a form of distributed +fsck. This is useful in situations where repositiories cannot be trusted to +continue to exist, and cannot be checked directly, but you'd still like to +keep track of their status. [[design/iabackup]] is one use case for this. + +By running a periodic fsck with the --distributed option, +the repositories can verify that they still exist and that the +information about their contents is still accurate. This is done by +doing an extra update of the location log each time a file is verified by +fsck to still be in the repository. + +The other option looks like --expire="30d somerepo:60d". It checks that +each specified repository has recorded a distributed fsck within the specified +time period. If not, the repository is dropped from the location tracking +log. Of course it can always update that later if it's really still around. + +Distributed fsck is not the default because those extra location log updates +increase the size of the git-annex branch. I did one thing to keep the size +increase small: An identical line is logged to for each key, including the +timestamp, so git's delta compression will work as well as is possible. But, +there's still commit and tree update overhead. + +Probably doesn't make sense to run distributed fscks too often for that and +other reasons. If the git-annex branch does get too large, there's always +`git annex forget` ... + +**(Update: This was later rethought and works much more efficiently now..)** diff --git a/doc/devblog/day_271__parallel_get_groundwork.mdwn b/doc/devblog/day_271__parallel_get_groundwork.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_271__parallel_get_groundwork.mdwn @@ -0,0 +1,32 @@ +I've started work on [[todo/parallel_get]]. +Today, laid the groundwork in two areas: + +1. Evalulated the ascii-progress haskell library. It can display + multiple progress bars in the terminal, portably, and its author + Pedro Tacla Yamada has kindly offered to improve it to meet + git-annex's needs. + + I ended up filing [10 issues](https://github.com/yamadapc/haskell-ascii-progress/issues) + on it today, around 3 of the are blockers for git-annex using it. + +2. Worked on making --quiet more quiet. Commands like rsync and wget + need to have thier progress output disabled when run in parallel. + + Didn't quite finish this yet. + +--- + +Yesterday I made some improvements to how git-annex behaves when it's +passed a massive number of directories or files on the command line. +Eg, when driven by xargs. There turned out to be some bugs in that +scenario. + +One problem one I kind of had to paper over. While git-annex get +normally is careful to get the files in the same order they were listed on +the command line, it becomes very expensive to expand directories using +git-ls-files, and reorder its output to preserve order, when a large number +offiles are passed on the command line. There was a O(N*M) time blowup. + +I worked around it by making it only preserve the order of the first 100 +files. Assumption being that if you're specifying so many files on the +command line, you probably have less of an attachment to their ordering. :) diff --git a/doc/devblog/day_272__forest_for_trees.mdwn b/doc/devblog/day_272__forest_for_trees.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_272__forest_for_trees.mdwn @@ -0,0 +1,13 @@ +Rethought distributed fsck. It's not really a fsck, but an expiration of +inactive repositories, where fscking is one kind of activity. That insight +let me reimplement it much more efficiently. Rather than updating all +the location logs to prove it was active, `git annex fsck` can simply and +inexpensively update an activity log. It's so cheap it'll do it by default. +The `git annex expire` command then reads the activity log and expires +(or unexpires) repositories that have not been active in the desired time +period. Expiring a repository simply marks it as dead. + +Yesterday, finished making --quiet really be quiet. That sounds easy, +but it took several hours. On the `concurrentprogress` branch, I have +ascii-progress hooked up and working, but it's not quite ready for prime +time. diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn --- a/doc/encryption.mdwn +++ b/doc/encryption.mdwn @@ -89,7 +89,8 @@ encrypted using that old key, and the user should re-encrypt everything.) (Because filenames are MAC'ed, a cipher still needs to be -generated (and encrypted to the given key IDs).) +generated (and encrypted to the given key IDs). It is only used for MHAC +encryption of filenames.) ## MAC algorithm diff --git a/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn b/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn @@ -0,0 +1,5 @@ +I am interested in using git-annex to manage git repositories, and I am wondering if it is possible and if anyone has experience with it? + +I have done some searching, and I know that many people have asked for support for a Dropbox-like workflow, where Git repositories are mirrored everywhere. I also know that no such support seems forthcoming, however this is not my goal. Rather, I would like to use git-annex to track the location of many repositories. I keep a lot of repositories and would like to offload them onto other storage devices and keep track of where each repository is stored. + +Perhaps entire Git repositories can be added as a single unit for tracking in git-annex? diff --git a/doc/forum/ga-ncdu.mdwn b/doc/forum/ga-ncdu.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/ga-ncdu.mdwn @@ -0,0 +1,5 @@ +I've just written a quick script which provides an output which ncdu will use. + +It uses the size from the key for apparent size and the size of the file (if present) for the disk size so 'a' toggles between total size of the annex and the actually used disk space. + +Is there anyone else interested in having this? If so, I'll tidy it up and release it :) diff --git a/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment b/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment new file mode 100644 --- /dev/null +++ b/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="andy" + subject="comment 1" + date="2015-03-28T22:03:53Z" + content=""" +I'd take a look. I just downloaded ncdu prompted by your comment, and that would be helpful to get sane output regarding my annexed files. +"""]] diff --git a/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment b/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment new file mode 100644 --- /dev/null +++ b/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="markusk" + subject="Sounds nice !" + date="2015-03-28T22:41:45Z" + content=""" + +I would be very glad if you would release it! +"""]] diff --git a/doc/forum/gadu_-_git-annex_disk_usage.mdwn b/doc/forum/gadu_-_git-annex_disk_usage.mdwn --- a/doc/forum/gadu_-_git-annex_disk_usage.mdwn +++ b/doc/forum/gadu_-_git-annex_disk_usage.mdwn @@ -1,4 +1,4 @@ -Based on the thread over at I decided to finally write a du like utility for git-annex. A 0.01 version is up over at . It works, but I intend to make it smarter about handling git repos and annexed files, as well as adding more of the options available in the standard du utility. +Based on the thread over at [[forum/__34__du__34___equivalent_on_an_annex__63__]] I decided to finally write a du like utility for git-annex. A 0.01 version is up over at . It works, but I intend to make it smarter about handling git repos and annexed files, as well as adding more of the options available in the standard du utility. Currently it will tally up the sizes of links that look like they are annexed files. I plan to make it actually interact with git and git-annex to verify the files are annexed and enable options like tallying files only from specific remotes, only missing files, not double counting files which are annexed multiple times but stored only once, etc... diff --git a/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn b/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn @@ -0,0 +1,9 @@ +as a followup to [[forum/remote-specific_meta-data/]], now i'm thinking of injecting data in the `git-annex` branch. i'm still a little hesitant because [[forum/optimising_lookupkey/]] mentions that i need to "delete or update `.git/annex/index`" when doing so. yet I also saw in the source code there is a `.git/annex/index.lock` file that is being used to avoid concurrent access to that file - is that something that should be watched out for? how do I manage that lockfile? is it mandatory to stage files into the index at first? + +maybe i'm just not familiar enough with how git operates internally myself. i've reviewed this section of the progit book again: http://git-scm.com/book/en/v2/Git-Internals-Git-Objects - and i feel i have again some handle on how to do those things, but it seems like a major hurdle to go through just to change a single setting in one git-annex metadata file... wouldn't `enableremote` be sufficient here? I figured to append custom fields to a remote, i could do: + + git annex enableremote myremoteuuid "$(git cat-file -p git-annex:remote.log | grep myremoteuuid) extratag=true" + +Would that work at all? Should i operate directly on the git-annex branch myself instead? + +Thanks! --[[anarcat]] diff --git a/doc/forum/two_lines_back_to_remote__63__.mdwn b/doc/forum/two_lines_back_to_remote__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/two_lines_back_to_remote__63__.mdwn @@ -0,0 +1,10 @@ +`git annex map` generates a diagram roughly like this: + + +------+ +-------+ + |laptop| -> |usbdisk| + | | <- | | + | | <- | | + +------+ +-------+ + +Why are there *two* lines back from the disk to the laptop? + diff --git a/doc/git-annex-assistant.mdwn b/doc/git-annex-assistant.mdwn --- a/doc/git-annex-assistant.mdwn +++ b/doc/git-annex-assistant.mdwn @@ -26,7 +26,7 @@ * `--startdelay=N` Wait N seconds before running the startup scan. This process can - be expensive and you may not want to run it immediatly upon login. + be expensive and you may not want to run it immediately upon login. * `--foreground` diff --git a/doc/git-annex-expire.mdwn b/doc/git-annex-expire.mdwn new file mode 100644 --- /dev/null +++ b/doc/git-annex-expire.mdwn @@ -0,0 +1,66 @@ +# NAME + +git-annex expire - expire inactive repositories + +# SYNOPSIS + +git annex expire `[repository:]time ...` + +# DESCRIPTION + +This command expires repositories that have not performed some activity +within a specified time period. A repository is expired by marking it as +dead. De-expiration is also done; if a dead repository performed some +activity recently, it is marked as semitrusted again. + +This can be useful when it's not possible to keep track of the state +of repositories manually. For example, a distributed network of +repositories where nobody can directly access all the repositories to +check their status. + +The repository can be specified using the name of a remote, +or the description or uuid of the repository. + +The time is in the form "60d" or "1y". A time of "never" will disable +expiration. + +If a time is specified without a repository, it is used as the default +value for all repositories. Note that the current repository is never +expired. + +# OPTIONS + +* `--no-act` + + Print out what would be done, but not not actually expite or unexpire + any repositories. + +* `--activity=Name` + + Specify the activity that a repository must have performed to avoid being + expired. The default is any activity. + + Currently, the only activity that can be performed to avoid expiration + is `git annex fsck`. Note that fscking a remote updates the + expiration of the remote repository, not the local repository. + + The first version of git-annex that recorded fsck activity was + 5.20150405. + +# SEE ALSO + +[[git-annex]](1) + +[[git-annex-fsck]](1) + +[[git-annex-schedule]](1) + +[[git-annex-dead]](1) + +[[git-annex-semitrust]](1) + +# AUTHOR + +Joey Hess + +Warning: Automatically converted into a man page by mdwn2man. Edit with care. diff --git a/doc/git-annex-importfeed.mdwn b/doc/git-annex-importfeed.mdwn --- a/doc/git-annex-importfeed.mdwn +++ b/doc/git-annex-importfeed.mdwn @@ -9,7 +9,7 @@ # DESCRIPTION Imports the contents of podcast feeds. Only downloads files whose -urls have not already been added to the repository before, so you can +content has not already been added to the repository before, so you can delete, rename, etc the resulting files and repeated runs won't duplicate them. @@ -17,11 +17,14 @@ are on a video hosting site, and the video is downloaded. This allows importing e.g., youtube playlists. +To make the import process add metadata to the imported files from the feed, +`git config annex.genmetadata true` + # OPTIONS * `--force` - Force downoading urls it's seen before. + Force downoading items it's seen before. * `--template` diff --git a/doc/git-annex-registerurl.mdwn b/doc/git-annex-registerurl.mdwn --- a/doc/git-annex-registerurl.mdwn +++ b/doc/git-annex-registerurl.mdwn @@ -15,7 +15,7 @@ If the key and url are not specified on the command line, they are instead read from stdin. Any number of lines can be provided in this -mode, each containing a key and url, sepearated by whitespace. +mode, each containing a key and url, separated by whitespace. # SEE ALSO diff --git a/doc/git-annex-unannex.mdwn b/doc/git-annex-unannex.mdwn --- a/doc/git-annex-unannex.mdwn +++ b/doc/git-annex-unannex.mdwn @@ -1,6 +1,6 @@ # NAME -git-annex unannex - undo accidential add command +git-annex unannex - undo accidental add command # SYNOPSIS diff --git a/doc/git-annex-version.mdwn b/doc/git-annex-version.mdwn --- a/doc/git-annex-version.mdwn +++ b/doc/git-annex-version.mdwn @@ -19,6 +19,12 @@ built. Therefore, "5.20150320-gdd35cf3" is a daily build, and "5.20150401" is an April 1st release made a bit later. +# OPTIONS + +* `--raw` + + Causes only git-annex's version to be output, and nothing else. + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -172,7 +172,7 @@ * `assistant` - Atomatically sync folders between devices. + Automatically sync folders between devices. See [[git-annex-assistant]](1) for details. @@ -302,6 +302,11 @@ See [[git-annex-fsck]](1) for details. +* `expire [repository:]time ...` + + Expires repositories that have not recently performed an activity + (such as a fsck). + * `unused` Checks the annex for data that does not correspond to any files present @@ -655,8 +660,7 @@ * `--quiet` - Avoid the default verbose display of what is done; only show errors - and progress displays. + Avoid the default verbose display of what is done; only show errors. * `--verbose` @@ -792,7 +796,8 @@ In particular, it stores year and month metadata, from the file's modification date. - When importfeed is used, it stores additional metadata from the feed. + When importfeed is used, it stores additional metadata from the feed, + such as the author, title, etc. * `annex.queuesize` diff --git a/doc/index.mdwn b/doc/index.mdwn --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -33,7 +33,7 @@ - +
[[!inline feeds=no template=bare pages=footer/column_a]][[!inline feeds=no template=bare pages=footer/column_b]][[!inline feeds=no template=bare pages=footer/column_b]]
diff --git a/doc/install/fromsource.mdwn b/doc/install/fromsource.mdwn --- a/doc/install/fromsource.mdwn +++ b/doc/install/fromsource.mdwn @@ -43,8 +43,8 @@ Inside the source tree, run: cabal configure -f"-assistant -webapp -webdav -pairing -xmpp -dns" - cabal install --only-dependencies - cabal build + cabal install -j -f"-assistant -webapp -webdav -pairing -xmpp -dns" --only-dependencies + cabal build -j PATH=$HOME/bin:$PATH cabal install --bindir=$HOME/bin @@ -62,8 +62,8 @@ Once the C libraries are installed, run inside the source tree: cabal configure - cabal install --only-dependencies - cabal build + cabal install -j --only-dependencies + cabal build -j PATH=$HOME/bin:$PATH cabal install --bindir=$HOME/bin diff --git a/doc/internals.mdwn b/doc/internals.mdwn --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -247,6 +247,14 @@ 42bf2035-0636-461d-a367-49e9dfd361dd fsck self 30m every day at any time; fsck 4b3ebc86-0faf-4892-83c5-ce00cbe30f0a 1h every year at any time timestamp=1385646997.053162s +## `activity.log` + +Used to record the times of activities, such as fscks. + +Example: + + 42bf2035-0636-461d-a367-49e9dfd361dd Fsck timestamp=1422387398.30395s + ## `transitions.log` Used to record transitions, eg by `git annex forget` diff --git a/doc/news/version_5.20150113.mdwn b/doc/news/version_5.20150113.mdwn deleted file mode 100644 --- a/doc/news/version_5.20150113.mdwn +++ /dev/null @@ -1,25 +0,0 @@ -git-annex 5.20150113 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * unlock: Don't allow unlocking files that have never been committed to git - before, to avoid an intractable problem that prevents the pre-commit - hook from telling if such a file is intended to be an annexed file or not. - * Avoid re-checksumming when migrating from hash to hashE backend. - Closes: #[774494](http://bugs.debian.org/774494) - * Fix build with process 1.2.1.0. - * Android: Provide a version built with -fPIE -pie to support Android 5.0. - * sync: Fix an edge case where syncing in a bare repository would try to - merge and so fail. - * Check git version at runtime, rather than assuming it will be the same - as the git version used at build time when running git-checkattr and - git-branch remove. - * Switch to using relative paths to the git repository. - - This allows the git repository to be moved while git-annex is running in - it, with fewer problems. - - On Windows, this avoids some of the problems with the absurdly small - MAX\_PATH of 260 bytes. In particular, git-annex repositories should - work in deeper/longer directory structures than before. - * Generate shorter keys for WORM and URL, avoiding keys that are longer - than used for SHA256, so as to not break on systems like Windows that - have very small maximum path length limits. - * Bugfix: A file named HEAD in the work tree could confuse some git commands - run by git-annex."""]] diff --git a/doc/news/version_5.20150406.mdwn b/doc/news/version_5.20150406.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20150406.mdwn @@ -0,0 +1,33 @@ +git-annex 5.20150406 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Prevent git-ls-files from double-expanding wildcards when an + unexpanded wildcard is passed to a git-annex command like add or find. + * Fix make build target. Thanks, Justin Geibel. + * Fix GETURLS in external special remote protocol to strip + downloader prefix from logged url info before checking for the + specified prefix. + * importfeed: Avoid downloading a redundant item from a feed whose + guid has been downloaded before, even when the url has changed. + * importfeed: Always store itemid in metadata; before this was only + done when annex.genmetadata was set. + * Relax debian package dependencies to git >= 1:1.8.1 rather + than needing >= 1:2.0. + * test: Fix --list-tests + * addurl --file: When used with a special remote that claims + urls and checks their contents, don't override the user's provided + filename with filenames that the special remote suggests. Also, + don't allow adding the url if the special remote says it contains + multiple files. + * import: --deduplicate and --cleanduplicates now output the keys + corresponding to duplicated files they process. + * expire: New command, for expiring inactive repositories. + * fsck: Record fsck activity for use by expire command. + * Fix truncation of parameters that could occur when using xargs git-annex. + * Significantly sped up processing of large numbers of directories + passed to a single git-annex command. + * version: Add --raw + * init: Improve fifo test to detect NFS systems that support fifos + but not well enough for sshcaching. + * --quiet now suppresses progress displays from eg, rsync. + (The option already suppressed git-annex's own built-in progress + displays.)"""]] diff --git a/doc/publicrepos.mdwn b/doc/publicrepos.mdwn --- a/doc/publicrepos.mdwn +++ b/doc/publicrepos.mdwn @@ -6,7 +6,7 @@ Various downloads of things produced by Joey Hess, including git-annex builds and videos. -* debconf-share +* debconf-share (currently unavailable via IPv4) `git clone http://annex.debconf.org/debconf-share/.git/` [DebConf](http://debconf.org/) Media, photos, videos, etc. @@ -18,12 +18,15 @@ * [ocharles's papers](https://github.com/ocharles/papers) Lots of CS papers read by [Oliver](http://ocharles.org.uk/blog/). +* [rejuvyesh's papers](https://github.com/rejuvyesh/papers) + Quite a few AI papers read by [rejuvyesh](http://rejuvyVesh.com) + * [MRI brain scan data](http://studyforrest.org/pages/access.html) `git clone http://psydata.ovgu.de/forrest_gump/.git studyforrest` High-resolution, ultra-highfield fMRI dataset on auditory perception. -* [rejuvyesh's papers](https://github.com/rejuvyesh/papers) - Quite a few AI papers read by [rejuvyesh](http://rejuvyesh.com) +* [gst-integration-testsuites](http://cgit.freedesktop.org/gstreamer/gst-integration-testsuites) + GStreamer integration testsuite media assets management This is a wiki -- add your own public repository to the list! See [[tips/centralized_git_repository_tutorial]]. diff --git a/doc/special_remotes/comment_23_aba4abb897c6ecc0c6ba72cbd21b9f4b._comment b/doc/special_remotes/comment_23_aba4abb897c6ecc0c6ba72cbd21b9f4b._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/comment_23_aba4abb897c6ecc0c6ba72cbd21b9f4b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnWvnTWY6LrcPB4BzYEBn5mRTpNhg5EtEg" + nickname="Bence" + subject="Amazon Cloud Drive support" + date="2015-03-28T10:35:47Z" + content=""" +Is there a special remote implementation for Amazon Cloud Drive? + +It's just became unlimited for a fair price: $60/year ( https://www.amazon.com/clouddrive/pricing ). + +http://techcrunch.com/2015/03/26/amazon-goes-after-dropbox-google-microsoft-with-unlimited-cloud-drive-storage/ +"""]] diff --git a/doc/todo/Output_key_with_import__47__deduplicate.mdwn b/doc/todo/Output_key_with_import__47__deduplicate.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/Output_key_with_import__47__deduplicate.mdwn @@ -0,0 +1,22 @@ +When using + + git annex import --deduplicate -- ~/directory/to/import + +would it be possible to output the keys for duplicate files? This would allow people to find where in the annex the files already exist. + +e.g. + + import ~/directory/to/import/001.txt ok + import ~/directory/to/import/002.txt ok + import ~/directory/to/import/002_d.txt (duplicate SHA256E-s261--fb1230987ac123098...) ok + import ~/directory/to/import/003.txt ok + +Then you could use + + git log -S SHA256E-s261--fb1230987ac123098... + +to find out where it is already. + +Not sure if that is the nicest layout.. (or what it might break). + +> [[done]] --[[Joey]] diff --git a/doc/todo/cheaper_global_fsck.mdwn b/doc/todo/cheaper_global_fsck.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/cheaper_global_fsck.mdwn @@ -0,0 +1,38 @@ +Global fsck updates all location log entries for a repo. This wastes disk +space. + +I realized now that it can be implemented w/o such waste. Probably cheaply +enough to be the default! + +What we need is a new log file, call it fscktimes.log. +This records the time of the last fsck of each repo. + +`git annex fsck --expire` no longer needs to look at the location log at +all. It can just check the repo's fscktimes.log entry. If the entry is +recent enough, we know that the repo has fscked recently, and its location +log is good, and nothing needs to be done. Otherwise, we know that the repo +has stopped fscking, and we simply expire *all* its location logs. + +Note that fscktime.log is only used by fsck; it does not impact git-annex +generally or make it slower. And, it's very low overhead to update the one +file. Repos could do a fsck --fast on a daily basis and not grow the +git-annex branch much. Maybe on an hourly basis even. + +(BTW, there is some overlap with the fsck.log file that is currently used to +hold the timestamp of the last local fsck. May be able to eliminate that +file too.) + +---- + +It might be worth making the fsck.log record --fast and full fscks +separately so we know the last of each for each repo. This would let +--expire require periodic full fscks and more frequent fast fscks. + +---- + +Hmm, --expire updates all the location logs when it thinks a repo has gone +missing. Why not just mark it dead? Again, this would save a lot of space! +It would complicate recovery if a repo had been offline and came back; it +would need to mark itself as not dead any longer. + +> [[done]] --[[Joey]] diff --git a/doc/todo/command_line_interface_for_required_content_setthings.mdwn b/doc/todo/command_line_interface_for_required_content_setthings.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/command_line_interface_for_required_content_setthings.mdwn @@ -0,0 +1,11 @@ +Someone in the forum noticed that `git annex wanted` +handles preferred content settings, but there is no analagous `git annex +required`. + +Probably worth adding that, although required content is not an often +used feature, and vicfg can already configure it. + +(I don't much like the `git annex required` name. Nor the `git annex wanted` +one when it comes to that. Oh well.) + +--[[Joey]] diff --git a/doc/todo/parallel_get.mdwn b/doc/todo/parallel_get.mdwn --- a/doc/todo/parallel_get.mdwn +++ b/doc/todo/parallel_get.mdwn @@ -71,3 +71,8 @@ This approach is just not as flexible or nice in general. See also: [[parallel_possibilities]] + +> I am looking at using the ascii-progress library for this. +> It has nice support for multiple progress bars, and is portable. +> I have filed 7 issues on it, around 4 of which need to get fixed before +> it's suitable for git-annex to use.. --[[Joey]] diff --git a/doc/todo/podcatching_handling_updated_files.mdwn b/doc/todo/podcatching_handling_updated_files.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/podcatching_handling_updated_files.mdwn @@ -0,0 +1,19 @@ +Files in feeds can be updated, and if this update includes changing the +url, `importfeed` will treat this as a new file. This results in `foo.mp3` +having a `2_foo.mp3` added next to it. + +This seems to happen especially commonly with feeds using FeedBurner. +Saw several with same size, different checksum and url. + +To detect this, `importfeed` could store the item's guid in the metadata +of the key. Where it currently builds a `Map URLString Key` of all +known items, it could instead build a `Map (Either URlString GUID) Key`. + +This would at least prevent the duplication, when the feed has guids. + +> [[done]] --[[Joey]] + +It would be even nicer if the old file could be updated with the new +content. But, since files can be moved around, deleted, tagged, etc, +that only seems practical at all if the file is still in the directory +where `importfeed` created it. diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20150327 +Version: 5.20150406 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess diff --git a/man/git-annex-.1 b/man/git-annex-.1 deleted file mode 100644 --- a/man/git-annex-.1 +++ /dev/null @@ -1,18 +0,0 @@ -.TH git-annex- 1 -.SH NAME -git\-annex \- -.PP -.SH SYNOPSIS -git annex \fB[path ...]\fP -.PP -.SH DESCRIPTION -.PP -.SH OPTIONS -.SH SEE ALSO -git\-annex(1) -.PP -.SH AUTHOR -Joey Hess -.PP -.PP - diff --git a/man/git-annex-assistant.1 b/man/git-annex-assistant.1 --- a/man/git-annex-assistant.1 +++ b/man/git-annex-assistant.1 @@ -22,7 +22,7 @@ .IP .IP "\fB\-\-startdelay=N\fP" Wait N seconds before running the startup scan. This process can -be expensive and you may not want to run it immediatly upon login. +be expensive and you may not want to run it immediately upon login. .IP .IP "\fB\-\-foreground\fP" Avoid forking to the background. diff --git a/man/git-annex-expire.1 b/man/git-annex-expire.1 new file mode 100644 --- /dev/null +++ b/man/git-annex-expire.1 @@ -0,0 +1,61 @@ +.TH git-annex-expire 1 +.SH NAME +git\-annex expire \- expire inactive repositories +.PP +.SH SYNOPSIS +git annex expire \fB[repository:]time ...\fP +.PP +.SH DESCRIPTION +This command expires repositories that have not performed some activity +within a specified time period. A repository is expired by marking it as +dead. De\-expiration is also done; if a dead repository performed some +activity recently, it is marked as semitrusted again. +.PP +This can be useful when it's not possible to keep track of the state +of repositories manually. For example, a distributed network of +repositories where nobody can directly access all the repositories to +check their status. +.PP +The repository can be specified using the name of a remote, +or the description or uuid of the repository. +.PP +The time is in the form "60d" or "1y". A time of "never" will disable +expiration. +.PP +If a time is specified without a repository, it is used as the default +value for all repositories. Note that the current repository is never +expired. +.PP +.SH OPTIONS +.IP "\fB\-\-no\-act\fP" +.IP +Print out what would be done, but not not actually expite or unexpire +any repositories. +.IP +.IP "\fB\-\-activity=Name\fP" +Specify the activity that a repository must have performed to avoid being +expired. The default is any activity. +.IP +Currently, the only activity that can be performed to avoid expiration +is \fBgit annex fsck\fP. Note that fscking a remote updates the +expiration of the remote repository, not the local repository. +.IP +The first version of git\-annex that recorded fsck activity was +5.20150405. +.IP +.SH SEE ALSO +git\-annex(1) +.PP +git\-annex\-fsck(1) +.PP +git\-annex\-schedule(1) +.PP +git\-annex\-dead(1) +.PP +git\-annex\-semitrust(1) +.PP +.SH AUTHOR +Joey Hess +.PP +.PP + diff --git a/man/git-annex-importfeed.1 b/man/git-annex-importfeed.1 --- a/man/git-annex-importfeed.1 +++ b/man/git-annex-importfeed.1 @@ -7,7 +7,7 @@ .PP .SH DESCRIPTION Imports the contents of podcast feeds. Only downloads files whose -urls have not already been added to the repository before, so you can +content has not already been added to the repository before, so you can delete, rename, etc the resulting files and repeated runs won't duplicate them. .PP @@ -15,10 +15,13 @@ are on a video hosting site, and the video is downloaded. This allows importing e.g., youtube playlists. .PP +To make the import process add metadata to the imported files from the feed, +\fBgit config annex.genmetadata true\fP +.PP .SH OPTIONS .IP "\fB\-\-force\fP" .IP -Force downoading urls it's seen before. +Force downoading items it's seen before. .IP .IP "\fB\-\-template\fP" Controls where the files are stored. diff --git a/man/git-annex-registerurl.1 b/man/git-annex-registerurl.1 --- a/man/git-annex-registerurl.1 +++ b/man/git-annex-registerurl.1 @@ -13,7 +13,7 @@ .PP If the key and url are not specified on the command line, they are instead read from stdin. Any number of lines can be provided in this -mode, each containing a key and url, sepearated by whitespace. +mode, each containing a key and url, separated by whitespace. .PP .SH SEE ALSO git\-annex(1) diff --git a/man/git-annex-unannex.1 b/man/git-annex-unannex.1 --- a/man/git-annex-unannex.1 +++ b/man/git-annex-unannex.1 @@ -1,6 +1,6 @@ .TH git-annex-unannex 1 .SH NAME -git\-annex unannex \- undo accidential add command +git\-annex unannex \- undo accidental add command .PP .SH SYNOPSIS git annex unannex \fB[path ...]\fP diff --git a/man/git-annex-version.1 b/man/git-annex-version.1 --- a/man/git-annex-version.1 +++ b/man/git-annex-version.1 @@ -17,6 +17,11 @@ built. Therefore, "5.20150320\-gdd35cf3" is a daily build, and "5.20150401" is an April 1st release made a bit later. .PP +.SH OPTIONS +.IP "\fB\-\-raw\fP" +.IP +Causes only git\-annex's version to be output, and nothing else. +.IP .SH SEE ALSO git\-annex(1) .PP diff --git a/man/git-annex.1 b/man/git-annex.1 --- a/man/git-annex.1 +++ b/man/git-annex.1 @@ -149,7 +149,7 @@ See git\-annex\-watch(1) for details. .IP .IP "\fBassistant\fP" -Atomatically sync folders between devices. +Automatically sync folders between devices. .IP See git\-annex\-assistant(1) for details. .IP @@ -260,6 +260,10 @@ .IP See git\-annex\-fsck(1) for details. .IP +.IP "\fBexpire [repository:]time ...\fP" +Expires repositories that have not recently performed an activity +(such as a fsck). +.IP .IP "\fBunused\fP" Checks the annex for data that does not correspond to any files present in any tag or branch, and prints a numbered list of the data. @@ -562,8 +566,7 @@ What is avoided depends on the command. .IP .IP "\fB\-\-quiet\fP" -Avoid the default verbose display of what is done; only show errors -and progress displays. +Avoid the default verbose display of what is done; only show errors. .IP .IP "\fB\-\-verbose\fP" Enable verbose display. @@ -680,7 +683,8 @@ In particular, it stores year and month metadata, from the file's modification date. .IP -When importfeed is used, it stores additional metadata from the feed. +When importfeed is used, it stores additional metadata from the feed, +such as the author, title, etc. .IP .IP "\fBannex.queuesize\fP" git\-annex builds a queue of git commands, in order to combine similar diff --git a/standalone/linux/skel/git-annex b/standalone/linux/skel/git-annex --- a/standalone/linux/skel/git-annex +++ b/standalone/linux/skel/git-annex @@ -28,7 +28,4 @@ export GIT_ANNEX_APP_BASE fi -GIT_ANNEX_PROGRAMPATH="$0" -export GIT_ANNEX_PROGRAMPATH - exec "$base/runshell" git-annex "$@"