packages feed

git-annex 5.20150327 → 5.20150406

raw patch · 104 files changed

+1484/−341 lines, 104 files

Files

Annex.hs view
@@ -142,7 +142,7 @@ 	, backends = [] 	, remotes = [] 	, remoteannexstate = M.empty-	, output = defaultMessageState+	, output = def 	, force = False 	, fast = False 	, daemon = False
Annex/Content.hs view
@@ -57,6 +57,7 @@ import Annex.Content.Direct import Annex.ReplaceFile import Utility.LockFile+import Messages.Progress  {- Checks if a given key's content is currently present. -} inAnnex :: Key -> Annex Bool@@ -555,12 +556,17 @@ downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool downloadUrl urls file = go =<< annexWebDownloadCommand <$> Annex.getGitConfig   where-	go Nothing = Url.withUrlOptions $ \uo ->-		anyM (\u -> Url.download u file uo) urls-	go (Just basecmd) = liftIO $ anyM (downloadcmd basecmd) urls+	go Nothing = do+		a <- ifM commandProgressDisabled+			( return Url.downloadQuiet+			, return Url.download+			)+		Url.withUrlOptions $ \uo ->+			anyM (\u -> a u file uo) urls+	go (Just basecmd) = anyM (downloadcmd basecmd) urls 	downloadcmd basecmd url =-		boolSystem "sh" [Param "-c", Param $ gencmd url basecmd]-			<&&> doesFileExist file+		progressCommand "sh" [Param "-c", Param $ gencmd url basecmd]+			<&&> liftIO (doesFileExist file) 	gencmd url = massReplace 		[ ("%file", shellEscape file) 		, ("%url", shellEscape url)
Annex/Fixup.hs view
@@ -11,6 +11,7 @@ import Git.Config import Types.GitConfig import qualified Git.Construct as Construct+import qualified Git.BuildVersion import Utility.Path import Utility.SafeCommand import Utility.Directory@@ -27,10 +28,19 @@  fixupRepo :: Repo -> GitConfig -> IO Repo fixupRepo r c = do-	r' <- fixupSubmodule r c+	let r' = disableWildcardExpansion r+	r'' <- fixupSubmodule r' c 	if annexDirect c-		then fixupDirect r'-		else return r'+		then fixupDirect r''+		else return r''++{- Disable git's built-in wildcard expansion, which is not wanted+ - when using it as plumbing by git-annex. -}+disableWildcardExpansion :: Repo -> Repo+disableWildcardExpansion r+	| Git.BuildVersion.older "1.8.1" = r+	| otherwise = r+		{ gitGlobalOpts = gitGlobalOpts r ++ [Param "--literal-pathspecs"] }  {- Direct mode repos have core.bare=true, but are not really bare.  - Fix up the Repo to be a non-bare repo, and arrange for git commands
Annex/Init.hs view
@@ -162,13 +162,17 @@ #else 	tmp <- fromRepo gitAnnexTmpMiscDir 	let f = tmp </> "gaprobe"+	let f2 = tmp </> "gaprobe2" 	createAnnexDirectory tmp 	liftIO $ do 		nukeFile f+		nukeFile f2 		ms <- tryIO $ do 			createNamedPipe f ownerReadMode+			createLink f f2 			getFileStatus f 		nukeFile f+		nukeFile f2 		return $ either (const False) isNamedPipe ms #endif
Assistant/Threads/Committer.hs view
@@ -290,7 +290,7 @@ 	-- files. The ls-files is run on a batch of files. 	findnew [] = return ([], noop) 	findnew pending@(exemplar:_) = do-		let segments = segmentXargs $ map changeFile pending+		let segments = segmentXargsUnordered $ map changeFile pending 		rs <- liftAnnex $ forM segments $ \fs -> 			inRepo (Git.LsFiles.notInRepo False fs) 		let (newfiles, cleanup) = foldl'@@ -457,7 +457,7 @@ 	 -} 	findopenfiles keysources = ifM crippledFileSystem 		( liftIO $ do-			let segments = segmentXargs $ map keyFilename keysources+			let segments = segmentXargsUnordered $ map keyFilename keysources 			concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) 		, do 			tmpdir <- fromRepo gitAnnexTmpMiscDir
Build/LinuxMkLibs.hs view
@@ -69,6 +69,8 @@ 		createSymbolicLink link (top </> exelink) 	writeFile exe $ unlines 		[ "#!/bin/sh"+		, "GIT_ANNEX_PROGRAMPATH=\"$0\""+		, "export GIT_ANNEX_PROGRAMPATH" 		, "exec \"$GIT_ANNEX_DIR/" ++ exelink ++ "\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_DIR/shimmed/" ++ base ++ "/" ++ base ++ "\" \"$@\"" 		] 	modifyFileMode exe $ addModes executeModes
CHANGELOG view
@@ -1,3 +1,39 @@+git-annex (5.20150406) unstable; urgency=medium++  * Prevent git-ls-files from double-expanding wildcards when an+    unexpanded wildcard is passed to a git-annex command like add or find.+  * Fix make build target. Thanks, Justin Geibel.+  * Fix GETURLS in external special remote protocol to strip+    downloader prefix from logged url info before checking for the+    specified prefix.+  * importfeed: Avoid downloading a redundant item from a feed whose+    guid has been downloaded before, even when the url has changed.+  * importfeed: Always store itemid in metadata; before this was only+    done when annex.genmetadata was set.+  * Relax debian package dependencies to git >= 1:1.8.1 rather+    than needing >= 1:2.0.+  * test: Fix --list-tests+  * addurl --file: When used with a special remote that claims+    urls and checks their contents, don't override the user's provided+    filename with filenames that the special remote suggests. Also,+    don't allow adding the url if the special remote says it contains+    multiple files.+  * import: --deduplicate and --cleanduplicates now output the keys+    corresponding to duplicated files they process.+  * expire: New command, for expiring inactive repositories.+  * fsck: Record fsck activity for use by expire command.+  * Fix truncation of parameters that could occur when using xargs git-annex.+  * Significantly sped up processing of large numbers of directories+    passed to a single git-annex command.+  * version: Add --raw+  * init: Improve fifo test to detect NFS systems that support fifos+    but not well enough for sshcaching.+  * --quiet now suppresses progress displays from eg, rsync.+    (The option already suppressed git-annex's own built-in progress+    displays.)++ -- Joey Hess <id@joeyh.name>  Mon, 06 Apr 2015 12:48:48 -0400+ git-annex (5.20150327) unstable; urgency=medium    * readpresentkey: New plumbing command for checking location log.
CmdLine/GitAnnex.hs view
@@ -45,6 +45,7 @@ import qualified Command.InitRemote import qualified Command.EnableRemote import qualified Command.Fsck+import qualified Command.Expire import qualified Command.Repair import qualified Command.Unused import qualified Command.DropUnused@@ -169,6 +170,7 @@ 	, Command.VCycle.cmd 	, Command.Fix.cmd 	, Command.Fsck.cmd+	, Command.Expire.cmd 	, Command.Repair.cmd 	, Command.Unused.cmd 	, Command.DropUnused.cmd
CmdLine/Seek.hs view
@@ -215,8 +215,8 @@  seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [FilePath] -> Annex [FilePath] seekHelper a params = do-	ll <- inRepo $ \g ->-		runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g) params+	ll <- inRepo $ \g -> concat <$> forM (segmentXargsOrdered params)+		(runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) 	forM_ (map fst $ filter (null . snd) $ zip params ll) $ \p -> 		unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ 			error $ p ++ " not found"
Command/AddUrl.hs view
@@ -64,25 +64,36 @@ 		r <- Remote.claimingUrl u 		if Remote.uuid r == webUUID || raw 			then void $ commandAction $ startWeb relaxed optfile pathdepth u-			else do-				pathmax <- liftIO $ fileNameLengthLimit "."-				let deffile = fromMaybe (urlString2file u pathdepth pathmax) optfile-				res <- tryNonAsync $ maybe-					(error $ "unable to checkUrl of " ++ Remote.name r)-					(flip id u)-					(Remote.checkUrl r)-				case res of-					Left e -> void $ commandAction $ do-						showStart "addurl" u-						warning (show e)-						next $ next $ return False-					Right (UrlContents sz mf) -> do-						void $ commandAction $-							startRemote r relaxed (maybe deffile fromSafeFilePath mf) u sz-					Right (UrlMulti l) ->-						forM_ l $ \(u', sz, f) ->-							void $ commandAction $-								startRemote r relaxed (deffile </> fromSafeFilePath f) u' sz+			else checkUrl r u optfile relaxed pathdepth++checkUrl :: Remote -> URLString -> Maybe FilePath -> Bool -> Maybe Int -> Annex ()+checkUrl r u optfile relaxed pathdepth = do+	pathmax <- liftIO $ fileNameLengthLimit "."+	let deffile = fromMaybe (urlString2file u pathdepth pathmax) optfile+	go deffile =<< maybe+		(error $ "unable to checkUrl of " ++ Remote.name r)+		(tryNonAsync . flip id u)+		(Remote.checkUrl r)+  where++	go _ (Left e) = void $ commandAction $ do+		showStart "addurl" u+		warning (show e)+		next $ next $ return False+	go deffile (Right (UrlContents sz mf)) = do+		let f = fromMaybe (maybe deffile fromSafeFilePath mf) optfile+		void $ commandAction $+			startRemote r relaxed f u sz+	go deffile (Right (UrlMulti l))+		| isNothing optfile =+			forM_ l $ \(u', sz, f) ->+				void $ commandAction $+					startRemote r relaxed (deffile </> fromSafeFilePath f) u' sz+		| otherwise = error $ unwords+			[ "That url contains multiple files according to the"+			, Remote.name r+			, " remote; cannot add it to a single file."+			]  startRemote :: Remote -> Bool -> FilePath -> URLString -> Maybe Integer -> CommandStart startRemote r relaxed file uri sz = do
+ Command/Expire.hs view
@@ -0,0 +1,106 @@+{- git-annex command+ -+ - Copyright 2015 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Expire where++import Common.Annex+import Command+import Logs.Activity+import Logs.UUID+import Logs.MapLog+import Logs.Trust+import Annex.UUID+import qualified Remote+import Utility.HumanTime++import Data.Time.Clock.POSIX+import qualified Data.Map as M++cmd :: [Command]+cmd = [withOptions [activityOption, noActOption] $ command "expire" paramExpire seek+	SectionMaintenance "expire inactive repositories"]++paramExpire :: String+paramExpire = (paramRepeating $ paramOptional paramRemote ++ ":" ++ paramTime)++activityOption :: Option+activityOption = fieldOption [] "activity" "Name" "specify activity"++noActOption :: Option+noActOption = flagOption [] "no-act" "don't really do anything"++seek :: CommandSeek+seek ps = do+	expire <- parseExpire ps+	wantact <- getOptionField activityOption (pure . parseActivity)+	noact <- getOptionFlag noActOption+	actlog <- lastActivities wantact+	u <- getUUID+	us <- filter (/= u) . M.keys <$> uuidMap+	descs <- uuidMap+	seekActions $ pure $ map (start expire noact actlog descs) us++start :: Expire -> Bool -> Log Activity -> M.Map UUID String -> UUID -> CommandStart+start (Expire expire) noact actlog descs u =+	case lastact of+		Just ent | notexpired ent -> checktrust (== DeadTrusted) $ do+			showStart "unexpire" desc+			showNote =<< whenactive+			unless noact $+				trustSet u SemiTrusted+		_ -> checktrust (/= DeadTrusted) $ do+			showStart "expire" desc+			showNote =<< whenactive+			unless noact $+				trustSet u DeadTrusted+  where+	lastact = changed <$> M.lookup u actlog+	whenactive = case lastact of+		Just (Date t) -> do+			d <- liftIO $ durationSince $ posixSecondsToUTCTime t+			return $ "last active: " ++ fromDuration d ++ " ago"+		_  -> return "no activity"+	desc = fromUUID u ++ " " ++ fromMaybe "" (M.lookup u descs)+	notexpired ent = case ent of+		Unknown -> False+		Date t -> case lookupexpire of+			Just (Just expiretime) -> t >= expiretime+			_ -> True+	lookupexpire = headMaybe $ catMaybes $+		map (`M.lookup` expire) [Just u, Nothing]+	checktrust want a = ifM (want <$> lookupTrust u)+		( do+			void a+			next $ next $ return True+		, stop+		)++data Expire = Expire (M.Map (Maybe UUID) (Maybe POSIXTime))++parseExpire :: [String] -> Annex Expire+parseExpire [] = error "Specify an expire time."+parseExpire ps = do+	now <- liftIO getPOSIXTime+	Expire . M.fromList <$> mapM (parse now) ps+  where+	parse now s = case separate (== ':') s of+		(t, []) -> return (Nothing, parsetime now t)+		(n, t) -> do+			r <- Remote.nameToUUID n+			return (Just r, parsetime now t)+	parsetime _ "never" = Nothing+	parsetime now s = case parseDuration s of+		Nothing -> error $ "bad expire time: " ++ s+		Just d -> Just (now - durationToPOSIXTime d)++parseActivity :: Maybe String -> Maybe Activity+parseActivity Nothing = Nothing+parseActivity (Just s) = case readish s of+	Nothing -> error $ "Unknown activity. Choose from: " ++ +		unwords (map show [minBound..maxBound :: Activity])+	Just v -> Just v+
Command/Fsck.hs view
@@ -23,6 +23,7 @@ import Annex.Link import Logs.Location import Logs.Trust+import Logs.Activity import Config.NumCopies import Annex.UUID import Utility.DataUnits@@ -74,41 +75,7 @@ 		(withFilesInGit $ whenAnnexed $ start from i) 		ps 	withFsckDb i FsckDb.closeDb--getIncremental :: UUID -> Annex Incremental-getIncremental u = do-	i <- maybe (return False) (checkschedule . parseDuration)-		=<< Annex.getField (optionName incrementalScheduleOption)-	starti <- Annex.getFlag (optionName startIncrementalOption)-	morei <- Annex.getFlag (optionName moreIncrementalOption)-	case (i, starti, morei) of-		(False, False, False) -> return NonIncremental-		(False, True, False) -> startIncremental-		(False ,False, True) -> contIncremental-		(True, False, False) ->-			maybe startIncremental (const contIncremental)-				=<< getStartTime u-		_ -> error "Specify only one of --incremental, --more, or --incremental-schedule"-  where-	startIncremental = do-		recordStartTime u-		ifM (FsckDb.newPass u)-			( StartIncremental <$> FsckDb.openDb u-			, error "Cannot start a new --incremental fsck pass; another fsck process is already running."-			)-	contIncremental = ContIncremental <$> FsckDb.openDb u--	checkschedule Nothing = error "bad --incremental-schedule value"-	checkschedule (Just delta) = do-		Annex.addCleanup FsckCleanup $ do-			v <- getStartTime u-			case v of-				Nothing -> noop-				Just started -> do-					now <- liftIO getPOSIXTime-					when (now - realToFrac started >= durationToPOSIXTime delta) $-						resetStartTime u-		return True+	recordActivity Fsck u  start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart start from inc file key = do@@ -421,8 +388,6 @@ 	return $ (if ok then "dropped from " else "failed to drop from ") 		++ Remote.name remote -data Incremental = StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle | NonIncremental- runFsck :: Incremental -> FilePath -> Key -> Annex Bool -> CommandStart runFsck inc file key a = ifM (needFsck inc key) 	( do@@ -497,3 +462,40 @@ #else 		fromfile >= fromstatus #endif++data Incremental = StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle | NonIncremental++getIncremental :: UUID -> Annex Incremental+getIncremental u = do+	i <- maybe (return False) (checkschedule . parseDuration)+		=<< Annex.getField (optionName incrementalScheduleOption)+	starti <- getOptionFlag startIncrementalOption+	morei <- getOptionFlag moreIncrementalOption+	case (i, starti, morei) of+		(False, False, False) -> return NonIncremental+		(False, True, False) -> startIncremental+		(False ,False, True) -> contIncremental+		(True, False, False) ->+			maybe startIncremental (const contIncremental)+				=<< getStartTime u+		_ -> error "Specify only one of --incremental, --more, or --incremental-schedule"+  where+	startIncremental = do+		recordStartTime u+		ifM (FsckDb.newPass u)+			( StartIncremental <$> FsckDb.openDb u+			, error "Cannot start a new --incremental fsck pass; another fsck process is already running."+			)+	contIncremental = ContIncremental <$> FsckDb.openDb u++	checkschedule Nothing = error "bad --incremental-schedule value"+	checkschedule (Just delta) = do+		Annex.addCleanup FsckCleanup $ do+			v <- getStartTime u+			case v of+				Nothing -> noop+				Just started -> do+					now <- liftIO getPOSIXTime+					when (now - realToFrac started >= durationToPOSIXTime delta) $+						resetStartTime u+		return True
Command/Import.hs view
@@ -15,6 +15,7 @@ import Backend import Remote import Types.KeySource+import Types.Key  cmd :: [Command] cmd = [withOptions opts $ notBareRepo $ command "import" paramPaths seek@@ -72,8 +73,8 @@ 		, stop 		)   where-	deletedup = do-		showNote "duplicate"+	deletedup k = do+		showNote $ "duplicate of " ++ key2file k 		liftIO $ removeFile srcfile 		next $ return True 	importfile = do@@ -88,17 +89,19 @@ 		| isDirectory s = notoverwriting "(is a directory)" 		| otherwise = ifM (Annex.getState Annex.force) 			( liftIO $ nukeFile destfile-			, notoverwriting "(use --force to override)"+			, notoverwriting "(use --force to override, or a duplication option such as --deduplicate to clean up)" 			) 	notoverwriting why = error $ "not overwriting existing " ++ destfile ++ " " ++ why 	checkdup dupa notdupa = do 		backend <- chooseBackend destfile 		let ks = KeySource srcfile srcfile Nothing 		v <- genKey ks backend-		isdup <- case v of-			Just (k, _) -> not . null <$> keyLocations k-			_ -> return False-		return $ if isdup then dupa else notdupa+		case v of+			Just (k, _) -> ifM (not . null <$> keyLocations k)+				( return (maybe Nothing (\a -> Just (a k)) dupa)+				, return notdupa+				)+			_ -> return notdupa 	pickaction = case mode of 		DeDuplicate -> checkdup (Just deletedup) (Just importfile) 		CleanDuplicates -> checkdup (Just deletedup) Nothing
Command/ImportFeed.hs view
@@ -73,7 +73,7 @@ 	v <- findDownloads url 	case v of 		[] -> do-			feedProblem url "bad feed content"+			feedProblem url "bad feed content; no enclosures to download" 			next $ return True 		l -> do 			ok <- and <$> mapM (performDownload opts cache) l@@ -96,22 +96,33 @@  data DownloadLocation = Enclosure URLString | QuviLink URLString +type ItemId = String+ data Cache = Cache 	{ knownurls :: S.Set URLString+	, knownitems :: S.Set ItemId 	, template :: Utility.Format.Format 	}  getCache :: Maybe String -> Annex Cache getCache opttemplate = ifM (Annex.getState Annex.force)-	( ret S.empty+	( ret S.empty S.empty 	, do 		showSideAction "checking known urls"-		ret =<< S.fromList <$> knownUrls+		(is, us) <- unzip <$> (mapM knownItems =<< knownUrls)+		ret (S.fromList us) (S.fromList (concat is)) 	)   where 	tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate-	ret s = return $ Cache s tmpl+	ret us is = return $ Cache us is tmpl +knownItems :: (Key, URLString) -> Annex ([ItemId], URLString)+knownItems (k, u) = do+	itemids <- S.toList . S.filter (/= noneValue) . S.map fromMetaValue +		. currentMetaDataValues itemIdField +		<$> getCurrentMetaData k+	return (itemids, u)+ findDownloads :: URLString -> Annex [ToDownload] findDownloads u = go =<< downloadFeed u   where@@ -191,12 +202,18 @@   where 	forced = Annex.getState Annex.force -	{- Avoids downloading any urls that are already known to be+	{- Avoids downloading any items that are already known to be 	 - associated with a file in the annex, unless forced. -} 	checkknown url a-		| S.member url (knownurls cache) = ifM forced (a, return True)+		| knownitemid || S.member url (knownurls cache)+			= ifM forced (a, return True) 		| otherwise = a +	knownitemid = case getItemId (item todownload) of+		-- only when it's a permalink+		Just (True, itemid) -> S.member itemid (knownitems cache)+		_ -> False+ 	rundownload url extension getter = do 		dest <- makeunique url (1 :: Integer) $ 			feedFile (template cache) todownload extension@@ -211,8 +228,10 @@ 						checkFeedBroken (feedurl todownload) 					else do 						forM_ ks $ \key ->-							whenM (annexGenMetaData <$> Annex.getGitConfig) $-								addMetaData key $ extractMetaData todownload+							ifM (annexGenMetaData <$> Annex.getGitConfig)+								( addMetaData key $ extractMetaData todownload+								, addMetaData key $ minimalMetaData todownload+								) 						showEndOk 						return True @@ -275,6 +294,12 @@ 	tometa (k, v) = (mkMetaFieldUnchecked k, S.singleton (toMetaValue v)) 	meta = MetaData $ M.fromList $ map tometa $ extractFields i +minimalMetaData :: ToDownload -> MetaData+minimalMetaData i = case getItemId (item i) of+	(Nothing) -> emptyMetaData+	(Just (_, itemid)) -> MetaData $ M.singleton itemIdField +		(S.singleton $ toMetaValue itemid)+ {- Extract fields from the feed and item, that are both used as metadata,  - and to generate the filename. -} extractFields :: ToDownload -> [(String, String)]@@ -296,11 +321,17 @@ 	feedauthor = getFeedAuthor $ feed i 	itemauthor = getItemAuthor $ item i +itemIdField :: MetaField+itemIdField = mkMetaFieldUnchecked "itemid"+ extractField :: String -> [Maybe String] -> (String, String)-extractField k [] = (k, "none")+extractField k [] = (k, noneValue) extractField k (Just v:_) 	| not (null v) = (k, v) extractField k (_:rest) = extractField k rest++noneValue :: String+noneValue = "none"  {- Called when there is a problem with a feed.  - Throws an error if the feed is broken, otherwise shows a warning. -}
Command/Info.hs view
@@ -198,6 +198,7 @@ 	[ remote_name 	, remote_description 	, remote_uuid+	, remote_trust 	, remote_cost 	, remote_type 	]@@ -265,6 +266,10 @@ remote_uuid :: Remote -> Stat remote_uuid r = simpleStat "uuid" $ pure $ 	fromUUID $ Remote.uuid r++remote_trust :: Remote -> Stat+remote_trust r = simpleStat "trust" $ lift $+	showTrustLevel <$> lookupTrust (Remote.uuid r)  remote_cost :: Remote -> Stat remote_cost r = simpleStat "cost" $ pure $
Command/TransferKeys.hs view
@@ -16,7 +16,7 @@ import Annex.Transfer import qualified Remote import Types.Key-import Utility.SimpleProtocol (ioHandles)+import Utility.SimpleProtocol (dupIoHandles) import Git.Types (RemoteName)  data TransferRequest = TransferRequest Direction Remote Key AssociatedFile@@ -30,7 +30,7 @@  start :: CommandStart start = do-	(readh, writeh) <- liftIO ioHandles+	(readh, writeh) <- liftIO dupIoHandles 	runRequests readh writeh runner 	stop   where
Command/Version.hs view
@@ -18,16 +18,28 @@ import qualified Backend  cmd :: [Command]-cmd = [noCommit $ noRepo startNoRepo $ dontCheck repoExists $+cmd = [withOptions [rawOption] $+	noCommit $ noRepo startNoRepo $ dontCheck repoExists $ 	command "version" paramNothing seek SectionQuery "show version info"] +rawOption :: Option+rawOption = flagOption [] "raw" "output only program version"+ seek :: CommandSeek-seek = withNothing start+seek = withNothing $ ifM (getOptionFlag rawOption) (startRaw, start) +startRaw :: CommandStart+startRaw = do+	liftIO $ do+		putStr SysConfig.packageversion+		hFlush stdout+	stop+ start :: CommandStart start = do 	v <- getVersion 	liftIO $ do+ 		showPackageVersion 		info "local repository version" $ fromMaybe "unknown" v 		info "supported repository version" supportedVersion
Logs.hs view
@@ -1,6 +1,6 @@ {- git-annex log file names  -- - Copyright 2013-2014 Joey Hess <id@joeyh.name>+ - Copyright 2013-2015 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -40,6 +40,7 @@ 	, preferredContentLog 	, requiredContentLog 	, scheduleLog+	, activityLog 	, differenceLog 	] @@ -84,8 +85,12 @@ scheduleLog :: FilePath scheduleLog = "schedule.log" +activityLog :: FilePath+activityLog = "activity.log"+ differenceLog :: FilePath differenceLog = "difference.log"+  {- The pathname of the location log file for a given key. -} locationLogFile :: GitConfig -> Key -> String
+ Logs/Activity.hs view
@@ -0,0 +1,37 @@+{- git-annex activity log+ -+ - Copyright 2015 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.Activity (+	Log,+	Activity(..),+	recordActivity,+	lastActivities,+) where++import Data.Time.Clock.POSIX++import Common.Annex+import qualified Annex.Branch+import Logs+import Logs.UUIDBased++data Activity = Fsck+	deriving (Eq, Read, Show, Enum, Bounded)++recordActivity :: Activity -> UUID -> Annex ()+recordActivity act uuid = do+	ts <- liftIO getPOSIXTime+	Annex.Branch.change activityLog $+		showLog id . changeLog ts uuid (show act) . parseLog readish++lastActivities :: Maybe Activity -> Annex (Log Activity)+lastActivities wantact = parseLog onlywanted <$> Annex.Branch.get activityLog+  where+	onlywanted s = case readish s of+		Just a | wanted a -> Just a+		_ -> Nothing+	wanted a = maybe True (a ==) wantact
Logs/Location.hs view
@@ -19,6 +19,7 @@ 	logChange, 	loggedLocations, 	loggedLocationsHistorical,+	locationLog, 	loggedKeys, 	loggedKeysFor, ) where@@ -39,24 +40,32 @@  {- Log a change in the presence of a key's value in a repository. -} logChange :: Key -> UUID -> LogStatus -> Annex ()-logChange key (UUID u) s = do+logChange = logChange' logNow++logChange' :: (LogStatus -> String -> Annex LogLine) -> Key -> UUID -> LogStatus -> Annex ()+logChange' mklog key (UUID u) s = do 	config <- Annex.getGitConfig-	addLog (locationLogFile config key) =<< logNow s u-logChange _ NoUUID _ = noop+	addLog (locationLogFile config key) =<< mklog s u+logChange' _ _ NoUUID _ = noop  {- Returns a list of repository UUIDs that, according to the log, have  - the value of a key. -} loggedLocations :: Key -> Annex [UUID]-loggedLocations = getLoggedLocations currentLog+loggedLocations = getLoggedLocations currentLogInfo  {- Gets the location log on a particular date. -} loggedLocationsHistorical :: RefDate -> Key -> Annex [UUID]-loggedLocationsHistorical = getLoggedLocations . historicalLog+loggedLocationsHistorical = getLoggedLocations . historicalLogInfo  getLoggedLocations :: (FilePath -> Annex [String]) -> Key -> Annex [UUID] getLoggedLocations getter key = do 	config <- Annex.getGitConfig-	map toUUID <$> (getter . locationLogFile config) key+	map toUUID <$> getter (locationLogFile config key)++locationLog :: Key -> Annex [LogLine]+locationLog key = do+	config <- Annex.getGitConfig+	currentLog (locationLogFile config key)  {- Finds all keys that have location log information.  - (There may be duplicate keys in the list.) -}
Logs/Presence.hs view
@@ -17,7 +17,8 @@ 	readLog, 	logNow, 	currentLog,-	historicalLog+	currentLogInfo,+	historicalLogInfo, ) where  import Data.Time.Clock.POSIX@@ -43,14 +44,17 @@ 	return $ LogLine now s i  {- Reads a log and returns only the info that is still in effect. -}-currentLog :: FilePath -> Annex [String]-currentLog file = map info . filterPresent <$> readLog file+currentLogInfo :: FilePath -> Annex [String]+currentLogInfo file = map info <$> currentLog file +currentLog :: FilePath -> Annex [LogLine]+currentLog file = filterPresent <$> readLog file+ {- Reads a historical version of a log and returns the info that was in  - effect at that time.   -  - The date is formatted as shown in gitrevisions man page.  -}-historicalLog :: RefDate -> FilePath -> Annex [String]-historicalLog refdate file = map info . filterPresent . parseLog+historicalLogInfo :: RefDate -> FilePath -> Annex [String]+historicalLogInfo refdate file = map info . filterPresent . parseLog 	<$> Annex.Branch.getHistorical refdate file
Logs/Web.hs view
@@ -21,7 +21,6 @@  import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M-import Data.Tuple.Utils  import Common.Annex import qualified Annex@@ -44,13 +43,15 @@   where 	go [] = return [] 	go (l:ls) = do-		us <- currentLog l+		us <- currentLogInfo l 		if null us 			then go ls 			else return us  getUrlsWithPrefix :: Key -> String -> Annex [URLString]-getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) <$> getUrls key+getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) +	. map (fst . getDownloader)+	<$> getUrls key  setUrlPresent :: UUID -> Key -> URLString -> Annex () setUrlPresent uuid key url = do@@ -68,7 +69,7 @@ 		logChange key uuid InfoMissing  {- Finds all known urls. -}-knownUrls :: Annex [URLString]+knownUrls :: Annex [(Key, URLString)] knownUrls = do 	{- Ensure the git-annex branch's index file is up-to-date and 	 - any journaled changes are reflected in it, since we're going@@ -78,10 +79,13 @@ 	Annex.Branch.withIndex $ do 		top <- fromRepo Git.repoPath 		(l, cleanup) <- inRepo $ Git.LsFiles.stagedDetails [top]-		r <- mapM (geturls . snd3) $ filter (isUrlLog . fst3) l+		r <- mapM getkeyurls l 		void $ liftIO cleanup 		return $ concat r   where+	getkeyurls (f, s, _) = case urlLogFileKey f of+		Just k -> zip (repeat k) <$> geturls s+		Nothing -> return [] 	geturls Nothing = return [] 	geturls (Just logsha) = getLog . L.unpack <$> catObject logsha
Makefile view
@@ -1,5 +1,5 @@ mans=$(shell find doc -maxdepth 1 -name git-annex*.mdwn | sed -e 's/^doc/man/' -e 's/\.mdwn/\.1/')-all=git-annex $(mans) docs+all=git-annex mans docs  CABAL?=cabal # set to "./Setup" if you lack a cabal program GHC?=ghc@@ -103,7 +103,7 @@ Build/LinuxMkLibs: Build/LinuxMkLibs.hs 	$(GHC) --make $@ -Wall -sdist: clean $(mans)+sdist: clean mans 	./Build/make-sdist.sh  # Upload to hackage.
Messages.hs view
@@ -10,9 +10,6 @@ 	showStart', 	showNote, 	showAction,-	showProgress,-	metered,-	meteredBytes, 	showSideAction, 	doSideAction, 	doQuietSideAction,@@ -33,28 +30,26 @@ 	showRaw, 	setupConsole, 	enableDebugOutput,-	disableDebugOutput+	disableDebugOutput,+	commandProgressDisabled, ) where  import Text.JSON-import Data.Progress.Meter-import Data.Progress.Tracker-import Data.Quantity import System.Log.Logger import System.Log.Formatter import System.Log.Handler (setFormatter) import System.Log.Handler.Simple -import Common hiding (handle)+import Common import Types import Types.Messages+import Messages.Internal import qualified Messages.JSON as JSON import Types.Key import qualified Annex-import Utility.Metered  showStart :: String -> FilePath -> Annex ()-showStart command file = handle (JSON.start command $ Just file) $+showStart command file = handleMessage (JSON.start command $ Just file) $ 	flushed $ putStr $ command ++ " " ++ file ++ " "  showStart' :: String -> Key -> Maybe FilePath -> Annex ()@@ -62,42 +57,12 @@ 	fromMaybe (key2file key) afile  showNote :: String -> Annex ()-showNote s = handle (JSON.note s) $+showNote s = handleMessage (JSON.note s) $ 	flushed $ putStr $ "(" ++ s ++ ") "  showAction :: String -> Annex () showAction s = showNote $ s ++ "..." -{- Progress dots. -}-showProgress :: Annex ()-showProgress = handle q $-	flushed $ putStr "."--{- Shows a progress meter while performing a transfer of a key.- - The action is passed a callback to use to update the meter. -}-metered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a-metered combinemeterupdate key a = go (keySize key)-  where-	go (Just size) = meteredBytes combinemeterupdate size a-	go _ = a (const noop)--{- Shows a progress meter while performing an action on a given number- - of bytes. -}-meteredBytes :: Maybe MeterUpdate -> Integer -> (MeterUpdate -> Annex a) -> Annex a-meteredBytes combinemeterupdate size a = withOutputType go-  where-	go NormalOutput = do-		progress <- liftIO $ newProgress "" size-		meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1)-		showOutput-		r <- a $ \n -> liftIO $ do-			setP progress $ fromBytesProcessed n-			displayMeter stdout meter-			maybe noop (\m -> m n) combinemeterupdate-		liftIO $ clearMeter stdout meter-		return r-	go _ = a (const noop)- showSideAction :: String -> Annex () showSideAction m = Annex.getState Annex.output >>= go   where@@ -108,7 +73,7 @@ 			Annex.changeState $ \s -> s { Annex.output = st' } 		| sideActionBlock st == InBlock = return () 		| otherwise = p-	p = handle q $ putStrLn $ "(" ++ m ++ "...)"+	p = handleMessage q $ putStrLn $ "(" ++ m ++ "...)" 			 showStoringStateAction :: Annex () showStoringStateAction = showSideAction "recording state in git"@@ -130,12 +95,13 @@   where 	set o = Annex.changeState $ \s -> s {  Annex.output = o } +{- Make way for subsequent output of a command. -} showOutput :: Annex ()-showOutput = handle q $-	putStr "\n"+showOutput = unlessM commandProgressDisabled $+	handleMessage q $ putStr "\n"  showLongNote :: String -> Annex ()-showLongNote s = handle (JSON.note s) $+showLongNote s = handleMessage (JSON.note s) $ 	putStrLn $ '\n' : indent s  showEndOk :: Annex ()@@ -145,7 +111,7 @@ showEndFail = showEndResult False  showEndResult :: Bool -> Annex ()-showEndResult ok = handle (JSON.end ok) $ putStrLn msg+showEndResult ok = handleMessage (JSON.end ok) $ putStrLn msg   where 	msg 		| ok = "ok"@@ -159,7 +125,7 @@  warning' :: String -> Annex () warning' w = do-	handle q $ putStr "\n"+	handleMessage q $ putStr "\n" 	liftIO $ do 		hFlush stdout 		hPutStrLn stderr w@@ -175,7 +141,7 @@  {- Shows a JSON fragment only when in json mode. -} maybeShowJSON :: JSON a => [(String, a)] -> Annex ()-maybeShowJSON v = handle (JSON.add v) q+maybeShowJSON v = handleMessage (JSON.add v) q  {- Shows a complete JSON value, only when in json mode. -} showFullJSON :: JSON a => [(String, a)] -> Annex Bool@@ -190,16 +156,16 @@  - This is only needed when showStart and showEndOk is not used. -} showCustom :: String -> Annex Bool -> Annex () showCustom command a = do-	handle (JSON.start command Nothing) q+	handleMessage (JSON.start command Nothing) q 	r <- a-	handle (JSON.end r) q+	handleMessage (JSON.end r) q  showHeader :: String -> Annex ()-showHeader h = handle q $+showHeader h = handleMessage q $ 	flushed $ putStr $ h ++ ": "  showRaw :: String -> Annex ()-showRaw s = handle q $ putStrLn s+showRaw s = handleMessage q $ putStrLn s  setupConsole :: IO () setupConsole = do@@ -219,18 +185,11 @@ disableDebugOutput :: IO () disableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel NOTICE -handle :: IO () -> IO () -> Annex ()-handle json normal = withOutputType go-  where-	go NormalOutput = liftIO normal-	go QuietOutput = q-	go JSONOutput = liftIO $ flushed json--q :: Monad m => m ()-q = noop--flushed :: IO () -> IO ()-flushed a = a >> hFlush stdout--withOutputType :: (OutputType -> Annex a) -> Annex a-withOutputType a = outputType <$> Annex.getState Annex.output >>= a+{- Should commands that normally output progress messages have that+ - output disabled? -}+commandProgressDisabled :: Annex Bool+commandProgressDisabled = withOutputType $ \t -> return $ case t of+	QuietOutput -> True+	ProgressOutput -> True+	JSONOutput -> True+	NormalOutput -> False
+ Messages/Internal.hs view
@@ -0,0 +1,30 @@+{- git-annex output messages+ -+ - Copyright 2010-2014 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Messages.Internal where++import Common+import Types+import Types.Messages+import qualified Annex++handleMessage :: IO () -> IO () -> Annex ()+handleMessage json normal = withOutputType go+  where+	go NormalOutput = liftIO normal+	go QuietOutput = q+	go ProgressOutput = q+	go JSONOutput = liftIO $ flushed json++q :: Monad m => m ()+q = noop++flushed :: IO () -> IO ()+flushed a = a >> hFlush stdout++withOutputType :: (OutputType -> Annex a) -> Annex a+withOutputType a = outputType <$> Annex.getState Annex.output >>= a
+ Messages/Progress.hs view
@@ -0,0 +1,88 @@+{- git-annex progress output+ -+ - Copyright 2010-2015 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Messages.Progress where++import Common+import Messages+import Messages.Internal+import Utility.Metered+import Types+import Types.Messages+import Types.Key++import Data.Progress.Meter+import Data.Progress.Tracker+import Data.Quantity++{- Shows a progress meter while performing a transfer of a key.+ - The action is passed a callback to use to update the meter. -}+metered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a+metered combinemeterupdate key a = go (keySize key)+  where+	go (Just size) = meteredBytes combinemeterupdate size a+	go _ = a (const noop)++{- Shows a progress meter while performing an action on a given number+ - of bytes. -}+meteredBytes :: Maybe MeterUpdate -> Integer -> (MeterUpdate -> Annex a) -> Annex a+meteredBytes combinemeterupdate size a = withOutputType go+  where+	go NormalOutput = do+		progress <- liftIO $ newProgress "" size+		meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1)+		showOutput+		r <- a $ \n -> liftIO $ do+			setP progress $ fromBytesProcessed n+			displayMeter stdout meter+			maybe noop (\m -> m n) combinemeterupdate+		liftIO $ clearMeter stdout meter+		return r+	go _ = a (const noop)++{- Progress dots. -}+showProgressDots :: Annex ()+showProgressDots = handleMessage q $+	flushed $ putStr "."++{- Runs a command, that may output progress to either stdout or+ - stderr, as well as other messages.+ -+ - In quiet mode, the output is suppressed, except for error messages.+ -}+progressCommand :: FilePath -> [CommandParam] -> Annex Bool+progressCommand cmd params = progressCommandEnv cmd params Nothing++progressCommandEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> Annex Bool+progressCommandEnv cmd params environ = ifM commandProgressDisabled+	( do+		oh <- mkOutputHandler+		liftIO $ demeterCommandEnv oh cmd params environ+	, liftIO $ boolSystemEnv cmd params environ+	)++mkOutputHandler :: Annex OutputHandler+mkOutputHandler = OutputHandler+	<$> commandProgressDisabled+	<*> mkStderrEmitter++mkStderrRelayer :: Annex (Handle -> IO ())+mkStderrRelayer = do+	quiet <- commandProgressDisabled+	emitter <- mkStderrEmitter+	return $ \h -> avoidProgress quiet h emitter++{- Generates an IO action that can be used to emit stderr.+ -+ - When a progress meter is displayed, this takes care to avoid+ - messing it up with interleaved stderr from a command.+ -}+mkStderrEmitter :: Annex (String -> IO ())+mkStderrEmitter = withOutputType go+  where+	go ProgressOutput = return $ \s -> hPutStrLn stderr ("E: " ++ s)+	go _ = return (hPutStrLn stderr)
Remote/BitTorrent.hs view
@@ -19,6 +19,7 @@ import Types.UrlContents import Types.CleanupActions import Types.Key+import Messages.Progress import Utility.Metered import Utility.Tmp import Backend.URL@@ -288,14 +289,15 @@ 	return (ps ++ opts)  runAria :: [CommandParam] -> Annex Bool-runAria ps = liftIO . boolSystem "aria2c" =<< ariaParams ps+runAria ps = progressCommand "aria2c" =<< ariaParams ps  -- Parse aria output to find "(n%)" and update the progress meter--- with it. The output is also output to stdout.+-- with it. ariaProgress :: Maybe Integer -> MeterUpdate -> [CommandParam] -> Annex Bool ariaProgress Nothing _ ps = runAria ps-ariaProgress (Just sz) meter ps =-	liftIO . commandMeter (parseAriaProgress sz) meter "aria2c"+ariaProgress (Just sz) meter ps = do+	oh <- mkOutputHandler+	liftIO . commandMeter (parseAriaProgress sz) oh meter "aria2c" 		=<< ariaParams ps  parseAriaProgress :: Integer -> ProgressParser
Remote/Bup.hs view
@@ -121,18 +121,22 @@ 	showOutput -- make way for bup output 	liftIO $ boolSystem "bup" $ bupParams command buprepo params -bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> Annex [CommandParam]-bupSplitParams r buprepo k src = do+bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> [CommandParam]+bupSplitParams r buprepo k src = 	let os = map Param $ remoteAnnexBupSplitOptions $ gitconfig r-	showOutput -- make way for bup output-	return $ bupParams "split" buprepo +	in bupParams "split" buprepo  		(os ++ [Param "-q", Param "-n", Param (bupRef k)] ++ src)  store :: Remote -> BupRepo -> Storer store r buprepo = byteStorer $ \k b p -> do-	params <- bupSplitParams r buprepo k []+	let params = bupSplitParams r buprepo k []+	showOutput -- make way for bup output 	let cmd = proc "bup" (toCommand params)-	liftIO $ withHandle StdinHandle createProcessSuccess cmd $ \h -> do+	runner <- ifM commandProgressDisabled+		( return feedWithQuietOutput+		, return (withHandle StdinHandle)+		)+	liftIO $ runner createProcessSuccess cmd $ \h -> do 		meteredWrite p h b 		return True
Remote/External.hs view
@@ -17,6 +17,7 @@ import Config import Remote.Helper.Special import Utility.Metered+import Messages.Progress import Logs.Transfer import Logs.PreferredContent.Raw import Logs.RemoteState@@ -26,6 +27,7 @@ import Creds  import Control.Concurrent.STM+import Control.Concurrent.Async import System.Log.Logger (debugM) import qualified Data.Map as M @@ -228,8 +230,7 @@ 	handleRemoteRequest (SETURIMISSING key uri) = 		withurl (SETURLMISSING key) uri 	handleRemoteRequest (GETURLS key prefix) = do-		mapM_ (send . VALUE . fst . getDownloader)-			=<< getUrlsWithPrefix key prefix+		mapM_ (send . VALUE) =<< getUrlsWithPrefix key prefix 		send (VALUE "") -- end of list 	handleRemoteRequest (DEBUG msg) = liftIO $ debugM "external" msg 	handleRemoteRequest (VERSION _) =@@ -324,21 +325,28 @@ {- Starts an external remote process running, but does not handle checking  - VERSION, etc. -} startExternal :: ExternalType -> Annex ExternalState-startExternal externaltype = liftIO $ do-	(Just hin, Just hout, _, pid) <- createProcess $ (proc cmd [])-		{ std_in = CreatePipe-		, std_out = CreatePipe-		, std_err = Inherit-		}-	fileEncoding hin-	fileEncoding hout-	checkearlytermination =<< getProcessExitCode pid-	return $ ExternalState-		{ externalSend = hin-		, externalReceive = hout-		, externalPid = pid-		, externalPrepared = Unprepared-		}+startExternal externaltype = do+	errrelayer <- mkStderrRelayer+	liftIO $ do+		(Just hin, Just hout, Just herr, pid) <- createProcess $+			(proc cmd [])+				{ std_in = CreatePipe+				, std_out = CreatePipe+				, std_err = CreatePipe+				}+		fileEncoding hin+		fileEncoding hout+		fileEncoding herr+		stderrelay <- async $ errrelayer herr+		checkearlytermination =<< getProcessExitCode pid+		return $ ExternalState+			{ externalSend = hin+			, externalReceive = hout+			, externalShutdown = do+				cancel stderrelay+				void $ waitForProcess pid+			, externalPrepared = Unprepared+			}   where 	cmd = externalRemoteProgram externaltype @@ -358,7 +366,7 @@ 		void $ atomically $ tryTakeTMVar v 		hClose $ externalSend st 		hClose $ externalReceive st-		void $ waitForProcess $ externalPid st+		externalShutdown st 	v = externalState external  externalRemoteProgram :: ExternalType -> String
Remote/External/Types.hs view
@@ -70,7 +70,7 @@ data ExternalState = ExternalState 	{ externalSend :: Handle 	, externalReceive :: Handle-	, externalPid :: ProcessHandle+	, externalShutdown :: IO () 	, externalPrepared :: PrepareStatus 	}
Remote/Git.hs view
@@ -542,7 +542,8 @@ 	cache st = Annex.changeState $ \s -> s 		{ Annex.remoteannexstate = M.insert (uuid r) st (Annex.remoteannexstate s) } 	go st a' = do-		(ret, st') <- liftIO $ Annex.run st $+		curro <- Annex.getState Annex.output+		(ret, st') <- liftIO $ Annex.run (st { Annex.output = curro }) $ 			catFileStop `after` a' 		cache st' 		return ret
Remote/Helper/Special.hs view
@@ -42,6 +42,7 @@ import Remote.Helper.Encryptable as X import Remote.Helper.Messages import Annex.Content+import Messages.Progress import qualified Git import qualified Git.Command import qualified Git.Construct
Remote/Helper/Ssh.hs view
@@ -17,6 +17,7 @@ import qualified CmdLine.GitAnnexShell.Fields as Fields import Types.Key import Remote.Helper.Messages+import Messages.Progress import Utility.Metered import Utility.Rsync import Types.Remote@@ -100,9 +101,14 @@ 	[]  rsyncHelper :: Maybe MeterUpdate -> [CommandParam] -> Annex Bool-rsyncHelper callback params = do+rsyncHelper m params = do 	showOutput -- make way for progress bar-	ifM (liftIO $ (maybe rsync rsyncProgress callback) params)+	a <- case m of+		Nothing -> return $ rsync params+		Just meter -> do+			oh <- mkOutputHandler+			return $ rsyncProgress oh meter params+	ifM (liftIO a) 		( return True 		, do 			showLongNote "rsync failed -- run git annex again to resume file transfer"
Remote/Hook.hs view
@@ -17,6 +17,7 @@ import Annex.UUID import Remote.Helper.Special import Utility.Env+import Messages.Progress  import qualified Data.Map as M @@ -113,7 +114,7 @@   where 	run command = do 		showOutput -- make way for hook output-		ifM (liftIO $ boolSystemEnv "sh" [Param "-c", Param command] =<< hookEnv action k f)+		ifM (progressCommandEnv "sh" [Param "-c", Param command] =<< liftIO (hookEnv action k f)) 			( a 			, do 				warning $ hook ++ " hook exited nonzero!"
Remote/Rsync.hs view
@@ -31,6 +31,7 @@ import Crypto import Utility.Rsync import Utility.CopyFile+import Messages.Progress import Utility.Metered import Utility.PID import Annex.Perms@@ -281,11 +282,15 @@ 	)  rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool-rsyncRemote direction o callback params = do+rsyncRemote direction o m params = do 	showOutput -- make way for progress bar-	liftIO $ (maybe rsync rsyncProgress callback) $-		opts ++ [Params "--progress"] ++ params+	case m of+		Nothing -> liftIO $ rsync ps+		Just meter -> do+			oh <- mkOutputHandler+			liftIO $ rsyncProgress oh meter ps   where+	ps = opts ++ [Params "--progress"] ++ params 	opts 		| direction == Download = rsyncDownloadOptions o 		| otherwise = rsyncUploadOptions o
RemoteDaemon/Core.hs view
@@ -28,7 +28,7 @@  runForeground :: IO () runForeground = do-	(readh, writeh) <- ioHandles+	(readh, writeh) <- dupIoHandles 	ichan <- newTChanIO :: IO (TChan Consumed) 	ochan <- newTChanIO :: IO (TChan Emitted)
Test.hs view
@@ -125,8 +125,8 @@  ingredients :: [Ingredient] ingredients =-	[ rerunningTests [consoleTestReporter]-	, listingTests+	[ listingTests+	, rerunningTests [consoleTestReporter] 	]  properties :: TestTree
Types/Messages.hs view
@@ -7,8 +7,10 @@  module Types.Messages where -data OutputType = NormalOutput | QuietOutput | JSONOutput+import Data.Default +data OutputType = NormalOutput | QuietOutput | ProgressOutput | JSONOutput+ data SideActionBlock = NoBlock | StartBlock | InBlock 	deriving (Eq) @@ -17,5 +19,6 @@ 	, sideActionBlock :: SideActionBlock 	} -defaultMessageState :: MessageState-defaultMessageState = MessageState NormalOutput NoBlock+instance Default MessageState+  where+	def = MessageState NormalOutput NoBlock
Upgrade/V2.hs view
@@ -16,6 +16,7 @@ import Annex.Content import Utility.Tmp import Logs+import Messages.Progress  olddir :: Git.Repo -> FilePath olddir g@@ -44,7 +45,7 @@ 	old <- fromRepo olddir  	Annex.Branch.create-	showProgress+	showProgressDots  	e <- liftIO $ doesDirectoryExist old 	when e $ do@@ -53,12 +54,12 @@ 		mapM_ (\f -> inject f f) =<< logFiles old  	saveState False-	showProgress+	showProgressDots  	when e $ do 		inRepo $ Git.Command.run [Param "rm", Param "-r", Param "-f", Param "-q", File old] 		unless bare $ inRepo gitAttributesUnWrite-	showProgress+	showProgressDots  	unless bare push
Utility/Gpg.hs view
@@ -87,7 +87,7 @@ pipeStrict :: [CommandParam] -> String -> IO String pipeStrict params input = do 	params' <- stdParams params-	withBothHandles createProcessSuccess (proc gpgcmd params') $ \(to, from) -> do+	withIOHandles createProcessSuccess (proc gpgcmd params') $ \(to, from) -> do 		hSetBinaryMode to True 		hSetBinaryMode from True 		hPutStr to input@@ -142,7 +142,7 @@ 	setup = liftIO . createProcess 	cleanup p (_, _, _, pid) = liftIO $ forceSuccessProcess p pid 	go p = do-		let (to, from) = bothHandles p+		let (to, from) = ioHandles p 		liftIO $ void $ forkIO $ do 			feeder to 			hClose to
Utility/Metered.hs view
@@ -1,4 +1,4 @@-{- Metered IO+{- Metered IO and actions  -  - Copyright 2012-2105 Joey Hess <id@joeyh.name>  -@@ -18,6 +18,7 @@ import System.Posix.Types import Data.Int import Data.Bits.Utils+import Control.Concurrent.Async  {- An action that can be run repeatedly, updating it on the bytes processed.  -@@ -145,8 +146,13 @@ 	k = 1024 	chunkOverhead = 2 * sizeOf (undefined :: Int) -- GHC specific +data OutputHandler = OutputHandler+	{ quietMode :: Bool+	, stderrHandler :: String -> IO ()+	}+ {- Parses the String looking for a command's progress output, and returns- - Maybe the number of bytes rsynced so far, and any any remainder of the+ - Maybe the number of bytes done so far, and any any remainder of the  - string that could be an incomplete progress output. That remainder  - should be prepended to future output, and fed back in. This interface  - allows the command's output to be read in any desired size chunk, or@@ -155,11 +161,15 @@ type ProgressParser = String -> (Maybe BytesProcessed, String)  {- Runs a command and runs a ProgressParser on its output, in order- - to update the meter. The command's output is also sent to stdout. -}-commandMeter :: ProgressParser -> MeterUpdate -> FilePath -> [CommandParam] -> IO Bool-commandMeter progressparser meterupdate cmd params = liftIO $ catchBoolIO $-	withHandle StdoutHandle createProcessSuccess p $-		feedprogress zeroBytesProcessed []+ - to update a meter.+ -}+commandMeter :: ProgressParser -> OutputHandler -> MeterUpdate -> FilePath -> [CommandParam] -> IO Bool+commandMeter progressparser oh meterupdate cmd params = catchBoolIO $+	withOEHandles createProcessSuccess p $ \(outh, errh) -> do+		ep <- async $ handlestderr errh+		op <- async $ feedprogress zeroBytesProcessed [] outh+		wait ep+		wait op   where 	p = proc cmd (toCommand params) @@ -168,8 +178,9 @@ 		if S.null b 			then return True 			else do-				S.hPut stdout b-				hFlush stdout+				unless (quietMode oh) $ do+					S.hPut stdout b+					hFlush stdout 				let s = w82s (S.unpack b) 				let (mbytes, buf') = progressparser (buf++s) 				case mbytes of@@ -178,3 +189,40 @@ 						when (bytes /= prev) $ 							meterupdate bytes 						feedprogress bytes buf' h++	handlestderr h = unlessM (hIsEOF h) $ do+		stderrHandler oh =<< hGetLine h+		handlestderr h++{- Runs a command, that may display one or more progress meters on+ - either stdout or stderr, and prevents the meters from being displayed.+ -+ - The other command output is handled as configured by the OutputHandler.+ -}+demeterCommand :: OutputHandler -> FilePath -> [CommandParam] -> IO Bool+demeterCommand oh cmd params = demeterCommandEnv oh cmd params Nothing++demeterCommandEnv :: OutputHandler -> FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool+demeterCommandEnv oh cmd params environ = catchBoolIO $+	withOEHandles createProcessSuccess p $ \(outh, errh) -> do+		ep <- async $ avoidProgress True errh $ stderrHandler oh+		op <- async $ avoidProgress True outh $ \l ->+			unless (quietMode oh) $+				putStrLn l+		wait ep+		wait op+		return True+  where+	p = (proc cmd (toCommand params))+		{ env = environ }++{- To suppress progress output, while displaying other messages,+ - filter out lines that contain \r (typically used to reset to the+ - beginning of the line when updating a progress display).+ -}+avoidProgress :: Bool -> Handle -> (String -> IO ()) -> IO ()+avoidProgress doavoid h emitter = unlessM (hIsEOF h) $ do+	s <- hGetLine h+	unless (doavoid && '\r' `elem` s) $+		emitter s+	avoidProgress doavoid h emitter
Utility/Path.hs view
@@ -170,17 +170,26 @@ 				== joinPath ["..", "..", "..", "..", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"]  {- Given an original list of paths, and an expanded list derived from it,- - generates a list of lists, where each sublist corresponds to one of the- - original paths. When the original path is a directory, any items- - in the expanded list that are contained in that directory will appear in- - its segment.+ - which may be arbitrarily reordered, generates a list of lists, where+ - each sublist corresponds to one of the original paths.+ -+ - When the original path is a directory, any items in the expanded list+ - that are contained in that directory will appear in its segment.+ -+ - The order of the original list of paths is attempted to be preserved in+ - the order of the returned segments. However, doing so has a O^NM+ - growth factor. So, if the original list has more than 100 paths on it,+ - we stop preserving ordering at that point. Presumably a user passing+ - that many paths in doesn't care too much about order of the later ones.  -} segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]] segmentPaths [] new = [new] segmentPaths [_] new = [new] -- optimisation-segmentPaths (l:ls) new = [found] ++ segmentPaths ls rest+segmentPaths (l:ls) new = found : segmentPaths ls rest   where-	(found, rest)=partition (l `dirContains`) new+	(found, rest) = if length ls < 100+		then partition (l `dirContains`) new+		else break (\p -> not (l `dirContains` p)) new  {- This assumes that it's cheaper to call segmentPaths on the result,  - than it would be to run the action separately with each path. In
Utility/Process.hs view
@@ -25,14 +25,16 @@ 	processTranscript, 	processTranscript', 	withHandle,-	withBothHandles,+	withIOHandles,+	withOEHandles, 	withQuietOutput,+	feedWithQuietOutput, 	createProcess, 	startInteractiveProcess, 	stdinHandle, 	stdoutHandle, 	stderrHandle,-	bothHandles,+	ioHandles, 	processHandle, 	devNull, ) where@@ -255,12 +257,12 @@ 			(stderrHandle, base { std_err = CreatePipe })  {- Like withHandle, but passes (stdin, stdout) handles to the action. -}-withBothHandles+withIOHandles 	:: CreateProcessRunner 	-> CreateProcess 	-> ((Handle, Handle) -> IO a) 	-> IO a-withBothHandles creator p a = creator p' $ a . bothHandles+withIOHandles creator p a = creator p' $ a . ioHandles   where 	p' = p 		{ std_in = CreatePipe@@ -268,6 +270,20 @@ 		, std_err = Inherit 		} +{- Like withHandle, but passes (stdout, stderr) handles to the action. -}+withOEHandles+	:: CreateProcessRunner+	-> CreateProcess+	-> ((Handle, Handle) -> IO a)+	-> IO a+withOEHandles creator p a = creator p' $ a . oeHandles+  where+	p' = p+		{ std_in = Inherit+		, std_out = CreatePipe+		, std_err = CreatePipe+		}+ {- Forces the CreateProcessRunner to run quietly;  - both stdout and stderr are discarded. -} withQuietOutput@@ -281,6 +297,21 @@ 		} 	creator p' $ const $ return () +{- Stdout and stderr are discarded, while the process is fed stdin+ - from the handle. -}+feedWithQuietOutput+	:: CreateProcessRunner+	-> CreateProcess+	-> (Handle -> IO a)+	-> IO a+feedWithQuietOutput creator p a = withFile devNull WriteMode $ \nullh -> do+	let p' = p+		{ std_in = CreatePipe+		, std_out = UseHandle nullh+		, std_err = UseHandle nullh+		}+	creator p' $ a . stdinHandle+ devNull :: FilePath #ifndef mingw32_HOST_OS devNull = "/dev/null"@@ -303,9 +334,12 @@ stderrHandle :: HandleExtractor stderrHandle (_, _, Just h, _) = h stderrHandle _ = error "expected stderrHandle"-bothHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle)-bothHandles (Just hin, Just hout, _, _) = (hin, hout)-bothHandles _ = error "expected bothHandles"+ioHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle)+ioHandles (Just hin, Just hout, _, _) = (hin, hout)+ioHandles _ = error "expected ioHandles"+oeHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle)+oeHandles (_, Just hout, Just herr, _) = (hout, herr)+oeHandles _ = error "expected oeHandles"  processHandle :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> ProcessHandle processHandle (_, _, _, pid) = pid
Utility/Rsync.hs view
@@ -92,13 +92,13 @@ 	| rsyncUrlIsShell s = False 	| otherwise = ':' `notElem` s -{- Runs rsync, but intercepts its progress output and updates a meter.- - The progress output is also output to stdout. +{- Runs rsync, but intercepts its progress output and updates a progress+ - meter.  -  - The params must enable rsync's --progress mode for this to work.  -}-rsyncProgress :: MeterUpdate -> [CommandParam] -> IO Bool-rsyncProgress meterupdate = commandMeter parseRsyncProgress meterupdate "rsync" . rsyncParamsFixup+rsyncProgress :: OutputHandler -> MeterUpdate -> [CommandParam] -> IO Bool+rsyncProgress oh meter = commandMeter parseRsyncProgress oh meter "rsync" . rsyncParamsFixup  {- Strategy: Look for chunks prefixed with \r (rsync writes a \r before  - the first progress output, and each thereafter). The first number
Utility/SafeCommand.hs view
@@ -101,14 +101,19 @@ prop_idempotent_shellEscape_multiword :: [String] -> Bool prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s -{- Segements a list of filenames into groups that are all below the manximum- - command-line length limit. Does not preserve order. -}-segmentXargs :: [FilePath] -> [[FilePath]]-segmentXargs l = go l [] 0 []+{- Segments a list of filenames into groups that are all below the maximum+ - command-line length limit. -}+segmentXargsOrdered :: [FilePath] -> [[FilePath]]+segmentXargsOrdered = reverse . map reverse . segmentXargsUnordered++{- Not preserving data is a little faster, and streams better when+ - there are a great many filesnames. -}+segmentXargsUnordered :: [FilePath] -> [[FilePath]]+segmentXargsUnordered l = go l [] 0 []   where-	go [] c _ r = c:r+	go [] c _ r = (c:r) 	go (f:fs) c accumlen r-		| len < maxlen && newlen > maxlen = go (f:fs) [] 0 (c:r)+		| newlen > maxlen && len < maxlen = go (f:fs) [] 0 (c:r) 		| otherwise = go fs (f:c) newlen r 	  where 		len = length f
Utility/SimpleProtocol.hs view
@@ -16,7 +16,7 @@ 	parse1, 	parse2, 	parse3,-	ioHandles,+	dupIoHandles, ) where  import Data.Char@@ -80,8 +80,8 @@  - will mess up the protocol. To avoid that, close stdin, and   - and duplicate stderr to stdout. Return two new handles  - that are duplicates of the original (stdin, stdout). -}-ioHandles :: IO (Handle, Handle)-ioHandles = do+dupIoHandles :: IO (Handle, Handle)+dupIoHandles = do 	readh <- hDuplicate stdin 	writeh <- hDuplicate stdout 	nullh <- openFile devNull ReadMode
Utility/Url.hs view
@@ -205,7 +205,7 @@ downloadQuiet = download' True  download' :: Bool -> URLString -> FilePath -> UrlOptions -> IO Bool-download' quiet url file uo = +download' quiet url file uo = do 	case parseURIRelaxed url of 		Just u 			| uriScheme u == "file:" -> do@@ -224,7 +224,7 @@ 	 -} #ifndef __ANDROID__ 	wgetparams = catMaybes-		[ if Build.SysConfig.wgetquietprogress+		[ if Build.SysConfig.wgetquietprogress && not quiet 			then Just $ Params "-q --show-progress" 			else Nothing 		, Just $ Params "--clobber -c -O"
debian/changelog view
@@ -1,3 +1,39 @@+git-annex (5.20150406) unstable; urgency=medium++  * Prevent git-ls-files from double-expanding wildcards when an+    unexpanded wildcard is passed to a git-annex command like add or find.+  * Fix make build target. Thanks, Justin Geibel.+  * Fix GETURLS in external special remote protocol to strip+    downloader prefix from logged url info before checking for the+    specified prefix.+  * importfeed: Avoid downloading a redundant item from a feed whose+    guid has been downloaded before, even when the url has changed.+  * importfeed: Always store itemid in metadata; before this was only+    done when annex.genmetadata was set.+  * Relax debian package dependencies to git >= 1:1.8.1 rather+    than needing >= 1:2.0.+  * test: Fix --list-tests+  * addurl --file: When used with a special remote that claims+    urls and checks their contents, don't override the user's provided+    filename with filenames that the special remote suggests. Also,+    don't allow adding the url if the special remote says it contains+    multiple files.+  * import: --deduplicate and --cleanduplicates now output the keys+    corresponding to duplicated files they process.+  * expire: New command, for expiring inactive repositories.+  * fsck: Record fsck activity for use by expire command.+  * Fix truncation of parameters that could occur when using xargs git-annex.+  * Significantly sped up processing of large numbers of directories+    passed to a single git-annex command.+  * version: Add --raw+  * init: Improve fifo test to detect NFS systems that support fifos+    but not well enough for sshcaching.+  * --quiet now suppresses progress displays from eg, rsync.+    (The option already suppressed git-annex's own built-in progress+    displays.)++ -- Joey Hess <id@joeyh.name>  Mon, 06 Apr 2015 12:48:48 -0400+ git-annex (5.20150327) unstable; urgency=medium    * readpresentkey: New plumbing command for checking location log.
debian/control view
@@ -75,7 +75,7 @@ 	lsof [!kfreebsd-i386 !kfreebsd-amd64 !hurd-any], 	ikiwiki, 	perlmagick,-	git (>= 1:2.0),+	git (>= 1:1.8.1), 	rsync, 	wget, 	curl,@@ -92,7 +92,7 @@ Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends},-	git (>= 1:2.0),+	git (>= 1:1.8.1), 	rsync, 	wget, 	curl,
doc/Android.mdwn view
@@ -39,7 +39,7 @@ `rsync`, and `gpg`.  To prevent the webapp from being automatically started-when a terminal window opens, go into the terminal preferences, to "Inital+when a terminal window opens, go into the terminal preferences, to "Initial Command", and clear out the default `git annex webapp` setting.  Or, if you'd like to run the assistant automatically, but not open the
doc/automatic_conflict_resolution.mdwn view
@@ -17,7 +17,7 @@  The "AAA" and "BBB" in the above example are essentially arbitrary (technically they are the MD5 checksum of the key). The automatic merge-conflict resoltuion is designed so that if two or more repositories both get+conflict resolution is designed so that if two or more repositories both get a merge conflict, and resolve it, the resolved repositories will not themselves conflict. This is why it doesn't use something nicer, like perhaps the name of the remote that the file came from.
doc/bugs/--list-tests_runs_tests.mdwn view
@@ -23,3 +23,6 @@ [snip all the passing tests] All 140 tests passed (305.69s) """]]++> [[fixed|done]] although I don't understand why tasty needs the+> `listingTests` ingredient to come first for it to work. --[[Joey]]
doc/bugs/GETURLS_doesn__39__t_return_URLs_if_prefix_is_provided.mdwn view
@@ -33,3 +33,5 @@  # End of transcript or log. """]]++> [[fixed|done]] --[[Joey]]
+ doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn view
@@ -0,0 +1,81 @@+### Please describe the problem.++Rebooted my laptop recently (after dunno how long of uptime) and annex assistant was shut off in previous uptime, but this time decided to leave it running. spotted that laptop is hot today to see that git-annex is busy.  And I am not sure what it is really doing:  top says++     3087 yoh       30  10  520700  21084  14748 S 100.0  0.1  80:50.60 /usr/bin/git-annex remotedaemon +++[[!format sh """++$> tail -20 /proc/3087/cwd/.git/annex/daemon.log++Please make sure you have the correct access rights+and the repository exists.+ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out+fatal: Could not read from remote repository.++Please make sure you have the correct access rights+and the repository exists.+ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out+fatal: Could not read from remote repository.++Please make sure you have the correct access rights+and the repository exists.+[2015-03-28 20:27:53 EDT] main: Syncing with debandy_Vault +hostname: Name or service not known+hostname: Name or service not known+To ssh://yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex/media/Vault/annex/+   8c181af..a0b0bd6  git-annex -> synced/git-annex+   8352761..537d4a6  master -> synced/master++"""]]++in webapp I saw it struggling to connect  to vagus which is offline, I turned it off, then reanabled syncing to above yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex .  I don't know if annex was busier or not before, but it is 100% busy now, but nothing seems to be done -- no traffic, no changing fd's for that annex process++strace shows busy reading from fd 24:++[[!format sh """+[pid  3110] read(24, "", 8096)          = 0+[pid  3110] read(24, "", 8096)          = 0+[pid  3110] read(24, "", 8096)          = 0+[pid  3110] read(24, "", 8096)          = 0+"""]]+++    $> ls -l /proc/3087/fd/24                      +    lr-x------ 1 yoh yoh 64 Mar 28 20:31 /proc/3087/fd/24 -> pipe:[794930]+++so what could it be doing?++[[!format sh """+ 2807 yoh        20   0  693M  110M 36948 S  1.4  0.7 11:15.32 ├─ /usr/bin/git-annex assistant --startdelay=5s+ 3359 yoh        39  19 20996  3412  3096 S  0.0  0.0  0:00.01 │  ├─ git --git-dir=.git --work-tree=. check-attr -z --stdin annex.backend annex.numcopies --+ 3140 yoh        20   0 21128  3504  3188 S  0.0  0.0  0:00.01 │  ├─ git --git-dir=.git --work-tree=. check-ignore -z --stdin --verbose --non-matching+ 3117 yoh        20   0  693M  110M 36948 S  0.0  0.7  0:00.00 │  ├─ /usr/bin/git-annex assistant --startdelay=5s+ 3108 yoh        20   0  693M  110M 36948 S  0.0  0.7  0:00.15 │  ├─ /usr/bin/git-annex assistant --startdelay=5s+ 3087 yoh        30  10  508M 21084 14748 S 97.9  0.1  1h29:16 │  ├─ /usr/bin/git-annex remotedaemon+24361 yoh        30  10  508M 21084 14748 S  0.0  0.1  0:00.00 │  │  ├─ /usr/bin/git-annex remotedaemon+19799 yoh        30  10     0     0     0 Z  0.0  0.0  0:00.01 │  │  ├─ ssh+19440 yoh        30  10  508M 21084 14748 S  0.0  0.1  0:00.00 │  │  ├─ /usr/bin/git-annex remotedaemon+ 3222 yoh        30  10 27368  5796  3456 S  0.0  0.0  0:00.00 │  │  ├─ git --git-dir=.git --work-tree=. cat-file --batch+ 3110 yoh        30  10  508M 21084 14748 R 97.5  0.1  1h29:10 │  │  ├─ /usr/bin/git-annex remotedaemon+ 3090 yoh        30  10  508M 21084 14748 S  0.0  0.1  0:00.98 │  │  ├─ /usr/bin/git-annex remotedaemon+ 3089 yoh        30  10  508M 21084 14748 S  0.0  0.1  0:00.03 │  │  └─ /usr/bin/git-annex remotedaemon+"""]]++how could I figure out what is doing??? meanwhile I have just sent STOP signal to 3087++### What version of git-annex are you using? On what operating system?++5.20150327+git27-g6af24b6-1++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]
+ doc/bugs/feeding_git_annex_with_xargs_can_fail.mdwn view
@@ -0,0 +1,15 @@+Feeding git-annex a long list off directories, eg with xargs can have+2 bad behaviors:++* git-annex runs git ls-files and passes it the same long list. But the git+  ls-files command is longer than the git-annex command often, so it gets+  truncated and some files are not processed.++  > fixed --[[Joey]]++* It can take a really long time for git-annex to chew through the+  git-ls-files results. There is probably an exponential blowup in the time+  relative to the number of parameters. Some of the stuff being done to+  preserve original ordering etc is likely at fault.++  > [[fixed|done]] --[[Joey]]
+ doc/bugs/importfeed_fails_with_local_file_urls.mdwn view
@@ -0,0 +1,28 @@+Hi,++I've a script which generates .rss files which reference local files with the file:// scheme. I can import the file:// urls with git annex addurl, but it fails with git annex importfeed:+++`$ git annex importfeed --fast file:///path/to/local/rss/file.rss`++`(checking known urls...)`++`importfeed file:///path/to/local/rss/file.rss`++`git-annex: /tmp/feed6757: openFile: resource busy (file is locked)`++If I try to import it with `$ git annex importfeed --fast /path/to/local/rss/file.rss` I get++`importfeed /path/to/local/rss/file.rss`++`  warning: bad feed content`++`ok`++But the directory stays empty.+++Is it possible to use local files in rss format with items which reference local files using the file:// scheme as input for importfeed?++Cheers,+Marco
+ doc/bugs/package_build_fails_with_missing_man_directory.mdwn view
@@ -0,0 +1,30 @@+When building a debian package, the build fails due to a missing man/ directory.++    ./Build/mdwn2man man/git-annex-vpop.1 1 doc/git-annex-vpop.mdwn > man/git-annex-vpop.1+    /bin/sh: 1: cannot create man/git-annex-vpop.1: Directory nonexistent++I was able to build the package with the following patch:++[[!format patch """+From: Justin Geibel <jtgeibel@gmail.com>+Date: Fri, 27 Mar 2015 16:21:13 -0400+Subject: Fix build of man pages++---+ Makefile | 2 +-+ 1 file changed, 1 insertion(+), 1 deletion(-)++diff --git a/Makefile b/Makefile+index d6fb1a1..b36cbcd 100644+--- a/Makefile++++ b/Makefile+@@ -1,5 +1,5 @@+ mans=$(shell find doc -maxdepth 1 -name git-annex*.mdwn | sed -e 's/^doc/man/' -e 's/\.mdwn/\.1/')+-all=git-annex $(mans) docs++all=git-annex mans docs+ + CABAL?=cabal # set to "./Setup" if you lack a cabal program+ GHC?=ghc+"""]]++> [[fixed|done]] --[[Joey]]
+ doc/bugs/unexpected_double_wildcard_expansion.mdwn view
@@ -0,0 +1,15 @@+From the forum, it seems that git-ls-files very unexpectedly expands+wildcards in filenames passed to it. (Not a documented or an expected+behavior.)++This causes problems when eg, the user does `git annex add *.jpeg` and that+matches no files, but there are some jpegs in subdirectories. git-ls-files+re-expands the wildcard and finds those.++Seems that the best fix is to make Git.LsFiles paper over this git+misfeature, by always escaping wildcards in paths passed+to git-ls-files. AFAIK, no callers of Git.LsFiles expect to provide it+wildcards, because I was completely surprised when I learned they were+expanded.. --[[Joey]]++> [[fixed|done]] --[[Joey]]
doc/chunking.mdwn view
@@ -13,7 +13,7 @@ initremote`, specifying the chunk size.   Good chunk sizes will depend on the remote, but a good starting place-is probably `1MiB`. Very large chunks are problimatic, both because+is probably `1MiB`. Very large chunks are problematic, both because git-annex needs to buffer one chunk in memory when uploading, and because a larger chunk will make resuming interrupted transfers less efficient. On the other hand, when a file is split into a great many chunks,
doc/design/assistant/xmpp_security.mdwn view
@@ -18,11 +18,11 @@     use SPEKE (or similar methods like J-PAKE) to generate a shared key.     Avoids active MITM attacks. Makes pairing harder, especially pairing     between one's own devices, since the passphrase has to be entered on-    all devices. Also problimatic when pairing more than 2 devices,+    all devices. Also problematic when pairing more than 2 devices,     especially when adding a device to the set later, since there     would then be multiple different keys in use.   * Rely on the user's gpg key, and do gpg key verification during XMPP-    pairing. Problimatic because who wants to put their gpg key on their+    pairing. Problematic because who wants to put their gpg key on their     phone? Also, require the users be in the WOT and be gpg literate.  Update: This seems unlikely to be worth doing. [[Telehash]] is better.
doc/design/iabackup.mdwn view
@@ -134,6 +134,8 @@ With that change, the server can check for files that not enough clients have verified they have recently, and distribute them to more clients. +(This is now implemented.)+ Note that bad actors can lie about this verification; it's not a proof they still have the file. But, a bad actor could prove they have a file, and refuse to give it back if the IA needed to restore the backup, too.
+ doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawlZF5AC-FSxwkiay5ZgEYZwUzN69Wa6PTE"+ nickname="Sunke"+ subject="filename from metadata?"+ date="2015-04-06T18:00:25Z"+ content="""+Hi everbody,+is it possible to use a metadata field for the filename in a+metadata driven view?++I am thinking of the following use case:++git annex metadata --set artist=Led\ Zeppelin --set album=Led\ Zeppelin\ IV --set title=04\ Stairway\ to\ heaven  some/weird/filename.mp3+git annex view --filename-from title artist=* album=*++result:+Led Zeppelin/Led Zeppelin IV/04 Stairway to heaven.mp3+"""]]
+ doc/devblog/day_268_stressed_out.mdwn view
@@ -0,0 +1,18 @@+While I plowed through a lot of backlog the past several days,+I still have some 120 messages piled deep.++That work did result in a number of improvements, culminating in a rather+rushed release of version 5.20150327 today, to fix a regression affecting+`git annex sync` when using the standalone linux tarballs. Unfortunately, I+then had to update those tarballs a second time after the release+as the first fix was incomplete.++And, I'm feeling super stressed out. At this point, I think I should step+away until the end of the month. Unfortunately, this will mean more backlog+later. Including lots of noise and hand-holding that I just don't seem to+have time for if I want to continue making forward progress.++Maybe I'll think of a way to deal with it while I'm away. Currently, all I+have is that I may have to start ignoring irc and the forum, and+de-prioritizing bug reports that don't have either a working reproduction+recipe or multiple independent confirmations that it's a real bug.
+ doc/devblog/day_269__wildcards_and_podcasts.mdwn view
@@ -0,0 +1,26 @@+Turns out that git has a feature I didn't know about; it will expand+wildcards and other stuff in filenames passed to many git commands. This is+on top of the shell's expansion.++That led to some broken behavior by `git annex add 'foo.*'`+and, it could lead to other probably unwanted behavior, like `git annex+drop 'foo[barred]'` dropping a file named `food` in addition to+`foo[barred]`++For now, I've disabled this git feature throughout git-annex. If you relied+on it for something, let me know, I might think about adding it back in+specific places where it makes sense.++----++Improved `git annex importfeed` to check the itemid of the feed and avoid+re-downloading a file with the same itemid. Before, it would add duplicate+files if a feed kept the itemid the same, but changed the url. This was+easier than expected because annex.genmetadata already caused the itemid+to be stored in the git-annex metadata. I just had to make it check the+itemid metadata, and set itemid even when annex.genmetadata isn't set.++----++Also got 4 other bug reports fixed, even though I feel I'm taking it easy+today. It's good to be relaxed again!
+ doc/devblog/day_270__distributed_fsck.mdwn view
@@ -0,0 +1,27 @@+Added two options to `git annex fsck` that allow for a form of distributed+fsck.  This is useful in situations where repositiories cannot be trusted to+continue to exist, and cannot be checked directly, but you'd still like to+keep track of their status. [[design/iabackup]] is one use case for this.++By running a periodic fsck with the --distributed option,+the repositories can verify that they still exist and that the+information about their contents is still accurate. This is done by+doing an extra update of the location log each time a file is verified by+fsck to still be in the repository.++The other option looks like --expire="30d somerepo:60d". It checks that+each specified repository has recorded a distributed fsck within the specified+time period. If not, the repository is dropped from the location tracking+log. Of course it can always update that later if it's really still around.++Distributed fsck is not the default because those extra location log updates+increase the size of the git-annex branch. I did one thing to keep the size+increase small: An identical line is logged to for each key, including the+timestamp, so git's delta compression will work as well as is possible. But,+there's still commit and tree update overhead. ++Probably doesn't make sense to run distributed fscks too often for that and+other reasons. If the git-annex branch does get too large, there's always+`git annex forget` ...++**(Update: This was later rethought and works much more efficiently now..)**
+ doc/devblog/day_271__parallel_get_groundwork.mdwn view
@@ -0,0 +1,32 @@+I've started work on [[todo/parallel_get]].+Today, laid the groundwork in two areas:++1. Evalulated the ascii-progress haskell library. It can display+   multiple progress bars in the terminal, portably, and its author+   Pedro Tacla Yamada has kindly offered to improve it to meet+   git-annex's needs.++   I ended up filing [10 issues](https://github.com/yamadapc/haskell-ascii-progress/issues)+   on it today, around 3 of the are blockers for git-annex using it.++2. Worked on making --quiet more quiet. Commands like rsync and wget+   need to have thier progress output disabled when run in parallel.++   Didn't quite finish this yet.++---++Yesterday I made some improvements to how git-annex behaves when it's+passed a massive number of directories or files on the command line.+Eg, when driven by xargs. There turned out to be some bugs in that+scenario.++One problem one I kind of had to paper over. While git-annex get+normally is careful to get the files in the same order they were listed on+the command line, it becomes very expensive to expand directories using+git-ls-files, and reorder its output to preserve order, when a large number+offiles are passed on the command line. There was a O(N*M) time blowup.++I worked around it by making it only preserve the order of the first 100+files. Assumption being that if you're specifying so many files on the+command line, you probably have less of an attachment to their ordering. :)
+ doc/devblog/day_272__forest_for_trees.mdwn view
@@ -0,0 +1,13 @@+Rethought distributed fsck. It's not really a fsck, but an expiration of+inactive repositories, where fscking is one kind of activity. That insight+let me reimplement it much more efficiently. Rather than updating all+the location logs to prove it was active, `git annex fsck` can simply and+inexpensively update an activity log. It's so cheap it'll do it by default.+The `git annex expire` command then reads the activity log and expires+(or unexpires) repositories that have not been active in the desired time+period. Expiring a repository simply marks it as dead.++Yesterday, finished making --quiet really be quiet. That sounds easy,+but it took several hours. On the `concurrentprogress` branch, I have+ascii-progress hooked up and working, but it's not quite ready for prime+time.
doc/encryption.mdwn view
@@ -89,7 +89,8 @@ encrypted using that old key, and the user should re-encrypt everything.)  (Because filenames are MAC'ed, a cipher still needs to be-generated (and encrypted to the given key IDs).)+generated (and encrypted to the given key IDs). It is only used for MHAC+encryption of filenames.)  ## MAC algorithm
+ doc/forum/Using_git-annex_to_manage_git_repositories.mdwn view
@@ -0,0 +1,5 @@+I am interested in using git-annex to manage git repositories, and I am wondering if it is possible and if anyone has experience with it?++I have done some searching, and I know that many people have asked for support for a Dropbox-like workflow, where Git repositories are mirrored everywhere.  I also know that no such support seems forthcoming, however this is not my goal.  Rather, I would like to use git-annex to track the location of many repositories.  I keep a lot of repositories and would like to offload them onto other storage devices and keep track of where each repository is stored.++Perhaps entire Git repositories can be added as a single unit for tracking in git-annex?
+ doc/forum/ga-ncdu.mdwn view
@@ -0,0 +1,5 @@+I've just written a quick script which provides an output which ncdu will use.++It uses the size from the key for apparent size and the size of the file (if present) for the disk size so 'a' toggles between total size of the annex and the actually used disk space.++Is there anyone else interested in having this? If so, I'll tidy it up and release it :)
+ doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment view
@@ -0,0 +1,7 @@+[[!comment format=mdwn+ username="andy"+ subject="comment 1"+ date="2015-03-28T22:03:53Z"+ content="""+I'd take a look. I just downloaded ncdu prompted by your comment, and that would be helpful to get sane output regarding my annexed files.+"""]]
+ doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="markusk"+ subject="Sounds nice !"+ date="2015-03-28T22:41:45Z"+ content="""++I would be very glad if you would release it!+"""]]
doc/forum/gadu_-_git-annex_disk_usage.mdwn view
@@ -1,4 +1,4 @@-Based on the thread over at <http://git-annex.branchable.com/forum/__34__du__34___equivalent_on_an_annex__63__/> I decided to finally write a du like utility for git-annex.  A 0.01 version is up over at <http://git-annex.mysteryvortex.com/git-annex-utils.html>.  It works, but I intend to make it smarter about handling git repos and annexed files, as well as adding more of the options available in the standard du utility.+Based on the thread over at [[forum/__34__du__34___equivalent_on_an_annex__63__]] I decided to finally write a du like utility for git-annex.  A 0.01 version is up over at <http://git-annex.mysteryvortex.com/git-annex-utils.html>.  It works, but I intend to make it smarter about handling git repos and annexed files, as well as adding more of the options available in the standard du utility.  Currently it will tally up the sizes of links that look like they are annexed files.  I plan to make it actually interact with git and git-annex to verify the files are annexed and enable options like tallying files only from specific remotes, only missing files, not double counting files which are annexed multiple times but stored only once, etc...
+ doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn view
@@ -0,0 +1,9 @@+as a followup to [[forum/remote-specific_meta-data/]], now i'm thinking of injecting data in the `git-annex` branch. i'm still a little hesitant because [[forum/optimising_lookupkey/]] mentions that i need to "delete or update `.git/annex/index`" when doing so. yet I also saw in the source code there is a `.git/annex/index.lock` file that is being used to avoid concurrent access to that file - is that something that should be watched out for? how do I manage that lockfile? is it mandatory to stage files into the index at first?++maybe i'm just not familiar enough with how git operates internally myself. i've reviewed this section of the progit book again: http://git-scm.com/book/en/v2/Git-Internals-Git-Objects - and i feel i have again some handle on how to do those things, but it seems like a major hurdle to go through just to change a single setting in one git-annex metadata file... wouldn't `enableremote` be sufficient here? I figured to append custom fields to a remote, i could do:++    git annex enableremote myremoteuuid "$(git cat-file -p git-annex:remote.log | grep myremoteuuid) extratag=true"++Would that work at all? Should i operate directly on the git-annex branch myself instead?++Thanks! --[[anarcat]]
+ doc/forum/two_lines_back_to_remote__63__.mdwn view
@@ -0,0 +1,10 @@+`git annex map` generates a diagram roughly like this:++    +------+    +-------++    |laptop| -> |usbdisk|+    |      | <- |       |+    |      | <- |       |+    +------+    +-------+++Why are there *two* lines back from the disk to the laptop?+
doc/git-annex-assistant.mdwn view
@@ -26,7 +26,7 @@ * `--startdelay=N`    Wait N seconds before running the startup scan. This process can-  be expensive and you may not want to run it immediatly upon login.+  be expensive and you may not want to run it immediately upon login.  * `--foreground`
+ doc/git-annex-expire.mdwn view
@@ -0,0 +1,66 @@+# NAME++git-annex expire - expire inactive repositories++# SYNOPSIS++git annex expire `[repository:]time ...`++# DESCRIPTION++This command expires repositories that have not performed some activity+within a specified time period. A repository is expired by marking it as+dead. De-expiration is also done; if a dead repository performed some+activity recently, it is marked as semitrusted again.++This can be useful when it's not possible to keep track of the state+of repositories manually. For example, a distributed network of+repositories where nobody can directly access all the repositories to+check their status.++The repository can be specified using the name of a remote,+or the description or uuid of the repository. ++The time is in the form "60d" or "1y". A time of "never" will disable+expiration.++If a time is specified without a repository, it is used as the default+value for all repositories. Note that the current repository is never+expired.++# OPTIONS++* `--no-act`++  Print out what would be done, but not not actually expite or unexpire+  any repositories.++* `--activity=Name`++  Specify the activity that a repository must have performed to avoid being+  expired. The default is any activity.++  Currently, the only activity that can be performed to avoid expiration+  is `git annex fsck`. Note that fscking a remote updates the+  expiration of the remote repository, not the local repository.++  The first version of git-annex that recorded fsck activity was+  5.20150405.++# SEE ALSO++[[git-annex]](1)++[[git-annex-fsck]](1)++[[git-annex-schedule]](1)++[[git-annex-dead]](1)++[[git-annex-semitrust]](1)++# AUTHOR++Joey Hess <id@joeyh.name>++Warning: Automatically converted into a man page by mdwn2man. Edit with care.
doc/git-annex-importfeed.mdwn view
@@ -9,7 +9,7 @@ # DESCRIPTION  Imports the contents of podcast feeds. Only downloads files whose-urls have not already been added to the repository before, so you can+content has not already been added to the repository before, so you can delete, rename, etc the resulting files and repeated runs won't duplicate them. @@ -17,11 +17,14 @@ are on a video hosting site, and the video is downloaded. This allows importing e.g., youtube playlists. +To make the import process add metadata to the imported files from the feed,+`git config annex.genmetadata true`+ # OPTIONS  * `--force` -  Force downoading urls it's seen before.+  Force downoading items it's seen before.  * `--template`
doc/git-annex-registerurl.mdwn view
@@ -15,7 +15,7 @@  If the key and url are not specified on the command line, they are instead read from stdin. Any number of lines can be provided in this-mode, each containing a key and url, sepearated by whitespace.+mode, each containing a key and url, separated by whitespace.  # SEE ALSO
doc/git-annex-unannex.mdwn view
@@ -1,6 +1,6 @@ # NAME -git-annex unannex - undo accidential add command+git-annex unannex - undo accidental add command  # SYNOPSIS
doc/git-annex-version.mdwn view
@@ -19,6 +19,12 @@ built. Therefore, "5.20150320-gdd35cf3" is a daily build, and "5.20150401" is an April 1st release made a bit later. +# OPTIONS++* `--raw`++  Causes only git-annex's version to be output, and nothing else.+ # SEE ALSO  [[git-annex]](1)
doc/git-annex.mdwn view
@@ -172,7 +172,7 @@  * `assistant` -  Atomatically sync folders between devices.+  Automatically sync folders between devices.    See [[git-annex-assistant]](1) for details. @@ -302,6 +302,11 @@    See [[git-annex-fsck]](1) for details. +* `expire [repository:]time ...`++  Expires repositories that have not recently performed an activity+  (such as a fsck).+ * `unused`    Checks the annex for data that does not correspond to any files present@@ -655,8 +660,7 @@  * `--quiet` -  Avoid the default verbose display of what is done; only show errors-  and progress displays.+  Avoid the default verbose display of what is done; only show errors.  * `--verbose` @@ -792,7 +796,8 @@   In particular, it stores year and month metadata, from the file's   modification date. -  When importfeed is used, it stores additional metadata from the feed.+  When importfeed is used, it stores additional metadata from the feed,+  such as the author, title, etc.  * `annex.queuesize`
doc/index.mdwn view
@@ -33,7 +33,7 @@ <table> <tr> <td width="50%" valign="top">[[!inline feeds=no template=bare pages=footer/column_a]]</td>-<td widtd="50%" valign="top">[[!inline feeds=no template=bare pages=footer/column_b]]</td>+<td width="50%" valign="top">[[!inline feeds=no template=bare pages=footer/column_b]]</td> </tr> </table>
doc/install/fromsource.mdwn view
@@ -43,8 +43,8 @@ Inside the source tree, run:  	cabal configure -f"-assistant -webapp -webdav -pairing -xmpp -dns"-	cabal install --only-dependencies-	cabal build+	cabal install -j -f"-assistant -webapp -webdav -pairing -xmpp -dns" --only-dependencies+	cabal build -j 	PATH=$HOME/bin:$PATH 	cabal install --bindir=$HOME/bin @@ -62,8 +62,8 @@ Once the C libraries are installed, run inside the source tree:  	cabal configure-	cabal install --only-dependencies-	cabal build+	cabal install -j --only-dependencies+	cabal build -j 	PATH=$HOME/bin:$PATH 	cabal install --bindir=$HOME/bin
doc/internals.mdwn view
@@ -247,6 +247,14 @@  	42bf2035-0636-461d-a367-49e9dfd361dd fsck self 30m every day at any time; fsck 4b3ebc86-0faf-4892-83c5-ce00cbe30f0a 1h every year at any time timestamp=1385646997.053162s +## `activity.log`++Used to record the times of activities, such as fscks.++Example:++	42bf2035-0636-461d-a367-49e9dfd361dd Fsck timestamp=1422387398.30395s+ ## `transitions.log`  Used to record transitions, eg by `git annex forget`
− doc/news/version_5.20150113.mdwn 
@@ -1,25 +0,0 @@-git-annex 5.20150113 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * unlock: Don't allow unlocking files that have never been committed to git-     before, to avoid an intractable problem that prevents the pre-commit-     hook from telling if such a file is intended to be an annexed file or not.-   * Avoid re-checksumming when migrating from hash to hashE backend.-     Closes: #[774494](http://bugs.debian.org/774494)-   * Fix build with process 1.2.1.0.-   * Android: Provide a version built with -fPIE -pie to support Android 5.0.-   * sync: Fix an edge case where syncing in a bare repository would try to-     merge and so fail.-   * Check git version at runtime, rather than assuming it will be the same-     as the git version used at build time when running git-checkattr and-     git-branch remove.-   * Switch to using relative paths to the git repository.-     - This allows the git repository to be moved while git-annex is running in-       it, with fewer problems.-     - On Windows, this avoids some of the problems with the absurdly small-       MAX\_PATH of 260 bytes. In particular, git-annex repositories should-       work in deeper/longer directory structures than before.-   * Generate shorter keys for WORM and URL, avoiding keys that are longer-     than used for SHA256, so as to not break on systems like Windows that-     have very small maximum path length limits.-   * Bugfix: A file named HEAD in the work tree could confuse some git commands-     run by git-annex."""]]
+ doc/news/version_5.20150406.mdwn view
@@ -0,0 +1,33 @@+git-annex 5.20150406 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * Prevent git-ls-files from double-expanding wildcards when an+     unexpanded wildcard is passed to a git-annex command like add or find.+   * Fix make build target. Thanks, Justin Geibel.+   * Fix GETURLS in external special remote protocol to strip+     downloader prefix from logged url info before checking for the+     specified prefix.+   * importfeed: Avoid downloading a redundant item from a feed whose+     guid has been downloaded before, even when the url has changed.+   * importfeed: Always store itemid in metadata; before this was only+     done when annex.genmetadata was set.+   * Relax debian package dependencies to git &gt;= 1:1.8.1 rather+     than needing &gt;= 1:2.0.+   * test: Fix --list-tests+   * addurl --file: When used with a special remote that claims+     urls and checks their contents, don't override the user's provided+     filename with filenames that the special remote suggests. Also,+     don't allow adding the url if the special remote says it contains+     multiple files.+   * import: --deduplicate and --cleanduplicates now output the keys+     corresponding to duplicated files they process.+   * expire: New command, for expiring inactive repositories.+   * fsck: Record fsck activity for use by expire command.+   * Fix truncation of parameters that could occur when using xargs git-annex.+   * Significantly sped up processing of large numbers of directories+     passed to a single git-annex command.+   * version: Add --raw+   * init: Improve fifo test to detect NFS systems that support fifos+     but not well enough for sshcaching.+   * --quiet now suppresses progress displays from eg, rsync.+     (The option already suppressed git-annex's own built-in progress+     displays.)"""]]
doc/publicrepos.mdwn view
@@ -6,7 +6,7 @@   Various downloads of things produced by Joey Hess, including git-annex   builds and videos. -* debconf-share  +* debconf-share (currently unavailable via IPv4)     `git clone http://annex.debconf.org/debconf-share/.git/`     [DebConf](http://debconf.org/) Media, photos, videos, etc. @@ -18,12 +18,15 @@ * [ocharles's papers](https://github.com/ocharles/papers)     Lots of CS papers read by [Oliver](http://ocharles.org.uk/blog/). +* [rejuvyesh's papers](https://github.com/rejuvyesh/papers)   +  Quite a few AI papers read by [rejuvyesh](http://rejuvyVesh.com)+ * [MRI brain scan data](http://studyforrest.org/pages/access.html)     `git clone http://psydata.ovgu.de/forrest_gump/.git studyforrest`     High-resolution, ultra-highfield fMRI dataset on auditory perception. -* [rejuvyesh's papers](https://github.com/rejuvyesh/papers)   -  Quite a few AI papers read by [rejuvyesh](http://rejuvyesh.com)+* [gst-integration-testsuites](http://cgit.freedesktop.org/gstreamer/gst-integration-testsuites)  +  GStreamer integration testsuite media assets management  This is a wiki -- add your own public repository to the list! See [[tips/centralized_git_repository_tutorial]].
+ doc/special_remotes/comment_23_aba4abb897c6ecc0c6ba72cbd21b9f4b._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnWvnTWY6LrcPB4BzYEBn5mRTpNhg5EtEg"+ nickname="Bence"+ subject="Amazon Cloud Drive support"+ date="2015-03-28T10:35:47Z"+ content="""+Is there a special remote implementation for Amazon Cloud Drive?++It's just became unlimited for a fair price: $60/year ( https://www.amazon.com/clouddrive/pricing ).++http://techcrunch.com/2015/03/26/amazon-goes-after-dropbox-google-microsoft-with-unlimited-cloud-drive-storage/+"""]]
+ doc/todo/Output_key_with_import__47__deduplicate.mdwn view
@@ -0,0 +1,22 @@+When using++    git annex import --deduplicate -- ~/directory/to/import++would it be possible to output the keys for duplicate files? This would allow people to find where in the annex the files already exist.++e.g.++    import ~/directory/to/import/001.txt ok+    import ~/directory/to/import/002.txt ok+    import ~/directory/to/import/002_d.txt (duplicate SHA256E-s261--fb1230987ac123098...) ok+    import ~/directory/to/import/003.txt ok++Then you could use++    git log -S SHA256E-s261--fb1230987ac123098...++to find out where it is already.++Not sure if that is the nicest layout.. (or what it might break).++> [[done]] --[[Joey]]
+ doc/todo/cheaper_global_fsck.mdwn view
@@ -0,0 +1,38 @@+Global fsck updates all location log entries for a repo. This wastes disk+space.++I realized now that it can be implemented w/o such waste. Probably cheaply+enough to be the default!++What we need is a new log file, call it fscktimes.log.+This records the time of the last fsck of each repo.++`git annex fsck --expire` no longer needs to look at the location log at+all. It can just check the repo's fscktimes.log entry. If the entry is+recent enough, we know that the repo has fscked recently, and its location+log is good, and nothing needs to be done. Otherwise, we know that the repo+has stopped fscking, and we simply expire *all* its location logs.++Note that fscktime.log is only used by fsck; it does not impact git-annex+generally or make it slower. And, it's very low overhead to update the one+file. Repos could do a fsck --fast on a daily basis and not grow the+git-annex branch much. Maybe on an hourly basis even.++(BTW, there is some overlap with the fsck.log file that is currently used to+hold the timestamp of the last local fsck. May be able to eliminate that+file too.)++----++It might be worth making the fsck.log record --fast and full fscks+separately so we know the last of each for each repo. This would let+--expire require periodic full fscks and more frequent fast fscks.++----++Hmm, --expire updates all the location logs when it thinks a repo has gone+missing. Why not just mark it dead? Again, this would save a lot of space!+It would complicate recovery if a repo had been offline and came back; it+would need to mark itself as not dead any longer.++> [[done]] --[[Joey]]
+ doc/todo/command_line_interface_for_required_content_setthings.mdwn view
@@ -0,0 +1,11 @@+Someone in the forum noticed that `git annex wanted`+handles preferred content settings, but there is no analagous `git annex+required`.++Probably worth adding that, although required content is not an often +used feature, and vicfg can already configure it.++(I don't much like the `git annex required` name. Nor the `git annex wanted`+one when it comes to that. Oh well.)++--[[Joey]]
doc/todo/parallel_get.mdwn view
@@ -71,3 +71,8 @@    This approach is just not as flexible or nice in general.  See also: [[parallel_possibilities]]++> I am looking at using the ascii-progress library for this.+> It has nice support for multiple progress bars, and is portable.+> I have filed 7 issues on it, around 4 of which need to get fixed before+> it's suitable for git-annex to use.. --[[Joey]]
+ doc/todo/podcatching_handling_updated_files.mdwn view
@@ -0,0 +1,19 @@+Files in feeds can be updated, and if this update includes changing the+url, `importfeed` will treat this as a new file. This results in `foo.mp3`+having a `2_foo.mp3` added next to it.++This seems to happen especially commonly with feeds using FeedBurner.+Saw several with same size, different checksum and url.++To detect this, `importfeed` could store the item's guid in the metadata+of the key. Where it currently builds a `Map URLString Key` of all+known items, it could instead build a `Map (Either URlString GUID) Key`.++This would at least prevent the duplication, when the feed has guids.++> [[done]] --[[Joey]]++It would be even nicer if the old file could be updated with the new+content. But, since files can be moved around, deleted, tagged, etc,+that only seems practical at all if the file is still in the directory+where `importfeed` created it.
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20150327+Version: 5.20150406 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name>
− man/git-annex-.1 
@@ -1,18 +0,0 @@-.TH git-annex- 1-.SH NAME-git\-annex  \- -.PP-.SH SYNOPSIS-git annex  \fB[path ...]\fP-.PP-.SH DESCRIPTION-.PP-.SH OPTIONS-.SH SEE ALSO-git\-annex(1)-.PP-.SH AUTHOR-Joey Hess <id@joeyh.name>-.PP-.PP-
man/git-annex-assistant.1 view
@@ -22,7 +22,7 @@ .IP .IP "\fB\-\-startdelay=N\fP" Wait N seconds before running the startup scan. This process can-be expensive and you may not want to run it immediatly upon login.+be expensive and you may not want to run it immediately upon login. .IP .IP "\fB\-\-foreground\fP" Avoid forking to the background.
+ man/git-annex-expire.1 view
@@ -0,0 +1,61 @@+.TH git-annex-expire 1+.SH NAME+git\-annex expire \- expire inactive repositories+.PP+.SH SYNOPSIS+git annex expire \fB[repository:]time ...\fP+.PP+.SH DESCRIPTION+This command expires repositories that have not performed some activity+within a specified time period. A repository is expired by marking it as+dead. De\-expiration is also done; if a dead repository performed some+activity recently, it is marked as semitrusted again.+.PP+This can be useful when it's not possible to keep track of the state+of repositories manually. For example, a distributed network of+repositories where nobody can directly access all the repositories to+check their status.+.PP+The repository can be specified using the name of a remote,+or the description or uuid of the repository. +.PP+The time is in the form "60d" or "1y". A time of "never" will disable+expiration.+.PP+If a time is specified without a repository, it is used as the default+value for all repositories. Note that the current repository is never+expired.+.PP+.SH OPTIONS+.IP "\fB\-\-no\-act\fP"+.IP+Print out what would be done, but not not actually expite or unexpire+any repositories.+.IP+.IP "\fB\-\-activity=Name\fP"+Specify the activity that a repository must have performed to avoid being+expired. The default is any activity.+.IP+Currently, the only activity that can be performed to avoid expiration+is \fBgit annex fsck\fP. Note that fscking a remote updates the+expiration of the remote repository, not the local repository.+.IP+The first version of git\-annex that recorded fsck activity was+5.20150405.+.IP+.SH SEE ALSO+git\-annex(1)+.PP+git\-annex\-fsck(1)+.PP+git\-annex\-schedule(1)+.PP+git\-annex\-dead(1)+.PP+git\-annex\-semitrust(1)+.PP+.SH AUTHOR+Joey Hess <id@joeyh.name>+.PP+.PP+
man/git-annex-importfeed.1 view
@@ -7,7 +7,7 @@ .PP .SH DESCRIPTION Imports the contents of podcast feeds. Only downloads files whose-urls have not already been added to the repository before, so you can+content has not already been added to the repository before, so you can delete, rename, etc the resulting files and repeated runs won't duplicate them. .PP@@ -15,10 +15,13 @@ are on a video hosting site, and the video is downloaded. This allows importing e.g., youtube playlists. .PP+To make the import process add metadata to the imported files from the feed,+\fBgit config annex.genmetadata true\fP+.PP .SH OPTIONS .IP "\fB\-\-force\fP" .IP-Force downoading urls it's seen before.+Force downoading items it's seen before. .IP .IP "\fB\-\-template\fP" Controls where the files are stored.
man/git-annex-registerurl.1 view
@@ -13,7 +13,7 @@ .PP If the key and url are not specified on the command line, they are instead read from stdin. Any number of lines can be provided in this-mode, each containing a key and url, sepearated by whitespace.+mode, each containing a key and url, separated by whitespace. .PP .SH SEE ALSO git\-annex(1)
man/git-annex-unannex.1 view
@@ -1,6 +1,6 @@ .TH git-annex-unannex 1 .SH NAME-git\-annex unannex \- undo accidential add command+git\-annex unannex \- undo accidental add command .PP .SH SYNOPSIS git annex unannex \fB[path ...]\fP
man/git-annex-version.1 view
@@ -17,6 +17,11 @@ built. Therefore, "5.20150320\-gdd35cf3" is a daily build, and "5.20150401" is an April 1st release made a bit later. .PP+.SH OPTIONS+.IP "\fB\-\-raw\fP"+.IP+Causes only git\-annex's version to be output, and nothing else.+.IP .SH SEE ALSO git\-annex(1) .PP
man/git-annex.1 view
@@ -149,7 +149,7 @@ See git\-annex\-watch(1) for details. .IP .IP "\fBassistant\fP"-Atomatically sync folders between devices.+Automatically sync folders between devices. .IP See git\-annex\-assistant(1) for details. .IP@@ -260,6 +260,10 @@ .IP See git\-annex\-fsck(1) for details. .IP+.IP "\fBexpire [repository:]time ...\fP"+Expires repositories that have not recently performed an activity+(such as a fsck).+.IP .IP "\fBunused\fP" Checks the annex for data that does not correspond to any files present in any tag or branch, and prints a numbered list of the data.@@ -562,8 +566,7 @@ What is avoided depends on the command. .IP .IP "\fB\-\-quiet\fP"-Avoid the default verbose display of what is done; only show errors-and progress displays.+Avoid the default verbose display of what is done; only show errors. .IP .IP "\fB\-\-verbose\fP" Enable verbose display.@@ -680,7 +683,8 @@ In particular, it stores year and month metadata, from the file's modification date. .IP-When importfeed is used, it stores additional metadata from the feed.+When importfeed is used, it stores additional metadata from the feed,+such as the author, title, etc. .IP .IP "\fBannex.queuesize\fP" git\-annex builds a queue of git commands, in order to combine similar
standalone/linux/skel/git-annex view
@@ -28,7 +28,4 @@ 	export GIT_ANNEX_APP_BASE fi -GIT_ANNEX_PROGRAMPATH="$0"-export GIT_ANNEX_PROGRAMPATH- exec "$base/runshell" git-annex "$@"