packages feed

git-annex 6.20160211 → 6.20160229

raw patch · 122 files changed

+1990/−501 lines, 122 filesdep ~network

Dependency ranges changed: network

Files

Annex/CatFile.hs view
@@ -10,6 +10,7 @@ 	catFileDetails, 	catObject, 	catTree,+	catCommit, 	catObjectDetails, 	catFileHandle, 	catFileStop,@@ -51,6 +52,11 @@ catTree ref = do 	h <- catFileHandle 	liftIO $ Git.CatFile.catTree h ref++catCommit :: Git.Ref -> Annex (Maybe Commit)+catCommit ref = do+	h <- catFileHandle+	liftIO $ Git.CatFile.catCommit h ref  catObjectDetails :: Git.Ref -> Annex (Maybe (L.ByteString, Sha, ObjectType)) catObjectDetails ref = do
Annex/FileMatcher.hs view
@@ -33,6 +33,7 @@  #ifdef WITH_MAGICMIME import Magic+import Utility.Env #endif  import Data.Either@@ -129,8 +130,15 @@ mkLargeFilesParser :: Annex (String -> [ParseResult]) mkLargeFilesParser = do #ifdef WITH_MAGICMIME-	magicmime <- liftIO $ magicOpen [MagicMimeType]-	liftIO $ magicLoadDefault magicmime+	magicmime <- liftIO $ catchMaybeIO $ do+		m <- magicOpen [MagicMimeType]+		liftIO $ do+			md <- getEnv "GIT_ANNEX_DIR"+			case md of+				Nothing -> magicLoadDefault m+				Just d -> magicLoad m+					(d </> "magic" </> "magic.mgc")+		return m #endif 	let parse = parseToken $ commonTokens #ifdef WITH_MAGICMIME
Annex/Index.hs view
@@ -5,19 +5,14 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE CPP #-}--module Annex.Index (-	withIndexFile,-	addGitEnv,-) where+module Annex.Index (withIndexFile) where  import qualified Control.Exception as E  import Annex.Common import Git.Types+import Git.Env import qualified Annex-import Utility.Env  {- Runs an action using a different git index file. -} withIndexFile :: FilePath -> Annex a -> Annex a@@ -30,23 +25,3 @@ 		a 	Annex.changeState $ \s -> s { Annex.repo = (Annex.repo s) { gitEnv = gitEnv g} } 	either E.throw return r--addGitEnv :: Repo -> String -> String -> IO Repo-addGitEnv g var val = do-	e <- maybe copyenv return (gitEnv g)-	let e' = addEntry var val e-	return $ g { gitEnv = Just e' }-  where-	copyenv = do-#ifdef __ANDROID__-		{- This should not be necessary on Android, but there is some-		 - weird getEnvironment breakage. See-		 - https://github.com/neurocyte/ghc-android/issues/7-		 - Use getEnv to get some key environment variables that-		 - git expects to have. -}-		let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME"-		let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k-		liftIO $ catMaybes <$> forM keyenv getEnvPair-#else-		liftIO getEnvironment-#endif
Annex/Ingest.hs view
@@ -1,6 +1,6 @@ {- git-annex content ingestion  -- - Copyright 2010-2015 Joey Hess <id@joeyh.name>+ - Copyright 2010-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -11,14 +11,17 @@ 	LockedDown(..), 	LockDownConfig(..), 	lockDown,+	ingestAdd, 	ingest, 	finishIngestDirect, 	finishIngestUnlocked, 	cleanOldKeys, 	addLink, 	makeLink,+	addUnlocked, 	restoreFile, 	forceParams,+	addAnnexedFile, ) where  import Annex.Common@@ -29,6 +32,7 @@ import Annex.Perms import Annex.Link import Annex.MetaData+import Annex.Version import Logs.Location import qualified Annex import qualified Annex.Queue@@ -111,11 +115,30 @@ 			, inodeCache = cache 			} -{- Ingests a locked down file into the annex.- -- - The file may be added to the git repository as a locked or an unlocked- - file. When unlocked, the work tree file is left alone. When locked, - - the work tree file is deleted, in preparation for adding the symlink.+{- Ingests a locked down file into the annex. Updates the work tree and+ - index. -}+ingestAdd :: Maybe LockedDown -> Annex (Maybe Key)+ingestAdd Nothing = return Nothing+ingestAdd ld@(Just (LockedDown cfg source)) = do+	(mk, mic) <- ingest ld+	case mk of+		Nothing -> return Nothing+		Just k -> do+			let f = keyFilename source+			if lockingFile cfg+				then do+					liftIO $ nukeFile f+					addLink f k mic+				else ifM isDirect+					( do+						l <- calcRepo $ gitAnnexLink f k+						stageSymlink f =<< hashSymlink l+					, stagePointerFile f =<< hashPointerFile k+					)+			return (Just k)++{- Ingests a locked down file into the annex. Does not update the working+ - tree or the index.  -} ingest :: Maybe LockedDown -> Annex (Maybe Key, Maybe InodeCache) ingest Nothing = return (Nothing, Nothing)@@ -141,7 +164,6 @@ 	golocked key mcache s = do 		catchNonAsync (moveAnnex key $ contentLocation source) 			(restoreFile (keyFilename source) key)-		liftIO $ nukeFile $ keyFilename source 		populateAssociatedFiles key source 		success key mcache s @@ -295,3 +317,50 @@ 	( return [Param "-f"] 	, return [] 	)++{- Whether a file should be added unlocked or not. Default is to not,+ - unless symlinks are not supported. annex.addunlocked can override that. -}+addUnlocked :: Annex Bool+addUnlocked = isDirect <||>+	(versionSupportsUnlockedPointers <&&>+	 ((not . coreSymlinks <$> Annex.getGitConfig) <||>+	  (annexAddUnlocked <$> Annex.getGitConfig)+	 )+	)++{- Adds a file to the work tree for the key, and stages it in the index.+ - The content of the key may be provided in a temp file, which will be+ - moved into place. -}+addAnnexedFile :: FilePath -> Key -> Maybe FilePath -> Annex ()+addAnnexedFile file key mtmp = ifM (addUnlocked <&&> not <$> isDirect)+	( do+		stagePointerFile file =<< hashPointerFile key+		Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file)+		case mtmp of+			Just tmp -> do+				moveAnnex key tmp+				linkunlocked+			Nothing -> ifM (inAnnex key)+				( linkunlocked+				, writepointer+				)+	, do+		addLink file key Nothing+		whenM isDirect $ do+			void $ addAssociatedFile key file+		case mtmp of+			Just tmp -> do+				{- For moveAnnex to work in direct mode, the+				 - symlink must already exist, so flush the queue. -}+				whenM isDirect $+					Annex.Queue.flush+				moveAnnex key tmp+			Nothing -> return ()+	)+  where+	writepointer = liftIO $ writeFile file (formatPointer key)+	linkunlocked = do+		r <- linkFromAnnex key file+		case r of+			LinkAnnexFailed -> writepointer+			_ -> return ()
Annex/Init.hs view
@@ -14,6 +14,7 @@ 	initialize', 	uninitialize, 	probeCrippledFileSystem,+	probeCrippledFileSystem', ) where  import Annex.Common@@ -36,11 +37,11 @@ import Annex.Hook import Annex.InodeSentinal import Upgrade+import Annex.Perms import qualified Database.Keys #ifndef mingw32_HOST_OS import Utility.UserInfo import Utility.FileMode-import Annex.Perms import System.Posix.User import qualified Utility.LockFile.Posix as Posix #endif@@ -91,7 +92,7 @@ 	whenM versionSupportsUnlockedPointers $ do 		configureSmudgeFilter 		Database.Keys.scanAssociatedFiles-	ifM (crippledFileSystem <&&> not <$> isBare)+	ifM (crippledFileSystem <&&> (not <$> isBare) <&&> (not <$> versionSupportsUnlockedPointers)) 		( do 			enableDirectMode 			setDirect True@@ -134,16 +135,20 @@  - or removing write access from files. -} probeCrippledFileSystem :: Annex Bool probeCrippledFileSystem = do+	tmp <- fromRepo gitAnnexTmpMiscDir+	createAnnexDirectory tmp+	liftIO $ probeCrippledFileSystem' tmp++probeCrippledFileSystem' :: FilePath -> IO Bool+probeCrippledFileSystem' tmp = do #ifdef mingw32_HOST_OS 	return True #else-	tmp <- fromRepo gitAnnexTmpMiscDir 	let f = tmp </> "gaprobe"-	createAnnexDirectory tmp-	liftIO $ writeFile f ""-	uncrippled <- liftIO $ probe f-	void $ liftIO $ tryIO $ allowWrite f-	liftIO $ removeFile f+	writeFile f ""+	uncrippled <- probe f+	void $ tryIO $ allowWrite f+	removeFile f 	return $ not uncrippled   where 	probe f = catchBoolIO $ do
Annex/Link.hs view
@@ -12,6 +12,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE CPP, BangPatterns #-}+ module Annex.Link where  import Annex.Common@@ -23,6 +25,7 @@ import Git.FilePath  import qualified Data.ByteString.Lazy as L+import Data.Int  type LinkTarget = String @@ -130,28 +133,50 @@  - Only looks at the first line, as pointer files can have subsequent  - lines. -} parseLinkOrPointer :: L.ByteString -> Maybe Key-parseLinkOrPointer = parseLinkOrPointer' . decodeBS . L.take maxsz+parseLinkOrPointer = parseLinkOrPointer' . decodeBS . L.take maxPointerSz   where-	{- Want to avoid buffering really big files in git into-	 - memory when reading files that may be pointers.-	 --	 - 8192 bytes is plenty for a pointer to a key.-	 - Pad some more to allow for any pointer files that might have-	 - lines after the key explaining what the file is used for. -}-	maxsz = 81920 +{- Want to avoid buffering really big files in git into+ - memory when reading files that may be pointers.+ -+ - 8192 bytes is plenty for a pointer to a key.+ - Pad some more to allow for any pointer files that might have+ - lines after the key explaining what the file is used for. -}+maxPointerSz :: Int64+maxPointerSz = 81920+ parseLinkOrPointer' :: String -> Maybe Key-parseLinkOrPointer' s = headMaybe (lines (fromInternalGitPath s)) >>= go+parseLinkOrPointer' = go . fromInternalGitPath . takeWhile (not . lineend)   where 	go l-		| isLinkToAnnex l = file2key $ takeFileName l+		| isLinkToAnnex l = fileKey $ takeFileName l 		| otherwise = Nothing+	lineend '\n' = True+	lineend '\r' = True+	lineend _ = False  formatPointer :: Key -> String formatPointer k = -	toInternalGitPath (pathSeparator:objectDir </> key2file k) ++ "\n"+	toInternalGitPath (pathSeparator:objectDir </> keyFile k) ++ "\n"  {- Checks if a file is a pointer to a key. -} isPointerFile :: FilePath -> IO (Maybe Key)-isPointerFile f = catchDefaultIO Nothing $ -	parseLinkOrPointer <$> L.readFile f+isPointerFile f = catchDefaultIO Nothing $ do+	b <- L.take maxPointerSz <$> L.readFile f+	let !mk = parseLinkOrPointer' (decodeBS b)+	return mk++{- Checks a symlink target or pointer file first line to see if it+ - appears to point to annexed content.+ -+ - We only look for paths inside the .git directory, and not at the .git+ - directory itself, because GIT_DIR may cause a directory name other+ - than .git to be used.+ -}+isLinkToAnnex :: FilePath -> Bool+isLinkToAnnex s = (pathSeparator:objectDir) `isInfixOf` s+#ifdef mingw32_HOST_OS+	-- '/' is still used inside pointer files on Windows, not the native+	-- '\'+	|| ('/':objectDir) `isInfixOf` s+#endif
Annex/Locations.hs view
@@ -63,7 +63,6 @@ 	gitAnnexSshDir, 	gitAnnexRemotesDir, 	gitAnnexAssistantDefaultDir,-	isLinkToAnnex, 	HashLevels(..), 	hashDirMixed, 	hashDirLower,@@ -385,15 +384,6 @@  - repositories, by default. -} gitAnnexAssistantDefaultDir :: FilePath gitAnnexAssistantDefaultDir = "annex"--{- Checks a symlink target to see if it appears to point to annexed content.- -- - We only look at paths inside the .git directory, and not at the .git- - directory itself, because GIT_DIR may cause a directory name other- - than .git to be used.- -}-isLinkToAnnex :: FilePath -> Bool-isLinkToAnnex s = (pathSeparator:objectDir) `isInfixOf` s  {- Sanitizes a String that will be used as part of a Key's keyName,  - dealing with characters that cause problems on substandard filesystems.
Annex/MetaData.hs view
@@ -1,6 +1,6 @@ {- git-annex metadata  -- - Copyright 2014 Joey Hess <id@joeyh.name>+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -8,6 +8,8 @@ module Annex.MetaData ( 	genMetaData, 	dateMetaData,+	parseModMeta,+	parseMetaDataMatcher, 	module X ) where @@ -17,6 +19,7 @@ import Annex.MetaData.StandardFields as X import Logs.MetaData import Annex.CatFile+import Utility.Glob  import qualified Data.Set as S import qualified Data.Map as M@@ -53,3 +56,37 @@   where 	isnew (f, _) = S.null (currentMetaDataValues f old) 	(y, m, _d) = toGregorian $ utctDay mtime++{- Parses field=value, field+=value, field-=value, field?=value -}+parseModMeta :: String -> Either String ModMeta+parseModMeta p = case lastMaybe f of+	Just '+' -> AddMeta <$> mkMetaField f' <*> v+	Just '-' -> DelMeta <$> mkMetaField f' <*> v+	Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v+	_ -> SetMeta <$> mkMetaField f <*> v+  where+	(f, sv) = separate (== '=') p+	f' = beginning f+	v = pure (toMetaValue sv)++{- Parses field=value, field<value, field<=value, field>value, field>=value -}+parseMetaDataMatcher :: String -> Either String (MetaField, MetaValue -> Bool)+parseMetaDataMatcher p = (,)+	<$> mkMetaField f+	<*> pure matcher+  where+	(f, op_v) = break (`elem` "=<>") p+	matcher = case op_v of+		('=':v) -> checkglob v+		('<':'=':v) -> checkcmp (<=) v+		('<':v) -> checkcmp (<) v+		('>':'=':v) -> checkcmp (>=) v+		('>':v) -> checkcmp (>) v+		_ -> checkglob ""+	checkglob v =+		let cglob = compileGlob v CaseInsensative+		in matchGlob cglob . fromMetaValue+	checkcmp cmp v v' = case (doubleval v, doubleval (fromMetaValue v')) of+		(Just d, Just d') -> d' `cmp` d+		_ -> False+	doubleval v = readish v :: Maybe Double
Annex/Ssh.hs view
@@ -34,7 +34,7 @@ import Annex.Path import Utility.Env import Types.CleanupActions-import Annex.Index (addGitEnv)+import Git.Env #ifndef mingw32_HOST_OS import Annex.Perms import Annex.LockPool
Annex/Transfer.hs view
@@ -91,8 +91,9 @@ 			return v   where #ifndef mingw32_HOST_OS-	prep tfile mode info = do+	prep tfile mode info = catchPermissionDenied (const prepfailed) $ do 		let lck = transferLockFile tfile+		createAnnexDirectory $ takeDirectory lck 		r <- tryLockExclusive (Just mode) lck 		case r of 			Nothing -> return (Nothing, True)@@ -104,9 +105,10 @@ 				, return (Nothing, True) 				) #else-	prep tfile _mode info = liftIO $ do+	prep tfile _mode info = catchPermissionDenied (const prepfailed) $ do 		let lck = transferLockFile tfile-		v <- catchMaybeIO $ lockExclusive lck+		createAnnexDirectory $ takeDirectory lck+		v <- catchMaybeIO $ liftIO $ lockExclusive lck 		case v of 			Nothing -> return (Nothing, False) 			Just Nothing -> return (Nothing, True)@@ -115,6 +117,8 @@ 					writeTransferInfoFile info tfile 				return (Just lockhandle, False) #endif+	prepfailed = return (Nothing, False)+ 	cleanup _ Nothing = noop 	cleanup tfile (Just lockhandle) = do 		let lck = transferLockFile tfile
Backend/Hash.hs view
@@ -5,7 +5,7 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE BangPatterns, CPP #-}+{-# LANGUAGE CPP #-}  module Backend.Hash ( 	backends,@@ -171,8 +171,9 @@ 	go (SkeinHash hashsize) = use (skeinHasher hashsize) 	 	use hasher = liftIO $ do-		!h <- hasher <$> L.readFile file-		return h+		h <- hasher <$> L.readFile file+		-- Force full evaluation so file is read and closed.+		return (length h `seq` h) 	 	usehasher hashsize = case shaHasher hashsize filesize of 		Left sha -> use sha
Build/BundledPrograms.hs view
@@ -17,7 +17,31 @@  -  - These may be just the command name, or the full path to it. -} bundledPrograms :: [FilePath]-bundledPrograms = catMaybes+bundledPrograms = preferredBundledPrograms ++ extraBundledPrograms++{- Programs that are only included in the bundle in case the system+ - doesn't have them. These come after the system PATH.+ -}+extraBundledPrograms :: [FilePath]+extraBundledPrograms = catMaybes+	-- The system gpg is probably better, because it may better+	-- integrate with the system gpg-agent, etc.+	-- On Windows, gpg is bundled with git for windows.+#ifndef mingw32_HOST_OS+	[ SysConfig.gpg+#else+	[+#endif+	]++{- Programs that should be preferred for use from the bundle, over+ - any that might be installed on the system otherwise. These come before+ - the system PATH.+ -+ - For example, git-annex is built for a specific version of git.+ -}+preferredBundledPrograms :: [FilePath]+preferredBundledPrograms = catMaybes 	[ Nothing #ifndef mingw32_HOST_OS 	-- git is not included in the windows bundle; git for windows is used@@ -56,7 +80,6 @@ #ifndef mingw32_HOST_OS 	-- All these utilities are included in git for Windows 	, ifset SysConfig.curl "curl"-	, SysConfig.gpg 	, SysConfig.sha1 	, SysConfig.sha256 	, SysConfig.sha512
Build/Standalone.hs view
@@ -26,6 +26,9 @@ progDir topdir = topdir </> "bin" #endif +extraProgDir :: FilePath -> FilePath+extraProgDir topdir = topdir </> "extra"+ installProg :: FilePath -> FilePath -> IO (FilePath, FilePath) installProg dir prog = searchPath prog >>= go   where@@ -41,7 +44,9 @@   where 	go [] = error "specify topdir" 	go (topdir:_) = do-		let dir = progDir topdir-		createDirectoryIfMissing True dir-		installed <- forM bundledPrograms $ installProg dir-		writeFile "tmp/standalone-installed" (show installed)+		installed <- forM+			[ (progDir topdir, preferredBundledPrograms)+			, (extraProgDir topdir, extraBundledPrograms) ] $ \(dir, progs) -> do+			createDirectoryIfMissing True dir+			forM progs $ installProg dir+		writeFile "tmp/standalone-installed" (show (concat installed))
CHANGELOG view
@@ -1,3 +1,55 @@+git-annex (6.20160229) unstable; urgency=medium++  * Update perlmagick build dependency. Closes: #789225+  * Fix memory leak in last release, which affected commands like+    git-annex status when a large non-annexed file is present in the work+    tree.+  * fsck: When the only copy of a file is in a dead repository, mention+    the repository.+  * info: Mention when run in a dead repository.+  * Linux and OSX standalone builds put the bundled gpg last in PATH,+    so any system gpg will be preferred over it.+  * Avoid crashing when built with MagicMime support, but when the magic+    database cannot be loaded.+  * Include magic database in the linux and OSX standalone builds.+  * Fix memory leak when hashing files, which triggered during fsck+    when an external hash program was not used.+    (This leak was introduced in version 6.20160114.)+  * Support --metadata field<number, --metadata field>number etc+    to match ranges of numeric values.+  * Similarly, support preferred content expressions like+    metadata=field<number and metadata=field>number+  * The pre-commit-annex hook script that automatically extracts+    metadata has been updated to also use exiftool.+    Thanks, Klaus Ethgen.++ -- Joey Hess <id@joeyh.name>  Mon, 29 Feb 2016 12:41:49 -0400++git-annex (6.20160217) unstable; urgency=medium++  * Support getting files from read-only repositories.+  * checkpresentkey: Allow to be run without an explicit remote.+  * checkpresentkey: Added --batch.+  * Work around problem with concurrent-output when in a non-unicode locale+    by avoiding use of it in such a locale. Instead -J will behave as if+    it was built without concurrent-output support in this situation.+  * Fix storing of filenames of v6 unlocked files when the filename is not+    representable in the current locale.+  * fsck: Detect and fix missing associated file mappings in v6 repositories.+  * fsck: Populate unlocked files in v6 repositories whose content is+    present in annex/objects but didn't reach the work tree.+  * When initializing a v6 repo on a crippled filesystem, don't force it+    into direct mode.+  * Windows: Fix v6 unlocked files to actually work.+  * add, addurl, import, importfeed: When in a v6 repository on a crippled+    filesystem, add files unlocked.+  * annex.addunlocked: New configuration setting, makes files always be+    added unlocked. (v6 only)+  * Improve format of v6 unlocked pointer files to support keys containing+    slashes.++ -- Joey Hess <id@joeyh.name>  Wed, 17 Feb 2016 14:48:51 -0400+ git-annex (6.20160211) unstable; urgency=medium    * annex.addsmallfiles: New option controlling what is done when
@@ -44,6 +44,11 @@            2010-2015 Joey Hess <id@joeyh.name> License: GPL-3+ +Files: doc/tips/automatically_adding_metadata/pre-commit-annex +Copyright: 2014 Joey Hess <id@joeyh.name>+           2016 Klaus Ethgen <Klaus@Ethgen.ch>+License: GPL-3++ Files: Utility/libmounts.c Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California            2001 David Rufino <daverufino@btinternet.com>
CmdLine/Action.hs view
@@ -53,7 +53,7 @@ commandAction :: CommandStart -> Annex () commandAction a = withOutputType go    where-	go (ConcurrentOutput n) = do+	go o@(ConcurrentOutput n _) = do 		ws <- Annex.getState Annex.workers 		(st, ws') <- if null ws 			then do@@ -63,7 +63,7 @@ 				l <- liftIO $ drainTo (n-1) ws 				findFreeSlot l 		w <- liftIO $ async-			$ snd <$> Annex.run st (inOwnConsoleRegion run)+			$ snd <$> Annex.run st (inOwnConsoleRegion o run) 		Annex.changeState $ \s -> s { Annex.workers = Right w:ws' } 	go _  =	run 	run = void $ includeCommandAction a@@ -155,9 +155,13 @@ allowConcurrentOutput a = go =<< Annex.getState Annex.concurrentjobs   where 	go Nothing = a-	go (Just n) = Regions.displayConsoleRegions $-		bracket_ (setup n) cleanup a-	setup = Annex.setOutput . ConcurrentOutput+	go (Just n) = ifM (liftIO concurrentOutputSupported)+		( Regions.displayConsoleRegions $+			goconcurrent (ConcurrentOutput n True)+		, goconcurrent (ConcurrentOutput n False)+		)+	goconcurrent o = bracket_ (setup o) cleanup a+	setup = Annex.setOutput 	cleanup = do 		finishCommandActions 		Annex.setOutput NormalOutput
Command/Add.hs view
@@ -12,12 +12,10 @@ import Logs.Location import Annex.Content import Annex.Content.Direct-import Annex.Link import qualified Annex import qualified Annex.Queue import qualified Database.Keys import Config-import Utility.InodeCache import Annex.FileMatcher import Annex.Version @@ -99,48 +97,42 @@ 		( do 			ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file 			case ms of-				Just s | isSymbolicLink s -> fixup key+				Just s | isSymbolicLink s -> fixuplink key 				_ -> ifM (sameInodeCache file =<< Database.Keys.getInodeCaches key) 						( stop, add ) 		, ifM isDirect 			( do 				ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file 				case ms of-					Just s | isSymbolicLink s -> fixup key+					Just s | isSymbolicLink s -> fixuplink key 					_ -> ifM (goodContent key file) 						( stop , add )-			, fixup key+			, fixuplink key 			) 		)-	fixup key = do+	fixuplink key = do 		-- the annexed symlink is present but not yet added to git 		showStart "add" file 		liftIO $ removeFile file-		whenM isDirect $-			void $ addAssociatedFile key file-		next $ next $ cleanup file key Nothing =<< inAnnex key+		next $ next $ do+			addLink file key Nothing+			cleanup key =<< inAnnex key  perform :: FilePath -> CommandPerform perform file = do-	lockingfile <- not <$> isDirect+	lockingfile <- not <$> addUnlocked 	let cfg = LockDownConfig 		{ lockingFile = lockingfile 		, hardlinkFileTmp = True 		}-	lockDown cfg file >>= ingest >>= go+	lockDown cfg file >>= ingestAdd >>= finish   where-	go (Just key, cache) = next $ cleanup file key cache True-	go (Nothing, _) = stop+	finish (Just key) = next $ cleanup key True+	finish Nothing = stop -cleanup :: FilePath -> Key -> Maybe InodeCache -> Bool -> CommandCleanup-cleanup file key mcache hascontent = do+cleanup :: Key -> Bool -> CommandCleanup+cleanup key hascontent = do 	maybeShowJSON [("key", key2file key)]-	ifM (isDirect <&&> pure hascontent)-		( do-			l <- calcRepo $ gitAnnexLink file key-			stageSymlink file =<< hashSymlink l-		, addLink file key mcache-		) 	when hascontent $ 		logStatus key InfoPresent 	return True
Command/AddUrl.hs view
@@ -12,7 +12,6 @@ import Command import Backend import qualified Annex-import qualified Annex.Queue import qualified Annex.Url as Url import qualified Backend.URL import qualified Remote@@ -24,8 +23,6 @@ import Logs.Web import Types.KeySource import Types.UrlContents-import Config-import Annex.Content.Direct import Annex.FileMatcher import Logs.Location import Utility.Metered@@ -363,13 +360,7 @@ 		when (isJust mtmp) $ 			logStatus key InfoPresent 		setUrlPresent u key url-		addLink file key Nothing-		whenM isDirect $ do-			void $ addAssociatedFile key file-			{- For moveAnnex to work in direct mode, the symlink-			 - must already exist, so flush the queue. -}-			Annex.Queue.flush-		maybe noop (moveAnnex key) mtmp+		addAnnexedFile file key mtmp  nodownload :: URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) nodownload url urlinfo file
Command/CheckPresentKey.hs view
@@ -1,6 +1,6 @@ {- git-annex command  -- - Copyright 2015 Joey Hess <id@joeyh.name>+ - Copyright 2015-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -9,29 +9,70 @@  import Command import qualified Remote-import Annex-import Types.Messages  cmd :: Command-cmd = noCommit $ +cmd = noCommit $ noMessages $ 	command "checkpresentkey" SectionPlumbing 		"check if key is present in remote"-		(paramPair paramKey paramRemote)-		(withParams seek)+		(paramPair paramKey (paramOptional paramRemote))+		(seek <$$> optParser) -seek :: CmdParams -> CommandSeek-seek = withWords start+data CheckPresentKeyOptions = CheckPresentKeyOptions+	{ params :: CmdParams+	, batchOption :: BatchMode+	} -start :: [String] -> CommandStart-start (ks:rn:[]) = do-	setOutput QuietOutput-	maybe (error "Unknown remote") (go <=< flip Remote.hasKey k)-		=<< Remote.byNameWithUUID (Just rn)+optParser :: CmdParamsDesc -> Parser CheckPresentKeyOptions+optParser desc = CheckPresentKeyOptions+	<$> cmdParams desc+	<*> parseBatchOption++seek :: CheckPresentKeyOptions -> CommandSeek+seek o = case batchOption o of+	NoBatch -> case params o of+		(ks:rn:[]) -> toRemote rn >>= (check ks . Just) >>= exitResult+		(ks:[]) -> check ks Nothing >>= exitResult+		_ -> wrongnumparams+	Batch -> do+		checker <- case params o of+			(rn:[]) -> toRemote rn >>= \r -> return (flip check (Just r))+			[] -> return (flip check Nothing)+			_ -> wrongnumparams+		batchInput Right $ checker >=> batchResult   where-	k = fromMaybe (error "bad key") (file2key ks)-	go (Right True) = liftIO exitSuccess-	go (Right False) = liftIO exitFailure-	go (Left e) = liftIO $ do-		hPutStrLn stderr e-		exitWith $ ExitFailure 100-start _ = error "Wrong number of parameters"+	wrongnumparams = error "Wrong number of parameters"+					+data Result = Present | NotPresent | CheckFailure String++check :: String -> Maybe Remote -> Annex Result+check ks mr = case mr of+	Nothing -> go Nothing =<< Remote.keyPossibilities k+	Just r -> go Nothing [r]+  where+	k = toKey ks+	go Nothing [] = return NotPresent+	go (Just e) [] = return $ CheckFailure e+	go olderr (r:rs) = do+		v <- Remote.hasKey r k+		case v of+			Right True -> return Present+			Right False -> go olderr rs+			Left e -> go (Just e) rs++exitResult :: Result -> Annex a+exitResult Present = liftIO exitSuccess+exitResult NotPresent = liftIO exitFailure+exitResult (CheckFailure msg) = liftIO $ do+	hPutStrLn stderr msg+	exitWith $ ExitFailure 100++batchResult :: Result -> Annex ()+batchResult Present = liftIO $ putStrLn "1"+batchResult _ = liftIO $ putStrLn "0"++toKey :: String -> Key+toKey = fromMaybe (error "Bad key") . file2key++toRemote :: String -> Annex Remote+toRemote rn = maybe (error "Unknown remote") return+	=<< Remote.byNameWithUUID (Just rn)
Command/Fsck.hs view
@@ -1,6 +1,6 @@ {- git-annex command  -- - Copyright 2010-2015 Joey Hess <id@joeyh.name>+ - Copyright 2010-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -15,7 +15,7 @@ import qualified Types.Backend import qualified Backend import Annex.Content-import Annex.Content.Direct+import qualified Annex.Content.Direct as Direct import Annex.Direct import Annex.Perms import Annex.Link@@ -25,6 +25,7 @@ import Logs.TimeStamp import Annex.NumCopies import Annex.UUID+import Annex.ReplaceFile import Utility.DataUnits import Config import Utility.HumanTime@@ -114,13 +115,13 @@  perform :: Key -> FilePath -> Backend -> NumCopies -> Annex Bool perform key file backend numcopies = do-	keystatus <- getKeyStatus key+	keystatus <- getKeyFileStatus key file 	check 		-- order matters 		[ fixLink key file 		, verifyLocationLog key keystatus file-		, verifyDirectMapping key file-		, verifyDirectMode key file+		, verifyAssociatedFiles key keystatus file+		, verifyWorkTree key file 		, checkKeySize key keystatus 		, checkBackend backend key keystatus (Just file) 		, checkKeyNumCopies key (Just file) numcopies@@ -261,30 +262,55 @@ 		showNote "fixing location log" 		updatestatus s -{- Ensures the direct mode mapping file is consistent. Each file- - it lists for the key should exist, and the specified file should be- - included in it.- -}-verifyDirectMapping :: Key -> FilePath -> Annex Bool-verifyDirectMapping key file = do-	whenM isDirect $ do-		fs <- addAssociatedFile key file+{- Verifies the associated file records. -}+verifyAssociatedFiles :: Key -> KeyStatus -> FilePath -> Annex Bool+verifyAssociatedFiles key keystatus file = do+	ifM isDirect (godirect, goindirect)+	return True+  where+	godirect = do+		fs <- Direct.addAssociatedFile key file 		forM_ fs $ \f ->  			unlessM (liftIO $ doesFileExist f) $-				void $ removeAssociatedFile key f-	return True+				void $ Direct.removeAssociatedFile key f+	goindirect = case keystatus of+		KeyUnlocked -> do+			f <- inRepo $ toTopFilePath file+			afs <- Database.Keys.getAssociatedFiles key+			unless (getTopFilePath f `elem` map getTopFilePath afs) $+				Database.Keys.addAssociatedFile key f+		_ -> return () -{- Ensures that files whose content is available are in direct mode. -}-verifyDirectMode :: Key -> FilePath -> Annex Bool-verifyDirectMode key file = do-	whenM (isDirect <&&> isJust <$> isAnnexLink file) $ do+verifyWorkTree :: Key -> FilePath -> Annex Bool+verifyWorkTree key file = do+	ifM isDirect ( godirect, goindirect )+	return True+  where+	{- Ensures that files whose content is available are in direct mode. -}+	godirect = whenM (isJust <$> isAnnexLink file) $ do 		v <- toDirectGen key file 		case v of 			Nothing -> noop 			Just a -> do 				showNote "fixing direct mode" 				a-	return True+	{- Make sure that a pointer file is replaced with its content,+	 - when the content is available. -}+	goindirect = do+		mk <- liftIO $ isPointerFile file+		case mk of+			Just k | k == key -> whenM (inAnnex key) $ do+				showNote "fixing worktree content"+				replaceFile file $ \tmp -> +					ifM (annexThin <$> Annex.getGitConfig)+						( void $ linkFromAnnex key tmp+						, do+							obj <- calcRepo $ gitAnnexLocation key+							void $ checkedCopyFile key obj tmp+							thawContent tmp+						)+				Database.Keys.storeInodeCaches key [file]+			_ -> return ()  {- The size of the data for a key is checked against the size encoded in  - the key's metadata, if available.@@ -346,9 +372,9 @@ 			, checkBackendOr badContent backend key content 			) 	go True = maybe nocheck checkdirect mfile-	checkdirect file = ifM (goodContent key file)+	checkdirect file = ifM (Direct.goodContent key file) 		( checkBackendOr' (badContentDirect file) backend key file-			(goodContent key file)+			(Direct.goodContent key file) 		, nocheck 		) 	nocheck = return True@@ -383,7 +409,9 @@ checkKeyNumCopies :: Key -> AssociatedFile -> NumCopies -> Annex Bool checkKeyNumCopies key afile numcopies = do 	let file = fromMaybe (key2file key) afile-	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< Remote.keyLocations key+	locs <- loggedLocations key+	(untrustedlocations, otherlocations) <- trustPartition UnTrusted locs+	(deadlocations, safelocations) <- trustPartition DeadTrusted otherlocations 	let present = NumCopies (length safelocations) 	if present < numcopies 		then ifM (pure (isNothing afile) <&&> checkDead key)@@ -391,29 +419,35 @@ 				showLongNote $ "This key is dead, skipping." 				return True 			, do-				ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations-				warning $ missingNote file present numcopies ppuuids+				untrusted <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations+				dead <- Remote.prettyPrintUUIDs "dead" deadlocations+				warning $ missingNote file present numcopies untrusted dead 				when (fromNumCopies present == 0 && isNothing afile) $ 					showLongNote "(Avoid this check by running: git annex dead --key )" 				return False 			) 		else return True -missingNote :: String -> NumCopies -> NumCopies -> String -> String-missingNote file (NumCopies 0) _ [] = -		"** No known copies exist of " ++ file-missingNote file (NumCopies 0) _ untrusted =+missingNote :: String -> NumCopies -> NumCopies -> String -> String -> String+missingNote file (NumCopies 0) _ [] dead = +		"** No known copies exist of " ++ file ++ honorDead dead+missingNote file (NumCopies 0) _ untrusted dead = 		"Only these untrusted locations may have copies of " ++ file ++ 		"\n" ++ untrusted ++-		"Back it up to trusted locations with git-annex copy."-missingNote file present needed [] =+		"Back it up to trusted locations with git-annex copy." ++ honorDead dead+missingNote file present needed [] _ = 		"Only " ++ show (fromNumCopies present) ++ " of " ++ show (fromNumCopies needed) ++  		" trustworthy copies exist of " ++ file ++ 		"\nBack it up with git-annex copy."-missingNote file present needed untrusted = -		missingNote file present needed [] +++missingNote file present needed untrusted dead = +		missingNote file present needed [] dead ++ 		"\nThe following untrusted locations may also have copies: " ++ 		"\n" ++ untrusted+	+honorDead :: String -> String+honorDead dead+	| null dead = ""+	| otherwise = "\nThese dead repositories used to have copies\n" ++ dead  {- Bad content is moved aside. -} badContent :: Key -> Annex String@@ -587,8 +621,17 @@ getKeyStatus key = ifM isDirect 	( return KeyUnlocked 	, catchDefaultIO KeyMissing $ do-		obj <- calcRepo $ gitAnnexLocation key-		unlocked <- ((> 1) . linkCount <$> liftIO (getFileStatus obj))-			<&&> (not . null <$> Database.Keys.getAssociatedFiles key)+		unlocked <- not . null <$> Database.Keys.getAssociatedFiles key 		return $ if unlocked then KeyUnlocked else KeyLocked 	)++getKeyFileStatus :: Key -> FilePath -> Annex KeyStatus+getKeyFileStatus key file = do+	s <- getKeyStatus key+	case s of+		KeyLocked -> catchDefaultIO KeyLocked $+			ifM (isJust <$> isAnnexLink file)+				( return KeyLocked+				, return KeyUnlocked+				)+		_ -> return s
Command/Info.hs view
@@ -22,6 +22,7 @@ import Utility.DataUnits import Utility.DiskFree import Annex.Content+import Annex.UUID import Logs.UUID import Logs.Trust import Logs.Location@@ -111,6 +112,9 @@  globalInfo :: InfoOptions -> Annex () globalInfo o = do+	u <- getUUID+	whenM ((==) DeadTrusted <$> lookupTrust u) $+		earlyWarning "Warning: This repository is currently marked as dead." 	stats <- selStats global_fast_stats global_slow_stats 	showCustom "info" $ do 		evalStateT (mapM_ showStat stats) (emptyStatInfo o)
Database/Keys.hs view
@@ -56,7 +56,7 @@ 	h <- getDbHandle 	withDbState h go   where-	go DbEmpty = return (mempty, DbEmpty)+	go DbUnavailable = return (mempty, DbUnavailable) 	go st@(DbOpen qh) = do 		liftIO $ H.flushDbQueue qh 		v <- a (SQL.ReadHandle qh)@@ -114,8 +114,8 @@  -} openDb :: Bool -> DbState -> Annex DbState openDb _ st@(DbOpen _) = return st-openDb False DbEmpty = return DbEmpty-openDb createdb _ = withExclusiveLock gitAnnexKeysDbLock $ do+openDb False DbUnavailable = return DbUnavailable+openDb createdb _ = catchPermissionDenied permerr $ withExclusiveLock gitAnnexKeysDbLock $ do 	dbdir <- fromRepo gitAnnexKeysDb 	let db = dbdir </> "db" 	dbexists <- liftIO $ doesFileExist db@@ -128,9 +128,14 @@ 			setAnnexDirPerm dbdir 			setAnnexFilePerm db 			open db-		(False, False) -> return DbEmpty+		(False, False) -> return DbUnavailable   where 	open db = liftIO $ DbOpen <$> H.openDbQueue db SQL.containedTable+	-- If permissions don't allow opening the database, treat it as if+	-- it does not exist.+	permerr e = case createdb of+		False -> return DbUnavailable+		True -> throwM e  addAssociatedFile :: Key -> TopFilePath -> Annex () addAssociatedFile k f = runWriterIO $ SQL.addAssociatedFile (toIKey k) f@@ -169,7 +174,7 @@ 	add h i k = liftIO $ flip SQL.queueDb h $  		void $ insertUnique $ SQL.Associated 			(toIKey k)-			(getTopFilePath $ Git.LsTree.file i)+			(toSFilePath $ getTopFilePath $ Git.LsTree.file i)  {- Stats the files, and stores their InodeCaches. -} storeInodeCaches :: Key -> [FilePath] -> Annex ()
Database/Keys/Handle.hs view
@@ -26,8 +26,8 @@ newtype DbHandle = DbHandle (MVar DbState)  -- The database can be closed or open, but it also may have been--- tried to open (for read) and didn't exist yet.-data DbState = DbClosed | DbOpen H.DbQueue | DbEmpty+-- tried to open (for read) and didn't exist yet or is not readable.+data DbState = DbClosed | DbOpen H.DbQueue | DbUnavailable  newDbHandle :: IO DbHandle newDbHandle = DbHandle <$> newMVar DbClosed
Database/Keys/SQL.hs view
@@ -26,7 +26,7 @@ share [mkPersist sqlSettings, mkMigrate "migrateKeysDb"] [persistLowerCase| Associated   key IKey-  file FilePath+  file SFilePath   KeyFileIndex key file   FileKeyIndex file key Content@@ -63,8 +63,10 @@ 	-- If the same file was associated with a different key before, 	-- remove that. 	delete $ from $ \r -> do-		where_ (r ^. AssociatedFile ==. val (getTopFilePath f) &&. not_ (r ^. AssociatedKey ==. val ik))-	void $ insertUnique $ Associated ik (getTopFilePath f)+		where_ (r ^. AssociatedFile ==. val af &&. not_ (r ^. AssociatedKey ==. val ik))+	void $ insertUnique $ Associated ik af+  where+	af = toSFilePath (getTopFilePath f)  {- Note that the files returned were once associated with the key, but  - some of them may not be any longer. -}@@ -73,21 +75,25 @@ 	l <- select $ from $ \r -> do 		where_ (r ^. AssociatedKey ==. val ik) 		return (r ^. AssociatedFile)-	return $ map (asTopFilePath . unValue) l+	return $ map (asTopFilePath . fromSFilePath . unValue) l  {- Gets any keys that are on record as having a particular associated file.  - (Should be one or none but the database doesn't enforce that.) -} getAssociatedKey :: TopFilePath -> ReadHandle -> IO [IKey] getAssociatedKey f = readDb $ do 	l <- select $ from $ \r -> do-		where_ (r ^. AssociatedFile ==. val (getTopFilePath f))+		where_ (r ^. AssociatedFile ==. val af) 		return (r ^. AssociatedKey) 	return $ map unValue l+  where+	af = toSFilePath (getTopFilePath f)  removeAssociatedFile :: IKey -> TopFilePath -> WriteHandle -> IO () removeAssociatedFile ik f = queueDb $  	delete $ from $ \r -> do-		where_ (r ^. AssociatedKey ==. val ik &&. r ^. AssociatedFile ==. val (getTopFilePath f))+		where_ (r ^. AssociatedKey ==. val ik &&. r ^. AssociatedFile ==. val af)+  where+	af = toSFilePath (getTopFilePath f)  addInodeCaches :: IKey -> [InodeCache] -> WriteHandle -> IO () addInodeCaches ik is = queueDb $
Database/Types.hs view
@@ -1,6 +1,6 @@ {- types for SQL databases  -- - Copyright 2015 Joey Hess <id@joeyh.name>+ - Copyright 2015-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -11,7 +11,9 @@  import Database.Persist.TH import Data.Maybe+import Data.Char +import Utility.PartialPrelude import Types.Key import Utility.InodeCache @@ -53,6 +55,41 @@ toSInodeCache = I . showInodeCache  fromSInodeCache :: SInodeCache -> InodeCache-fromSInodeCache (I s) = fromMaybe (error $ "bad serialied InodeCache " ++ s) (readInodeCache s)+fromSInodeCache (I s) = fromMaybe (error $ "bad serialized InodeCache " ++ s) (readInodeCache s)  derivePersistField "SInodeCache"++-- A serialized FilePath.+--+-- Not all unicode characters round-trip through sqlite. In particular,+-- surrigate code points do not. So, escape the FilePath. But, only when+-- it contains such characters.+newtype SFilePath = SFilePath String++-- Note that Read instance does not work when used in any kind of complex+-- data structure.+instance Read SFilePath where+	readsPrec _ s = [(SFilePath s, "")]++instance Show SFilePath where+	show (SFilePath s) = s++toSFilePath :: FilePath -> SFilePath+toSFilePath s@('"':_) = SFilePath (show s)+toSFilePath s+	| any needsescape s = SFilePath (show s)+	| otherwise = SFilePath s+  where+	needsescape c = case generalCategory c of+		Surrogate -> True+		PrivateUse -> True+		NotAssigned -> True+		_ -> False++fromSFilePath :: SFilePath -> FilePath+fromSFilePath (SFilePath s@('"':_)) =+	fromMaybe (error "bad serialized SFilePath " ++ s) (readish s)+fromSFilePath (SFilePath s) = s++derivePersistField "SFilePath"+
Git/Branch.hs view
@@ -23,7 +23,7 @@  - branch is not created yet. So, this also looks at show-ref HEAD  - to double-check.  -}-current :: Repo -> IO (Maybe Git.Ref)+current :: Repo -> IO (Maybe Branch) current r = do 	v <- currentUnsafe r 	case v of@@ -35,7 +35,7 @@ 				)  {- The current branch, which may not really exist yet. -}-currentUnsafe :: Repo -> IO (Maybe Git.Ref)+currentUnsafe :: Repo -> IO (Maybe Branch) currentUnsafe r = parse . firstLine 	<$> pipeReadStrict [Param "symbolic-ref", Param "-q", Param $ fromRef Git.Ref.headRef] r   where@@ -144,25 +144,31 @@ 		pipeReadStrict [Param "write-tree"] repo 	ifM (cancommit tree) 		( do-			sha <- getSha "commit-tree" $-				pipeWriteRead ([Param "commit-tree", Param (fromRef tree)] ++ ps) sendmsg repo+			sha <- commitTree commitmode message parentrefs tree repo 			update branch sha repo 			return $ Just sha 		, return Nothing 		)   where-	ps = applyCommitMode commitmode $-		map Param $ concatMap (\r -> ["-p", fromRef r]) parentrefs 	cancommit tree 		| allowempty = return True 		| otherwise = case parentrefs of 			[p] -> maybe False (tree /=) <$> Git.Ref.tree p repo 			_ -> return True-	sendmsg = Just $ flip hPutStr message  commitAlways :: CommitMode -> String -> Branch -> [Ref] -> Repo -> IO Sha commitAlways commitmode message branch parentrefs repo = fromJust 	<$> commit commitmode True message branch parentrefs repo++commitTree :: CommitMode -> String -> [Ref] -> Ref -> Repo -> IO Sha+commitTree commitmode message parentrefs tree repo =+	getSha "commit-tree" $+		pipeWriteRead ([Param "commit-tree", Param (fromRef tree)] ++ ps)+			sendmsg repo+  where+	ps = applyCommitMode commitmode $+		map Param $ concatMap (\r -> ["-p", fromRef r]) parentrefs+	sendmsg = Just $ flip hPutStr message  {- A leading + makes git-push force pushing a branch. -} forcePush :: String -> String
Git/CatFile.hs view
@@ -1,6 +1,6 @@ {- git cat-file interface  -- - Copyright 2011, 2013 Joey Hess <id@joeyh.name>+ - Copyright 2011-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -13,6 +13,7 @@ 	catFile, 	catFileDetails, 	catTree,+	catCommit, 	catObject, 	catObjectDetails, ) where@@ -20,6 +21,10 @@ import System.IO import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L+import qualified Data.ByteString.Lazy.Char8 as L8+import qualified Data.Map as M+import Data.String+import Data.Char import Data.Tuple.Utils import Numeric import System.Posix.Types@@ -110,3 +115,43 @@ 		let (modestr, file) = separate (== ' ') (decodeBS b) 		in (file, readmode modestr) 	readmode = fromMaybe 0 . fmap fst . headMaybe . readOct++catCommit :: CatFileHandle -> Ref -> IO (Maybe Commit)+catCommit h commitref = go <$> catObjectDetails h commitref+  where+	go (Just (b, _, CommitObject)) = parseCommit b+	go _ = Nothing++parseCommit :: L.ByteString -> Maybe Commit+parseCommit b = Commit+	<$> (extractSha . L8.unpack =<< field "tree")+	<*> (parsemetadata <$> field "author")+	<*> (parsemetadata <$> field "committer")+	<*> Just (L8.unpack $ L.intercalate (L.singleton nl) message)+  where+	field n = M.lookup (fromString n) fields+	fields = M.fromList ((map breakfield) header)+	breakfield l =+		let (k, sp_v) = L.break (== sp) l+		in (k, L.drop 1 sp_v)+	(header, message) = separate L.null ls+	ls = L.split nl b++	-- author and committer lines have the form: "name <email> date"+	-- The email is always present, even if empty "<>"+	parsemetadata l = CommitMetaData+		{ commitName = whenset $ L.init name_sp+		, commitEmail = whenset email+		, commitDate = whenset $ L.drop 2 gt_sp_date+		}+	  where+		(name_sp, rest) = L.break (== lt) l+		(email, gt_sp_date) = L.break (== gt) (L.drop 1 rest)+		whenset v+			| L.null v = Nothing+			| otherwise = Just (L8.unpack v)++	nl = fromIntegral (ord '\n')+	sp = fromIntegral (ord ' ')+	lt = fromIntegral (ord '<')+	gt = fromIntegral (ord '>')
+ Git/Env.hs view
@@ -0,0 +1,57 @@+{- Adjusting the environment while running git commands.+ -+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++{-# LANGUAGE CPP #-}++module Git.Env where++import Git+import Git.Types+import Utility.Env+#ifdef __ANDROID__+import Common+#endif++{- Adjusts the gitEnv of a Repo. Copies the system environment if the repo+ - does not have any gitEnv yet. -}+adjustGitEnv :: Repo -> ([(String, String)] -> [(String, String)]) -> IO Repo+adjustGitEnv g adj = do+	e <- maybe copyenv return (gitEnv g)+	let e' = adj e+	return $ g { gitEnv = Just e' }+  where+	copyenv = do+#ifdef __ANDROID__+		{- This should not be necessary on Android, but there is some+		 - weird getEnvironment breakage. See+		 - https://github.com/neurocyte/ghc-android/issues/7+		 - Use getEnv to get some key environment variables that+		 - git expects to have. -}+		let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME"+		let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k+		catMaybes <$> forM keyenv getEnvPair+#else+		getEnvironment+#endif++addGitEnv :: Repo -> String -> String -> IO Repo+addGitEnv g var val = adjustGitEnv g (addEntry var val)++{- Use with any action that makes a commit to set metadata. -}+commitWithMetaData :: CommitMetaData -> CommitMetaData -> (Repo -> IO a) -> Repo -> IO a+commitWithMetaData authormetadata committermetadata a g =+	a =<< adjustGitEnv g adj+  where+	adj = mkadj "AUTHOR" authormetadata+		. mkadj "COMMITTER" committermetadata+	mkadj p md = go "NAME" commitName+		. go "EMAIL" commitEmail+		. go "DATE" commitDate+	  where+		go s getv = case getv md of+			Nothing -> id+			Just v -> addEntry ("GIT_" ++ p ++ "_" ++ s) v
Git/FilePath.hs view
@@ -31,7 +31,7 @@  {- A FilePath, relative to the top of the git repository. -} newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath }-	deriving (Show)+	deriving (Show, Eq)  {- Path to a TopFilePath, within the provided git repo. -} fromTopFilePath :: TopFilePath -> Git.Repo -> FilePath
Git/LsTree.hs view
@@ -5,12 +5,15 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE BangPatterns #-}+ module Git.LsTree ( 	TreeItem(..), 	lsTree,+	lsTree', 	lsTreeParams, 	lsTreeFiles,-	parseLsTree+	parseLsTree, ) where  import Common@@ -33,8 +36,11 @@ {- Lists the complete contents of a tree, recursing into sub-trees,  - with lazy output. -} lsTree :: Ref -> Repo -> IO ([TreeItem], IO Bool)-lsTree t repo = do-	(l, cleanup) <- pipeNullSplit (lsTreeParams t []) repo+lsTree = lsTree' []++lsTree' :: [CommandParam] -> Ref -> Repo -> IO ([TreeItem], IO Bool)+lsTree' ps t repo = do+	(l, cleanup) <- pipeNullSplit (lsTreeParams t ps) repo 	return (map parseLsTree l, cleanup)  lsTreeParams :: Ref -> [CommandParam] -> [CommandParam]@@ -64,16 +70,18 @@  - (The --long format is not currently supported.) -} parseLsTree :: String -> TreeItem parseLsTree l = TreeItem -	{ mode = fst $ Prelude.head $ readOct m+	{ mode = smode 	, typeobj = t 	, sha = Ref s-	, file = asTopFilePath $ Git.Filename.decode f+	, file = sfile 	}   where 	-- l = <mode> SP <type> SP <sha> TAB <file> 	-- All fields are fixed, so we can pull them out of 	-- specific positions in the line. 	(m, past_m) = splitAt 7 l-	(t, past_t) = splitAt 4 past_m-	(s, past_s) = splitAt shaSize $ Prelude.tail past_t-	f = Prelude.tail past_s+	(!t, past_t) = splitAt 4 past_m+	(!s, past_s) = splitAt shaSize $ Prelude.tail past_t+	!f = Prelude.tail past_s+	!smode = fst $ Prelude.head $ readOct m+	!sfile = asTopFilePath $ Git.Filename.decode f
+ Git/Tree.hs view
@@ -0,0 +1,180 @@+{- git trees+ -+ - Copyright 2016 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++{-# LANGUAGE BangPatterns #-}++module Git.Tree (+	Tree(..),+	TreeContent(..),+	getTree,+	recordTree,+	TreeItem(..),+	adjustTree,+) where++import Common+import Git+import Git.FilePath+import Git.Types+import Git.Command+import Git.Sha+import qualified Git.LsTree as LsTree+import qualified Utility.CoProcess as CoProcess++import Numeric+import System.Posix.Types+import Control.Monad.IO.Class++newtype Tree = Tree [TreeContent]+	deriving (Show)++data TreeContent+	= TreeBlob TopFilePath FileMode Sha+	-- A subtree that is already recorded in git, with a known sha.+	| RecordedSubTree TopFilePath Sha [TreeContent]+	-- A subtree that has not yet been recorded in git.+	| NewSubTree TopFilePath [TreeContent]+	deriving (Show)++{- Gets the Tree for a Ref. -}+getTree :: Ref -> Repo -> IO Tree+getTree r repo = do+	(l, cleanup) <- lsTreeWithObjects r repo+	let !t = either (\e -> error ("ls-tree parse error:" ++ e)) id+		(extractTree l)+	void cleanup+	return t++lsTreeWithObjects :: Ref -> Repo -> IO ([LsTree.TreeItem], IO Bool)+lsTreeWithObjects = LsTree.lsTree' [Param "-t"]++newtype MkTreeHandle = MkTreeHandle CoProcess.CoProcessHandle++withMkTreeHandle :: (MonadIO m, MonadMask m) => Repo -> (MkTreeHandle -> m a) -> m a+withMkTreeHandle repo a = bracketIO setup cleanup (a . MkTreeHandle)+  where+	setup = CoProcess.rawMode =<< gitCoProcessStart False ps repo+	ps = [Param "mktree", Param "--batch", Param "-z"]+	cleanup = CoProcess.stop++{- Records a Tree in the Repo, returning its Sha.+ - + - Efficiently handles subtrees, by only recording ones that have not+ - already been recorded before. And even when many subtrees need to be+ - recorded, it's done with a single call to git mktree, using its batch+ - interface.+ -}+recordTree :: Tree -> Repo -> IO Sha+recordTree t repo = withMkTreeHandle repo $ \h -> recordTree' h t++recordTree' :: MkTreeHandle -> Tree -> IO Sha+recordTree' h (Tree l) = mkTree h =<< mapM (recordSubTree h) l++{- Note that the returned RecordedSubTree does not have its [TreeContent]+ - list populated. This is a memory optimisation, since the list is not+ - used. -}+recordSubTree :: MkTreeHandle -> TreeContent -> IO TreeContent+recordSubTree h (NewSubTree d l) = do+	sha <- mkTree h =<< mapM (recordSubTree h) l+	return (RecordedSubTree d sha [])+recordSubTree _ alreadyrecorded = return alreadyrecorded+ +mkTree :: MkTreeHandle -> [TreeContent] -> IO Sha+mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive+  where+	send h = do+		forM_ l $ \i -> hPutStr h $ case i of+			TreeBlob f fm s -> mkTreeOutput fm BlobObject s f+			RecordedSubTree f s _ -> mkTreeOutput 0o040000 TreeObject s f+			NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree"+		hPutStr h "\NUL" -- signal end of tree to --batch+	receive h = getSha "mktree" (hGetLine h)++mkTreeOutput :: FileMode -> ObjectType -> Sha -> TopFilePath -> String+mkTreeOutput fm ot s f = concat+	[ showOct fm ""+	, " "+	, show ot+	, " "+	, fromRef s+	, "\t"+	, takeFileName (getTopFilePath f)+	, "\NUL"+	]++data TreeItem = TreeItem TopFilePath FileMode Sha+	deriving (Eq)++{- Applies an adjustment to items in a tree.+ -+ - While less flexible than using getTree and recordTree, this avoids+ - buffering the whole tree in memory.+ -}+adjustTree :: (MonadIO m, MonadMask m) => (TreeItem -> m (Maybe TreeItem)) -> Ref -> Repo -> m Sha+adjustTree adjust r repo = withMkTreeHandle repo $ \h -> do+	(l, cleanup) <- liftIO $ lsTreeWithObjects r repo+	(l', _, _) <- go h False [] topTree l+	sha <- liftIO $ mkTree h l'+	void $ liftIO cleanup+	return sha+  where+	go _ wasmodified c _ [] = return (c, wasmodified, [])+	go h wasmodified c intree (i:is)+		| intree i =+			case readObjectType (LsTree.typeobj i) of+				Just BlobObject -> do+					let ti = TreeItem (LsTree.file i) (LsTree.mode i) (LsTree.sha i)+					v <- adjust ti+					case v of+						Nothing -> go h True c intree is+						Just ti'@(TreeItem f m s) ->+							let !modified = wasmodified || ti' /= ti+							    blob = TreeBlob f m s+							in go h modified (blob:c) intree is+				Just TreeObject -> do+					(sl, modified, is') <- go h False [] (subTree i) is+					subtree <- if modified+						then liftIO $ recordSubTree h $ NewSubTree (LsTree.file i) sl+						else return $ RecordedSubTree (LsTree.file i) (LsTree.sha i) [] +					let !modified' = modified || wasmodified+					go h modified' (subtree : c) intree is'+				_ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"")+		| otherwise = return (c, wasmodified, i:is)++{- Assumes the list is ordered, with tree objects coming right before their+ - contents. -}+extractTree :: [LsTree.TreeItem] -> Either String Tree+extractTree l = case go [] topTree l of+	Right (t, []) -> Right (Tree t)+	Right _ -> parseerr "unexpected tree form"+	Left e -> parseerr e+  where+	go t _ [] = Right (t, [])+	go t intree (i:is)+		| intree i = +			case readObjectType (LsTree.typeobj i) of+				Just BlobObject ->+					let b = TreeBlob (LsTree.file i) (LsTree.mode i) (LsTree.sha i)+					in go (b:t) intree is+				Just TreeObject -> case go [] (subTree i) is of+					Right (subtree, is') ->+						let st = RecordedSubTree (LsTree.file i) (LsTree.sha i) subtree+						in go (st:t) intree is'+					Left e -> Left e+				_ -> parseerr ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"")+		| otherwise = Right (t, i:is)+	parseerr = Left++type InTree = LsTree.TreeItem -> Bool++topTree :: InTree+topTree = notElem '/' . getTopFilePath . LsTree.file++subTree :: LsTree.TreeItem -> InTree+subTree t =+	let prefix = getTopFilePath (LsTree.file t) ++ "/"+	in (\i -> prefix `isPrefixOf` getTopFilePath (LsTree.file i))
Git/Types.hs view
@@ -11,7 +11,6 @@ import qualified Data.Map as M import System.Posix.Types import Utility.SafeCommand-import Utility.URI ()  {- Support repositories on local disk, and repositories accessed via an URL.  -@@ -98,3 +97,23 @@ toBlobType 0o100755 = Just ExecutableBlob toBlobType 0o120000 = Just SymlinkBlob toBlobType _ = Nothing++fromBlobType :: BlobType -> FileMode+fromBlobType FileBlob = 0o100644+fromBlobType ExecutableBlob = 0o100755+fromBlobType SymlinkBlob = 0o120000++data Commit = Commit+	{ commitTree :: Sha+	, commitAuthorMetaData :: CommitMetaData+	, commitCommitterMetaData :: CommitMetaData+	, commitMessage :: String+	}+	deriving (Show)++data CommitMetaData = CommitMetaData+	{ commitName :: Maybe String+	, commitEmail :: Maybe String+	, commitDate :: Maybe String -- In raw git form, "epoch -tzoffset"+	}+	deriving (Show)
Limit.hs view
@@ -23,6 +23,7 @@ import Types.Group import Types.FileMatcher import Types.MetaData+import Annex.MetaData import Logs.MetaData import Logs.Group import Logs.Unused@@ -97,14 +98,15 @@ 	go (MatchingInfo af _ _ _) = matchGlob cglob <$> getInfo af  #ifdef WITH_MAGICMIME-matchMagic :: Magic -> MkLimit Annex-matchMagic magic glob = Right $ const go+matchMagic :: Maybe Magic -> MkLimit Annex+matchMagic (Just magic) glob = Right $ const go   where  	cglob = compileGlob glob CaseSensative -- memoized 	go (MatchingKey _) = pure False 	go (MatchingFile fi) = liftIO $ catchBoolIO $ 		matchGlob cglob <$> magicFile magic (matchFile fi) 	go (MatchingInfo _ _ _ mimeval) = matchGlob cglob <$> getInfo mimeval+matchMagic Nothing _ = Left "unable to load magic database; \"mimetype\" cannot be used" #endif  {- Adds a limit to skip files not believed to be present@@ -277,14 +279,12 @@ addMetaData = addLimit . limitMetaData  limitMetaData :: MkLimit Annex-limitMetaData s = case parseMetaData s of+limitMetaData s = case parseMetaDataMatcher s of 	Left e -> Left e-	Right (f, v) ->-		let cglob = compileGlob (fromMetaValue v) CaseInsensative-		in Right $ const $ checkKey (check f cglob)+	Right (f, matching) -> Right $ const $ checkKey (check f matching)   where-	check f cglob k = not . S.null -		. S.filter (matchGlob cglob . fromMetaValue) +	check f matching k = not . S.null +		. S.filter matching 		. metaDataValues f <$> getCurrentMetaData k  addTimeLimit :: String -> Annex ()
Makefile view
@@ -147,6 +147,8 @@ 	install -d "$(LINUXSTANDALONE_DEST)/git-core" 	(cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x) 	install -d "$(LINUXSTANDALONE_DEST)/templates"+	install -d "$(LINUXSTANDALONE_DEST)/magic"+	cp /usr/share/file/magic.mgc "$(LINUXSTANDALONE_DEST)/magic" 	 	./Build/LinuxMkLibs "$(LINUXSTANDALONE_DEST)" 	@@ -199,6 +201,12 @@  	(cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x) 	install -d "$(OSXAPP_BASE)/templates"+	install -d "$(OSXAPP_BASE)/magic"+	if [ -e "$(OSX_MAGIC_FILE)" ]; then \+		cp "$(OSX_MAGIC_FILE)" "$(OSXAPP_BASE)/magic/magic.mgc"; \+	else \+		echo "** OSX_MAGIC_FILE not set; not including it" >&2; \+	fi  	# OSX looks in man dir nearby the bin 	$(MAKE) install-mans DESTDIR="$(OSXAPP_BASE)/.." SHAREDIR="" PREFIX=""@@ -251,16 +259,19 @@ 	$(MAKE) android 	$(MAKE) -C standalone/android -# We bypass cabal, and only run the main ghc --make command for a-# fast development built.-fast: dist/caballog-	@$$(grep 'ghc --make' dist/caballog | head -n 1 | sed -e 's/-package-id [^ ]*//g' -e 's/-hide-all-packages//') -O0 -j -dynamic+# Bypass cabal, and only run the main ghc --make command for a+# faster development build.+fast: dist/cabalbuild+	@sh dist/cabalbuild 	@ln -sf dist/build/git-annex/git-annex git-annex 	@$(MAKE) tags >/dev/null 2>&1 & +dist/cabalbuild: dist/caballog+	grep 'ghc --make' dist/caballog | tail -n 1 > dist/cabalbuild+	 dist/caballog: git-annex.cabal 	$(BUILDER) configure -f"-Production" -O0 --enable-executable-dynamic-	$(BUILDER) build -v2 | tee $@+	$(BUILDER) build -v2 --ghc-options="-O0 -j" | tee dist/caballog  # Hardcoded command line to make hdevtools start up and work. # You will need some memory. It's worth it.
Messages.hs view
@@ -212,7 +212,7 @@ 	QuietOutput -> True 	JSONOutput -> True 	NormalOutput -> False-	ConcurrentOutput _ -> True+	ConcurrentOutput {} -> True  {- Use to show a message that is displayed implicitly, and so might be  - disabled when running a certian command that needs more control over its
Messages/Concurrent.hs view
@@ -1,26 +1,28 @@-{-# OPTIONS_GHC -fno-warn-orphans #-}- {- git-annex output messages, including concurrent output to display regions  -- - Copyright 2010-2015 Joey Hess <id@joeyh.name>+ - Copyright 2010-2016 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}  {-# LANGUAGE CPP #-}+{-# OPTIONS_GHC -fno-warn-orphans #-}  module Messages.Concurrent where  import Annex+import Types.Messages  #ifdef WITH_CONCURRENTOUTPUT import Common-import Types.Messages import qualified System.Console.Concurrent as Console import qualified System.Console.Regions as Regions import Control.Concurrent.STM import qualified Data.Text as T+#ifndef mingw32_HOST_OS+import GHC.IO.Encoding #endif+#endif  {- Outputs a message in a concurrency safe way.  -@@ -29,9 +31,14 @@  - When built without concurrent-output support, the fallback action is run  - instead.  -}-concurrentMessage :: Bool -> String -> Annex () -> Annex ()+concurrentMessage :: OutputType -> Bool -> String -> Annex () -> Annex ()+concurrentMessage o iserror msg fallback  #ifdef WITH_CONCURRENTOUTPUT-concurrentMessage iserror msg _ = go =<< consoleRegion <$> Annex.getState Annex.output+	| concurrentOutputEnabled o =+		go =<< consoleRegion <$> Annex.getState Annex.output+#endif+	| otherwise = fallback+#ifdef WITH_CONCURRENTOUTPUT   where 	go Nothing 		| iserror = liftIO $ Console.errorConcurrent msg@@ -48,9 +55,6 @@ 			rl <- takeTMVar Regions.regionList 			putTMVar Regions.regionList 				(if r `elem` rl then rl else r:rl)--#else-concurrentMessage _ _ fallback = fallback #endif  {- Runs an action in its own dedicated region of the console.@@ -62,21 +66,25 @@  - When not at a console, a region is not displayed until the action is  - complete.  -}-inOwnConsoleRegion :: Annex a -> Annex a+inOwnConsoleRegion :: OutputType -> Annex a -> Annex a+inOwnConsoleRegion o a #ifdef WITH_CONCURRENTOUTPUT-inOwnConsoleRegion a = do-	r <- mkregion-	setregion (Just r)-	eret <- tryNonAsync a `onException` rmregion r-	case eret of-		Left e -> do-			-- Add error message to region before it closes.-			concurrentMessage True (show e) noop-			rmregion r-			throwM e-		Right ret -> do-			rmregion r-			return ret+	| concurrentOutputEnabled o = do+		r <- mkregion+		setregion (Just r)+		eret <- tryNonAsync a `onException` rmregion r+		case eret of+			Left e -> do+				-- Add error message to region before it closes.+				concurrentMessage o True (show e) noop+				rmregion r+				throwM e+			Right ret -> do+				rmregion r+				return ret+#endif+	| otherwise = a+#ifdef WITH_CONCURRENTOUTPUT   where 	-- The region is allocated here, but not displayed until  	-- a message is added to it. This avoids unnecessary screen@@ -94,8 +102,6 @@ 			unless (T.null t) $ 				Console.bufferOutputSTM h t 			Regions.closeConsoleRegion r-#else-inOwnConsoleRegion = id #endif  {- The progress region is displayed inline with the current console region. -}@@ -108,3 +114,24 @@ instance Regions.LiftRegion Annex where 	liftRegion = liftIO . atomically #endif++{- The concurrent-output library uses Text, which bypasses the normal use+ - of the fileSystemEncoding to roundtrip invalid characters, when in a+ - non-unicode locale. Work around that problem by avoiding using+ - concurrent output when not in a unicode locale. -}+concurrentOutputSupported :: IO Bool+#ifdef WITH_CONCURRENTOUTPUT+#ifndef mingw32_HOST_OS+concurrentOutputSupported = do+	enc <- getLocaleEncoding+	return ("UTF" `isInfixOf` textEncodingName enc)+#else+concurrentOutputSupported = return True -- Windows is always unicode+#endif+#else+concurrentOutputSupported = return False+#endif++concurrentOutputEnabled :: OutputType -> Bool+concurrentOutputEnabled (ConcurrentOutput _ b) = b+concurrentOutputEnabled _ = False
Messages/Internal.hs view
@@ -21,13 +21,13 @@ 	go NormalOutput = liftIO $ 		flushed $ putStr s 	go QuietOutput = q-	go (ConcurrentOutput _) = concurrentMessage False s q+	go o@(ConcurrentOutput {}) = concurrentMessage o False s q 	go JSONOutput = liftIO $ flushed json  outputError :: String -> Annex () outputError s = withOutputType go   where-	go (ConcurrentOutput _) = concurrentMessage True s (go NormalOutput)+	go o@(ConcurrentOutput {}) = concurrentMessage o True s (go NormalOutput) 	go _ = liftIO $ do 		hFlush stdout 		hPutStr stderr s
Messages/Progress.hs view
@@ -45,17 +45,17 @@ 			maybe noop (\m -> m n) combinemeterupdate 		liftIO $ clearMeter stdout meter 		return r+	go size o@(ConcurrentOutput {}) #if WITH_CONCURRENTOUTPUT-	go size (ConcurrentOutput _) = withProgressRegion $ \r -> do-		(progress, meter) <- mkmeter size-		a $ \n -> liftIO $ do-			setP progress $ fromBytesProcessed n-			s <- renderMeter meter-			Regions.setConsoleRegion r ("\n" ++ s)-			maybe noop (\m -> m n) combinemeterupdate-#else-	go _ (ConcurrentOutput _) = nometer+		| concurrentOutputEnabled o = withProgressRegion $ \r -> do+			(progress, meter) <- mkmeter size+			a $ \n -> liftIO $ do+				setP progress $ fromBytesProcessed n+				s <- renderMeter meter+				Regions.setConsoleRegion r ("\n" ++ s)+				maybe noop (\m -> m n) combinemeterupdate #endif+		| otherwise = nometer  	mkmeter size = do 		progress <- liftIO $ newProgress "" size@@ -69,14 +69,14 @@ concurrentMetered :: Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a concurrentMetered combinemeterupdate key a = withOutputType go   where-	go (ConcurrentOutput _) = metered combinemeterupdate key a+	go (ConcurrentOutput {}) = metered combinemeterupdate key a 	go _ = a (fromMaybe nullMeterUpdate combinemeterupdate)  {- Poll file size to display meter, but only for concurrent output. -} concurrentMeteredFile :: FilePath -> Maybe MeterUpdate -> Key -> Annex a -> Annex a concurrentMeteredFile file combinemeterupdate key a = withOutputType go   where-	go (ConcurrentOutput _) = metered combinemeterupdate key $ \p ->+	go (ConcurrentOutput {}) = metered combinemeterupdate key $ \p -> 		watchFileSize file p a 	go _ = a @@ -120,6 +120,6 @@ mkStderrEmitter = withOutputType go   where #ifdef WITH_CONCURRENTOUTPUT-	go (ConcurrentOutput _) = return Console.errorConcurrent+	go o | concurrentOutputEnabled o = return Console.errorConcurrent #endif 	go _ = return (hPutStrLn stderr)
Test.hs view
@@ -71,6 +71,7 @@ import qualified Annex.Link import qualified Annex.Init import qualified Annex.CatFile+import qualified Annex.Path import qualified Annex.View import qualified Annex.View.ViewedFile import qualified Logs.View@@ -102,22 +103,25 @@  optParser :: Parser TestOptions optParser = TestOptions-	<$> suiteOptionParser ingredients (tests mempty)+	<$> suiteOptionParser ingredients (tests False mempty) 	<*> switch 		( long "keep-failures" 		<> help "preserve repositories on test failure"-		)+	)  runner :: Maybe (TestOptions -> IO ())-runner = Just $ \opts -> case tryIngredients ingredients (tastyOptionSet opts) (tests opts) of-	Nothing -> error "No tests found!?"-	Just act -> ifM act-		( exitSuccess-		, do-			putStrLn "  (This could be due to a bug in git-annex, or an incompatability"-			putStrLn "   with utilities, such as git, installed on this system.)"-			exitFailure-		)+runner = Just $ \opts -> do+	ensuretmpdir+	crippledfilesystem <- Annex.Init.probeCrippledFileSystem' tmpdir+	case tryIngredients ingredients (tastyOptionSet opts) (tests crippledfilesystem opts) of+		Nothing -> error "No tests found!?"+		Just act -> ifM act+			( exitSuccess+			, do+				putStrLn "  (This could be due to a bug in git-annex, or an incompatability"+				putStrLn "   with utilities, such as git, installed on this system.)"+				exitFailure+			)  ingredients :: [Ingredient] ingredients =@@ -125,19 +129,19 @@ 	, rerunningTests [consoleTestReporter] 	] -tests :: TestOptions -> TestTree-tests opts = testGroup "Tests" $ properties :+tests :: Bool -> TestOptions -> TestTree+tests crippledfilesystem opts = testGroup "Tests" $ properties : 	map (\(d, te) -> withTestMode te (unitTests d)) testmodes   where-	testmodes =-		[ ("v6 unlocked", (testMode opts "6") { unlockedFiles = True })-		, ("v6 locked", testMode opts "6")-		, ("v5", testMode opts "5")-#ifndef mingw32_HOST_OS-		-- Windows will only use direct mode, so don't test twice.-		, ("v5 direct", (testMode opts "5") { forceDirect = True })-#endif+	testmodes = catMaybes+		[ Just ("v6 unlocked", (testMode opts "6") { unlockedFiles = True })+		, unlesscrippled ("v5", testMode opts "5")+		, unlesscrippled ("v6 locked", testMode opts "6")+		, Just ("v5 direct", (testMode opts "5") { forceDirect = True }) 		]+	unlesscrippled v+		| crippledfilesystem = Nothing+		| otherwise = Just v  properties :: TestTree properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck"@@ -323,12 +327,12 @@ test_import = intmpclonerepo $ Utility.Tmp.withTmpDir "importtest" $ \importdir -> do 	(toimport1, importf1, imported1) <- mktoimport importdir "import1" 	git_annex "import" [toimport1] @? "import failed"-	annexed_present_locked imported1+	annexed_present_imported imported1 	checkdoesnotexist importf1  	(toimport2, importf2, imported2) <- mktoimport importdir "import2" 	git_annex "import" [toimport2] @? "import of duplicate failed"-	annexed_present_locked imported2+	annexed_present_imported imported2 	checkdoesnotexist importf2  	(toimport3, importf3, imported3) <- mktoimport importdir "import3"@@ -348,11 +352,11 @@ 	 	(toimport5, importf5, imported5) <- mktoimport importdir "import5" 	git_annex "import" ["--duplicate", toimport5] @? "import --duplicate failed"-	annexed_present_locked imported5+	annexed_present_imported imported5 	checkexists importf5 	 	git_annex "drop" ["--force", imported1, imported2, imported5] @? "drop failed"-	annexed_notpresent_locked imported2+	annexed_notpresent_imported imported2 	(toimportdup, importfdup, importeddup) <- mktoimport importdir "importdup" 	git_annex "import" ["--clean-duplicates", toimportdup]  		@? "import of missing duplicate with --clean-duplicates failed"@@ -364,6 +368,14 @@ 		let importf = subdir </> "f" 		writeFile (importdir </> importf) (content importf) 		return (importdir </> subdir, importdir </> importf, importf)+	annexed_present_imported f = ifM (annexeval Config.crippledFileSystem)+		( annexed_present_unlocked f+		, annexed_present_locked f+		)+	annexed_notpresent_imported f = ifM (annexeval Config.crippledFileSystem)+		( annexed_notpresent_unlocked f+		, annexed_notpresent_locked f+		)  test_reinject :: Assertion test_reinject = intmpclonerepoInDirect $ do@@ -373,8 +385,11 @@ 	key <- Types.Key.key2file <$> getKey backendSHA1 tmp 	git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed" 	annexed_present sha1annexedfile-	git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed for dup"-	annexed_present_locked sha1annexedfiledup+	-- fromkey can't be used on a crippled filesystem, since it makes a+	-- symlink+	unlessM (annexeval Config.crippledFileSystem) $ do+		git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed for dup"+		annexed_present_locked sha1annexedfiledup   where 	tmp = "tmpfile" @@ -814,16 +829,20 @@ 	checkunused [] "after dropunused" 	not <$> git_annex "dropunused" ["--force", "10", "501"] @? "dropunused failed to fail on bogus numbers" -	-- unused used to miss renamed symlinks that were not staged-	-- and pointed at annexed content, and think that content was unused-	writeFile "unusedfile" "unusedcontent"-	git_annex "add" ["unusedfile"] @? "add of unusedfile failed"-	unusedfilekey <- getKey backendSHA256E "unusedfile"-	renameFile "unusedfile" "unusedunstagedfile"-	boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed"-	checkunused [] "with unstaged link"-	removeFile "unusedunstagedfile"-	checkunused [unusedfilekey] "with renamed link deleted"+	-- Unused used to miss renamed symlinks that were not staged+	-- and pointed at annexed content, and think that content was unused.+	-- This is only relevant when using locked files; if the file is+	-- unlocked, the work tree file has the content, and there's no way+	-- to associate it with the key.+	unlessM (unlockedFiles <$> getTestMode) $ do+		writeFile "unusedfile" "unusedcontent"+		git_annex "add" ["unusedfile"] @? "add of unusedfile failed"+		unusedfilekey <- getKey backendSHA256E "unusedfile"+		renameFile "unusedfile" "unusedunstagedfile"+		boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed"+		checkunused [] "with unstaged link"+		removeFile "unusedunstagedfile"+		checkunused [unusedfilekey] "with renamed link deleted"  	-- unused used to miss symlinks that were deleted or modified 	-- manually@@ -1269,7 +1288,7 @@  - lost track of whether a file was a symlink.   -} test_conflict_resolution_symlink_bit :: Assertion-test_conflict_resolution_symlink_bit =+test_conflict_resolution_symlink_bit = unlessM (unlockedFiles <$> getTestMode) $ 	withtmpclonerepo $ \r1 -> 		withtmpclonerepo $ \r2 -> 			withtmpclonerepo $ \r3 -> do@@ -1542,9 +1561,10 @@  	{- Regression test for Windows bug where symlinks were not 	 - calculated correctly for files in subdirs. -}-	git_annex "sync" [] @? "sync failed"-	l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo")-	"../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l)+	unlessM (unlockedFiles <$> getTestMode) $ do+		git_annex "sync" [] @? "sync failed"+		l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo")+		"../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l)  	createDirectory "dir2" 	writeFile ("dir2" </> "foo") $ content annexedfile@@ -1577,7 +1597,8 @@ {- Runs git-annex and returns its output. -} git_annex_output :: String -> [String] -> IO String git_annex_output command params = do-	got <- Utility.Process.readProcess "git-annex" (command:params)+	pp <- Annex.Path.programPath+	got <- Utility.Process.readProcess pp (command:params) 	-- Since the above is a separate process, code coverage stats are 	-- not gathered for things run in it. 	-- Run same command again, to get code coverage.@@ -1748,12 +1769,17 @@ 				removeDirectoryRecursive dir 	 checklink :: FilePath -> Assertion-checklink f = do-	s <- getSymbolicLinkStatus f+checklink f = 	-- in direct mode, it may be a symlink, or not, depending 	-- on whether the content is present. 	unlessM (annexeval Config.isDirect) $-		isSymbolicLink s @? f ++ " is not a symlink"+		ifM (annexeval Config.crippledFileSystem)+			( (isJust <$> annexeval (Annex.Link.getAnnexLinkTarget f))+				@? f ++ " is not a (crippled) symlink"+			, do+				s <- getSymbolicLinkStatus f+				isSymbolicLink s @? f ++ " is not a symlink"+			)  checkregularfile :: FilePath -> Assertion checkregularfile f = do@@ -1855,8 +1881,10 @@ 	)  annexed_present_locked :: FilePath -> Assertion-annexed_present_locked = runchecks-	[checklink, checkcontent, checkunwritable, inlocationlog]+annexed_present_locked f = ifM (annexeval Config.crippledFileSystem)+	( runchecks [checklink, inlocationlog] f+	, runchecks [checklink, checkcontent, checkunwritable, inlocationlog] f+	)  annexed_present_unlocked :: FilePath -> Assertion annexed_present_unlocked = runchecks
Types/GitConfig.hs view
@@ -69,6 +69,7 @@ 	, annexVerify :: Bool 	, annexPidLock :: Bool 	, annexPidLockTimeout :: Seconds+	, annexAddUnlocked :: Bool 	, coreSymlinks :: Bool 	, coreSharedRepository :: SharedRepository 	, gcryptId :: Maybe String@@ -118,6 +119,7 @@ 	, annexPidLock = getbool (annex "pidlock") False 	, annexPidLockTimeout = Seconds $ fromMaybe 300 $ 		getmayberead (annex "pidlocktimeout")+	, annexAddUnlocked = getbool (annex "addunlocked") False 	, coreSymlinks = getbool "core.symlinks" True 	, coreSharedRepository = getSharedRepository r 	, gcryptId = getmaybe "core.gcrypt-id"
Types/Messages.hs view
@@ -15,7 +15,7 @@ import System.Console.Regions (ConsoleRegion) #endif -data OutputType = NormalOutput | QuietOutput | ConcurrentOutput Int | JSONOutput+data OutputType = NormalOutput | QuietOutput | ConcurrentOutput Int Bool | JSONOutput 	deriving (Show)  data SideActionBlock = NoBlock | StartBlock | InBlock
Types/MetaData.hs view
@@ -36,8 +36,6 @@ 	metaDataValues, 	ModMeta(..), 	modMeta,-	parseModMeta,-	parseMetaData, 	prop_metadata_sane, 	prop_metadata_serialize ) where@@ -238,26 +236,6 @@ modMeta m (MaybeSetMeta f v) 	| S.null (currentMetaDataValues f m) = updateMetaData f v emptyMetaData 	| otherwise = emptyMetaData--{- Parses field=value, field+=value, field-=value, field?=value -}-parseModMeta :: String -> Either String ModMeta-parseModMeta p = case lastMaybe f of-	Just '+' -> AddMeta <$> mkMetaField f' <*> v-	Just '-' -> DelMeta <$> mkMetaField f' <*> v-	Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v-	_ -> SetMeta <$> mkMetaField f <*> v-  where-	(f, sv) = separate (== '=') p-	f' = beginning f-	v = pure (toMetaValue sv)--{- Parses field=value -}-parseMetaData :: String -> Either String (MetaField, MetaValue)-parseMetaData p = (,)-	<$> mkMetaField f-	<*> pure (toMetaValue v)-  where-	(f, v) = separate (== '=') p  {- Avoid putting too many fields in the map; extremely large maps make  - the seriaization test slow due to the sheer amount of data.
Upgrade/V1.hs view
@@ -13,6 +13,7 @@  import Annex.Common import Annex.Content+import Annex.Link import Logs.Presence import qualified Annex.Queue import qualified Git
Utility/Exception.hs view
@@ -21,7 +21,8 @@ 	tryNonAsync, 	tryWhenExists, 	catchIOErrorType,-	IOErrorType(..)+	IOErrorType(..),+	catchPermissionDenied, ) where  import Control.Monad.Catch as X hiding (Handler)@@ -97,3 +98,6 @@ 	onlymatching e 		| ioeGetErrorType e == errtype = onmatchingerr e 		| otherwise = throwM e++catchPermissionDenied :: MonadCatch m => (IOException -> m a) -> m a -> m a+catchPermissionDenied = catchIOErrorType PermissionDenied
Utility/Format.hs view
@@ -103,7 +103,7 @@ {- Decodes a C-style encoding, where \n is a newline, \NNN is an octal  - encoded character, and \xNN is a hex encoded character.  -}-decode_c :: FormatString -> FormatString+decode_c :: FormatString -> String decode_c [] = [] decode_c s = unescape ("", s)   where@@ -141,14 +141,14 @@ 	handle n = ("", n)  {- Inverse of decode_c. -}-encode_c :: FormatString -> FormatString+encode_c :: String -> FormatString encode_c = encode_c' (const False)  {- Encodes more strictly, including whitespace. -}-encode_c_strict :: FormatString -> FormatString+encode_c_strict :: String -> FormatString encode_c_strict = encode_c' isSpace -encode_c' :: (Char -> Bool) -> FormatString -> FormatString+encode_c' :: (Char -> Bool) -> String -> FormatString encode_c' p = concatMap echar   where 	e c = '\\' : [c]
Utility/Mounts.hsc view
@@ -1,5 +1,7 @@ {- Interface to mtab (and fstab)  - + - Deprecated; moving to mountpoints library on hackage.+ -   - Derived from hsshellscript, originally written by  - Volker Wysk <hsss@volker-wysk.de>  - @@ -7,6 +9,7 @@  - Joey Hess <id@joeyh.name>  -  - Licensed under the GNU LGPL version 2.1 or higher.+ -  -}  {-# LANGUAGE ForeignFunctionInterface #-}@@ -34,7 +37,7 @@ 	{ mnt_fsname :: String 	, mnt_dir :: FilePath 	, mnt_type :: String-	} deriving (Read, Show, Eq, Ord)+	} deriving (Show, Eq, Ord)  #ifndef __ANDROID__
− Utility/URI.hs 
@@ -1,18 +0,0 @@-{- Network.URI- -- - Copyright 2014 Joey Hess <id@joeyh.name>- -- - License: BSD-2-clause- -}--{-# LANGUAGE CPP #-}--module Utility.URI where---- Old versions of network lacked an Ord for URI-#if ! MIN_VERSION_network(2,4,0)-import Network.URI--instance Ord URI where-	a `compare` b = show a `compare` show b-#endif
Utility/UserInfo.hs view
@@ -58,6 +58,6 @@ #ifndef mingw32_HOST_OS 	go [] = extract <$> (getUserEntryForID =<< getEffectiveUserID) #else-	go [] = error $ "environment not set: " ++ show envvars+	go [] = extract <$> error ("environment not set: " ++ show envvars) #endif 	go (v:vs) = maybe (go vs) return =<< getEnv v
debian/changelog view
@@ -1,3 +1,55 @@+git-annex (6.20160229) unstable; urgency=medium++  * Update perlmagick build dependency. Closes: #789225+  * Fix memory leak in last release, which affected commands like+    git-annex status when a large non-annexed file is present in the work+    tree.+  * fsck: When the only copy of a file is in a dead repository, mention+    the repository.+  * info: Mention when run in a dead repository.+  * Linux and OSX standalone builds put the bundled gpg last in PATH,+    so any system gpg will be preferred over it.+  * Avoid crashing when built with MagicMime support, but when the magic+    database cannot be loaded.+  * Include magic database in the linux and OSX standalone builds.+  * Fix memory leak when hashing files, which triggered during fsck+    when an external hash program was not used.+    (This leak was introduced in version 6.20160114.)+  * Support --metadata field<number, --metadata field>number etc+    to match ranges of numeric values.+  * Similarly, support preferred content expressions like+    metadata=field<number and metadata=field>number+  * The pre-commit-annex hook script that automatically extracts+    metadata has been updated to also use exiftool.+    Thanks, Klaus Ethgen.++ -- Joey Hess <id@joeyh.name>  Mon, 29 Feb 2016 12:41:49 -0400++git-annex (6.20160217) unstable; urgency=medium++  * Support getting files from read-only repositories.+  * checkpresentkey: Allow to be run without an explicit remote.+  * checkpresentkey: Added --batch.+  * Work around problem with concurrent-output when in a non-unicode locale+    by avoiding use of it in such a locale. Instead -J will behave as if+    it was built without concurrent-output support in this situation.+  * Fix storing of filenames of v6 unlocked files when the filename is not+    representable in the current locale.+  * fsck: Detect and fix missing associated file mappings in v6 repositories.+  * fsck: Populate unlocked files in v6 repositories whose content is+    present in annex/objects but didn't reach the work tree.+  * When initializing a v6 repo on a crippled filesystem, don't force it+    into direct mode.+  * Windows: Fix v6 unlocked files to actually work.+  * add, addurl, import, importfeed: When in a v6 repository on a crippled+    filesystem, add files unlocked.+  * annex.addunlocked: New configuration setting, makes files always be+    added unlocked. (v6 only)+  * Improve format of v6 unlocked pointer files to support keys containing+    slashes.++ -- Joey Hess <id@joeyh.name>  Wed, 17 Feb 2016 14:48:51 -0400+ git-annex (6.20160211) unstable; urgency=medium    * annex.addsmallfiles: New option controlling what is done when
debian/control view
@@ -75,7 +75,7 @@ 	libghc-magic-dev, 	lsof [linux-any], 	ikiwiki,-	perlmagick,+	libimage-magick-perl, 	git (>= 1:1.8.1), 	rsync, 	wget,
debian/copyright view
@@ -44,6 +44,11 @@            2010-2015 Joey Hess <id@joeyh.name> License: GPL-3+ +Files: doc/tips/automatically_adding_metadata/pre-commit-annex +Copyright: 2014 Joey Hess <id@joeyh.name>+           2016 Klaus Ethgen <Klaus@Ethgen.ch>+License: GPL-3++ Files: Utility/libmounts.c Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California            2001 David Rufino <daverufino@btinternet.com>
@@ -1,3 +1,5 @@+[[!meta title="hard links not synced"]]+ ### Please describe the problem.  Direct mode repositories seem to initially ignore hard linked files and then when changes are done to them sync them as separate files. However, changes to one file are only propagated to that file and not to any of the others that are hardlinked to it.
+ doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn view
@@ -0,0 +1,107 @@+### Please describe the problem.++I noticed three tests failed when running "git annex test", and it seems +it's because I have `annex.backends` set to `SHA256` in `~/.gitconfig`. +When I disable that option, all tests succeed.++### What steps will reproduce the problem?++Add++    [annex]+      backends = SHA256++to `~/.gitconfig`.++### What version of git-annex are you using? On what operating system?++Newest git-annex (6.20160211 amd64) from downloads.kitenet.net.++- Debian GNU/Linux 8.3 (64 bit). Installed yesterday, so it's pretty pristine.+- git version 2.7.1.287.g4943984 (Newest version from 'master' in +  git.git)++The first version with this problem is 6.20160114. 5.20151218 works.++### Please provide any additional information below.++[[!format text """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++$ git annex test+Tests++[Removed 188 lines]++    migrate (via gitattributes):                          /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+OK (3.11s)+    unused:                                               /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+FAIL (1.84s)+      unused keys differ after origin branches are gone+      expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+       but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+    describe:                                             /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+OK (0.84s)++[Removed 1108 lines]++    migrate (via gitattributes):                          /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+OK (3.34s)+    unused:                                               /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+FAIL (1.82s)+      unused keys differ after origin branches are gone+      expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+       but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+    describe:                                             /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'+OK (0.78s)++[Removed 1062 lines]++    migrate (via gitattributes):                          OK (2.64s)+    unused:                                               FAIL (1.53s)+      unused keys differ after origin branches are gone+      expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+       but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}]+    describe:                                             OK (0.62s)++[Removed 1667 lines]++3 out of 269 tests failed (640.58s)+  (This could be due to a bug in git-annex, or an incompatability+   with utilities, such as git, installed on this system.)+$++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++Roses are red<br/>+Violets are blue<br/>+git-annex is awesome<br/>+and so are you++`;-)`++But bloody hell, it's hard to get this thing to build. As you've +mentioned earlier, building on Debian 7 isn't supported anymore, so I +installed Debian 8.3 yesterday for the sole purpose of compiling git-annex, +but still no luck:++    [32 of 32] Compiling Main             ( dist/setup/setup.hs, dist/setup/Main.o )+    Linking ./dist/setup/setup ...+    unrecognized option `--extra-prog-path=/home/sunny/.cabal/bin'+    Makefile:19: recipe for target 'Build/SysConfig.hs' failed+    make: *** [Build/SysConfig.hs] Error 1++I was hoping to get it to build so I could find the exact commit that +introduced this, and also provide some patches with some functionality +I've been thinking of. Is building on Debian 8.3 (jessie) supported? I +found a bug report from you on +<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=778987> that seems to +be related, but I don't know if that's the case. The newest version I'm +able to build is 5.20150219, somewhere after that various things start +to fail.
doc/bugs/Unable_to_parallel_fsck.mdwn view
@@ -81,3 +81,8 @@   [[!meta title="-J can crash on displaying filenames not supported by current locale"]]++> I've worked around this by detecting the non-unicode locale and avoiding+> the fancy concurrent output which needs it. So -J will work, just not+> with concurrent progress. I think this is the best that can be done+> reasonably, so [[done]]. --[[Joey]]
doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn view
@@ -100,3 +100,4 @@ [[!meta title="direct mode mappings scale badly with thousands of identical files"]]  [[!tag confirmed]]+[[!meta tag=deprecateddirectmode]]
@@ -0,0 +1,8 @@+Hi,++Because git annex doesn't annex symlinks, it is not possible to copy files from the a repository with a simple cp/rsync dereferencing each files. If we do this as of today, we would lose the original symlink information.+Would it be possible to change this behavior in the future, at least with an option?++Thanks++> Not going to happen, sorry. [[done]] --[[Joey]]
doc/bugs/cannot_remove___96__.t__96___directory.mdwn view
@@ -32,3 +32,6 @@  On /tmp the self-test works. Maybe it is related to NFS? +> Turns out that git-annex was keeping files in .t open in some+> circumstances after deleting them. I have fixed some of this. Possibly+> not all. --[[Joey]]
+ doc/bugs/checksum_loads_whole_file_into_memory.mdwn view
@@ -0,0 +1,24 @@+Using eg, fsck with the MD5 backend loads whole files into memory.++May only happen for very large files (40 gb) or in other specific+circumstances, including ghc version used for buildd etc.++Observed with 6.20160217-g95bbdb8, linux standalone amd64.++Not observed with 5.20151218-g5008846.++Commit 7482853ddddc21f2696dcfbc82d737f03032134a may be relevant,+but I don't understand how yet. A small test program using the same+code doesn't exhibit the problem, even when built in the identical build+environment as the 6.20160217-g95bbdb8 that has the problem.++> Update: Reverted 7482853ddddc21f2696dcfbc82d737f03032134a and indeed the+> problem got fixed. But, reverting that commit breaks the test suite on+> windows and has a FD leak, so is not desirable. This needs more+> investigation. --[[Joey]]++>> I see it now, the checksum is a String and it was only forced to WHNF,+>> so the hashing didn't fully complete and the file got buffered.+>> Probably only occurred when fscking, and not when adding a file,+>> due to differing use patterns of the checksum.+>> [[fixed|done]] --[[Joey]]
doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn view
@@ -43,3 +43,5 @@ * [[forum/git-status_typechange_in_direct_mode/]]  [[!meta title="git annex lock --force deletes only copy of content after interrupted switch to direct mode"]++[[!meta tag=deprecateddirectmode]]
doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn view
@@ -58,3 +58,4 @@ """]]  [[!tag moreinfo]]+[[!meta tag=deprecateddirectmode]]
doc/bugs/direct_mode_merge_interrupt.mdwn view
@@ -52,3 +52,5 @@ > then run mergeDirectCleanup to recover, before any commits can be made > from the inconsistent state. This approach seems to get complicated > quickly.. --[[Joey]]++[[!meta tag=deprecateddirectmode]]
doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn view
@@ -35,4 +35,4 @@ Alternatively, git-annex could learn/probe the full set of characters not allowed in filenames, and examine merges before performing them, and refuse to do anything if the merge added an illegal filename.a  [[!tag confirmed]]-+[[!meta tag=deprecateddirectmode]]
+ doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn view
@@ -0,0 +1,22 @@+[[!format sh """+$> git annex version                    +git-annex version: 6.20160213+gitg9597a21-1~ndall+1+...+$> git annex get -J 5 .+get docs/freesurfer.groupanalysis.ppt (from origin...) (checksum...) ok+get docs/freesurfer.future_directions.2007.ppt (from origin...) (checksum...) ok+get docs/freesurfer.groupanalysis.short.ppt (from origin...) (checksum...) ok+get distribution/trctrain/trctraindata.tar.gz (from origin...) +47%          80.2MB/s 3s+47%          80.2MB/s 3s+get docs/freesurfer.inferring_architectonics.ppt (from origin...) +33%           8.1MB/s 4s+33%           8.1MB/s 4s+get docs/freesurfer.intro.2011.ppt (from origin...) +64%           7.3MB/s 1s+64%           7.3MB/s 1s+get docs/freesurfer.intro.mmclass.ppt (from origin...)+# End of transcript or log.+"""]]++[[!meta author=yoh]]
doc/bugs/git_annex_get_fails_from_read-only_repository.mdwn view
@@ -42,3 +42,5 @@ ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)  Yes! :-) Despite this, everything works like a charm with several repos on different OSs with lots of data. Thanks for this great tool!++> [[fixed|done]] --[[Joey]]
doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn view
@@ -35,3 +35,4 @@ I wonder if this would have side effects, or if there are other places in the code where skip-worktree files would need to be handled, though. I'm particularly motivated to solve this, so let me know if it doesn't look like it would get looked at right away, and I'll have an excuse to get a Haskell dev environment setup again and shake the rust off.  [[!tag confirmed]]+[[!meta tag=deprecateddirectmode]]
doc/bugs/put_gpg_last_in_OSX_dmg_PATH.mdwn view
@@ -6,3 +6,5 @@ it at the end of PATH, not the front. So system one is used if available.  This should also be considered for the linux standalone builds.++> [[done]] for both OSX and linux. --[[Joey]]
+ doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn view
@@ -0,0 +1,9 @@+### Please describe the problem.++Initially generated an annex while having a restrictive umask 077.  Then cloned that repository to another host for public consumption so directory had proper/good permissions set allowing group to access.  And everything is accessible but not the load which I 'annex get'ed.  Key directories were readable but not the content.  I guess there is somewhere 'preserve permissions' setting for rsync/scp which imho shouldn't be there and content should inherit local/environment settings++### What version of git-annex are you using? On what operating system?++6.20160208+gitg1ac9034-1~ndall+1++[[!meta author=yoh]]
@@ -88,3 +88,7 @@  # End of transcript or log. """]]++> Closing as I think we're in agreement this is the right behavior.+> But, see <http://git-annex.branchable.com/todo/hide_missing_files/>.+> [[done]] --[[Joey]]
+ doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.++Although probably not an annex issue but thought to let you know since you might see+a quick resolution on your end++Here is the log from datalad:++[[!format sh """+2016-02-25 22:53:32,886 [DEBUG] Running: ['git', '-c', 'receive.autogc=false', '-c', 'annex.alwayscommit=false', 'annex', 'add', '--debug', '--json', '-c', 'annex.largefiles=exclude=CHANGES* and exclude=README* and exclude=*.[mc] and exclude=dataset*.json and (exclude=*.txt or include=*/*.txt)  and (exclude=*.json or include=*/*.json) and (exclude=*.tsv or include=*/*.tsv)', 'README.txt'] (cmd.py:351)+2016-02-25 22:53:32,979 [ERROR] stderr| /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' (cmd.py:351)+++or outside (file is already under annex)++$> git annex --debug add -c 'annex.largefiles=exclude=CHANGES* and exclude=README*' README.txt+/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'++"""]]++annex is up to date: 6.20160225+gitg229db26-1~ndall+1++edit1: that is happening on jessie with file 1:5.22+15-2+deb8u1 if that is relevant+[[!meta author=yoh]]++> [[fixed|done]] --[[Joey]]
doc/design/adjusted_branches.mdwn view
@@ -20,30 +20,158 @@  [[!toc]] +## filtering++	master           adjusted/master+	A+	|--------------->A'+	|                |++When generating commit A', reuse the date of A and use a standard author,+committer, and message. This means that two users with the adjusted branch+checked out and using the same filters will get identical shas for A', and+so can collaborate on them.++## commit++When committing changes, a commit is made as usual to the adjusted branch.+So, the user can `git commit` as usual. This does not touch the+original branch yet. ++Then we need to get from that commit to one with the filters reversed,+which should be the same as if the adjusted branch had not been used.+This commit gets added onto the original branch.++So, the branches would look like this:++	master           adjusted/master+	A+	|--------------->A'+	|                |+	|                C (new commit)+	B < - - - - - - -+	|                +	|--------------->B'+	|                |++Note particularly that B does not have A' or C in its history;+the adjusted branch is not evident from outside.++Also note that B gets filtered and the adjusted branch is rebased on top of+it, so C does not remain in the adjusted branch history either. This will+make other checkouts that are in the same adjusted branch end up with the+same B' commit when they pull B.++It might be useful to have a post-commit hook that generates B and B'+and updates the branches. And/or `git-annex sync` could do it.++There may be multiple commits made to the adjusted branch before any get+applied back to the original branch. This is handled by reverse filtering+one at a time and rebasing the others on top.++	master           adjusted/master+	A+	|--------------->A'+	|                |+	|                C1+	|                |+	|                C2+++	master           adjusted/master+	A+	|--------------->A'+	|                |+	|                C1+	B1< - - - - - - -+	|+	|--------------->B1'+	|                |+	|                C2'+	B2< - - - - - - -+	|+	|--------------->B2'+++[WORKTREE: A pre-commit hook would be needed to update the staged changes, +reversing the filter before the commit is made. All the other complications+above are avoided.]+ ## merge -When merging changes from a remote, apply the filter to the head of the-remote branch, resulting in a commit with its changes. Merge in that-commit. Note that it's possible to control the metadata of the commit such-that 2 users who have the same adjusted branch checked out, both generate-the same commit sha.+This would be done by `git annex merge` and `git annex sync`, with the goal+of merging origin/master into master, and updating adjusted/master. -This would be done by `git annex merge` and `git annex sync`.+Note that the adjusted files db needs to be updated to reflect the changes+that are merged in, for object add/remove to work as described below. +When merging, there should never be any commits present on the+adjusted/master branch that have not yet been filtered over to the master+branch. If there are any such commits, just filter them into master before+beginning the merge. There may be staged changes, or changes in the work tree.++First filter the new commit:++	origin/master    adjusted/master+	A+	|--------------->A'+	|                |+	|                |+	B+	|                +	|---------->B'++Then, merge that into adjusted/master:++	origin/master    adjusted/master+	A+	|--------------->A'+	|                |+	|                |+	B                |+	|                |+	|----------->B'->B''++That merge will take care of updating the work tree.++(What if there is a merge conflict between A' and B'? Normally such a merge+conflict should only affect the work tree/index, so can be resolved without+making a commit, but B'' may end up being made to resolve a merge+conflict.)++Once the merge is done, we have a commit B'' on adjusted/master. To finish,+adjust that commit so it does not have adjusted/master as its parent.++	origin/master    adjusted/master+	A+	|--------------->A'+	|                |+	|                |+	B+	|                +	|--------------->B''+	|                |++Finally, update master to point to B''.++Notice how similar this is to the commit graph. So, "fast-forward" +merging the same B commit from origin/master will lead to an identical+sha for B' as the original committer got.+ Since the adjusted/master branch is not present on the remote, if the user does a `git pull`, it won't merge in changes from origin/master. Which is good because the filter needs to be applied first. -[WORKTREE: `git pull` would update the work tree, and may lead to conflicts-between the adjusted work tree and pulled changes. A post-merge hook would-be needed to re-adjust the work tree, and there would be a window where eg,-not present files would appear in the work tree.]- However, if the user does `git merge origin/master`, they'll get into a state where the filter has not been applied. The post-merge hook could be used to clean up after that. Or, let the user foot-shoot this way; they can always reset back once they notice the mistake. +[WORKTREE: `git pull` would update the work tree, and may lead to conflicts+between the adjusted work tree and pulled changes. A post-merge hook would+be needed to re-adjust the work tree, and there would be a window where eg,+not present files would appear in the work tree.]+ ## annex object add/remove  When objects are added/removed from the annex, the associated file has to@@ -57,60 +185,30 @@  [WORKTREE: Simply adjust the work tree (and index) per the filter.] -## commit--When committing changes, a commit is made as usual to the adjusted branch.-So, the user can `git commit` (or `git annex sync`). This does not touch-the original branch yet. --Then we need to get from that commit to one with the filters reversed,-which should be the same as if the adjusted branch had not been used.-This commit gets added onto the original branch.--So, the branches would look like this:--        master           adjusted/master-        A ---filter----> A-        |                |-        |                A'-        |                |-        |                B'-        B <--rev filter- |-        |                B-        | ---filter----> |-        |                B''--Note particularly that B does not have A' in its history; the adjusted-branch is not evident from outside. So, we need a way to detect commits-like A'.--Also note that B gets merged back to the adjusted branch, re-applying the-filter. This will make other checkouts that are in the same adjusted branch-end up with the same B'' commit when they pull B.--It might be useful to have a post-commit hook that generates the-reverse-filtered commit and updates the original branch. And/or-`git-annex sync` could do it.--[WORKTREE: A pre-commit hook would be needed to update the staged changes, -reversing the filter before the commit is made. All the other complications-above are avoided.]- ## reverse filtering  Reversing filter #1 would mean only converting pointer files to symlinks when the file was originally a symlink. This is problimatic when a file is renamed. Would it be ok, if foo is renamed to bar and bar is committed, for it to be committed as an unlocked file, even if foo was-originally locked?+originally locked? Probably.  Reversing filter #2 would mean not deleting removed files whose content was not present. When the commit includes deletion of files that were removed due to their content not being present, those deletions are not propigated. When the user deletes an unlocked file, the content is still present in annex, so reversing the filter should propigate the file-deletion.+deletion.  +What if an object was sent to the annex (or removed from the annex)+after the commit and before the reverse filtering? This would cause the+reverse filter to draw the wrong conclusion. Maybe look at a list of what+objects were not present when applying the filter, and use that to decide+which to not delete when reversing it?++So, a reverse filter may need some state that was collected when running+the filter forwards, in order to decide what to do.+ ## push  The new master branch can then be pushed out to remotes. The@@ -178,6 +276,8 @@ For example, it's not currently possible to update a view branch with changes fetched from a remote, and this could get us there. +This would need the reverse filter to be able to change metadata.+ [WORKTREE: Wouldn't be able to integrate, unless view branches are changed into adjusted view worktrees.] @@ -220,5 +320,3 @@ 	-- Generate a version of the commit made on the filter branch 	-- with the filtering of modified files reversed. 	unfilteredCommit :: Filter -> Git.Commit -> Git.Commit--	isFilteredCommit :: Git.Commit -> Bool
doc/design/assistant/polls/prioritizing_special_remotes.mdwn view
@@ -6,7 +6,7 @@ Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 75 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]]+[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 76 "My phone (or MP3 player)" 27 "Tahoe-LAFS" 16 "OpenStack SWIFT" 37 "Google Drive"]]  This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they
+ doc/devblog/day_362__encoding_fun.mdwn view
@@ -0,0 +1,19 @@+This was one of those days where I somehow end up dealing with tricky+filename encoding problems all day.++First, worked around inability for concurrent-output to display unicode+characters when in a non-unicode locale. The normal trick that git-annex+uses doesn't work in this case. Since it only affected -J, I decided to+make git-annex detect the problem and make -J behave as if it was not built+with the concurrent-output feature. So, it just doesn't display concurrent+output, which is better than crashing with an encoding error.++The other problem affects v6 repos only. Seems that not all Strings will+round trip through a persistent sqlite database. In particular, unicode+surrogate characters are replaced with garbage. This is really [a bug in+persistent](https://github.com/yesodweb/persistent/issues/540).+But, for git-annex's purposes, it was possible to work around it,+by detecting such Strings and serializing them differently.++Then I had to enhance `git annex fsck` to fix up repositories that were+affected by that problem.
+ doc/devblog/day_363__snow_day.mdwn view
@@ -0,0 +1,11 @@+Made a `no-cbits` branch that removes several things that use C code and+the FFI. I moved one of them out to a new haskell library,+<http://hackage.haskell.org/package/mountpoints>. Others were replaced with+other existing libraries. This will simplify git-annex's build process, and+more library use is good. Planning to merge this branch in a week or two.++v6 unlocked files don't work on Windows. I had assumed that since the build+was succeeding, the test suite was passing there. But, it turns out the+test suite was failing and somehow not failing the build. Have now fixed+several problems with v6 on Windows. Still a couple test suite problems to+address.
+ doc/devblog/day_364__more_v6_unlocked.mdwn view
@@ -0,0 +1,8 @@+In a v6 repository on a filesystem not supporting symlinks,+it makes sense for commands like `git annex add` and `git annex import`+to add the files unlocked, since locked files are not usable there.+After implementing that, I also added an `annex.addunlocked` config setting,+so that the same behavior can be configured in other repositories.++Rest of the day was spent fixing up the test suite's v6 repository tests+to work on FAT and Windows.
+ doc/devblog/day_365__some_kind_of_milestone.mdwn view
@@ -0,0 +1,6 @@+Should mention that there was a release two days ago. The main reason for+the timing of that release is because the Linux wstandalone builds include+glibc, which recently had a nasty security hole and had to be updated.++Today, fixed a memory leak, and worked on getting caught up with backlog,+which now stands at 112 messages.
+ doc/devblog/day_366__starting_adjusted_branches.mdwn view
@@ -0,0 +1,33 @@+Getting started on [[design/adjusted_branches]], taking a top-down and+bottom-up approach. Yesterday I worked on improving the design. Today,+built a `git mktree` interface that supports recursive tree generation and+filtering, which is the low-level core of what's needed to implement the+adjusted branches.++To test that, wrote a fun program that generates a git tree with all+the filenames reversed.++[[!format haskell """+import Git.Tree+import Git.CurrentRepo+import Git.FilePath+import Git.Types+import System.FilePath++main = do+        r <- Git.CurrentRepo.get+        (Tree t, cleanup) <- getTree (Ref "HEAD") r+        print =<< recordTree r (Tree (map reverseTree t))+        cleanup++reverseTree :: TreeContent -> TreeContent+reverseTree (TreeBlob f m s) = TreeBlob (reverseFile f) m s+reverseTree (RecordedSubTree f s l) = NewSubTree (reverseFile f) (map reverseTree l)++reverseFile :: TopFilePath -> TopFilePath+reverseFile = asTopFilePath . joinPath . map reverse . splitPath . getTopFilePath+"""]]++Also, fixed problems with the Android, Windows, and OSX builds today.+Made a point release of the OSX dmg, because the last several releases+of it will SIGILL on some hardware.
+ doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn view
@@ -0,0 +1,18 @@+Now I have a proof of concept [[design/adjusted_branches]]+implementation, that creates a branch where all locked files+are adjusted to be unlocked. It works!++Building the adjusted branch is pretty fast; around 2 thousand files+per second. And, I have a trick in my back pocket that could double that+speed. It's important this be quite fast, because it'll be done often.++Checking out the adjusted branch can be bit slow though, since git runs+`git annex smudge` once per unlocked file. So that might need to be+optimised somehow. On the other hand, this should be done only rarely.++I like that it generates reproducible git commits so the same adjustments+of the same branch will always have the same sha, no matter when and where+it's done. Implementing that involved parsing git commit objects.++Next step will be merging pulled changes into the adjusted branch, while+maintaining the desired adjustments.
+ doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn view
@@ -0,0 +1,3 @@+I am trying to use the git-annex assistant to automatically sync photos from my Lightroom catalog on Mac OS X to a remote archive server. I hit a snag due to a race condition combined with some peculiar behavior of Adobe Lightroom: it dereferences symlinks and uses the name of the target when importing (see https://forums.adobe.com/thread/568863?tstart=0) I observe the following problematic workflow: 1. Files are imported using the original filename. 2. The assistant eventually moves the content to .git/annex/objects and creates symlinks to replace the originals. 3. The photos no longer appear in the Library, however Lightroom will not allow me to re-import as it thinks the files have already been imported and are duplicates.++I know a few other folks are using git-annex and Lightroom together, but I cannot tell if they are using the assistant or manually managing the annex. This implementation is for my wife who is not technical enough to use git on the command line so that's out of the question. It seems like direct mode is my best option for now, however, I'd like to avoid it since it's being deprecated. Is there some other option that I'm missing? Is anyone else out there successfully using the assistant and Lightroom together? I'd love some advice.
+ doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn view
@@ -0,0 +1,3 @@+I've been using git-annex to manage a repo of ~4000 files (an MP3 collection). I used to use direct mode, but switched to v6 and am using unlocked files instead. However, even small commits have been taking 10-20 minutes. Looking at `ps` output shows `git-annex smudge` processes running against all of the files in the repo, one by one.++Is there any way I can speed things up?
+ doc/forum/Multiple_remotes_with_the_same_path.mdwn view
@@ -0,0 +1,67 @@+This is a followup to <https://git-annex.branchable.com/forum/basic_usage_questions/> – I'm actually testing the "two repos at same URL" situation now, with git-annex 6.20160211 and 6.20160221.++So I have a git-annex repo on two hosts at `rain:~/Attic/Software`, `frost:~/Attic/Software`, and a third clone at `/mnt/portable_HD/Attic/Software`. This means that the repo on the portable HD has two remotes with identical paths, but corresponding to different repositories:++    rain    /home/grawity/Attic/Software+    frost   /home/grawity/Attic/Software++[I was told earlier](/forum/basic_usage_questions) that this configuration would work fine and git-annex would not get confused. However, that doesn't seem to be the case (unless I misunderstood what it considers to be the "right thing"?) I see that whenever I run a command like `git annex info` on the portable\_HD repo, it overrides `remote.{rain,frost}.annex-uuid` with whatever UUID it sees right now – resulting in `git annex info` output such as:++    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master+    ┘ git config -l | grep annex-uuid+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3+    remote.rain.annex-uuid=3e342a37-6c35-40e5-99a1-1f140e6c363d    <--+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873+    +    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master+    ┘ git annex info --fast --verbose --debug+    repository mode: indirect+    trusted repositories: [2016-02-29 08:09:29.284136] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]+    [2016-02-29 08:09:29.286872] process done ExitSuccess+    [2016-02-29 08:09:29.286954] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]+    [2016-02-29 08:09:29.289167] process done ExitSuccess+    [2016-02-29 08:09:29.289316] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..fc873fc472a8c5d0db3c91a3868d9adc072f7076","-n1","--pretty=%H"]+    [2016-02-29 08:09:29.291836] process done ExitSuccess+    [2016-02-29 08:09:29.291941] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..23c27db2f2a02515ba39e7d0bb8653fa786b6aea","-n1","--pretty=%H"]+    [2016-02-29 08:09:29.294762] process done ExitSuccess+    [2016-02-29 08:09:29.294869] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..690a90da7fc28be67d5053c0a6c3050cca614eaa","-n1","--pretty=%H"]+    [2016-02-29 08:09:29.301018] process done ExitSuccess+    [2016-02-29 08:09:29.30112] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..94375fb892d1c9f5b70bdd5917c85c5c18a13168","-n1","--pretty=%H"]+    [2016-02-29 08:09:29.3031] process done ExitSuccess+    [2016-02-29 08:09:29.303182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4a7f17e644e86776b510606beb6f3cb3819d8256","-n1","--pretty=%H"]+    [2016-02-29 08:09:29.304916] process done ExitSuccess+    [2016-02-29 08:09:29.305455] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]+    [2016-02-29 08:09:29.308946] read: git ["config","--null","--list"]+    [2016-02-29 08:09:29.312347] process done ExitSuccess+    [2016-02-29 08:09:29.312939] read: git ["config","--null","--list"]+    [2016-02-29 08:09:29.3179] process done ExitSuccess+    [2016-02-29 08:09:29.318494] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","remote.rain.annex-uuid","524b8690-9b3e-48d4-b1a8-c0edb35c1ccf"]+    [2016-02-29 08:09:29.320648] process done ExitSuccess+    [2016-02-29 08:09:29.320735] read: git ["config","--null","--list"]+    [2016-02-29 08:09:29.327697] process done ExitSuccess+    0+    semitrusted repositories: 9+    	00000000-0000-0000-0000-000000000001 -- web+     	00000000-0000-0000-0000-000000000002 -- bittorrent+     	0ebc2083-f95e-4637-bd3e-09db8471daf3 -- origin+     	3e14eb23-1b16-4b52-8798-56efb550ab00 -- [vol4_grimoire]:/Attic/Software [here]+     	3e342a37-6c35-40e5-99a1-1f140e6c363d -- grawity@rain:~/Attic/Software              <--+     	524b8690-9b3e-48d4-b1a8-c0edb35c1ccf -- grawity@frost:~/Downloads/Software [rain]  <--+     	b9c0c485-07e5-4166-b5ac-ba971faab98a -- [vol3_tombstone]:/Attic/Software+     	c3c6dd39-ebc7-475a-9992-f99ca01a7f3a -- grawity@wolke:~/Attic/Software+     	ed83c81c-1a95-4acb-b76c-b6724fe88873 -- fs1:/Attic/Software [fs1]+    untrusted repositories: 0+    transfers in progress: none+    available local disk space: 920.96 gigabytes (+1 megabyte reserved)+    +    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master+    ┘ git config -l | grep annex-uuid+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3+    remote.rain.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf    <--+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873++Notice how the `[rain]` remote tag now shows up next to the wrong remote. Among other things, **this means I cannot use `git annex find --in frost --not --in rain`** and similar commands (unless I specify all the UUIDs by hand). And even though I trust git-annex to not lose any data, this behavior is a bit confusing.++Is it possible that git-annex could detect such situations and avoid updating an UUID if it's _already attached_ to another remote?
+ doc/forum/The_future_of_Direct_Mode.mdwn view
@@ -0,0 +1,32 @@+Hello everyone. I could use some pointers from experienced users. Here is my use-case:++I have a number of portable storage devices where I store media files. They serve a number of purposes:++  1. as archives and backup where I put away the data I am not using to allow for more free space on my work PC;++  2. as storage for the media I play on a media playing device (that only supports Windows filesystems);++  3. as transport medium to share files with other people (that are Windows-users).++I wanted to use git-annex to manage/organize/locate all these devices, but without breaking too much my system. Most files are not super-important, and I intend to keep multiple copies of those that are.+The main issue is that points 2 and 3 require that the real files need to be accessible directly. I *could* work around this using an extra device, but that would be much less practical, as I would need to go to the PC and transfer the large files over USB, so I am trying to avoid that.++Here is what I learned so far:++  - Indirect mode is OK on my PC. No problem there. However, it won't work on VFAT or NTFS, so I can't use it on the portable devices.++  - Direct mode seems to work fine on the portable devices, but is now deprecated. It won't work on the V6 repositories.++  - The new V6 repositories allows for the unlocking of files, replacing the link-like files (on Windows filesystems) with the real files, that allows me to browse them on the media player (by path and name). I did not find a way to keep them always unlocked by default, so I guess I would need to unlock them manually. Also, I would be keeping an extra copy of each file, since `annex.thin` won't work. In addition, as I don't expect to be editing any these files (not on these devices, at least), the loss of half the storage capacity is wasteful and undesirable.++  - Special remotes `directory` and `rsync` do not keep the original structure of the system, and for this reason do not fit for my purposes.++For now I can create V5 repositories and use direct mode on the external devices. I think this could work quite well, but I have the following questions:++  1. Am I missing something that could help me?++  2. Will using V5 repositories be somewhat future-proof? I mean, will I be able to create V5 repositories in the future? How much of a bad idea would it be to start using git-annex depending on "old versions"? I understand that direct mode will continue to be supported, but deprecation indicates it is a bad idea.++  3. I am guessing that "direct mode" will not be making a comeback, but it does have its uses — however, I did not expect that my case was an unusual one. Is there anything on V6 repositories that would make new version more adequate to my requirements? I would not mind having the files read-only or facing similar restrictions if it helped simplify things.++Thank you for your time.
doc/git-annex-add.mdwn view
@@ -21,7 +21,8 @@  Large files are added to the annex in locked form, which prevents further modification of their content unless unlocked by [[git-annex-unlock]](1).-(This is not the case however when a repository is in direct mode.)+(This is not the case however when a repository is in a filesystem not+supporting symlinks, or is in direct mode.) To add a file to the annex in unlocked form, `git add` can be used instead  (that only works when the repository has annex.version 6 or higher).
doc/git-annex-checkpresentkey.mdwn view
@@ -4,16 +4,28 @@  # SYNOPSIS -git annex checkpresentkey `key remote`+git annex checkpresentkey `key` `[remote]`  # DESCRIPTION  This plumbing-level command verifies if the specified key's content is present in the specified remote. +When no remote is specified, it verifies if the key's content is present+somewhere, checking accessible remotes until it finds the content.+ Exits 0 if the content is verified present, or 1 if it is verified to not-be present. If there is a problem checking the remote, the special-exit code 100 is used, and an error message is output to stderr.+be present. If there is a problem, the special exit code 100 is used,+and an error message is output to stderr.++# OPTIONS++* `--batch`++  Enables batch mode. In this mode, the `key` is not specified at the+  command line, but the `remote` may still be. Lines containing keys are+  read from stdin, and a line is output with "1" if the key is verified to+  be present, and "0" otherwise.  # SEE ALSO
doc/git-annex-matching-options.mdwn view
@@ -115,6 +115,15 @@   matches the glob. The values of metadata fields are matched case   insensitively. +* `--metadata field<number` / `--metadata field>number`+* `--metadata field<=number` / `--metadata field>=number`++  Matches only files that have a metadata field attached with a value that+  is a number and is less than or greater than the specified number.++  (Note that you will need to quote the second parameter to avoid+  the shell doing redirection.)+ * `--want-get`    Matches files that the preferred content settings for the repository
doc/git-annex-preferred-content.mdwn view
@@ -119,6 +119,15 @@    To match author metadata, use `metadata=author=*Smith` +* `metadata=field<number` / `metadata=field>number` +* `metadata=field<=number` / `metadata=field>=number`++  Matches only files that have a metadata field attached with a value that+  is a number and is less than or greater than the specified number.++  To match PDFs with between 100 and 200 pages (assuming something has set+  that metadata), use `metadata=pagecount>=100 and metadata=pagecount<=200`+ * `present`    Makes content be wanted if it's present, but not otherwise.
doc/git-annex-view.mdwn view
@@ -27,9 +27,9 @@ These location fields can be used the same as other metadata to construct the view.   -For example, `/=podcasts` will only include files from the podcasts-directory in the view, while `podcasts/=*` will preserve the-subdirectories of the podcasts directory in the view.+For example, `/=foo` will only include files from the foo+directory in the view, while `foo/=*` will preserve the+subdirectories of the foo directory in the view.  # SEE ALSO
doc/git-annex.mdwn view
@@ -823,6 +823,15 @@   should be checked into git by `git annex add`. Defaults to true;   set to false to instead make small files be skipped. +* `annex.addunlocked`++  Set to true to make commands like `git-annex add` that add files to the+  repository add them in unlocked form. The default is to add files in+  locked form. This only has effect in version 6 repositories.++  When a repository has core.symlinks set to false, it implicitly+  sets annex.addunlocked to true.+ * `annex.numcopies`    This is a deprecated setting. You should instead use the
− doc/install/Homebrew-cask.mdwn 
@@ -1,3 +0,0 @@-[Homebrew Cask](http://caskroom.io) has a [cask](https://github.com/caskroom/homebrew-cask/blob/master/Casks/git-annex.rb) for git-annex.--Homebrew cask users can simply run `brew cask install git-annex` to install git-annex and git-annex-shell from the OSX binary images.
+ doc/install/OSX/Homebrew-cask.mdwn view
@@ -0,0 +1,3 @@+[Homebrew Cask](http://caskroom.io) has a [cask](https://github.com/caskroom/homebrew-cask/blob/master/Casks/git-annex.rb) for git-annex.++Homebrew cask users can simply run `brew cask install git-annex` to install git-annex and git-annex-shell from the OSX binary images.
− doc/news/version_5.20151218.mdwn 
@@ -1,10 +0,0 @@-git-annex 5.20151218 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Add S3 features to git-annex version output.-   * webdav: When testing the WebDAV server, send a file with content.-     The empty file it was sending tickled bugs in some php WebDAV server.-   * fsck: Failed to honor annex.diskreserve when checking a remote.-   * Debian: Build depend on concurrent-output.-   * Fix insecure temporary permissions when git-annex repair is used in-     in a corrupted git repository.-   * Fix potential denial of service attack when creating temp dirs."""]]
+ doc/news/version_6.20160217.mdwn view
@@ -0,0 +1,22 @@+git-annex 6.20160217 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * Support getting files from read-only repositories.+   * checkpresentkey: Allow to be run without an explicit remote.+   * checkpresentkey: Added --batch.+   * Work around problem with concurrent-output when in a non-unicode locale+     by avoiding use of it in such a locale. Instead -J will behave as if+     it was built without concurrent-output support in this situation.+   * Fix storing of filenames of v6 unlocked files when the filename is not+     representable in the current locale.+   * fsck: Detect and fix missing associated file mappings in v6 repositories.+   * fsck: Populate unlocked files in v6 repositories whose content is+     present in annex/objects but didn't reach the work tree.+   * When initializing a v6 repo on a crippled filesystem, don't force it+     into direct mode.+   * Windows: Fix v6 unlocked files to actually work.+   * add, addurl, import, importfeed: When in a v6 repository on a crippled+     filesystem, add files unlocked.+   * annex.addunlocked: New configuration setting, makes files always be+     added unlocked. (v6 only)+   * Improve format of v6 unlocked pointer files to support keys containing+     slashes."""]]
+ doc/news/version_6.20160229.mdwn view
@@ -0,0 +1,24 @@+git-annex 6.20160229 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * Update perlmagick build dependency. Closes: #[789225](http://bugs.debian.org/789225)+   * Fix memory leak in last release, which affected commands like+     git-annex status when a large non-annexed file is present in the work+     tree.+   * fsck: When the only copy of a file is in a dead repository, mention+     the repository.+   * info: Mention when run in a dead repository.+   * Linux and OSX standalone builds put the bundled gpg last in PATH,+     so any system gpg will be preferred over it.+   * Avoid crashing when built with MagicMime support, but when the magic+     database cannot be loaded.+   * Include magic database in the linux and OSX standalone builds.+   * Fix memory leak when hashing files, which triggered during fsck+     when an external hash program was not used.+     (This leak was introduced in version 6.20160114.)+   * Support --metadata field&lt;number, --metadata field&gt;number etc+     to match ranges of numeric values.+   * Similarly, support preferred content expressions like+     metadata=field&lt;number and metadata=field&gt;number+   * The pre-commit-annex hook script that automatically extracts+     metadata has been updated to also use exiftool.+     Thanks, Klaus Ethgen."""]]
doc/preferred_content.mdwn view
@@ -67,3 +67,4 @@ * "metadata=" 5.20140221 * "lackingcopies=", "approxlackingcopies=", "unused=" 5.20140127 * "inpreferreddir=" 4.20130501+* "metadata=field&lt;number" etc 6.20160227
doc/related_software.mdwn view
@@ -28,6 +28,11 @@   uses git-annex to "create a two-way, distributed content distribution   network for communities with poor connexions to the internet". +* [The Japanese American Legacy Project](http://www.densho.org/)+  uses git-annex to manage upwards of 100 terabytes of collections,+  transporting them from small cultural heritage sites on USB drives.+  User interface is a [Django web app](https://github.com/densho).+ * [[forum/gadu_-_git-annex_disk_usage]] is a du like utility that   is git-annex aware.
doc/tips/automatically_adding_metadata.mdwn view
@@ -2,22 +2,47 @@ metadata attached to them.  To make git-annex automatically set the year and month when adding files,-run `git config annex.genmetadata true`.+run: `git config annex.genmetadata true` +## git commit hook+ A git commit hook can be set up to extract lots of metadata from files-like photos, mp3s, etc.+like photos, mp3s, etc. Whenever annexed files are committed, their+metadata will be extracted and stored. -1. Install the `extract` utility, from <http://www.gnu.org/software/libextractor/>  -   `apt-get install extract`-2. Download [[pre-commit-annex]] and install it in your git-annex repository-   as `.git/hooks/pre-commit-annex`.  -   Remember to make the script executable!-3. Run: `git config metadata.extract "artist album title camera_make video_dimensions"`+Download [[pre-commit-annex]] and install it in your git-annex repository+as `.git/hooks/pre-commit-annex`  +Remember to make the script executable! `chmod +x .git/hooks/pre-commit-annex` -Now any fields you list in metadata.extract to will be extracted and-stored when files are committed.+### using extract +The git commit hook can use extract to get metadata.++Install it from <http://www.gnu.org/software/libextractor/>  +`apt-get install extract`++Configure which metadata fields to ask extract for: `git config metadata.extract "artist album title camera_make video_dimensions"`+ To get a list of all possible fields, run: `extract -L | sed 's/ /_/g'`++### using exiftool++The git commit hook can also use exiftool to get metadata.++Install it from <http://owl.phy.queensu.ca/~phil/exiftool/>  +`apt-get install libimage-exiftool-perl`++Configure which metadata fields to ask exiftool for: `git config metadata.exiftool "Model ImageSize FocusRange GPSAltitude GPSCoordinates"`++To get a list of all possible fields, run: `exiftool -list`++### using both extract and exiftool++If you want some metadata that extract knows about, and other metadata+that exiftool knows about, just install them both, and set both+`metadata.extract` and `metadata.exiftool`.++### overwriting existing metadata  By default, if a git-annex already has a metadata field for a file, its value will not be overwritten with metadata taken from files.
doc/tips/automatically_adding_metadata/pre-commit-annex view
@@ -1,72 +1,118 @@-#!/bin/sh+#! /bin/sh #+# Copyright (C) 2014 Joey Hess <id@joeyh.name>+# Copyright (C) 2016 Klaus Ethgen <Klaus@Ethgen.ch>+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU General Public License+# along with this program.  If not, see <http://www.gnu.org/licenses/>.+# # This script can be used to add git-annex metadata to files when they're # committed. It is typically installed as .git/hooks/pre-commit-annex # # You can also run this script by hand, passing it the names of files # already checked into git-annex, and it will extract/refresh the git-annex # metadata from the files.-#-# Copyright 2014 Joey Hess <id@joeyh.name>-# License: GPL-3+ -extract="$(git config metadata.extract || true)" -want="$(perl -e 'print (join("|", map {s/_/ /g; "^$_ - "} (split " ", shift())))' "$extract")"+tool="$(git config metadata.tool || :)"+if [ -z "$tool" ]; then+	tool=extract+fi+case "$tool" in+	exiftool)+		tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"+	;;+	*)+		tool_exec="$tool"+	;;+esac -if [ -z "$want" ]; then+extract_fields="$(git config metadata.extract || :)"+if [ -n "$extract_fields" ]; then+	tools=extract+	extract_want="^($(echo "$extract_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"+fi+exiftool_fields="$(git config metadata.exiftool || :)"+if [ -n "$exiftool_fields" ]; then+	tools="exiftool $tools"+	exiftool_want="^($(echo "$exiftool_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"+fi+if [ -z "$tools" ]; then 	exit 0 fi -case "$(git config --bool metadata.overwrite || true)" in+case "$(git config --bool metadata.overwrite || :)" in 	true)-		overwrite=1+		equal="=" 	;; 	*)-		overwrite=""+		equal="?=" 	;; esac -addmeta () {-	file="$1"-	field="$2"-	value="$3"-	afield="$(echo "$field" | tr ' ' _)"-	if [ "$overwrite" ]; then-		p="$afield=$value"--	else-		p="$afield?=$value"-	fi-	git -c annex.alwayscommit=false annex metadata "$file" -s "$p" --quiet-}- if git rev-parse --verify HEAD >/dev/null 2>&1; then-	against=HEAD+	against="HEAD" else 	# Initial commit: diff against an empty tree object-	against=4b825dc642cb6eb9a060e54bf8d69288fbee4904+	against="4b825dc642cb6eb9a060e54bf8d69288fbee4904" fi -IFS="-"+addmeta() {+	file="$1"+	field="$2"+	value="$3"+	afield="$(echo "$field" | tr ' ' '_')"+	git -c annex.alwayscommit=false annex metadata \+		--set "$afield$equal$value" --quiet -- "$file"+} -process () {+process() { 	if [ -e "$f" ]; then 		echo "adding metadata for $f"-		for l in $(extract "$f" | egrep "$want"); do-			field="${l%% - *}"-			value="${l#* - }"-			addmeta "$f" "$field" "$value"+		for tool in $tools; do+			case "$tool" in+				exiftool)+					tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"+				;;+				*)+					tool_exec="$tool"+				;;+			esac+			LC_ALL=C $tool_exec "./$f" | eval egrep --text -i \""\$${tool}_want"\" | while read line; do+				case "$tool" in+					extract)+						field="${line%% - *}"+						value="${line#* - }"+					;;+					exiftool)+						field="${line%%: *}"+						value="${line#*: }"+					;;+				esac+				+				if [ -n "$value" ]; then+					addmeta "$f" "$field" "$value"+				fi+			done 		done 	fi }  if [ -n "$*" ]; then-	for f in $@; do+	for f in "$@"; do 		process "$f" 	done else-	for f in $(git diff-index --name-only --cached $against); do+	for f in "$(git diff-index --name-only --cached $against)"; do 		process "$f" 	done fi
doc/tips/unlocked_files.mdwn view
@@ -88,6 +88,13 @@ which is what gets committed to git. All the regular git-annex commands (get, drop, etc) can be used on unlocked files too. +[[!template id=note text="""+By default, git-annex commands will add files in locked mode,+unless used on a filesystem that does not support symlinks, when unlocked+mode is used. To make them always use unlocked mode, run:+`git config annex.addunlocked true`+"""]]+ A v6 repository can contain both locked and unlocked files. You can switch  a file back and forth using the `git annex lock` and `git annex unlock` commands. This changes what's stored in git between a git-annex symlink
+ doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn view
@@ -0,0 +1,4 @@+I have four git-annex repositories and I often use the option "Switch repository" in the webapp, but this option is just above the "Restart daemon" option and I have clicked it quite a few time unintentionally.+Could it be possible to add a confirmation dialog just like in the "Shutdown daemon" option?++> spacer added; [[done]]  --[[Joey]]
doc/todo/Improve_direct_mode_using_copy_on_write.mdwn view
@@ -40,3 +40,5 @@ Looking at the code it would be preferable to exec directly to `cp`, see [copy_range() on LWN](http://lwn.net/Articles/550621/) and this [more recent article about splice() on LWN](http://lwn.net/Articles/567086/)  Also, `cp --reflink` fall back to copy when copy-on-write is not available while `cp --reflink=always` do not.++> The new v6 repository mode works this way. [[done]] --[[Joey]]
+ doc/todo/add_magicmime_support_to_OSX_dmg.mdwn view
@@ -0,0 +1,3 @@+The OSX .dmg is built without the MagicMime build flag. Turning it on will+take some work to ship a copy of the magic database inside the dmg.+--[[Joey]]
doc/todo/checkpresentkey_without_explicit_remote.mdwn view
@@ -1,3 +1,5 @@ While being asked to check if file is available from "[datalad-archives]" remote I need to check if the archive's key available.  Ideally I wish I could ask through the ongoing interaction protocol, but if not, I could use smth like 'git annex checkpresentkey' but that one demands specification also of a remote which to check.  In my case I just want to know if that key is available from any remote, so I could confirm that the file is still present in our archives remote, i.e. that it could be retrieved later on  [[!meta author=yoh]]++> [[done]]] --[[Joey]]
doc/todo/cloning_direct_mode_repo_over_http.mdwn view
@@ -33,3 +33,4 @@ the direct mode database either way!  --[[Joey]]+[[!meta tag=deprecateddirectmode]]
doc/todo/git_annex_push.mdwn view
@@ -1,1 +1,6 @@ A common use case for me is to use annex as really just an addition to git to store additional content.  What I am ending up with is two stage procedure to submit my changes to our shared repository:  git push REMOTE; git annex copy --to=REMOTE . IMHO it would only be logical if there was "git annex push REMOTE [GITPUSHOPTIONS]" which would be merely an alias for "git push REMOTE [GITPUSHOPTIONS]; git annex copy --to=REMOTE" but which will make use of annex for such common usecase simple(r)++> After this was opened, some options were added, so use:+> `git-annex sync --no-pull --content`+> +> [[done]] --[[Joey]]
+ doc/todo/import_--reinject.mdwn view
@@ -0,0 +1,5 @@+There's `git annex reinject <src> <dst>` for re-adding one file's contents to the local annex. But what if I have a whole bunch of files, and want git-annex itself to decide whether & where it needs to reinject them? (And if the file doesn't need to be reinjected, it would remain in its original place.)++None of the `git annex import` modes work properly in this case. By default, importing adds another, unnecessary copy of the imported file (which I have to `rm` after importing). The `--clean-duplicates` mode seems close, but it insists on verifying the content in other repositories rather than just reinjecting it locally. (Let's assume that the main reason I'm trying to reinject is that I cannot access other repos.)++So I'm hoping for something like `git annex import --reinject <src>...`. Or are there other existing ways to achieve the same? I couldn't find any.
+ doc/todo/make_annex_info_more_efficient.mdwn view
@@ -0,0 +1,3 @@+ATM it takes about a minute for 'git annex info' on a sizeable but not huge repository with only ~450 files under annex but a few thousand of files  (~7000) in the tree.  I am not quite sure why it takes that long since it seems to care only about annexed files.  Also it might be of benefit to parallelize some traversal operations to take advantage of multiple cpu/cores++[[!meta author=yoh]]
+ doc/todo/metadata_--batch.mdwn view
@@ -0,0 +1,1 @@+[[!meta author=yoh]]
doc/todo/smudge.mdwn view
@@ -32,6 +32,10 @@ * Eventually (but not yet), make v6 the default for new repositories.   Note that the assistant forces repos into direct mode; that will need to   be changed then, and it should enable annex.thin instead.+* annex.thin doesn't work on crippled filesystems, so changing to v6+  unlocked files on such a FS always doubles disk use from direct mode.+  Do something about this? Could use windows hard link equivilant.. Or,+  could only store the file content in the work tree and not annex/objects. * Later still, remove support for direct mode, and enable automatic   v5 to v6 upgrades.
doc/todo/unwanted_repository_version_upgrades.mdwn view
@@ -23,3 +23,11 @@ """]]  I'm guessing that one of the machines with access to this repository was running a newer version of git-annex, and that the repository was upgraded in the course of some action.++> I took this onboard with v6; upgrade to it is currently optional and will+> only become the default once enough years have passed that it's+> reasonable to assume most people have git-annex 6.x installed.+> +> I think at this point it's reasonable to assume most people have+> git-annex 5.x installed, so there's no point in trying to change to v3 to+> v5 forced upgrade at this point. So, [[done]] --[[Joey]]
ghci view
@@ -1,4 +1,4 @@ #!/bin/sh # ghci using objects built by cabal make dist/caballog-$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//; s/-O//') $@+$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//') $@
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 6.20160211+Version: 6.20160229 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name>@@ -130,7 +130,7 @@   if flag(network-uri)     Build-Depends: network-uri (>= 2.6), network (>= 2.6)   else-    Build-Depends: network (< 2.6), network (>= 2.0)+    Build-Depends: network (< 2.6), network (>= 2.4)    if flag(Cryptonite)     Build-Depends: cryptonite@@ -230,8 +230,9 @@     CPP-Options: -DWITH_TORRENTPARSER    if flag(MagicMime)-    Build-Depends: magic-    CPP-Options: -DWITH_MAGICMIME+    if (! os(windows))+      Build-Depends: magic+      CPP-Options: -DWITH_MAGICMIME    if flag(ConcurrentOutput)     Build-Depends: concurrent-output (>= 1.6)
man/git-annex-add.1 view
@@ -19,7 +19,8 @@ .PP Large files are added to the annex in locked form, which prevents further modification of their content unless unlocked by git-annex\-unlock(1).-(This is not the case however when a repository is in direct mode.)+(This is not the case however when a repository is in a filesystem not+supporting symlinks, or is in direct mode.) To add a file to the annex in unlocked form, \fBgit add\fP can be used instead  (that only works when the repository has annex.version 6 or higher). .PP
man/git-annex-checkpresentkey.1 view
@@ -3,16 +3,27 @@ git-annex-checkpresentkey \- check if key is present in remote .PP .SH SYNOPSIS-git annex checkpresentkey \fBkey remote\fP+git annex checkpresentkey \fBkey\fP \fB[remote]\fP .PP .SH DESCRIPTION This plumbing\-level command verifies if the specified key's content is present in the specified remote. .PP+When no remote is specified, it verifies if the key's content is present+somewhere, checking accessible remotes until it finds the content.+.PP Exits 0 if the content is verified present, or 1 if it is verified to not-be present. If there is a problem checking the remote, the special-exit code 100 is used, and an error message is output to stderr.+be present. If there is a problem, the special exit code 100 is used,+and an error message is output to stderr. .PP+.SH OPTIONS+.IP "\fB\-\-batch\fP"+.IP+Enables batch mode. In this mode, the \fBkey\fP is not specified at the+command line, but the \fBremote\fP may still be. Lines containing keys are+read from stdin, and a line is output with "1" if the key is verified to+be present, and "0" otherwise.+.IP .SH SEE ALSO git-annex(1) .PP
man/git-annex-matching-options.1 view
@@ -101,6 +101,14 @@ matches the glob. The values of metadata fields are matched case insensitively. .IP+.IP "\fB\-\-metadata field<number\fP / \fB\-\-metadata field>number\fP"+.IP "\fB\-\-metadata field<=number\fP / \fB\-\-metadata field>=number\fP"+Matches only files that have a metadata field attached with a value that+is a number and is less than or greater than the specified number.+.IP+(Note that you will need to quote the second parameter to avoid+the shell doing redirection.)+.IP .IP "\fB\-\-want\-get\fP" Matches files that the preferred content settings for the repository make it want to get. Note that this will match even files that are
man/git-annex-preferred-content.1 view
@@ -107,6 +107,14 @@ .IP To match author metadata, use \fBmetadata=author=*Smith\fP .IP+.IP "\fBmetadata=field<number\fP / \fBmetadata=field>number\fP "+.IP "\fBmetadata=field<=number\fP / \fBmetadata=field>=number\fP"+Matches only files that have a metadata field attached with a value that+is a number and is less than or greater than the specified number.+.IP+To match PDFs with between 100 and 200 pages (assuming something has set+that metadata), use \fBmetadata=pagecount>=100 and metadata=pagecount<=200\fP+.IP .IP "\fBpresent\fP" Makes content be wanted if it's present, but not otherwise. .IP
man/git-annex-view.1 view
@@ -25,9 +25,9 @@ These location fields can be used the same as other metadata to construct the view. .PP-For example, \fB/=podcasts\fP will only include files from the podcasts-directory in the view, while \fBpodcasts/=*\fP will preserve the-subdirectories of the podcasts directory in the view.+For example, \fB/=foo\fP will only include files from the foo+directory in the view, while \fBfoo/=*\fP will preserve the+subdirectories of the foo directory in the view. .PP .SH SEE ALSO git-annex(1)
man/git-annex.1 view
@@ -705,6 +705,14 @@ should be checked into git by \fBgit annex add\fP. Defaults to true; set to false to instead make small files be skipped. .IP+.IP "\fBannex.addunlocked\fP"+Set to true to make commands like \fBgit-annex add\fP that add files to the+repository add them in unlocked form. The default is to add files in+locked form. This only has effect in version 6 repositories.+.IP+When a repository has core.symlinks set to false, it implicitly+sets annex.addunlocked to true.+.IP .IP "\fBannex.numcopies\fP" This is a deprecated setting. You should instead use the \fBgit annex numcopies\fP command to configure how many copies of files
standalone/android/install-haskell-packages view
@@ -65,7 +65,7 @@ installgitannexdeps () { 	pushd ../.. 	ln -sf standalone/android/cabal.config-	cabal install --only-dependencies "$@" # --force-reinstalls --reinstall+	cabal install --only-dependencies --flags="-magicmime -concurrent-output" "$@" # --force-reinstalls --reinstall 	rm -f cabal.config 	popd }
standalone/licences.gz view

binary file changed (55832 → 55882 bytes)

standalone/linux/skel/runshell view
@@ -81,10 +81,10 @@ fi  # Put our binaries first, to avoid issues with out of date or incompatable-# system binaries.+# system binaries. Extra binaries come after system path. ORIG_PATH="$PATH" export ORIG_PATH-PATH="$base/bin:$PATH"+PATH="$base/bin:$PATH:$base/extra" export PATH  # These env vars are used by the shim wrapper around each binary.
standalone/osx/git-annex.app/Contents/MacOS/runshell view
@@ -61,10 +61,10 @@ fi  # Put our binaries first, to avoid issues with out of date or incompatable-# system binaries.+# system binaries. Extra binaries come after system path. ORIG_PATH="$PATH" export ORIG_PATH-PATH="$bundle:$PATH"+PATH="$bundle:$PATH:$bundle/extra" export PATH  ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"@@ -77,7 +77,11 @@ GIT_TEMPLATE_DIR="$bundle/templates" export GIT_TEMPLATE_DIR -# Indicate which variables were exported above.+GIT_ANNEX_DIR="$bundle"+export GIT_ANNEX_DIR++# Indicate which variables were exported above and should be cleaned+# when running non-bundled programs. GIT_ANNEX_STANDLONE_ENV="PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR" export GIT_ANNEX_STANDLONE_ENV
standalone/windows/build.sh view
@@ -101,10 +101,5 @@ export PATH mkdir -p c:/WINDOWS/Temp/git-annex-test/ cd c:/WINDOWS/Temp/git-annex-test/-if withcyg git-annex.exe test; then-	rm -rf .t-else-	rm -rf .t-	echo "Test suite failure; failing build!"-	false-fi+rm -rf .t+withcyg git-annex.exe test
templates/controlmenu.hamlet view
@@ -7,6 +7,7 @@     <a href="@{RepositorySwitcherR}">       <span .glyphicon .glyphicon-folder-close>       \ Switch repository+  <li>&nbsp;</li>   <li>     <a href="@{RestartR}">       <span .glyphicon .glyphicon-repeat>