packages feed

git-annex 5.20150930 → 5.20151019

raw patch · 140 files changed

+2782/−607 lines, 140 filesdep ~process

Dependency ranges changed: process

Files

Annex.hs view
@@ -13,6 +13,7 @@ 	new, 	run, 	eval,+	makeRunner, 	getState, 	changeState, 	withState,@@ -202,6 +203,13 @@ eval s a = do 	mvar <- newMVar s 	runReaderT (runAnnex a) mvar++{- Makes a runner action, that allows diving into IO and from inside+ - the IO action, running an Annex action. -}+makeRunner :: Annex (Annex a -> IO a)+makeRunner = do+	mvar <- ask+	return $ \a -> runReaderT (runAnnex a) mvar  getState :: (AnnexState -> v) -> Annex v getState selector = do
Annex/AutoMerge.hs view
@@ -77,6 +77,10 @@  - In direct mode, the work tree is not touched here; files are staged to  - the index, and written to the gitAnnexMergeDir, for later handling by  - the direct mode merge code.+ -+ - Returns false when there are no merge conflicts to resolve.+ - A git merge can fail for other reasons, and this allows detecting+ - such failures.  -} resolveMerge :: Maybe Git.Ref -> Git.Ref -> Annex Bool resolveMerge us them = do
Annex/Branch.hs view
@@ -18,6 +18,7 @@ 	get, 	getHistorical, 	change,+	maybeChange, 	commit, 	forceCommit, 	files,@@ -224,7 +225,15 @@  - modifes the current content of the file on the branch.  -} change :: FilePath -> (String -> String) -> Annex ()-change file a = lockJournal $ \jl -> a <$> getLocal file >>= set jl file+change file f = lockJournal $ \jl -> f <$> getLocal file >>= set jl file++{- Applies a function which can modify the content of a file, or not. -}+maybeChange :: FilePath -> (String -> Maybe String) -> Annex ()+maybeChange file f = lockJournal $ \jl -> do+	v <- getLocal file+	case f v of+		Just v' | v' /= v -> set jl file v'+		_ -> noop  {- Records new content of a file into the journal -} set :: JournalLocked -> FilePath -> String -> Annex ()
Annex/Content.hs view
@@ -12,11 +12,14 @@ 	inAnnex', 	inAnnexSafe, 	inAnnexCheck,-	lockContent,+	lockContentShared,+	lockContentForRemoval,+	ContentRemovalLock, 	getViaTmp,-	getViaTmpChecked,-	getViaTmpUnchecked,-	prepGetViaTmpChecked,+	getViaTmp',+	checkDiskSpaceToGet,+	VerifyConfig(..),+	Types.Remote.unVerified, 	prepTmp, 	withTmp, 	checkDiskSpace,@@ -62,6 +65,11 @@ import Annex.ReplaceFile import Utility.LockPool import Messages.Progress+import qualified Types.Remote+import qualified Types.Backend+import qualified Backend+import Types.NumCopies+import Annex.UUID  {- Checks if a given key's content is currently present. -} inAnnex :: Key -> Annex Bool@@ -161,84 +169,176 @@ contentLockFile key = Just <$> calcRepo (gitAnnexContentLock key) #endif -newtype ContentLock = ContentLock Key+{- Prevents the content from being removed while the action is running.+ - Uses a shared lock.+ -+ - Does not actually check if the content is present. Use inAnnex for that.+ - However, since the contentLockFile is the content file in indirect mode,+ - if the content is not present, locking it will fail.+ -+ - If locking fails, throws an exception rather than running the action.+ -+ - Note that, in direct mode, nothing prevents the user from directly+ - editing or removing the content, even while it's locked by this.+ -}+lockContentShared :: Key -> (VerifiedCopy -> Annex a) -> Annex a+lockContentShared key a = lockContentUsing lock key $ do+	u <- getUUID+	withVerifiedCopy LockedCopy u (return True) a+  where+#ifndef mingw32_HOST_OS+	lock contentfile Nothing = liftIO $ tryLockShared Nothing contentfile+	lock _ (Just lockfile) = posixLocker tryLockShared lockfile+#else+	lock = winLocker lockShared+#endif -{- Content is exclusively locked while running an action that might remove- - it. (If the content is not present, no locking is done.)+{- Exclusively locks content, while performing an action that+ - might remove it.  -}-lockContent :: Key -> (ContentLock -> Annex a) -> Annex a-lockContent key a = do+lockContentForRemoval :: Key -> (ContentRemovalLock -> Annex a) -> Annex a+lockContentForRemoval key a = lockContentUsing lock key $ +	a (ContentRemovalLock key)+  where+#ifndef mingw32_HOST_OS+	{- Since content files are stored with the write bit disabled, have+	 - to fiddle with permissions to open for an exclusive lock. -}+	lock contentfile Nothing = bracket_+		(thawContent contentfile)+		(freezeContent contentfile)+		(liftIO $ tryLockExclusive Nothing contentfile)+	lock _ (Just lockfile) = posixLocker tryLockExclusive lockfile+#else+	lock = winLocker lockExclusive+#endif++{- Passed the object content file, and maybe a separate lock file to use,+ - when the content file itself should not be locked. -}+type ContentLocker = FilePath -> Maybe LockFile -> Annex (Maybe LockHandle)++#ifndef mingw32_HOST_OS+posixLocker :: (Maybe FileMode -> LockFile -> IO (Maybe LockHandle)) -> LockFile -> Annex (Maybe LockHandle)+posixLocker takelock lockfile = do+	mode <- annexFileMode+	modifyContent lockfile $+		liftIO $ takelock (Just mode) lockfile+	+#else+winLocker :: (LockFile -> IO (Maybe LockHandle)) -> ContentLocker+winLocker takelock _ (Just lockfile) = do+	modifyContent lockfile $+		void $ liftIO $ tryIO $+			writeFile lockfile ""+	liftIO $ takelock lockfile+-- never reached; windows always uses a separate lock file+winLocker _ _ Nothing = return Nothing+#endif++lockContentUsing :: ContentLocker -> Key -> Annex a -> Annex a+lockContentUsing locker key a = do 	contentfile <- calcRepo $ gitAnnexLocation key 	lockfile <- contentLockFile key 	bracket 		(lock contentfile lockfile) 		(unlock lockfile)-		(const $ a $ ContentLock key )+		(const a)   where 	alreadylocked = error "content is locked"-	cleanuplockfile lockfile = modifyContent lockfile $-		void $ liftIO $ tryIO $-			nukeFile lockfile-#ifndef mingw32_HOST_OS-	{- Since content files are stored with the write bit disabled, have-	 - to fiddle with permissions to open for an exclusive lock. -}-	lock contentfile Nothing = trylock $ bracket_-		(thawContent contentfile)-		(freezeContent contentfile)+	failedtolock e = error $ "failed to lock content: " ++ show e++	lock contentfile lockfile = 		(maybe alreadylocked return -			=<< liftIO (tryLockExclusive Nothing contentfile))-	lock _ (Just lockfile) = trylock $ do-		mode <- annexFileMode-		maybe alreadylocked return -			=<< modifyContent lockfile-				(liftIO $ tryLockExclusive (Just mode) lockfile)+			=<< locker contentfile lockfile)+		`catchIO` failedtolock++#ifndef mingw32_HOST_OS 	unlock mlockfile lck = do 		maybe noop cleanuplockfile mlockfile 		liftIO $ dropLock lck-	-	failedtolock e = error $ "failed to lock content: " ++ show e-	trylock locker = locker `catchIO` failedtolock #else-	lock _ (Just lockfile) = do-		modifyContent lockfile $-			void $ liftIO $ tryIO $-				writeFile lockfile ""-		maybe alreadylocked (return . Just)-			=<< liftIO (lockExclusive lockfile)-	-- never reached; windows always uses a separate lock file-	lock _ Nothing = return Nothing-	unlock mlockfile mlockhandle = do-		liftIO $ maybe noop dropLock mlockhandle+	unlock mlockfile lck = do+		-- Can't delete a locked file on Windows+		liftIO $ dropLock lck 		maybe noop cleanuplockfile mlockfile #endif -{- Runs an action, passing it a temporary filename to get,- - and if the action succeeds, moves the temp file into - - the annex as a key's content. -}-getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool-getViaTmp = getViaTmpChecked (return True)+	cleanuplockfile lockfile = modifyContent lockfile $+		void $ liftIO $ tryIO $+			nukeFile lockfile +{- Runs an action, passing it the temp file to get,+ - and if the action succeeds, verifies the file matches+ - the key and moves the file into the annex as a key's content. -}+getViaTmp :: VerifyConfig -> Key -> (FilePath -> Annex (Bool, Types.Remote.Verification)) -> Annex Bool+getViaTmp v key action = checkDiskSpaceToGet key False $+	getViaTmp' v key action+ {- Like getViaTmp, but does not check that there is enough disk space  - for the incoming key. For use when the key content is already on disk  - and not being copied into place. -}-getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool-getViaTmpUnchecked = finishGetViaTmp (return True)+getViaTmp' :: VerifyConfig -> Key -> (FilePath -> Annex (Bool, Types.Remote.Verification)) -> Annex Bool+getViaTmp' v key action = do+	tmpfile <- prepTmp key+	(ok, verification) <- action tmpfile+	if ok+		then ifM (verifyKeyContent v verification key tmpfile)+			( do+				moveAnnex key tmpfile+				logStatus key InfoPresent+				return True+			, do+				warning "verification of content failed"+				liftIO $ nukeFile tmpfile+				return False+			)+		-- On transfer failure, the tmp file is left behind, in case+		-- caller wants to resume its transfer+		else return False -getViaTmpChecked :: Annex Bool -> Key -> (FilePath -> Annex Bool) -> Annex Bool-getViaTmpChecked check key action = -	prepGetViaTmpChecked key False $-		finishGetViaTmp check key action+{- Verifies that a file is the expected content of a key.+ - Configuration can prevent verification, for either a+ - particular remote or always.+ -+ - Most keys have a known size, and if so, the file size is checked.+ -+ - When the key's backend allows verifying the content (eg via checksum),+ - it is checked. + -}+verifyKeyContent :: VerifyConfig -> Types.Remote.Verification -> Key -> FilePath -> Annex Bool+verifyKeyContent _ Types.Remote.Verified _ _ = return True+verifyKeyContent v Types.Remote.UnVerified k f = ifM (shouldVerify v)+	( verifysize <&&> verifycontent+	, return True+	)+  where+	verifysize = case Types.Key.keySize k of+		Nothing -> return True+		Just size -> do+			size' <- liftIO $ catchDefaultIO 0 $ getFileSize f+			return (size' == size)+	verifycontent = case Types.Backend.verifyKeyContent =<< Backend.maybeLookupBackendName (Types.Key.keyBackendName k) of+		Nothing -> return True+		Just verifier -> verifier k f -{- Prepares to download a key via a tmp file, and checks that there is- - enough free disk space.+data VerifyConfig = AlwaysVerify | NoVerify | RemoteVerify Remote | DefaultVerify++shouldVerify :: VerifyConfig -> Annex Bool+shouldVerify AlwaysVerify = return True+shouldVerify NoVerify = return False+shouldVerify DefaultVerify = annexVerify <$> Annex.getGitConfig+shouldVerify (RemoteVerify r) = shouldVerify DefaultVerify+	<&&> pure (remoteAnnexVerify (Types.Remote.gitconfig r))++{- Checks if there is enough free disk space to download a key+ - to its temp file.  -  - When the temp file already exists, count the space it is using as  - free, since the download will overwrite it or resume.  -  - Wen there's enough free space, runs the download action.  -}-prepGetViaTmpChecked :: Key -> a -> Annex a -> Annex a-prepGetViaTmpChecked key unabletoget getkey = do+checkDiskSpaceToGet :: Key -> a -> Annex a -> Annex a+checkDiskSpaceToGet key unabletoget getkey = do 	tmp <- fromRepo $ gitAnnexTmpObjectLocation key  	e <- liftIO $ doesFileExist tmp@@ -253,19 +353,6 @@ 		, return unabletoget 		) -finishGetViaTmp :: Annex Bool -> Key -> (FilePath -> Annex Bool) -> Annex Bool-finishGetViaTmp check key action = do-	tmpfile <- prepTmp key-	ifM (action tmpfile <&&> check)-		( do-			moveAnnex key tmpfile-			logStatus key InfoPresent-			return True-		-- the tmp file is left behind, in case caller wants-		-- to resume its transfer-		, return False-		)- prepTmp :: Key -> Annex FilePath prepTmp key = do 	tmp <- fromRepo $ gitAnnexTmpObjectLocation key@@ -460,8 +547,8 @@  - In direct mode, deletes the associated files or files, and replaces  - them with symlinks.  -}-removeAnnex :: ContentLock -> Annex ()-removeAnnex (ContentLock key) = withObjectLoc key remove removedirect+removeAnnex :: ContentRemovalLock -> Annex ()+removeAnnex (ContentRemovalLock key) = withObjectLoc key remove removedirect   where 	remove file = cleanObjectLoc key $ do 		secureErase file
Annex/Direct.hs view
@@ -173,16 +173,19 @@  	withIndexFile tmpi $ do 		merged <- stageMerge d branch commitmode-		r <- if merged+		ok <- if merged 			then return True 			else resolvemerge-		mergeDirectCleanup d (fromMaybe Git.Sha.emptyTree oldref)-		mergeDirectCommit merged startbranch branch commitmode--		liftIO $ whenM (doesFileExist tmpi) $-			rename tmpi reali--		return r+		if ok+			then do+				mergeDirectCleanup d (fromMaybe Git.Sha.emptyTree oldref)+				mergeDirectCommit merged startbranch branch commitmode+				liftIO $ whenM (doesFileExist tmpi) $+					rename tmpi reali+			else do+				liftIO $ nukeFile tmpi+		liftIO $ removeDirectoryRecursive d+		return ok   where 	exclusively = withExclusiveLock gitAnnexMergeLock @@ -231,9 +234,7 @@ 	canff = maybe (return False) (\o -> inRepo $ Git.Branch.fastForwardable o branch) old  mergeDirectCleanup :: FilePath -> Git.Ref -> Annex ()-mergeDirectCleanup d oldref = do-	updateWorkTree d oldref False-	liftIO $ removeDirectoryRecursive d+mergeDirectCleanup d oldref = updateWorkTree d oldref False  {- Updates the direct mode work tree to reflect the changes staged in the  - index by a git command, that was run in a temporary work tree.
Annex/Drop.hs view
@@ -30,20 +30,22 @@  - The UUIDs are ones where the content is believed to be present.  - The Remote list can include other remotes that do not have the content;  - only ones that match the UUIDs will be dropped from.- - If allowed to drop fromhere, that drop will be tried first.  -- - A remote can be specified that is known to have the key. This can be- - used an an optimisation when eg, a key has just been uploaded to a- - remote.+ - If allowed to drop fromhere, that drop will be done last. This is done+ - because local drops do not need any LockedCopy evidence, and so dropping+ - from local last allows the content to be removed from more remotes.  -+ - A VerifiedCopy can be provided as an optimisation when eg, a key+ - has just been uploaded to a remote.+ -  - In direct mode, all associated files are checked, and only if all  - of them are unwanted are they dropped.  -  - The runner is used to run commands, and so can be either callCommand  - or commandAction.  -}-handleDropsFrom :: [UUID] -> [Remote] -> Reason -> Bool -> Key -> AssociatedFile -> Maybe Remote -> (CommandStart -> CommandCleanup) -> Annex ()-handleDropsFrom locs rs reason fromhere key afile knownpresentremote runner = do+handleDropsFrom :: [UUID] -> [Remote] -> Reason -> Bool -> Key -> AssociatedFile -> [VerifiedCopy] -> (CommandStart -> CommandCleanup) -> Annex ()+handleDropsFrom locs rs reason fromhere key afile preverified runner = do 	fs <- ifM isDirect 		( do 			l <- associatedFilesRelative key@@ -53,8 +55,8 @@ 		, return $ maybeToList afile 		) 	n <- getcopies fs-	if fromhere && checkcopies n Nothing-		then go fs rs =<< dropl fs n+	void $ if fromhere && checkcopies n Nothing+		then go fs rs n >>= dropl fs 		else go fs rs n   where 	getcopies fs = do@@ -79,12 +81,12 @@ 		| S.member u untrusted = v 		| otherwise = decrcopies v Nothing -	go _ [] _ = noop+	go _ [] n = pure n 	go fs (r:rest) n 		| uuid r `S.notMember` slocs = go fs rest n 		| checkcopies n (Just $ Remote.uuid r) = 			dropr fs r n >>= go fs rest-		| otherwise = noop+		| otherwise = pure n  	checkdrop fs n u a 		| null fs = check $ -- no associated files; unused content@@ -112,7 +114,7 @@ 			)  	dropl fs n = checkdrop fs n Nothing $ \numcopies ->-		Command.Drop.startLocal afile numcopies key knownpresentremote+		Command.Drop.startLocal afile numcopies key preverified  	dropr fs r n  = checkdrop fs n (Just $ Remote.uuid r) $ \numcopies -> 		Command.Drop.startRemote afile numcopies key r
Annex/Init.hs view
@@ -209,7 +209,7 @@ initSharedClone :: Bool -> Annex () initSharedClone False = return () initSharedClone True = do-	showSideAction "Repository was cloned with --shared; setting annex.hardlink=true and making repository untrusted."+	showLongNote "Repository was cloned with --shared; setting annex.hardlink=true and making repository untrusted." 	u <- getUUID 	trustSet u UnTrusted 	setConfig (annexConfig "hardlink") (Git.Config.boolConfig True)
Annex/NumCopies.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable #-}+ module Annex.NumCopies ( 	module Types.NumCopies, 	module Logs.NumCopies,@@ -15,8 +17,9 @@ 	defaultNumCopies, 	numCopiesCheck, 	numCopiesCheck',-	verifyEnoughCopies,-	knownCopies,+	verifyEnoughCopiesToDrop,+	verifiableCopies,+	UnVerifiedCopy(..), ) where  import Common.Annex@@ -26,9 +29,14 @@ import Logs.Trust import Annex.CheckAttr import qualified Remote-import Annex.UUID+import qualified Types.Remote as Remote import Annex.Content+import Annex.UUID +import Control.Exception+import qualified Control.Monad.Catch as M+import Data.Typeable+ defaultNumCopies :: NumCopies defaultNumCopies = NumCopies 1 @@ -77,7 +85,11 @@  {- Checks if numcopies are satisfied for a file by running a comparison  - between the number of (not untrusted) copies that are- - belived to exist, and the configured value. -}+ - belived to exist, and the configured value.+ -+ - This is good enough for everything except dropping the file, which+ - requires active verification of the copies.+ -} numCopiesCheck :: FilePath -> Key -> (Int -> Int -> v) -> Annex v numCopiesCheck file key vs = do 	have <- trustExclude UnTrusted =<< Remote.keyLocations key@@ -88,60 +100,118 @@ 	NumCopies needed <- getFileNumCopies file 	return $ length have `vs` needed +data UnVerifiedCopy = UnVerifiedRemote Remote | UnVerifiedHere+	deriving (Ord, Eq)+ {- Verifies that enough copies of a key exist amoung the listed remotes,- - priting an informative message if not.+ - to safely drop it, running an action with a proof if so, and+ - printing an informative message if not.  -}-verifyEnoughCopies +verifyEnoughCopiesToDrop 	:: String -- message to print when there are no known locations 	-> Key+	-> Maybe ContentRemovalLock 	-> NumCopies 	-> [UUID] -- repos to skip considering (generally untrusted remotes)-	-> [UUID] -- repos that are trusted or already verified to have it-	-> [Remote] -- remotes to check to see if they have it-	-> Annex Bool-verifyEnoughCopies nolocmsg key need skip trusted tocheck = -	helper [] [] (nub trusted) (nub tocheck)+	-> [VerifiedCopy] -- copies already verified to exist+	-> [UnVerifiedCopy] -- places to check to see if they have copies+	-> (SafeDropProof -> Annex a) -- action to perform the drop+	-> Annex a -- action to perform when unable to drop+	-> Annex a+verifyEnoughCopiesToDrop nolocmsg key removallock need skip preverified tocheck dropaction nodropaction = +	helper [] [] preverified (nub tocheck)   where-	helper bad missing have []-		| NumCopies (length have) >= need = return True-		| otherwise = do-			notEnoughCopies key need have (skip++missing) bad nolocmsg-			return False-	helper bad missing have (r:rs)-		| NumCopies (length have) >= need = return True-		| otherwise = do-			let u = Remote.uuid r-			let duplicate = u `elem` have-			haskey <- Remote.hasKey r key-			case (duplicate, haskey) of-				(False, Right True)  -> helper bad missing (u:have) rs-				(False, Left _)      -> helper (r:bad) missing have rs-				(False, Right False) -> helper bad (u:missing) have rs-				_                    -> helper bad missing have rs+	helper bad missing have [] = do+		p <- liftIO $ mkSafeDropProof need have removallock+		case p of+			Right proof -> dropaction proof+			Left stillhave -> do+				notEnoughCopies key need stillhave (skip++missing) bad nolocmsg+				nodropaction+	helper bad missing have (c:cs)+		| isSafeDrop need have removallock = do+			p <- liftIO $ mkSafeDropProof need have removallock+			case p of+				Right proof -> dropaction proof+				Left stillhave -> helper bad missing stillhave (c:cs)+		| otherwise = case c of+			UnVerifiedHere -> lockContentShared key contverified+			UnVerifiedRemote r -> checkremote r contverified $ do+				haskey <- Remote.hasKey r key+				case haskey of+					Right True  -> helper bad missing (mkVerifiedCopy RecentlyVerifiedCopy r : have) cs+					Left _      -> helper (r:bad) missing have cs+					Right False -> helper bad (Remote.uuid r:missing) have cs+		  where+			contverified vc = helper bad missing (vc : have) cs -notEnoughCopies :: Key -> NumCopies -> [UUID] -> [UUID] -> [Remote] -> String -> Annex ()+	checkremote r cont fallback = case Remote.lockContent r of+		Just lockcontent -> do+			-- The remote's lockContent will throw an exception+			-- when it is unable to lock, in which case the+			-- fallback should be run. +			--+			-- On the other hand, the continuation could itself+			-- throw an exception (ie, the eventual drop action+			-- fails), and in this case we don't want to run the+			-- fallback since part of the drop action may have+			-- already been performed.+			--+			-- Differentiate between these two sorts+			-- of exceptions by using DropException.+			let a = lockcontent key $ \v -> +				cont v `catchNonAsync` (throw . DropException)+			a `M.catches`+				[ M.Handler (\ (e :: AsyncException) -> throwM e)+				, M.Handler (\ (DropException e') -> throwM e')+				, M.Handler (\ (_e :: SomeException) -> fallback)+				]+		Nothing -> fallback++data DropException = DropException SomeException+	deriving (Typeable, Show)++instance Exception DropException++notEnoughCopies :: Key -> NumCopies -> [VerifiedCopy] -> [UUID] -> [Remote] -> String -> Annex () notEnoughCopies key need have skip bad nolocmsg = do 	showNote "unsafe"-	showLongNote $-		"Could only verify the existence of " ++-		show (length have) ++ " out of " ++ show (fromNumCopies need) ++ -		" necessary copies"+	if length have < fromNumCopies need+		then showLongNote $+			"Could only verify the existence of " +++			show (length have) ++ " out of " ++ show (fromNumCopies need) ++ +			" necessary copies"+		else do+			showLongNote "Unable to lock down 1 copy of file that is required to safely drop it."+			showLongNote "(This could have happened because of a concurrent drop, or because a remote has too old a version of git-annex-shell installed.)" 	Remote.showTriedRemotes bad-	Remote.showLocations True key (have++skip) nolocmsg+	Remote.showLocations True key (map toUUID have++skip) nolocmsg -{- Cost ordered lists of remotes that the location log indicates- - may have a key.+{- Finds locations of a key that can be used to get VerifiedCopies,+ - in order to allow dropping the key.  -- - Also returns a list of UUIDs that are trusted to have the key- - (some may not have configured remotes). If the current repository- - currently has the key, and is not untrusted, it is included in this list.+ - Provide a list of UUIDs that the key is being dropped from.+ - The returned lists will exclude any of those UUIDs.+ -+ - The return lists also exclude any repositories that are untrusted,+ - since those should not be used for verification.+ -+ - The UnVerifiedCopy list is cost ordered.+ - The VerifiedCopy list contains repositories that are trusted to+ - contain the key.  -}-knownCopies :: Key -> Annex ([Remote], [UUID])-knownCopies key = do-	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key+verifiableCopies :: Key -> [UUID] -> Annex ([UnVerifiedCopy], [VerifiedCopy])+verifiableCopies key exclude = do+	locs <- Remote.keyLocations key+	(remotes, trusteduuids) <- Remote.remoteLocations locs+		=<< trustGet Trusted+	untrusteduuids <- trustGet UnTrusted+	let exclude' = exclude ++ untrusteduuids+	let remotes' = Remote.remotesWithoutUUID remotes (exclude' ++ trusteduuids)+	let verified = map (mkVerifiedCopy TrustedCopy) $+		filter (`notElem` exclude') trusteduuids 	u <- getUUID-	trusteduuids' <- ifM (inAnnex key <&&> (<= SemiTrusted) <$> lookupTrust u)-		( pure (u:trusteduuids)-		, pure trusteduuids-		)-	return (remotes, trusteduuids')+	let herec = if u `elem` locs && u `notElem` exclude'+		then [UnVerifiedHere]+		else []+	return (herec ++ map UnVerifiedRemote remotes', verified)
Annex/Perms.hs view
@@ -24,8 +24,6 @@ import qualified Annex import Config -import System.Posix.Types- withShared :: (SharedRepository -> Annex a) -> Annex a withShared a = a =<< coreSharedRepository <$> Annex.getGitConfig
Annex/Transfer.hs view
@@ -5,7 +5,7 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE CPP #-}+{-# LANGUAGE CPP, FlexibleInstances #-}  module Annex.Transfer ( 	module X,@@ -24,20 +24,31 @@ import Annex.Perms import Utility.Metered import Utility.LockPool+import Types.Remote (Verification(..))  import Control.Concurrent -type TransferAction = MeterUpdate -> Annex Bool- type TransferObserver = Bool -> Transfer -> TransferInfo -> Annex () +class Observable a where+	observeBool :: a -> Bool+	observeFailure :: a++instance Observable Bool where+	observeBool = id+	observeFailure = False++instance Observable (Bool, Verification) where+	observeBool = fst+	observeFailure = (False, UnVerified)+ noObserver :: TransferObserver noObserver _ _ _ = noop -upload :: UUID -> Key -> AssociatedFile -> RetryDecider -> TransferObserver -> TransferAction -> NotifyWitness -> Annex Bool+upload :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> TransferObserver -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v upload u key f d o a _witness = runTransfer (Transfer Upload u key) f d o a -download :: UUID -> Key -> AssociatedFile -> RetryDecider -> TransferObserver -> TransferAction -> NotifyWitness -> Annex Bool+download :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> TransferObserver -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v download u key f d o a _witness = runTransfer (Transfer Download u key) f d o a  {- Runs a transfer action. Creates and locks the lock file while the@@ -52,7 +63,7 @@  - An upload can be run from a read-only filesystem, and in this case  - no transfer information or lock file is used.  -}-runTransfer :: Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> TransferAction -> Annex Bool+runTransfer :: Observable v => Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> (MeterUpdate -> Annex v) -> Annex v runTransfer = runTransfer' False  {- Like runTransfer, but ignores any existing transfer lock file for the@@ -60,10 +71,10 @@  -  - Note that this may result in confusing progress meter display in the  - webapp, if multiple processes are writing to the transfer info file. -}-alwaysRunTransfer :: Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> TransferAction -> Annex Bool+alwaysRunTransfer :: Observable v => Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> (MeterUpdate -> Annex v) -> Annex v alwaysRunTransfer = runTransfer' True -runTransfer' :: Bool -> Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> TransferAction -> Annex Bool+runTransfer' :: Observable v => Bool -> Transfer -> Maybe FilePath -> RetryDecider -> TransferObserver -> (MeterUpdate -> Annex v) -> Annex v runTransfer' ignorelock t file shouldretry transferobserver transferaction = do 	info <- liftIO $ startTransferInfo file 	(meter, tfile, metervar) <- mkProgressUpdater t info@@ -72,12 +83,12 @@ 	if inprogress && not ignorelock 		then do 			showNote "transfer already in progress, or unable to take transfer lock"-			return False+			return observeFailure 		else do-			ok <- retry info metervar $ transferaction meter+			v <- retry info metervar $ transferaction meter 			liftIO $ cleanup tfile lck-			transferobserver ok t info-			return ok+			transferobserver (observeBool v) t info+			return v   where #ifndef mingw32_HOST_OS 	prep tfile mode info = do@@ -128,7 +139,7 @@ 				let newinfo = oldinfo { bytesComplete = Just b } 				if shouldretry oldinfo newinfo 					then retry newinfo metervar run-					else return False+					else return observeFailure 	getbytescomplete metervar 		| transferDirection t == Upload = 			liftIO $ readMVar metervar
Annex/UUID.hs view
@@ -15,6 +15,7 @@ 	getUUID, 	getRepoUUID, 	getUncachedUUID,+	isUUIDConfigured, 	prepUUID, 	genUUID, 	genUUIDInNameSpace,@@ -81,6 +82,13 @@  getUncachedUUID :: Git.Repo -> UUID getUncachedUUID = toUUID . Git.Config.get key ""+  where+	(ConfigKey key) = configkey++-- Does the repo's config have a key for the UUID?+-- True even when the key has no value.+isUUIDConfigured :: Git.Repo -> Bool+isUUIDConfigured = isJust . Git.Config.getMaybe key    where 	(ConfigKey key) = configkey
Assistant/Drop.hs view
@@ -15,11 +15,12 @@ import Annex.Drop (handleDropsFrom, Reason) import Logs.Location import CmdLine.Action+import Types.NumCopies  {- Drop from local and/or remote when allowed by the preferred content and  - numcopies settings. -}-handleDrops :: Reason -> Bool -> Key -> AssociatedFile -> Maybe Remote -> Assistant ()-handleDrops reason fromhere key f knownpresentremote = do+handleDrops :: Reason -> Bool -> Key -> AssociatedFile -> [VerifiedCopy] -> Assistant ()+handleDrops reason fromhere key f preverified = do 	syncrs <- syncDataRemotes <$> getDaemonStatus 	locs <- liftAnnex $ loggedLocations key-	liftAnnex $ handleDropsFrom locs syncrs reason fromhere key f knownpresentremote callCommandAction+	liftAnnex $ handleDropsFrom locs syncrs reason fromhere key f preverified callCommandAction
Assistant/Threads/Committer.hs view
@@ -479,7 +479,7 @@ 			void $ if present 				then queueTransfers "new file created" Next k (Just f) Upload 				else queueTransfers "new or renamed file wanted" Next k (Just f) Download-			handleDrops "file renamed" present k (Just f) Nothing+			handleDrops "file renamed" present k (Just f) []   where 	f = changeFile change checkChangeContent _ = noop
Assistant/Threads/SanityChecker.hs view
@@ -191,7 +191,7 @@ 	void $ liftAnnex $ setUnusedKeys unused 	forM_ unused $ \k -> do 		unlessM (queueTransfers "unused" Later k Nothing Upload) $-			handleDrops "unused" True k Nothing Nothing+			handleDrops "unused" True k Nothing []  	return True   where
Assistant/Threads/TransferScanner.hs view
@@ -157,7 +157,7 @@ 		present <- liftAnnex $ inAnnex key 		liftAnnex $ handleDropsFrom locs syncrs 			"expensive scan found too many copies of object"-			present key (Just f) Nothing callCommandAction+			present key (Just f) [] callCommandAction 		liftAnnex $ do 			let slocs = S.fromList locs 			let use a = return $ mapMaybe (a key slocs) syncrs
Assistant/TransferSlots.hs view
@@ -30,6 +30,7 @@ import Annex.Wanted import Annex.Path import Utility.Batch+import Types.NumCopies  import qualified Data.Map as M  import qualified Control.Exception as E@@ -160,7 +161,7 @@ 					("object uploaded to " ++ show remote) 					True (transferKey t) 					(associatedFile info)-					(Just remote)+					[mkVerifiedCopy RecentlyVerifiedCopy remote] 			void recordCommit 		, whenM (liftAnnex $ isNothing <$> checkTransfer t) $ 			void $ removeTransfer t@@ -225,7 +226,7 @@   where 	dodrops fromhere = handleDrops 		("drop wanted after " ++ describeTransfer t info)-		fromhere (transferKey t) (associatedFile info) Nothing+		fromhere (transferKey t) (associatedFile info) [] finishedTransfer _ _ = noop  {- Pause a running transfer. -}
Assistant/Unused.hs view
@@ -77,7 +77,7 @@ 	forM_ oldkeys $ \k -> do 		debug ["removing old unused key", key2file k] 		liftAnnex $ do-			lockContent k removeAnnex+			lockContentForRemoval k removeAnnex 			logStatus k InfoMissing   where 	boundry = durationToPOSIXTime <$> duration
Assistant/Upgrade.hs view
@@ -97,14 +97,14 @@ 		, transferKey = k 		} 	cleanup = liftAnnex $ do-		lockContent k removeAnnex+		lockContentForRemoval k removeAnnex 		setUrlMissing webUUID k u 		logStatus k InfoMissing  {- Called once the download is done.  - Passed an action that can be used to clean up the downloaded file.  -- - Fsck the key to verify the download.+ - Verifies the content of the downloaded key.  -} distributionDownloadComplete :: GitAnnexDistribution -> FilePath -> Assistant () -> Transfer -> Assistant () distributionDownloadComplete d dest cleanup t @@ -117,9 +117,9 @@ 	k = distributionKey d 	fsckit f = case Backend.maybeLookupBackendName (Types.Key.keyBackendName k) of 		Nothing -> return $ Just f-		Just b -> case Types.Backend.fsckKey b of+		Just b -> case Types.Backend.verifyKeyContent b of 			Nothing -> return $ Just f-			Just a -> ifM (a k f)+			Just verifier -> ifM (verifier k f) 				( return $ Just f 				, return Nothing 				)
Assistant/WebApp/Configurators/AWS.hs view
@@ -13,12 +13,12 @@ import Assistant.WebApp.MakeRemote #ifdef WITH_S3 import qualified Remote.S3 as S3-#endif-import qualified Remote.Glacier as Glacier-import qualified Remote.Helper.AWS as AWS import Logs.Remote import qualified Remote import qualified Types.Remote as Remote+#endif+import qualified Remote.Glacier as Glacier+import qualified Remote.Helper.AWS as AWS import Types.Remote (RemoteConfig) import Types.StandardGroups import Creds@@ -76,8 +76,10 @@ 	storageclasses :: [(Text, StorageClass)] 	storageclasses = 		[ ("Standard redundancy", StandardRedundancy)+#ifdef WITH_S3 #if MIN_VERSION_aws(0,13,0) 		, ("Infrequent access (cheaper for backups and archives)", StandardInfrequentAccess)+#endif #endif 		, ("Reduced redundancy (costs less)", ReducedRedundancy) 		]
Assistant/WebApp/Configurators/Edit.hs view
@@ -19,8 +19,8 @@ import Assistant.Sync import Assistant.Alert import qualified Assistant.WebApp.Configurators.AWS as AWS-import qualified Assistant.WebApp.Configurators.IA as IA #ifdef WITH_S3+import qualified Assistant.WebApp.Configurators.IA as IA import qualified Remote.S3 as S3 #endif import qualified Remote
Assistant/WebApp/Configurators/IA.hs view
@@ -15,15 +15,15 @@ import qualified Remote.S3 as S3 import qualified Remote.Helper.AWS as AWS import Assistant.WebApp.MakeRemote-#endif import qualified Remote import qualified Types.Remote as Remote import Types.StandardGroups-import Types.Remote (RemoteConfig) import Logs.Remote+import Assistant.Gpg+#endif+import Types.Remote (RemoteConfig) import qualified Annex.Url as Url import Creds-import Assistant.Gpg  import qualified Data.Text as T import qualified Data.Map as M
Assistant/WebApp/Configurators/Ssh.hs view
@@ -366,12 +366,6 @@  - Depending on the SshInput, avoids using a password, or uses a  - cached password. ssh is coaxed to use git-annex as SSH_ASKPASS  - to get the password.- -- - Note that ssh will only use SSH_ASKPASS when DISPLAY is set and there- - is no controlling terminal. On Unix, that is set up when the assistant- - starts, by calling createSession. On Windows, all of stdin, stdout, and- - stderr must be disconnected from the terminal. This is accomplished- - by always providing input on stdin.  -} sshAuthTranscript :: SshInput -> [String] -> (Maybe String) -> Assistant (String, Bool) sshAuthTranscript sshinput opts input = case inputAuthMethod sshinput of@@ -384,9 +378,21 @@ 	login = getLogin sshinput 	geti f = maybe "" T.unpack (f sshinput) -	go extraopts environ = processTranscript' "ssh" (extraopts ++ opts) environ $-		Just (fromMaybe "" input)+	go extraopts environ = processTranscript' +		(askPass environ) "ssh" (extraopts ++ opts)+		-- Always provide stdin, even when empty.+		(Just (fromMaybe "" input)) + 	{- ssh will only use SSH_ASKPASS when DISPLAY is set and there+	 - is no controlling terminal. -}+	askPass environ p = p+		{ env = environ+#if MIN_VERSION_process(1,3,0)+		, detach_console = True+		, new_session = True+#endif+		}+ 	setupAskPass = do 		program <- liftIO programPath 		v <- getCachedCred login@@ -399,10 +405,6 @@ 				let environ' = addEntries 					[ ("SSH_ASKPASS", program) 					, (sshAskPassEnv, passfile)-					-- ssh does not use SSH_ASKPASS-					-- unless DISPLAY is set, and-					-- there is no controlling-					-- terminal. 					, ("DISPLAY", ":0") 					] environ 				go [passwordprompts 1] (Just environ')
Backend/Hash.hs view
@@ -53,7 +53,7 @@ genBackend hash = Backend 	{ name = hashName hash 	, getKey = keyValue hash-	, fsckKey = Just $ checkKeyChecksum hash+	, verifyKeyContent = Just $ checkKeyChecksum hash 	, canUpgradeKey = Just needsUpgrade 	, fastMigrate = Just trivialMigrate 	, isStableKey = const True@@ -116,7 +116,7 @@ 		case (mstat, fast) of 			(Just stat, False) -> do 				filesize <- liftIO $ getFileSize' file stat-				showSideAction "checksum"+				showAction "checksum" 				check <$> hashFile hash file filesize 			_ -> return True 	expected = keyHash key
Backend/URL.hs view
@@ -22,7 +22,7 @@ backend = Backend 	{ name = "URL" 	, getKey = const $ return Nothing-	, fsckKey = Nothing+	, verifyKeyContent = Nothing 	, canUpgradeKey = Nothing 	, fastMigrate = Nothing 	-- The content of an url can change at any time, so URL keys are
Backend/WORM.hs view
@@ -21,7 +21,7 @@ backend = Backend 	{ name = "WORM" 	, getKey = keyValue-	, fsckKey = Nothing+	, verifyKeyContent = Nothing 	, canUpgradeKey = Nothing 	, fastMigrate = Nothing 	, isStableKey = const True
Build/EvilLinker.hs view
@@ -21,6 +21,7 @@  import Utility.Monad import Utility.Process hiding (env)+import qualified Utility.Process import Utility.Env  data CmdParams = CmdParams@@ -126,7 +127,7 @@ 	putStrLn $ unwords [c, show ps] 	systemenviron <- getEnvironment 	let environ' = fromMaybe [] environ ++ systemenviron-	out@(_, ok) <- processTranscript' c ps (Just environ') Nothing+	out@(_, ok) <- processTranscript' (\p -> p { Utility.Process.env = Just environ' }) c ps Nothing 	putStrLn $ unwords [c, "finished", show ok] 	return out
CHANGELOG view
@@ -1,3 +1,59 @@+git-annex (5.20151019) unstable; urgency=medium++  * Fix a longstanding, but unlikely to occur bug, where dropping+    a file from a remote could race with other drops of the same file,+    and result in all copies of its content being lost.+  * git-annex-shell: Added lockcontent command, to prevent dropping of+    a key's content. This is necessary due to the above bugfix.+  * In some cases, the above bugfix changes what git-annex allows you to+    drop:+    - When a file is present in several special remotes,+      but not in any accessible git repositories, dropping it from one of+      the special remotes will now fail. Instead, the file has to be+      moved from one of the special remotes to the git repository, and can+      then safely be dropped from the git repository.+    - If a git remote has too old a version of git-annex-shell installed,+      git-annex won't trust it to hold onto a copy of a file when dropping+      that file from the local git repository.+  * Changed drop ordering when using git annex sync --content or the+    assistant, to drop from remotes first and from the local repo last.+    This works better with the behavior changes to drop in many cases.+  * Do verification of checksums of annex objects downloaded from remotes.+  * When annex objects are received into git repositories from other git+    repos, their checksums are verified then too.+  * To get the old, faster, behavior of not verifying checksums, set+    annex.verify=false, or remote.<name>.annex-verify=false.+  * setkey, rekey: These commands also now verify that the provided file+    matches the expected checksum of the key, unless annex.verify=false.+  * reinject: Already verified content; this can now be disabled by+    setting annex.verify=false.+  * sync, merge, assistant: When git merge failed for a reason other+    than a conflicted merge, such as a crippled filesystem not allowing+    particular characters in filenames, git-annex would make a merge commit+    that could omit such files or otherwise be bad. Fixed by aborting the+    whole merge process when git merge fails for any reason other than a+    merge conflict.+  * Allow building with S3 disabled again.+  * Ported disk free space checking code to work on Solaris.+  * Windows webapp: Fix support for entering password when setting+    up a ssh remote.+  * copy --auto was checking the wrong repo's preferred content.+    (--from was checking what --to should, and vice-versa.)+    Fixed this bug, which was introduced in version 5.20150727.+  * Avoid unncessary write to the location log when a file is unlocked+    and then added back with unchanged content.+  * S3: Fix support for using https.+  * Avoid displaying network transport warning when a ssh remote+    does not yet have an annex.uuid set.+  * Debian: Add torrent library to build-depends as it's packaged now,+    and stop recommending bittornado | bittorrent.+  * Debian: Remove build dependency on transformers library, as it is now+    included in ghc.+  * Debian: Remote menu file, since a desktop file is provided and+    lintian says there can be only one.++ -- Joey Hess <id@joeyh.name>  Mon, 19 Oct 2015 13:59:01 -0400+ git-annex (5.20150930) unstable; urgency=medium    * Added new linux standalone "ancient" build to support kernels
CmdLine/GitAnnexShell.hs view
@@ -20,6 +20,7 @@  import qualified Command.ConfigList import qualified Command.InAnnex+import qualified Command.LockContent import qualified Command.DropKey import qualified Command.RecvKey import qualified Command.SendKey@@ -32,6 +33,7 @@ cmds_readonly = 	[ Command.ConfigList.cmd 	, gitAnnexShellCheck Command.InAnnex.cmd+	, gitAnnexShellCheck Command.LockContent.cmd 	, gitAnnexShellCheck Command.SendKey.cmd 	, gitAnnexShellCheck Command.TransferInfo.cmd 	, gitAnnexShellCheck Command.NotifyChanges.cmd
Command/AddUrl.hs view
@@ -154,7 +154,7 @@ 			-- so that the remote knows what url it 			-- should use to download it. 			setTempUrl urlkey loguri-			let downloader = Remote.retrieveKeyFile r urlkey (Just file)+			let downloader = \dest p -> fst <$> Remote.retrieveKeyFile r urlkey (Just file) dest p 			ret <- downloadWith downloader urlkey (Remote.uuid r) loguri file 			removeTempUrl urlkey 			return ret@@ -247,7 +247,7 @@ 			 - it later. -} 			urlinfo <- Url.withUrlOptions (Url.getUrlInfo videourl) 			let sizedkey = addSizeUrlKey urlinfo key-			prepGetViaTmpChecked sizedkey Nothing $ do+			checkDiskSpaceToGet sizedkey Nothing $ do 				tmp <- fromRepo $ gitAnnexTmpObjectLocation key 				showOutput 				ok <- Transfer.notifyTransfer Transfer.Download (Just file) $@@ -305,7 +305,7 @@  - stable. -} downloadWith :: (FilePath -> MeterUpdate -> Annex Bool) -> Key -> UUID -> URLString -> FilePath -> Annex (Maybe Key) downloadWith downloader dummykey u url file =-	prepGetViaTmpChecked dummykey Nothing $ do+	checkDiskSpaceToGet dummykey Nothing $ do 		tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey 		ifM (runtransfer tmp) 			( do
Command/Copy.hs view
@@ -52,7 +52,7 @@ 		| autoMode o = want <||> numCopiesCheck file key (<) 		| otherwise = return True 	want = case Command.Move.fromToOptions (moveOptions o) of-		ToRemote _ -> -			wantGet False (Just key) (Just file)-		FromRemote dest -> (Remote.uuid <$> getParsed dest) >>=+		ToRemote dest -> (Remote.uuid <$> getParsed dest) >>= 			wantSend False (Just key) (Just file)+		FromRemote _ ->+			wantGet False (Just key) (Just file)
Command/Drop.hs view
@@ -20,6 +20,7 @@ import Annex.Wanted import Annex.Notification +import System.Log.Logger (debugM) import qualified Data.Set as S  cmd :: Command@@ -64,11 +65,11 @@ 	checkDropAuto (autoMode o) from afile key $ \numcopies -> 		stopUnless (want from) $ 			case from of-				Nothing -> startLocal afile numcopies key Nothing+				Nothing -> startLocal afile numcopies key [] 				Just remote -> do 					u <- getUUID 					if Remote.uuid remote == u-						then startLocal afile numcopies key Nothing+						then startLocal afile numcopies key [] 						else startRemote afile numcopies key remote 	  where 		want from@@ -78,35 +79,31 @@ startKeys :: DropOptions -> Key -> CommandStart startKeys o key = start' o key Nothing -startLocal :: AssociatedFile -> NumCopies -> Key -> Maybe Remote -> CommandStart-startLocal afile numcopies key knownpresentremote = stopUnless (inAnnex key) $ do+startLocal :: AssociatedFile -> NumCopies -> Key -> [VerifiedCopy] -> CommandStart+startLocal afile numcopies key preverified = stopUnless (inAnnex key) $ do 	showStart' "drop" key afile-	next $ performLocal key afile numcopies knownpresentremote+	next $ performLocal key afile numcopies preverified  startRemote :: AssociatedFile -> NumCopies -> Key -> Remote -> CommandStart startRemote afile numcopies key remote = do 	showStart' ("drop " ++ Remote.name remote) key afile 	next $ performRemote key afile numcopies remote --- Note that lockContent is called before checking if the key is present--- on enough remotes to allow removal. This avoids a scenario where two--- or more remotes are trying to remove a key at the same time, and each--- see the key is present on the other.-performLocal :: Key -> AssociatedFile -> NumCopies -> Maybe Remote -> CommandPerform-performLocal key afile numcopies knownpresentremote = lockContent key $ \contentlock -> do-	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key-	let trusteduuids' = case knownpresentremote of-		Nothing -> trusteduuids-		Just r -> Remote.uuid r:trusteduuids-	untrusteduuids <- trustGet UnTrusted-	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids'++untrusteduuids)+performLocal :: Key -> AssociatedFile -> NumCopies -> [VerifiedCopy] -> CommandPerform+performLocal key afile numcopies preverified = lockContentForRemoval key $ \contentlock -> do 	u <- getUUID-	ifM (canDrop u key afile numcopies trusteduuids' tocheck [])-		( do+	(tocheck, verified) <- verifiableCopies key [u]+	doDrop u (Just contentlock) key afile numcopies [] (preverified ++ verified) tocheck+		( \proof -> do+			liftIO $ debugM "drop" $ unwords+				[ "Dropping from here"+				, "proof:"+				, show proof+				] 			removeAnnex contentlock 			notifyDrop afile True 			next $ cleanupLocal key-		, do+		, do  			notifyDrop afile False 			stop 		)@@ -117,14 +114,19 @@ 	-- places assumed to have the key, and places to check. 	-- When the local repo has the key, that's one additional copy, 	-- as long as the local repo is not untrusted.-	(remotes, trusteduuids) <- knownCopies key-	let have = filter (/= uuid) trusteduuids-	untrusteduuids <- trustGet UnTrusted-	let tocheck = filter (/= remote) $-		Remote.remotesWithoutUUID remotes (have++untrusteduuids)-	stopUnless (canDrop uuid key afile numcopies have tocheck [uuid]) $ do-		ok <- Remote.removeKey remote key-		next $ cleanupRemote key remote ok+	(tocheck, verified) <- verifiableCopies key [uuid]+	doDrop uuid Nothing key afile numcopies [uuid] verified tocheck+		( \proof -> do +			liftIO $ debugM "drop" $ unwords+				[ "Dropping from remote"+				, show remote+				, "proof:"+				, show proof+				]+			ok <- Remote.removeKey remote key+			next $ cleanupRemote key remote ok+		, stop+		)   where 	uuid = Remote.uuid remote @@ -139,30 +141,42 @@ 		Remote.logStatus remote key InfoMissing 	return ok -{- Checks specified remotes to verify that enough copies of a key exist to- - allow it to be safely removed (with no data loss). Can be provided with- - some locations where the key is known/assumed to be present.+{- Before running the dropaction, checks specified remotes to+ - verify that enough copies of a key exist to allow it to be+ - safely removed (with no data loss).  -  - Also checks if it's required content, and refuses to drop if so.  -  - --force overrides and always allows dropping.  -}-canDrop :: UUID -> Key -> AssociatedFile -> NumCopies -> [UUID] -> [Remote] -> [UUID] -> Annex Bool-canDrop dropfrom key afile numcopies have check skip = +doDrop+	:: UUID+	-> Maybe ContentRemovalLock+	-> Key+	-> AssociatedFile+	-> NumCopies+	-> [UUID]+	-> [VerifiedCopy]+	-> [UnVerifiedCopy]+	-> (Maybe SafeDropProof -> CommandPerform, CommandPerform)+	-> CommandPerform+doDrop dropfrom contentlock key afile numcopies skip preverified check (dropaction, nodropaction) =  	ifM (Annex.getState Annex.force)-		( return True-			, ifM (checkRequiredContent dropfrom key afile-				<&&> verifyEnoughCopies nolocmsg key numcopies skip have check-				)-				( return True-				, do-					hint-					return False-				)+		( dropaction Nothing+		, ifM (checkRequiredContent dropfrom key afile)+			( verifyEnoughCopiesToDrop nolocmsg key +				contentlock numcopies+				skip preverified check+					(dropaction . Just)+					(forcehint nodropaction)+			, stop+			) 		)   where 	nolocmsg = "Rather than dropping this file, try using: git annex move"-	hint = showLongNote "(Use --force to override this check, or adjust numcopies.)"+	forcehint a = do+		showLongNote "(Use --force to override this check, or adjust numcopies.)"+		a  checkRequiredContent :: UUID -> Key -> AssociatedFile -> Annex Bool checkRequiredContent u k afile =
Command/DropKey.hs view
@@ -31,7 +31,7 @@ 	next $ perform key  perform :: Key -> CommandPerform-perform key = lockContent key $ \contentlock -> do+perform key = lockContentForRemoval key $ \contentlock -> do 	removeAnnex contentlock 	next $ cleanup key
Command/DropUnused.hs view
@@ -44,7 +44,7 @@ 	Just r -> do 		showAction $ "from " ++ Remote.name r 		Command.Drop.performRemote key Nothing numcopies r-	Nothing -> Command.Drop.performLocal key Nothing numcopies Nothing+	Nothing -> Command.Drop.performLocal key Nothing numcopies []  performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform performOther filespec key = do
Command/Fsck.hs view
@@ -161,7 +161,7 @@ 			( return (Just True) 			, ifM (Annex.getState Annex.fast) 				( return Nothing-				, Just <$>+				, Just . fst <$> 					Remote.retrieveKeyFile remote key Nothing tmp dummymeter 				) 			)@@ -349,12 +349,15 @@ checkBackendOr bad backend key file = 	checkBackendOr' bad backend key file (return True) +-- The postcheck action is run after the content is verified,+-- in order to detect situations where the file is changed while being+-- verified (particularly in direct mode). checkBackendOr' :: (Key -> Annex String) -> Backend -> Key -> FilePath -> Annex Bool -> Annex Bool checkBackendOr' bad backend key file postcheck =-	case Types.Backend.fsckKey backend of+	case Types.Backend.verifyKeyContent backend of 		Nothing -> return True-		Just a -> do-			ok <- a key file+		Just verifier -> do+			ok <- verifier key file 			ifM postcheck 				( do 					unless ok $ do
Command/Get.hs view
@@ -68,17 +68,16 @@ 		next a  perform :: Key -> AssociatedFile -> CommandPerform-perform key afile = stopUnless (getViaTmp key $ getKeyFile key afile) $+perform key afile = stopUnless (getKey key afile) $ 	next $ return True -- no cleanup needed  {- Try to find a copy of the file in one of the remotes,  - and copy it to here. -}-getKeyFile :: Key -> AssociatedFile -> FilePath -> Annex Bool-getKeyFile key afile dest = getKeyFile' key afile dest-	=<< Remote.keyPossibilities key+getKey :: Key -> AssociatedFile -> Annex Bool+getKey key afile = getKey' key afile =<< Remote.keyPossibilities key -getKeyFile' :: Key -> AssociatedFile -> FilePath -> [Remote] -> Annex Bool-getKeyFile' key afile dest = dispatch+getKey' :: Key -> AssociatedFile -> [Remote] -> Annex Bool+getKey' key afile = dispatch   where 	dispatch [] = do 		showNote "not available"@@ -102,6 +101,9 @@ 		| Remote.hasKeyCheap r = 			either (const False) id <$> Remote.hasKey r key 		| otherwise = return True-	docopy r = download (Remote.uuid r) key afile noRetry noObserver $ \p -> do-		showAction $ "from " ++ Remote.name r-		Remote.retrieveKeyFile r key afile dest p+	docopy r witness = getViaTmp (RemoteVerify r) key $ \dest ->+		download (Remote.uuid r) key afile noRetry noObserver +			(\p -> do+				showAction $ "from " ++ Remote.name r+				Remote.retrieveKeyFile r key afile dest p+			) witness
Command/Import.hs view
@@ -19,8 +19,6 @@ import Types.Key import Annex.CheckIgnore import Annex.NumCopies-import Types.TrustLevel-import Logs.Trust  cmd :: Command cmd = withGlobalOptions fileMatchingOptions $ notBareRepo $@@ -83,7 +81,7 @@   where 	deletedup k = do 		showNote $ "duplicate of " ++ key2file k-		ifM (verifiedExisting k destfile)+		verifyExisting k destfile 			( do 				liftIO $ removeFile srcfile 				next $ return True@@ -134,13 +132,12 @@ 		SkipDuplicates -> checkdup Nothing (Just importfile) 		_ -> return (Just importfile) -verifiedExisting :: Key -> FilePath -> Annex Bool-verifiedExisting key destfile = do+verifyExisting :: Key -> FilePath -> (CommandPerform, CommandPerform) -> CommandPerform+verifyExisting key destfile (yes, no) = do 	-- Look up the numcopies setting for the file that it would be 	-- imported to, if it were imported. 	need <- getFileNumCopies destfile -	(remotes, trusteduuids) <- knownCopies key-	untrusteduuids <- trustGet UnTrusted-	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)-	verifyEnoughCopies [] key need [] trusteduuids tocheck+	(tocheck, preverified) <- verifiableCopies key []+	verifyEnoughCopiesToDrop [] key Nothing need [] preverified tocheck+		(const yes) no
Command/ImportFeed.hs view
@@ -115,7 +115,7 @@ getCache opttemplate = ifM (Annex.getState Annex.force) 	( ret S.empty S.empty 	, do-		showSideAction "checking known urls"+		showAction "checking known urls" 		(is, us) <- unzip <$> (mapM knownItems =<< knownUrls) 		ret (S.fromList us) (S.fromList (concat is)) 	)
+ Command/LockContent.hs view
@@ -0,0 +1,46 @@+{- git-annex-shell command+ -+ - Copyright 2015 Joey Hess <id@joeyh.name>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.LockContent where++import Common.Annex+import Command+import Annex.Content+import Types.Key+import Remote.Helper.Ssh (contentLockedMarker)++cmd :: Command+cmd = noCommit $ +	command "lockcontent" SectionPlumbing +		"locks key's content in the annex, preventing it being dropped"+		paramKey+		(withParams seek)++seek :: CmdParams -> CommandSeek+seek = withWords start++-- First, lock the content. Then, make sure the content is actually+-- present, and print out a "1". Wait for the caller to send a line before+-- dropping the lock.+start :: [String] -> CommandStart+start [ks] = do+	ok <- lockContentShared k (const locksuccess)+		`catchNonAsync` (const $ return False)+	liftIO $ if ok+		then exitSuccess+		else exitFailure+  where+	k = fromMaybe (error "bad key") (file2key ks)+	locksuccess = ifM (inAnnex k)+		( liftIO $ do+			putStrLn contentLockedMarker+			hFlush stdout+			_ <- getLine+			return True+		, return False+		)+start _ = error "Specify exactly 1 key."
Command/Mirror.hs view
@@ -65,7 +65,7 @@ 			Right False -> ifM (inAnnex key) 				( do 					numcopies <- getnumcopies-					Command.Drop.startLocal afile numcopies key Nothing+					Command.Drop.startLocal afile numcopies key [] 				, stop 				)   where
Command/Move.hs view
@@ -1,6 +1,6 @@ {- git-annex command  -- - Copyright 2010-2013 Joey Hess <id@joeyh.name>+ - Copyright 2010-2015 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -16,7 +16,10 @@ import Annex.UUID import Annex.Transfer import Logs.Presence+import Annex.NumCopies +import System.Log.Logger (debugM)+ cmd :: Command cmd = withGlobalOptions (jobsOption : annexedMatchingOptions) $ 	command "move" SectionCommon@@ -123,7 +126,7 @@ 			finish   where 	finish-		| move = lockContent key $ \contentlock -> do+		| move = lockContentForRemoval key $ \contentlock -> do 			removeAnnex contentlock 			next $ Command.Drop.cleanupLocal key 		| otherwise = next $ return True@@ -166,9 +169,22 @@ 	go = notifyTransfer Download afile $  		download (Remote.uuid src) key afile noRetry noObserver $ \p -> do 			showAction $ "from " ++ Remote.name src-			getViaTmp key $ \t -> Remote.retrieveKeyFile src key afile t p+			getViaTmp (RemoteVerify src) key $ \t ->+				Remote.retrieveKeyFile src key afile t p 	dispatch _ False = stop -- failed 	dispatch False True = next $ return True -- copy complete-	dispatch True True = do -- finish moving+	-- Finish by dropping from remote, taking care to verify that+	-- the copy here has not been lost somehow. +	-- (NumCopies is 1 since we're moving.)+	dispatch True True = verifyEnoughCopiesToDrop "" key Nothing+		(NumCopies 1) [] [] [UnVerifiedHere] dropremote faileddropremote+	dropremote proof = do+		liftIO $ debugM "drop" $ unwords+			[ "Dropping from remote"+			, show src+			, "proof:"+			, show proof+			] 		ok <- Remote.removeKey src key 		next $ Command.Drop.cleanupRemote key src ok+	faileddropremote = error "Unable to drop from remote."
Command/NumCopies.hs view
@@ -39,7 +39,7 @@ 	Annex.setOutput QuietOutput 	v <- getGlobalNumCopies 	case v of-		Just n -> liftIO $ print $ fromNumCopies n+		Just n -> liftIO $ putStrLn $ show $ fromNumCopies n 		Nothing -> do 			liftIO $ putStrLn "global numcopies is not set" 			old <- deprecatedNumCopies
Command/ReKey.hs view
@@ -52,7 +52,7 @@ {- Make a hard link to the old key content (when supported),  - to avoid wasting disk space. -} linkKey :: Key -> Key -> Annex Bool-linkKey oldkey newkey = getViaTmpUnchecked newkey $ \tmp -> do+linkKey oldkey newkey = getViaTmp' DefaultVerify newkey $ \tmp -> unVerified $ do 	src <- calcRepo $ gitAnnexLocation oldkey 	liftIO $ ifM (doesFileExist tmp) 		( return True
Command/RecvKey.hs view
@@ -16,9 +16,6 @@ import Logs.Transfer import Command.SendKey (fieldTransfer) import qualified CmdLine.GitAnnexShell.Fields as Fields-import qualified Types.Key-import qualified Types.Backend-import qualified Backend  cmd :: Command cmd = noCommit $ command "recvkey" SectionPlumbing @@ -29,8 +26,12 @@ seek = withKeys start  start :: Key -> CommandStart-start key = fieldTransfer Download key $ \_p ->-	ifM (getViaTmp key go)+start key = fieldTransfer Download key $ \_p -> do+	-- Always verify content when a direct mode repo is sending a file,+	-- as the file could change while being transferred.+	fromdirect <- isJust <$> Fields.getField Fields.direct+	let verify = if fromdirect then AlwaysVerify else DefaultVerify+	ifM (getViaTmp verify key go) 		( do 			-- forcibly quit after receiving one key, 			-- and shutdown cleanly@@ -39,46 +40,7 @@ 		, return False 		)   where-	go tmp = do+	go tmp = unVerified $ do 		opts <- filterRsyncSafeOptions . maybe [] words 			<$> getField "RsyncOptions"-		ok <- liftIO $ rsyncServerReceive (map Param opts) tmp--		-- The file could have been received with permissions that-		-- do not allow reading it, so this is done before the-		-- directcheck.-		freezeContent tmp--		if ok-			then ifM (isJust <$> Fields.getField Fields.direct)-				( directcheck tmp-				, return True-				)-			else return False-	{- If the sending repository uses direct mode, the file-	 - it sends could be modified as it's sending it. So check-	 - that the right size file was received, and that the key/value-	 - Backend is happy with it. -}-	directcheck tmp = do-		oksize <- case Types.Key.keySize key of-		        Nothing -> return True-		        Just size -> do-				size' <- liftIO $ getFileSize tmp-				return $ size == size'-		if oksize-			then case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of-				Nothing -> do-					warning "recvkey: received key from direct mode repository using unknown backend; cannot check; discarding"-					return False-				Just backend -> maybe (return True) runfsck-					(Types.Backend.fsckKey backend)-			else do-				warning "recvkey: received key with wrong size; discarding"-				return False-	  where-		runfsck check = ifM (check key tmp)-			( return True-			, do-				warning "recvkey: received key from direct mode repository seems to have changed as it was transferred; discarding"-				return False-			)+		liftIO $ rsyncServerReceive (map Param opts) tmp
Command/Reinject.hs view
@@ -11,8 +11,6 @@ import Command import Logs.Location import Annex.Content-import qualified Command.Fsck-import qualified Backend  cmd :: Command cmd = command "reinject" SectionUtility @@ -36,29 +34,19 @@ start _ = error "specify a src file and a dest file"  perform :: FilePath -> FilePath -> Key -> CommandPerform-perform src dest key = do-	{- Check the content before accepting it. -}-	v <- Backend.getBackend dest key-	case v of-		Nothing -> stop-		Just backend ->-			ifM (Command.Fsck.checkKeySizeOr reject key src-				<&&> Command.Fsck.checkBackendOr reject backend key src)-				( do-					unlessM move $ error "mv failed!"-					next $ cleanup key-				, error "not reinjecting"-				)+perform src _dest key = ifM move+	( next $ cleanup key+	, error "failed"+	)   where-	-- the file might be on a different filesystem,+	-- The file might be on a different filesystem, 	-- so moveFile is used rather than simply calling-	-- moveToObjectDir; disk space is also-	-- checked this way.-	move = getViaTmp key $ \tmp ->+	-- moveToObjectDir; disk space is also checked this way,+	-- and the file's content is verified to match the key.+	move = getViaTmp DefaultVerify key $ \tmp -> unVerified $ 		liftIO $ catchBoolIO $ do 			moveFile src tmp 			return True-	reject = const $ return "wrong file?"  cleanup :: Key -> CommandCleanup cleanup key = do
Command/SetKey.hs view
@@ -35,7 +35,7 @@ 	-- the file might be on a different filesystem, so moveFile is used 	-- rather than simply calling moveAnnex; disk space is also 	-- checked this way.-	ok <- getViaTmp key $ \dest ->+	ok <- getViaTmp DefaultVerify key $ \dest -> unVerified $ 		if dest /= file 			then liftIO $ catchBoolIO $ do 				moveFile file dest
Command/Sync.hs view
@@ -37,7 +37,7 @@ import Config import Annex.Wanted import Annex.Content-import Command.Get (getKeyFile')+import Command.Get (getKey') import qualified Command.Move import Logs.Location import Annex.Drop@@ -153,11 +153,10 @@ remoteBranch :: Remote -> Git.Ref -> Git.Ref remoteBranch remote = Git.Ref.underBase $ "refs/remotes/" ++ Remote.name remote +-- Do automatic initialization of remotes when possible when getting remote+-- list. syncRemotes :: [String] -> Annex [Remote]-syncRemotes ps = do-	-- Get remote list first, doing automatic initialization-	-- of remotes when possible.-	syncRemotes' ps =<< Remote.remoteList' True+syncRemotes ps = syncRemotes' ps =<< Remote.remoteList' True  syncRemotes' :: [String] -> [Remote] -> Annex [Remote] syncRemotes' ps remotelist = ifM (Annex.getState Annex.fast) ( nub <$> pickfast , wanted )@@ -460,8 +459,8 @@ 		-- includeCommandAction for drops, 		-- because a failure to drop does not mean 		-- the sync failed.-		handleDropsFrom locs' rs "unwanted" True k af-			Nothing callCommandAction+		handleDropsFrom locs' rs "unwanted" True k af []+			callCommandAction 	 	return (got || not (null putrs))   where@@ -476,7 +475,7 @@ 		) 	get have = includeCommandAction $ do 		showStart' "get" k af-		next $ next $ getViaTmp k $ \dest -> getKeyFile' k af dest have+		next $ next $ getKey' k af have  	wantput r 		| Remote.readonly r || remoteAnnexReadOnly (Remote.gitconfig r) = return False
Command/TestRemote.hs view
@@ -14,7 +14,7 @@ import qualified Types.Remote as Remote import Types import Types.Key (key2file, keyBackendName, keySize)-import Types.Backend (getKey, fsckKey)+import Types.Backend (getKey, verifyKeyContent) import Types.KeySource import Annex.Content import Backend@@ -63,7 +63,7 @@ start basesz name = do 	showStart "testremote" name 	r <- either error id <$> Remote.byName' name-	showSideAction "generating test keys"+	showAction "generating test keys" 	fast <- Annex.getState Annex.fast 	ks <- mapM randKey (keySizes basesz fast) 	rs <- catMaybes <$> mapM (adjustChunkSize r) (chunkSizes basesz fast)@@ -120,7 +120,7 @@ 	, check "storeKey when already present" store 	, present True 	, check "retrieveKeyFile" $ do-		lockContent k removeAnnex+		lockContentForRemoval k removeAnnex 		get 	, check "fsck downloaded object" fsck 	, check "retrieveKeyFile resume from 33%" $ do@@ -130,20 +130,20 @@ 			sz <- hFileSize h 			L.hGet h $ fromInteger $ sz `div` 3 		liftIO $ L.writeFile tmp partial-		lockContent k removeAnnex+		lockContentForRemoval k removeAnnex 		get 	, check "fsck downloaded object" fsck 	, check "retrieveKeyFile resume from 0" $ do 		tmp <- prepTmp k 		liftIO $ writeFile tmp ""-		lockContent k removeAnnex+		lockContentForRemoval k removeAnnex 		get 	, check "fsck downloaded object" fsck 	, check "retrieveKeyFile resume from end" $ do 		loc <- Annex.calcRepo (gitAnnexLocation k) 		tmp <- prepTmp k 		void $ liftIO $ copyFileExternal CopyAllMetaData loc tmp-		lockContent k removeAnnex+		lockContentForRemoval k removeAnnex 		get 	, check "fsck downloaded object" fsck 	, check "removeKey when present" remove@@ -156,10 +156,10 @@ 		(== Right b) <$> Remote.hasKey r k 	fsck = case maybeLookupBackendName (keyBackendName k) of 		Nothing -> return True-		Just b -> case fsckKey b of+		Just b -> case verifyKeyContent b of 			Nothing -> return True-			Just fscker -> fscker k (key2file k)-	get = getViaTmp k $ \dest ->+			Just verifier -> verifier k (key2file k)+	get = getViaTmp (RemoteVerify r) k $ \dest -> 		Remote.retrieveKeyFile r k Nothing dest nullMeterUpdate 	store = Remote.storeKey r k Nothing nullMeterUpdate 	remove = Remote.removeKey r k@@ -173,10 +173,10 @@ 	, check (`notElem` [Right True, Right False]) "checkPresent" $ 		Remote.checkPresent r k 	, check (== Right False) "retrieveKeyFile" $-		getViaTmp k $ \dest ->+		getViaTmp (RemoteVerify r) k $ \dest -> 			Remote.retrieveKeyFile r k Nothing dest nullMeterUpdate 	, check (== Right False) "retrieveKeyFileCheap" $-		getViaTmp k $ \dest ->+		getViaTmp (RemoteVerify r) k $ \dest -> unVerified $ 			Remote.retrieveKeyFileCheap r k Nothing dest 	]   where@@ -189,7 +189,7 @@ cleanup :: [Remote] -> [Key] -> Bool -> CommandCleanup cleanup rs ks ok = do 	forM_ rs $ \r -> forM_ ks (Remote.removeKey r)-	forM_ ks $ \k -> lockContent k removeAnnex+	forM_ ks $ \k -> lockContentForRemoval k removeAnnex 	return ok  chunkSizes :: Int -> Bool -> [Int]
Command/TransferKey.hs view
@@ -61,7 +61,8 @@ fromPerform :: Key -> AssociatedFile -> Remote -> CommandPerform fromPerform key file remote = go Upload file $ 	download (uuid remote) key file forwardRetry noObserver $ \p ->-		getViaTmp key $ \t -> Remote.retrieveKeyFile remote key file t p+		getViaTmp (RemoteVerify remote) key $ +			\t -> Remote.retrieveKeyFile remote key file t p  go :: Direction -> AssociatedFile -> (NotifyWitness -> Annex Bool) -> CommandPerform go direction file a = notifyTransfer direction file a >>= liftIO . exitBool
Command/TransferKeys.hs view
@@ -43,7 +43,8 @@ 				return ok 		| otherwise = notifyTransfer direction file $ 			download (Remote.uuid remote) key file forwardRetry observer $ \p ->-				getViaTmp key $ \t -> Remote.retrieveKeyFile remote key file t p+				getViaTmp (RemoteVerify remote) key $ \t -> +					Remote.retrieveKeyFile remote key file t p 	 	observer False t tinfo = recordFailedTransfer t tinfo 	observer True _ _ = noop
Command/Uninit.hs view
@@ -105,7 +105,7 @@ 	go c [] = return c 	go c (k:ks) = ifM (inAnnexCheck k $ liftIO . enoughlinks) 		( do-			lockContent k removeAnnex+			lockContentForRemoval k removeAnnex 			go c ks 		, go (k:c) ks 		)
Command/View.hs view
@@ -39,7 +39,7 @@  perform :: View -> CommandPerform perform view = do-	showSideAction "searching"+	showAction "searching" 	next $ checkoutViewBranch view applyView  paramView :: String
Database/Handle.hs view
@@ -137,7 +137,8 @@ 	res <- newEmptyMVar 	putMVar jobs $ QueryJob $ 		liftIO . putMVar res =<< tryNonAsync a-	either throwIO return =<< takeMVar res+	(either throwIO return =<< takeMVar res)+		`catchNonAsync` (const $ error "sqlite query crashed")  closeDb :: DbHandle -> IO () closeDb h@(DbHandle worker jobs _) = do
Git/FileMode.hs view
@@ -11,8 +11,6 @@  import Utility.FileMode -import System.PosixCompat.Types- symLinkMode :: FileMode symLinkMode = 40960
Logs/Location.hs view
@@ -48,7 +48,7 @@ logChange' :: (LogStatus -> String -> Annex LogLine) -> Key -> UUID -> LogStatus -> Annex () logChange' mklog key (UUID u) s = do 	config <- Annex.getGitConfig-	addLog (locationLogFile config key) =<< mklog s u+	maybeAddLog (locationLogFile config key) =<< mklog s u logChange' _ _ NoUUID _ = noop  {- Returns a list of repository UUIDs that, according to the log, have
Logs/Presence.hs view
@@ -4,7 +4,7 @@  - a way that can be union merged.  -  - A line of the log will look like: "date N INFO"- - Where N=1 when the INFO is present, and 0 otherwise.+ - Where N=1 when the INFO is present, 0 otherwise.  -   - Copyright 2010-2014 Joey Hess <id@joeyh.name>  -@@ -14,6 +14,7 @@ module Logs.Presence ( 	module X, 	addLog,+	maybeAddLog, 	readLog, 	logNow, 	currentLog,@@ -28,9 +29,20 @@ import qualified Annex.Branch import Git.Types (RefDate) +{- Adds a LogLine to the log, removing any LogLines that are obsoleted by+ - adding it. -} addLog :: FilePath -> LogLine -> Annex ()-addLog file line = Annex.Branch.change file $ \s -> +addLog file line = Annex.Branch.change file $ \s -> 	showLog $ compactLog (line : parseLog s)++{- When a LogLine already exists with the same status and info, but an+ - older timestamp, that LogLine is preserved, rather than updating the log+ - with a newer timestamp.+ -}+maybeAddLog :: FilePath -> LogLine -> Annex ()+maybeAddLog file line = Annex.Branch.maybeChange file $ \s -> do+	m <- insertNewStatus line $ logMap $ parseLog s+	return $ showLog $ mapLog m  {- Reads a log file.  - Note that the LogLines returned may be in any order. -}
Logs/Presence/Pure.hs view
@@ -61,20 +61,38 @@ {- Compacts a set of logs, returning a subset that contains the current  - status. -} compactLog :: [LogLine] -> [LogLine]-compactLog = M.elems . foldr mapLog M.empty+compactLog = mapLog . logMap  type LogMap = M.Map String LogLine -{- Inserts a log into a map of logs, if the log has better (ie, newer)- - information than the other logs in the map -}-mapLog :: LogLine -> LogMap -> LogMap-mapLog l m-	| better = M.insert i l m-	| otherwise = m+mapLog :: LogMap -> [LogLine]+mapLog = M.elems++logMap :: [LogLine] -> LogMap+logMap = foldr insertNewerLogLine M.empty++insertBetter :: (LogLine -> Bool) -> LogLine -> LogMap -> Maybe LogMap+insertBetter betterthan l m+	| better = Just (M.insert i l m)+	| otherwise = Nothing   where-	better = maybe True newer $ M.lookup i m-	newer l' = date l' <= date l+	better = maybe True betterthan (M.lookup i m) 	i = info l++{- Inserts a log into a map of logs, if the log has newer+ - information than the other logs in the map for the same info. -}+insertNewerLogLine :: LogLine -> LogMap -> LogMap+insertNewerLogLine l m = fromMaybe m $ insertBetter newer l m+  where+	newer l' = date l' <= date l++{- Inserts the log unless there's already one in the map with+ - the same status for its info, in which case there's no need to+ - change anything, to avoid log churn. -}+insertNewStatus :: LogLine -> LogMap -> Maybe LogMap+insertNewStatus l m  = insertBetter diffstatus l m+  where+	diffstatus l' = status l' /= status l  instance Arbitrary LogLine where 	arbitrary = LogLine
Remote.hs view
@@ -40,7 +40,7 @@ 	remotesWithoutUUID, 	keyLocations, 	keyPossibilities,-	keyPossibilitiesTrusted,+	remoteLocations, 	nameToUUID, 	nameToUUID', 	showTriedRemotes,@@ -142,7 +142,7 @@ 	matching r = n == name r  noRemoteUUIDMsg :: Remote -> String-noRemoteUUIDMsg r = "cannot determine uuid for " ++ name r+noRemoteUUIDMsg r = "cannot determine uuid for " ++ name r ++ " (perhaps you need to run \"git annex sync\"?)"  {- Looks up a remote by name (or by UUID, or even by description),  - and returns its UUID. Finds even repositories that are not@@ -260,33 +260,26 @@  - may have a key.  -} keyPossibilities :: Key -> Annex [Remote]-keyPossibilities key = fst <$> keyPossibilities' key []--{- Cost ordered lists of remotes that the location log indicates- - may have a key.- -- - Also returns a list of UUIDs that are trusted to have the key- - (some may not have configured remotes).- -}-keyPossibilitiesTrusted :: Key -> Annex ([Remote], [UUID])-keyPossibilitiesTrusted key = keyPossibilities' key =<< trustGet Trusted--keyPossibilities' :: Key -> [UUID] -> Annex ([Remote], [UUID])-keyPossibilities' key trusted = do+keyPossibilities key = do 	u <- getUUID- 	-- uuids of all remotes that are recorded to have the key-	validuuids <- filter (/= u) <$> keyLocations key+	locations <- filter (/= u) <$> keyLocations key+	fst <$> remoteLocations locations [] -	-- note that validuuids is assumed to not have dups-	let validtrusteduuids = validuuids `intersect` trusted+{- Given a list of locations of a key, and a list of all+ - trusted repositories, generates a cost-ordered list of+ - remotes that contain the key, and a list of trusted locations of the key.+ -}+remoteLocations :: [UUID] -> [UUID] -> Annex ([Remote], [UUID])+remoteLocations locations trusted = do+	let validtrustedlocations = nub locations `intersect` trusted  	-- remotes that match uuids that have the key 	allremotes <- filter (not . remoteAnnexIgnore . gitconfig) 		<$> remoteList-	let validremotes = remotesWithUUID allremotes validuuids+	let validremotes = remotesWithUUID allremotes locations -	return (sortBy (comparing cost) validremotes, validtrusteduuids)+	return (sortBy (comparing cost) validremotes, validtrustedlocations)  {- Displays known locations of a key. -} showLocations :: Bool -> Key -> [UUID] -> String -> Annex ()
Remote/BitTorrent.hs view
@@ -58,6 +58,7 @@ 		, retrieveKeyFile = downloadKey 		, retrieveKeyFileCheap = downloadKeyCheap 		, removeKey = dropKey+		, lockContent = Nothing 		, checkPresent = checkKey 		, checkPresentCheap = False 		, whereisKey = Nothing@@ -76,8 +77,8 @@ 		, checkUrl = Just checkTorrentUrl 		} -downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool-downloadKey key _file dest p = +downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)+downloadKey key _file dest p = unVerified $ 	get . map (torrentUrlNum . fst . getDownloader) =<< getBitTorrentUrls key   where 	get [] = do
Remote/Bup.hs view
@@ -58,6 +58,7 @@ 		, retrieveKeyFile = retreiveKeyFileDummy 		, retrieveKeyFileCheap = retrieveCheap buprepo 		, removeKey = removeKeyDummy+		, lockContent = Nothing 		, checkPresent = checkPresentDummy 		, checkPresentCheap = bupLocal buprepo 		, whereisKey = Nothing
Remote/Ddar.hs view
@@ -57,6 +57,7 @@ 		, retrieveKeyFile = retreiveKeyFileDummy 		, retrieveKeyFileCheap = retrieveCheap 		, removeKey = removeKeyDummy+		, lockContent = Nothing 		, checkPresent = checkPresentDummy 		, checkPresentCheap = ddarLocal ddarrepo 		, whereisKey = Nothing
Remote/Directory.hs view
@@ -55,6 +55,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap dir chunkconfig 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = True 			, whereisKey = Nothing
Remote/External.hs view
@@ -81,6 +81,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = \_ _ _ -> return False 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = towhereis
Remote/GCrypt.hs view
@@ -111,6 +111,7 @@ 		, retrieveKeyFile = retreiveKeyFileDummy 		, retrieveKeyFileCheap = \_ _ _ -> return False 		, removeKey = removeKeyDummy+		, lockContent = Nothing 		, checkPresent = checkPresentDummy 		, checkPresentCheap = repoCheap r 		, whereisKey = Nothing
Remote/Git.hs view
@@ -53,9 +53,11 @@ import Creds import Annex.CatFile import Messages.Progress+import Types.NumCopies  import Control.Concurrent import Control.Concurrent.MSampleVar+import Control.Concurrent.Async import qualified Data.Map as M import Network.URI @@ -142,6 +144,7 @@ 			, retrieveKeyFile = copyFromRemote new 			, retrieveKeyFileCheap = copyFromRemoteCheap new 			, removeKey = dropKey new+			, lockContent = Just (lockKey new) 			, checkPresent = inAnnex new 			, checkPresentCheap = repoCheap r 			, whereisKey = Nothing@@ -218,7 +221,7 @@ 		v <- Git.Config.fromPipe r cmd params 		case v of 			Right (r', val) -> do-				when (getUncachedUUID r' == NoUUID && not (null val)) $ do+				unless (isUUIDConfigured r' || null val) $ do 					warningIO $ "Failed to get annex.uuid configuration of repository " ++ Git.repoDescribe r 					warningIO $ "Instead, got: " ++ show val 					warningIO $ "This is unexpected; please check the network transport!"@@ -350,7 +353,7 @@ 			commitOnCleanup r $ onLocal r $ do 				ensureInitialized 				whenM (Annex.Content.inAnnex key) $ do-					Annex.Content.lockContent key+					Annex.Content.lockContentForRemoval key 						Annex.Content.removeAnnex 					logStatus key InfoMissing 					Annex.Content.saveState True@@ -358,14 +361,72 @@ 	| Git.repoIsHttp (repo r) = error "dropping from http remote not supported" 	| otherwise = commitOnCleanup r $ Ssh.dropKey (repo r) key +lockKey :: Remote -> Key -> (VerifiedCopy -> Annex r) -> Annex r+lockKey r key callback+	| not $ Git.repoIsUrl (repo r) =+		guardUsable (repo r) failedlock $ do+			inorigrepo <- Annex.makeRunner+			-- Lock content from perspective of remote,+			-- and then run the callback in the original+			-- annex monad, not the remote's.+			onLocal r $ +				Annex.Content.lockContentShared key $ \vc ->+					ifM (Annex.Content.inAnnex key)+						( liftIO $ inorigrepo $ callback vc+						, failedlock+						)+	| Git.repoIsSsh (repo r) = do+		showLocking r+		Just (cmd, params) <- Ssh.git_annex_shell (repo r) "lockcontent"+			[Param $ key2file key] []+		(Just hin, Just hout, Nothing, p) <- liftIO $ +			withFile devNull WriteMode $ \nullh ->+				createProcess $+					 (proc cmd (toCommand params))+						{ std_in = CreatePipe+						, std_out = CreatePipe+						, std_err = UseHandle nullh+						}+		-- Wait for either the process to exit, or for it to+		-- indicate the content is locked.+		v <- liftIO $ race +			(waitForProcess p)+			(hGetLine hout)+		let signaldone = void $ tryNonAsync $ liftIO $ do+			hPutStrLn hout ""+			hFlush hout+			hClose hin+			hClose hout+			void $ waitForProcess p+		let checkexited = not . isJust <$> getProcessExitCode p+		case v of+			Left _exited -> do+				showNote "lockcontent failed"+				liftIO $ do+					hClose hin+					hClose hout+				failedlock+			Right l +				| l == Ssh.contentLockedMarker -> bracket_+					noop+					signaldone +					(withVerifiedCopy LockedCopy r checkexited callback)+				| otherwise -> do+					showNote "lockcontent failed"+					signaldone+					failedlock+	| otherwise = failedlock+  where+	failedlock = error "can't lock content"+ {- Tries to copy a key's content from a remote's annex to a file. -}-copyFromRemote :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool+copyFromRemote :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) copyFromRemote r key file dest p = parallelMetered (Just p) key file $ 	copyFromRemote' r key file dest -copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool+copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) copyFromRemote' r key file dest meterupdate-	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) (return False) $ do+	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) (unVerified (return False)) $ do 		params <- Ssh.rsyncParams r Download 		u <- getUUID 		hardlink <- wantHardLink@@ -374,17 +435,17 @@ 			ensureInitialized 			v <- Annex.Content.prepSendAnnex key 			case v of-				Nothing -> return False+				Nothing -> return (False, UnVerified) 				Just (object, checksuccess) -> do-					copier <- mkCopier hardlink params object dest+					copier <- mkCopier hardlink params 					runTransfer (Transfer Download u key)-						file noRetry noObserver copier-						<&&> checksuccess-	| Git.repoIsSsh (repo r) = feedprogressback $ \feeder -> do+						file noRetry noObserver +						(\p -> copier object dest p checksuccess)+	| Git.repoIsSsh (repo r) = unVerified $ feedprogressback $ \feeder -> do 		direct <- isDirect 		Ssh.rsyncHelper (Just feeder)  			=<< Ssh.rsyncParamsRemote direct r Download key dest file-	| Git.repoIsHttp (repo r) = Annex.Content.downloadUrl (keyUrls r key) dest+	| Git.repoIsHttp (repo r) = unVerified $ Annex.Content.downloadUrl (keyUrls r key) dest 	| otherwise = error "copying from non-ssh, non-http remote not supported"   where 	{- Feed local rsync's progress info back to the remote,@@ -461,8 +522,8 @@ 			) 	| Git.repoIsSsh (repo r) = 		ifM (Annex.Content.preseedTmp key file)-			( parallelMetered Nothing key af $-				copyFromRemote' r key af file+			( fst <$> parallelMetered Nothing key af+				(copyFromRemote' r key af file) 			, return False 			) 	| otherwise = return False@@ -500,10 +561,12 @@ 			( return True 			, do 				ensureInitialized+				copier <- mkCopier hardlink params+				let verify = Annex.Content.RemoteVerify r 				runTransfer (Transfer Download u key) file noRetry noObserver $ const $ 					Annex.Content.saveState True `after`-						Annex.Content.getViaTmpChecked (liftIO checksuccessio) key-							(\dest -> mkCopier hardlink params object dest >>= \a -> a p)+						Annex.Content.getViaTmp verify key+							(\dest -> copier object dest p (liftIO checksuccessio)) 			)  fsckOnRemote :: Git.Repo -> [CommandParam] -> Annex (IO Bool)@@ -615,19 +678,34 @@ wantHardLink :: Annex Bool wantHardLink = (annexHardLink <$> Annex.getGitConfig) <&&> (not <$> isDirect) +-- Copies from src to dest, updating a meter. If the copy finishes+-- successfully, calls a final check action, which must also success, or+-- returns false.+-- -- If either the remote or local repository wants to use hard links,--- the copier will do so, falling back to copying.-mkCopier :: Bool -> [CommandParam] -> FilePath -> FilePath -> Annex (MeterUpdate -> Annex Bool)-mkCopier remotewanthardlink rsyncparams object dest = do-	let copier = rsyncOrCopyFile rsyncparams object dest+-- the copier will do so (falling back to copying if a hard link cannot be+-- made).+--+-- When a hard link is created, returns Verified; the repo being linked+-- from is implicitly trusted, so no expensive verification needs to be+-- done.+type Copier = FilePath -> FilePath -> MeterUpdate -> Annex Bool -> Annex (Bool, Verification)++mkCopier :: Bool -> [CommandParam] -> Annex Copier+mkCopier remotewanthardlink rsyncparams = do+	let copier = \src dest p check -> unVerified $+		rsyncOrCopyFile rsyncparams src dest p <&&> check #ifndef mingw32_HOST_OS 	localwanthardlink <- wantHardLink-	let linker = createLink object dest >> return True+	let linker = \src dest -> createLink src dest >> return True 	ifM (pure (remotewanthardlink || localwanthardlink) <&&> not <$> isDirect)-		( return $ \m -> liftIO (catchBoolIO linker)-			<||> copier m+		( return $ \src dest p check ->+			ifM (liftIO (catchBoolIO (linker src dest)))+				( return (True, Verified)+				, copier src dest p check+				) 		, return copier 		) #else-	return copier+	return $ if remotewanthardlink then copier else copier #endif
Remote/Glacier.hs view
@@ -55,6 +55,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap this 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = Nothing
Remote/Helper/Messages.hs view
@@ -13,20 +13,23 @@ import qualified Git import qualified Types.Remote as Remote -class Checkable a where-	descCheckable :: a -> String+class Describable a where+	describe :: a -> String -instance Checkable Git.Repo where-	descCheckable = Git.repoDescribe+instance Describable Git.Repo where+	describe = Git.repoDescribe -instance Checkable (Remote.RemoteA a) where-	descCheckable = Remote.name+instance Describable (Remote.RemoteA a) where+	describe = Remote.name -instance Checkable String where-	descCheckable = id+instance Describable String where+	describe = id -showChecking :: Checkable a => a -> Annex ()-showChecking v = showAction $ "checking " ++ descCheckable v+showChecking :: Describable a => a -> Annex ()+showChecking v = showAction $ "checking " ++ describe v -cantCheck :: Checkable a => a -> e-cantCheck v = error $ "unable to check " ++ descCheckable v+cantCheck :: Describable a => a -> e+cantCheck v = error $ "unable to check " ++ describe v++showLocking :: Describable a => a -> Annex ()+showLocking v = showAction $ "locking " ++ describe v
Remote/Helper/Special.hs view
@@ -122,8 +122,8 @@  -} storeKeyDummy :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool storeKeyDummy _ _ _ = return False-retreiveKeyFileDummy :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool-retreiveKeyFileDummy _ _ _ _ = return False+retreiveKeyFileDummy :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)+retreiveKeyFileDummy _ _ _ _ = unVerified (return False) removeKeyDummy :: Key -> Annex Bool removeKeyDummy _ = return False checkPresentDummy :: Key -> Annex Bool@@ -156,7 +156,7 @@   where 	encr = baser 		{ storeKey = \k f p -> cip >>= storeKeyGen k f p-		, retrieveKeyFile = \k f d p -> cip >>= retrieveKeyFileGen k f d p+		, retrieveKeyFile = \k f d p -> cip >>= unVerified . retrieveKeyFileGen k f d p 		, retrieveKeyFileCheap = \k f d -> cip >>= maybe 			(retrieveKeyFileCheap baser k f d) 			-- retrieval of encrypted keys is never cheap
Remote/Helper/Ssh.hs view
@@ -173,3 +173,8 @@ 		| direction == Download = remoteAnnexRsyncDownloadOptions gc 		| otherwise = remoteAnnexRsyncUploadOptions gc 	gc = gitconfig r++-- Used by git-annex-shell lockcontent to indicate the content is+-- successfully locked.+contentLockedMarker :: String+contentLockedMarker = "OK"
Remote/Hook.hs view
@@ -49,6 +49,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap hooktype 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = Nothing
Remote/Rsync.hs view
@@ -70,6 +70,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap o 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = Nothing
Remote/S3.hs view
@@ -21,8 +21,8 @@ import qualified Data.Map as M import Data.Char import Network.Socket (HostName)-import Network.HTTP.Conduit (Manager, newManager, closeManager)-import Network.HTTP.Client (defaultManagerSettings, managerResponseTimeout, responseStatus, responseBody, RequestBody(..))+import Network.HTTP.Conduit (Manager, newManager)+import Network.HTTP.Client (managerResponseTimeout, responseStatus, responseBody, RequestBody(..)) import Network.HTTP.Types import Control.Monad.Trans.Resource import Control.Monad.Catch@@ -48,7 +48,7 @@ import Utility.DataUnits import Annex.Content import Annex.Url (withUrlOptions)-import Utility.Url (checkBoth)+import Utility.Url (checkBoth, managerSettings, closeManager)  type BucketName = String @@ -81,6 +81,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = Just (getWebUrls info)@@ -417,7 +418,7 @@ 		Nothing -> a Nothing   where 	s3cfg = s3Configuration c-	httpcfg = defaultManagerSettings+	httpcfg = managerSettings 		{ managerResponseTimeout = Nothing }  s3Configuration :: RemoteConfig -> S3.S3Configuration AWS.NormalQuery
Remote/Tahoe.hs view
@@ -72,6 +72,7 @@ 		, retrieveKeyFile = retrieve u hdl 		, retrieveKeyFileCheap = \_ _ _ -> return False 		, removeKey = remove+		, lockContent = Nothing 		, checkPresent = checkKey u hdl 		, checkPresentCheap = False 		, whereisKey = Nothing@@ -116,8 +117,8 @@ 		(return False) 		(\cap -> storeCapability u k cap >> return True) -retrieve :: UUID -> TahoeHandle -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool-retrieve u hdl k _f d _p = go =<< getCapability u k+retrieve :: UUID -> TahoeHandle -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)+retrieve u hdl k _f d _p = unVerified $ go =<< getCapability u k   where 	go Nothing = return False 	go (Just cap) = liftIO $ requestTahoe hdl "get" [Param cap, File d]
Remote/Web.hs view
@@ -52,6 +52,7 @@ 		, retrieveKeyFile = downloadKey 		, retrieveKeyFileCheap = downloadKeyCheap 		, removeKey = dropKey+		, lockContent = Nothing 		, checkPresent = checkKey 		, checkPresentCheap = False 		, whereisKey = Nothing@@ -70,8 +71,8 @@ 		, checkUrl = Nothing 		} -downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool-downloadKey key _file dest _p = get =<< getWebUrls key+downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)+downloadKey key _file dest _p = unVerified $ get =<< getWebUrls key   where 	get [] = do 		warning "no known url"
Remote/WebDAV.hs view
@@ -60,6 +60,7 @@ 			, retrieveKeyFile = retreiveKeyFileDummy 			, retrieveKeyFileCheap = retrieveCheap 			, removeKey = removeKeyDummy+			, lockContent = Nothing 			, checkPresent = checkPresentDummy 			, checkPresentCheap = False 			, whereisKey = Nothing@@ -184,7 +185,7 @@  -} testDav :: URLString -> Maybe CredPair -> Annex () testDav url (Just (u, p)) = do-	showSideAction "testing WebDAV server"+	showAction "testing WebDAV server" 	test $ liftIO $ evalDAVT url $ do 		prepDAV user pass 		makeParentDirs
Test.hs view
@@ -487,15 +487,20 @@ test_preferred_content :: Assertion test_preferred_content = intmpclonerepo $ do 	annexed_notpresent annexedfile-	-- get --auto only looks at numcopies when preferred content is not+	-- get/copy --auto looks only at numcopies when preferred content is not 	-- set, and with 1 copy existing, does not get the file. 	git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed with default preferred content" 	annexed_notpresent annexedfile+	git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed with default preferred content"+	annexed_notpresent annexedfile  	git_annex "wanted" [".", "standard"] @? "set expression to standard failed" 	git_annex "group" [".", "client"] @? "set group to standard failed" 	git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed for client" 	annexed_present annexedfile+	git_annex "drop" [annexedfile] @? "drop of file failed"+	git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed for client"+	annexed_present annexedfile 	git_annex "ungroup" [".", "client"] @? "ungroup failed"  	git_annex "wanted" [".", "standard"] @? "set expression to standard failed"@@ -505,8 +510,10 @@ 	annexed_present annexedfile 	git_annex "drop" [annexedfile] @? "drop of file failed" 	annexed_notpresent annexedfile-	-- get --auto with manual does not get the file+	-- copy/get --auto with manual does not get the file 	git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed with manual preferred content"+	annexed_notpresent annexedfile+	git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed with manual preferred content" 	annexed_notpresent annexedfile 	git_annex "ungroup" [".", "client"] @? "ungroup failed"
Types/Backend.hs view
@@ -15,8 +15,8 @@ data BackendA a = Backend 	{ name :: String 	, getKey :: KeySource -> a (Maybe Key) -	-- Checks the content of a key.-	, fsckKey :: Maybe (Key -> FilePath -> a Bool)+	-- Verifies the content of a key.+	, verifyKeyContent :: Maybe (Key -> FilePath -> a Bool) 	-- Checks if a key can be upgraded to a better form. 	, canUpgradeKey :: Maybe (Key -> Bool) 	-- Checks if there is a fast way to migrate a key to a different
Types/GitConfig.hs view
@@ -61,6 +61,7 @@ 	, annexHardLink :: Bool 	, annexDifferences :: Differences 	, annexUsedRefSpec :: Maybe RefSpec+	, annexVerify :: Bool 	, coreSymlinks :: Bool 	, coreSharedRepository :: SharedRepository 	, gcryptId :: Maybe String@@ -103,6 +104,7 @@ 	, annexDifferences = getDifferences r 	, annexUsedRefSpec = either (const Nothing) Just . parseRefSpec  		=<< getmaybe (annex "used-refspec")+	, annexVerify = getbool (annex "verify") True 	, coreSymlinks = getbool "core.symlinks" True 	, coreSharedRepository = getSharedRepository r 	, gcryptId = getmaybe "core.gcrypt-id"@@ -129,6 +131,7 @@ 	, remoteAnnexIgnore :: Bool 	, remoteAnnexSync :: Bool 	, remoteAnnexReadOnly :: Bool+	, remoteAnnexVerify :: Bool 	, remoteAnnexTrustLevel :: Maybe String 	, remoteAnnexStartCommand :: Maybe String 	, remoteAnnexStopCommand :: Maybe String@@ -164,6 +167,7 @@ 	, remoteAnnexIgnore = getbool "ignore" False 	, remoteAnnexSync = getbool "sync" True 	, remoteAnnexReadOnly = getbool "readonly" False+	, remoteAnnexVerify = getbool "verify" True 	, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" 	, remoteAnnexStartCommand = notempty $ getmaybe "start-command" 	, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
Types/NumCopies.hs view
@@ -1,14 +1,178 @@-{- git-annex numcopies type+{- git-annex numcopies types  -- - Copyright 2014 Joey Hess <id@joeyh.name>+ - Copyright 2014-2015 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU GPL version 3 or higher.  -} -module Types.NumCopies where+module Types.NumCopies (+	NumCopies(..),+	fromNumCopies,+	VerifiedCopy(..),+	checkVerifiedCopy,+	invalidateVerifiedCopy,+	strongestVerifiedCopy,+	deDupVerifiedCopies,+	mkVerifiedCopy,+	invalidatableVerifiedCopy,+	withVerifiedCopy,+	isSafeDrop,+	SafeDropProof,+	mkSafeDropProof,+	ContentRemovalLock(..),+) where +import Types.UUID+import Types.Key+import Utility.Exception (bracketIO)+import Utility.Monad++import qualified Data.Map as M+import Control.Concurrent.MVar+import Control.Monad.Catch (MonadMask)+import Control.Monad.IO.Class (MonadIO)+import Control.Monad+ newtype NumCopies = NumCopies Int-	deriving (Ord, Eq)+	deriving (Ord, Eq, Show)  fromNumCopies :: NumCopies -> Int fromNumCopies (NumCopies n) = n++-- Indicates that a key's content is exclusively+-- locked locally, pending removal.+newtype ContentRemovalLock = ContentRemovalLock Key+	deriving (Show)++-- A verification that a copy of a key exists in a repository.+data VerifiedCopy+	{- Represents a recent verification that a copy of an+	 - object exists in a repository with the given UUID. -}+	= RecentlyVerifiedCopy V+	{- Use when a repository cannot be accessed, but it's+	 - a trusted repository, which is on record as containing a key+	 - and is presumably not going to lose its copy. -}+	| TrustedCopy V+ 	{- The strongest proof of the existence of a copy.+	 - Until its associated action is called to unlock it,+	 - the copy is locked in the repository and is guaranteed+	 - not to be removed by any git-annex process. -}+	| LockedCopy V+	deriving (Show)++data V = V+	{ _getUUID :: UUID+	, _checkVerifiedCopy :: IO Bool+	, _invalidateVerifiedCopy :: IO ()+	}++instance Show V where+	show v = show (_getUUID v)++instance ToUUID VerifiedCopy where+	toUUID = _getUUID . toV+	+toV :: VerifiedCopy -> V+toV (TrustedCopy v) = v+toV (RecentlyVerifiedCopy v) = v+toV (LockedCopy v) = v++-- Checks that it's still valid.+checkVerifiedCopy :: VerifiedCopy -> IO Bool+checkVerifiedCopy = _checkVerifiedCopy . toV++invalidateVerifiedCopy :: VerifiedCopy -> IO ()+invalidateVerifiedCopy = _invalidateVerifiedCopy . toV++strongestVerifiedCopy :: VerifiedCopy -> VerifiedCopy -> VerifiedCopy+strongestVerifiedCopy a@(LockedCopy _) _ = a+strongestVerifiedCopy _ b@(LockedCopy _) = b+strongestVerifiedCopy a@(TrustedCopy _) _ = a+strongestVerifiedCopy _ b@(TrustedCopy _) = b+strongestVerifiedCopy a@(RecentlyVerifiedCopy _) _ = a++-- Retains stronger verifications over weaker for the same uuid.+deDupVerifiedCopies :: [VerifiedCopy] -> [VerifiedCopy]+deDupVerifiedCopies l = M.elems $+	M.fromListWith strongestVerifiedCopy (zip (map toUUID l) l)++mkVerifiedCopy :: ToUUID u => (V -> VerifiedCopy) -> u -> VerifiedCopy+mkVerifiedCopy mk u = mk $ V (toUUID u) (return True) (return ())++invalidatableVerifiedCopy :: ToUUID u => (V -> VerifiedCopy) -> u -> IO Bool -> IO VerifiedCopy+invalidatableVerifiedCopy mk u check = do+	v <- newEmptyMVar+	let invalidate = do+		_ <- tryPutMVar v ()+		return ()+	let check' = isEmptyMVar v <&&> check+	return $ mk $ V (toUUID u) check' invalidate++-- Constructs a VerifiedCopy, and runs the action, ensuring that the+-- verified copy is invalidated when the action returns, or on error.+withVerifiedCopy +	:: (Monad m, MonadMask m, MonadIO m, ToUUID u)+	=> (V -> VerifiedCopy)+	-> u+	-> IO Bool+	-> (VerifiedCopy -> m a)+	-> m a+withVerifiedCopy mk u check = bracketIO setup cleanup+  where+	setup = invalidatableVerifiedCopy mk u check+	cleanup = invalidateVerifiedCopy++{- Check whether enough verification has been done of copies to allow+ - dropping content safely.+ -+ - This is carefully balanced to prevent data loss when there are races+ - between concurrent drops of the same content in different repos,+ - without requiring impractical amounts of locking.+ -+ - In particular, concurrent drop races may cause the number of copies+ - to fall below NumCopies, but it will never fall below 1.+ -}+isSafeDrop :: NumCopies -> [VerifiedCopy] -> Maybe ContentRemovalLock -> Bool+{- When a ContentRemovalLock is provided, the content is being+ - dropped from the local repo. That lock will prevent other git repos+ - that are concurrently dropping from using the local copy as a VerifiedCopy.+ - So, no additional locking is needed; all we need is verifications+ - of any kind of N other copies of the content. -}+isSafeDrop (NumCopies n) l (Just (ContentRemovalLock _)) = +	length (deDupVerifiedCopies l) >= n+{- Dropping from a remote repo.+ -+ - Unless numcopies is 0, at least one LockedCopy or TrustedCopy is required.+ - A LockedCopy prevents races between concurrent drops from+ - dropping the last copy, no matter what.+ -+ - The other N-1 copies can be less strong verifications, like+ - RecentlyVerifiedCopy. While those are subject to concurrent drop races,+ - and so could be dropped all at once, causing numcopies to be violated,+ - this is the best that can be done without requiring that + - all special remotes support locking.+ -}+isSafeDrop (NumCopies n) l Nothing+	| n == 0 = True+	| otherwise = and+		[ length (deDupVerifiedCopies l) >= n+		, any fullVerification l+		]++fullVerification :: VerifiedCopy -> Bool+fullVerification (LockedCopy _) = True+fullVerification (TrustedCopy _) = True+fullVerification (RecentlyVerifiedCopy _) = False++-- A proof that it's currently safe to drop an object.+data SafeDropProof = SafeDropProof NumCopies [VerifiedCopy] (Maybe ContentRemovalLock)+	deriving (Show)++-- Make sure that none of the VerifiedCopies have become invalidated+-- before constructing proof.+mkSafeDropProof :: NumCopies -> [VerifiedCopy] -> Maybe ContentRemovalLock -> IO (Either [VerifiedCopy] SafeDropProof)+mkSafeDropProof need have removallock = do+	stillhave <- filterM checkVerifiedCopy have+	return $ if isSafeDrop need stillhave removallock+		then Right (SafeDropProof need stillhave removallock)+		else Left stillhave
Types/Remote.hs view
@@ -7,12 +7,16 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE RankNTypes #-}+ module Types.Remote 	( RemoteConfigKey 	, RemoteConfig 	, RemoteTypeA(..) 	, RemoteA(..) 	, Availability(..)+	, Verification(..)+	, unVerified 	) 	where @@ -26,6 +30,7 @@ import Types.Availability import Types.Creds import Types.UrlContents+import Types.NumCopies import Config.Cost import Utility.Metered import Git.Types@@ -64,14 +69,20 @@ 	-- all of its contents have been transferred. 	storeKey :: Key -> AssociatedFile -> MeterUpdate -> a Bool, 	-- Retrieves a key's contents to a file.-	-- (The MeterUpdate does not need to be used if it retrieves-	-- directly to the file, and not to an intermediate file.)-	retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> a Bool,+	-- (The MeterUpdate does not need to be used if it writes+	-- sequentially to the file.)+	retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> a (Bool, Verification), 	-- Retrieves a key's contents to a tmp file, if it can be done cheaply. 	-- It's ok to create a symlink or hardlink. 	retrieveKeyFileCheap :: Key -> AssociatedFile -> FilePath -> a Bool,-	-- removes a key's contents (succeeds if the contents are not present)+	-- Removes a key's contents (succeeds if the contents are not present) 	removeKey :: Key -> a Bool,+	-- Uses locking to prevent removal of a key's contents,+	-- thus producing a VerifiedCopy, which is passed to the callback.+	-- If unable to lock, does not run the callback, and throws an+	-- error.+	-- This is optional; remotes do not have to support locking.+	lockContent :: forall r. Maybe (Key -> (VerifiedCopy -> a r) -> a r), 	-- Checks if a key is present in the remote. 	-- Throws an exception if the remote cannot be accessed. 	checkPresent :: Key -> a Bool,@@ -122,3 +133,15 @@  instance Ord (RemoteA a) where 	compare = comparing uuid++instance ToUUID (RemoteA a) where+	toUUID = uuid++-- Use Verified when the content of a key is verified as part of a+-- transfer, and so a separate verification step is not needed.+data Verification = UnVerified | Verified++unVerified :: Monad m => m Bool -> m (Bool, Verification)+unVerified a = do+	ok <- a+	return (ok, UnVerified)
Types/UUID.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+ module Types.UUID where  import qualified Data.Map as M@@ -19,9 +21,15 @@ fromUUID (UUID u) = u fromUUID NoUUID = "" -toUUID :: String -> UUID-toUUID [] = NoUUID-toUUID s = UUID s+class ToUUID a where+	toUUID :: a -> UUID++instance ToUUID UUID where+	toUUID = id++instance ToUUID String where+	toUUID [] = NoUUID+	toUUID s = UUID s  isUUID :: String -> Bool isUUID = isJust . U.fromString
Utility/FileMode.hs view
@@ -7,7 +7,10 @@  {-# LANGUAGE CPP #-} -module Utility.FileMode where+module Utility.FileMode (+	module Utility.FileMode,+	FileMode,+) where  import System.IO import Control.Monad
Utility/LockFile/Posix.hs view
@@ -9,6 +9,7 @@ 	LockHandle, 	lockShared, 	lockExclusive,+	tryLockShared, 	tryLockExclusive, 	checkLocked, 	getLockStatus,@@ -36,31 +37,43 @@ lockExclusive :: Maybe FileMode -> LockFile -> IO LockHandle lockExclusive = lock WriteLock +-- Tries to take a shared lock, but does not block.+tryLockShared :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle)+tryLockShared = tryLock ReadLock+ -- Tries to take an exclusive lock, but does not block. tryLockExclusive :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle)-tryLockExclusive mode lockfile = do-	l <- openLockFile mode lockfile-	v <- tryIO $ setLock l (WriteLock, AbsoluteSeek, 0, 0)-	case v of-		Left _ -> do-			closeFd l-			return Nothing-		Right _ -> return $ Just $ LockHandle l+tryLockExclusive = tryLock WriteLock  -- Setting the FileMode allows creation of a new lock file. -- If it's Nothing then this only succeeds when the lock file already exists. lock :: LockRequest -> Maybe FileMode -> LockFile -> IO LockHandle lock lockreq mode lockfile = do-	l <- openLockFile mode lockfile+	l <- openLockFile lockreq mode lockfile 	waitToSetLock l (lockreq, AbsoluteSeek, 0, 0) 	return (LockHandle l) +-- Tries to take an lock, but does not block.+tryLock :: LockRequest -> Maybe FileMode -> LockFile -> IO (Maybe LockHandle)+tryLock lockreq mode lockfile = do+	l <- openLockFile lockreq mode lockfile+	v <- tryIO $ setLock l (lockreq, AbsoluteSeek, 0, 0)+	case v of+		Left _ -> do+			closeFd l+			return Nothing+		Right _ -> return $ Just $ LockHandle l+ -- Close on exec flag is set so child processes do not inherit the lock.-openLockFile :: Maybe FileMode -> LockFile -> IO Fd-openLockFile filemode lockfile = do-	l <- openFd lockfile ReadWrite filemode defaultFileFlags+openLockFile :: LockRequest -> Maybe FileMode -> LockFile -> IO Fd+openLockFile lockreq filemode lockfile = do+	l <- openFd lockfile openfor filemode defaultFileFlags 	setFdOption l CloseOnExec True 	return l+  where+	openfor = case lockreq of+		ReadLock -> ReadOnly+		_ -> ReadWrite  -- Returns Nothing when the file doesn't exist, for cases where -- that is different from it not being locked.@@ -81,7 +94,7 @@ getLockStatus' :: LockFile -> IO (Maybe (Maybe ProcessID)) getLockStatus' lockfile = go =<< catchMaybeIO open   where-	open = openFd lockfile ReadOnly Nothing defaultFileFlags+	open = openLockFile ReadLock Nothing lockfile 	go Nothing = return Nothing 	go (Just h) = do 		v <- getLock h (ReadLock, AbsoluteSeek, 0, 0)
Utility/LockPool/Posix.hs view
@@ -6,9 +6,11 @@  -}  module Utility.LockPool.Posix (+	P.LockFile, 	LockHandle, 	lockShared, 	lockExclusive,+	tryLockShared, 	tryLockExclusive, 	checkLocked, 	getLockStatus,@@ -35,11 +37,19 @@ 	(P.waitTakeLock P.lockPool file LockShared) 	(F.lockShared mode file) +-- Takes an exclusive lock, blocking until the lock is available. lockExclusive :: Maybe FileMode -> LockFile -> IO LockHandle lockExclusive mode file = makeLockHandle 	(P.waitTakeLock P.lockPool file LockExclusive) 	(F.lockExclusive mode file) +-- Tries to take a shared lock, but does not block.+tryLockShared :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle)+tryLockShared mode file = tryMakeLockHandle+	(P.tryTakeLock P.lockPool file LockShared)+	(F.tryLockShared mode file)++-- Tries to take an exclusive lock, but does not block. tryLockExclusive :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) tryLockExclusive mode file = tryMakeLockHandle 	(P.tryTakeLock P.lockPool file LockExclusive)
Utility/LockPool/Windows.hs view
@@ -6,6 +6,7 @@  -}  module Utility.LockPool.Windows (+	P.LockFile, 	LockHandle, 	lockShared, 	lockExclusive,
Utility/Process.hs view
@@ -172,22 +172,21 @@ -- returns a transcript combining its stdout and stderr, and -- whether it succeeded or failed. processTranscript :: String -> [String] -> (Maybe String) -> IO (String, Bool)-processTranscript cmd opts = processTranscript' cmd opts Nothing+processTranscript = processTranscript' id -processTranscript' :: String -> [String] -> Maybe [(String, String)] -> (Maybe String) -> IO (String, Bool)-processTranscript' cmd opts environ input = do+processTranscript' :: (CreateProcess -> CreateProcess) -> String -> [String] -> Maybe String -> IO (String, Bool)+processTranscript' modproc cmd opts input = do #ifndef mingw32_HOST_OS {- This implementation interleves stdout and stderr in exactly the order  - the process writes them. -} 	(readf, writef) <- System.Posix.IO.createPipe 	readh <- System.Posix.IO.fdToHandle readf 	writeh <- System.Posix.IO.fdToHandle writef-	p@(_, _, _, pid) <- createProcess $+	p@(_, _, _, pid) <- createProcess $ modproc $ 		(proc cmd opts) 			{ std_in = if isJust input then CreatePipe else Inherit 			, std_out = UseHandle writeh 			, std_err = UseHandle writeh-			, env = environ 			} 	hClose writeh @@ -199,12 +198,11 @@ 	return (transcript, ok) #else {- This implementation for Windows puts stderr after stdout. -}-	p@(_, _, _, pid) <- createProcess $+	p@(_, _, _, pid) <- createProcess $ modproc $ 		(proc cmd opts) 			{ std_in = if isJust input then CreatePipe else Inherit 			, std_out = CreatePipe 			, std_err = CreatePipe-			, env = environ 			}  	getout <- mkreader (stdoutHandle p)
Utility/SimpleProtocol.hs view
@@ -77,7 +77,7 @@  {- When a program speaks a simple protocol over stdio, any other output  - to stdout (or anything that attempts to read from stdin)- - will mess up the protocol. To avoid that, close stdin, and + - will mess up the protocol. To avoid that, close stdin,  - and duplicate stderr to stdout. Return two new handles  - that are duplicates of the original (stdin, stdout). -} dupIoHandles :: IO (Handle, Handle)
Utility/Url.hs view
@@ -11,6 +11,8 @@ {-# LANGUAGE FlexibleContexts #-}  module Utility.Url (+	closeManager,+	managerSettings, 	URLString, 	UserAgent, 	UrlOptions,@@ -31,12 +33,29 @@ import qualified Build.SysConfig  import Network.URI-import Network.HTTP.Conduit import Network.HTTP.Types import qualified Data.CaseInsensitive as CI import qualified Data.ByteString as B import qualified Data.ByteString.UTF8 as B8+import Control.Monad.Trans.Resource+import Network.HTTP.Conduit hiding (closeManager) +-- closeManager is needed with older versions of http-client,+-- but not new versions, which warn about using it. Urgh.+#if ! MIN_VERSION_http_client(0,4,18)+import Network.HTTP.Client (closeManager)+#else+closeManager :: Manager -> IO ()+closeManager _ = return ()+#endif++managerSettings :: ManagerSettings+#if MIN_VERSION_http_conduit(2,1,7)+managerSettings = tlsManagerSettings+#else+managerSettings = conduitManagerSettings+#endif+ type URLString = String  type Headers = [String]@@ -164,16 +183,18 @@ 	firstheader h = headMaybe . map snd . 		filter (\p -> fst p == h) . responseHeaders -	existsconduit req = withManager $ \mgr -> do+	existsconduit req = do+		mgr <- newManager managerSettings 		let req' = headRequest (applyRequest uo req)-		resp <- http req' mgr-		-- forces processing the response before the-		-- manager is closed-		ret <- liftIO $ if responseStatus resp == ok200-			then found-				(extractlen resp)-				(extractfilename resp)-			else dne+		ret <- runResourceT $ do+			resp <- http req' mgr+			-- forces processing the response before the+			-- manager is closed+			liftIO $ if responseStatus resp == ok200+				then found+					(extractlen resp)+					(extractfilename resp)+				else dne 		liftIO $ closeManager mgr 		return ret
Utility/libdiskfree.c view
@@ -13,23 +13,26 @@ # include <sys/mount.h> # define STATCALL statfs # define STATSTRUCT statfs64+# define BSIZE f_bsize #else #if defined (__FreeBSD__) # include <sys/param.h> # include <sys/mount.h> # define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */ # define STATSTRUCT statfs+# define BSIZE f_bsize #else #if defined __ANDROID__ # warning free space checking code not available for Android # define UNKNOWN #else-#if defined (__linux__) || defined (__FreeBSD_kernel__)-/* Linux or Debian kFreeBSD */+#if defined (__linux__) || defined (__FreeBSD_kernel__) || (defined (__SVR4) && defined (__sun))+/* Linux or Debian kFreeBSD or Solaris */ /* This is a POSIX standard, so might also work elsewhere too. */ # include <sys/statvfs.h> # define STATCALL statvfs # define STATSTRUCT statvfs+# define BSIZE f_frsize #else # warning free space checking code not available for this OS # define UNKNOWN@@ -65,7 +68,7 @@ 			v = 0; 	} -	blocksize = buf.f_bsize;+	blocksize = buf.BSIZE; 	return v * blocksize; #endif }
debian/changelog view
@@ -1,3 +1,59 @@+git-annex (5.20151019) unstable; urgency=medium++  * Fix a longstanding, but unlikely to occur bug, where dropping+    a file from a remote could race with other drops of the same file,+    and result in all copies of its content being lost.+  * git-annex-shell: Added lockcontent command, to prevent dropping of+    a key's content. This is necessary due to the above bugfix.+  * In some cases, the above bugfix changes what git-annex allows you to+    drop:+    - When a file is present in several special remotes,+      but not in any accessible git repositories, dropping it from one of+      the special remotes will now fail. Instead, the file has to be+      moved from one of the special remotes to the git repository, and can+      then safely be dropped from the git repository.+    - If a git remote has too old a version of git-annex-shell installed,+      git-annex won't trust it to hold onto a copy of a file when dropping+      that file from the local git repository.+  * Changed drop ordering when using git annex sync --content or the+    assistant, to drop from remotes first and from the local repo last.+    This works better with the behavior changes to drop in many cases.+  * Do verification of checksums of annex objects downloaded from remotes.+  * When annex objects are received into git repositories from other git+    repos, their checksums are verified then too.+  * To get the old, faster, behavior of not verifying checksums, set+    annex.verify=false, or remote.<name>.annex-verify=false.+  * setkey, rekey: These commands also now verify that the provided file+    matches the expected checksum of the key, unless annex.verify=false.+  * reinject: Already verified content; this can now be disabled by+    setting annex.verify=false.+  * sync, merge, assistant: When git merge failed for a reason other+    than a conflicted merge, such as a crippled filesystem not allowing+    particular characters in filenames, git-annex would make a merge commit+    that could omit such files or otherwise be bad. Fixed by aborting the+    whole merge process when git merge fails for any reason other than a+    merge conflict.+  * Allow building with S3 disabled again.+  * Ported disk free space checking code to work on Solaris.+  * Windows webapp: Fix support for entering password when setting+    up a ssh remote.+  * copy --auto was checking the wrong repo's preferred content.+    (--from was checking what --to should, and vice-versa.)+    Fixed this bug, which was introduced in version 5.20150727.+  * Avoid unncessary write to the location log when a file is unlocked+    and then added back with unchanged content.+  * S3: Fix support for using https.+  * Avoid displaying network transport warning when a ssh remote+    does not yet have an annex.uuid set.+  * Debian: Add torrent library to build-depends as it's packaged now,+    and stop recommending bittornado | bittorrent.+  * Debian: Remove build dependency on transformers library, as it is now+    included in ghc.+  * Debian: Remote menu file, since a desktop file is provided and+    lintian says there can be only one.++ -- Joey Hess <id@joeyh.name>  Mon, 19 Oct 2015 13:59:01 -0400+ git-annex (5.20150930) unstable; urgency=medium    * Added new linux standalone "ancient" build to support kernels
debian/control view
@@ -19,7 +19,6 @@ 	libghc-quickcheck2-dev, 	libghc-monad-control-dev (>= 0.3), 	libghc-exceptions-dev (>= 0.6),-	libghc-transformers-dev, 	libghc-unix-compat-dev, 	libghc-dlist-dev, 	libghc-uuid-dev,@@ -71,6 +70,7 @@ 	libghc-tasty-quickcheck-dev, 	libghc-tasty-rerun-dev, 	libghc-optparse-applicative-dev (>= 0.11.0),+	libghc-torrent-dev, 	lsof [linux-any], 	ikiwiki, 	perlmagick,@@ -103,7 +103,6 @@ 	git-remote-gcrypt (>= 0.20130908-6), 	nocache, 	aria2,-	bittornado | bittorrent, Suggests: 	graphviz, 	bup,
− debian/menu 
@@ -1,2 +0,0 @@-?package(git-annex):needs="X11" section="Applications/Network/File Transfer" \-	title="git-annex assistant" command="git-annex webapp"
+ doc/bugs/A_weird___34__.__34___repository_shows_up_when_switching.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.+When switching between my 2 repositories I now see a "." repository, see screenshot: http://screencast.com/t/0wxugJ9P +If I chose to switch to it, I get this error: http://screencast.com/t/5mndGNlhh8oN++### What steps will reproduce the problem?+Not sure if this is a real bug, maybe I screwed up?++### What version of git-annex are you using? On what operating system?+Version: 5.20150929-g7010007 +Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA TorrentParser Database+MAC OSX 10.10.5++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+there is nothing relevant to this error in there unfortunately.++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++
+ doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set.mdwn view
@@ -0,0 +1,38 @@+### Please describe the problem.++Downloaded git-annex (version for android 5.0) from the website. Upon opening the app for the first time and setting up a repository at /sdcard/annex, the terminal gave out the error.++### What version of git-annex are you using? On what operating system?++Nexus 4 running Cyanogenmod 12.1 (Android 5.1) build 20151007 and 20150901++### Please provide any additional information below.+Output to the terminal window:+[[!format sh """+WARNING: linker: git-annex has text relocations. This is wating memory and prevents security hardening. Please fix.+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!++Detected a filesystem without fifo support.++Disabling ssh connection caching.++Detected a crippled filesystem.++Enabling direct mode.+fatal: ../../../../sdcard/annex: '../../../../sdcard/annex' is outside of repository+(recording state in git...)+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+__bionic_open_tzdata_path: ANDROID_ROOT not set!+"""]]+I can add remote repositories and they show up nicely in the webapp, but no files are ever downloaded.++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++I have a couple of repositories atm, one with my music, another that backs up our family pictures for the family and uses Amazon S3 as a backup.
doc/bugs/Interrupted_command_broke_encfs_repository.mdwn view
@@ -52,3 +52,5 @@  ### Please provide any additional information below. If you ask for additional information, I will gladly provide it.++> Incremented my `encfs_is_shite` counter; [[done]] --[[Joey]]
+ doc/bugs/SSL_repos_does_not_show_up_in_Assistant.mdwn view
@@ -0,0 +1,27 @@+### Please describe the problem.++I have recently switched to self hosted git. Gitbucket over SSL. When I swithced origin from rsync ssh to ssl url assistant stopped showing origin in the repositories list it also does not sync to it when a file changes. (It does sync to it every 5 6 hours.). Running sync on the command line does sync to the repo. Authentication is done using netrc file. Replacing the ssl based url to ssh based url, repo shows up in the list and assistant syncs to it.+++### What steps will reproduce the problem?++Replacing the ssh url with ssl url.+++### What version of git-annex are you using? On what operating system?++5.20150823-geb17375++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++
+ doc/bugs/STANDARD__95__IA_for_S3_remote_not_working.mdwn view
@@ -0,0 +1,60 @@+### Please describe the problem.++Cannot set S3 remote to use Infrequently accessed.++### What steps will reproduce the problem?++git annex enableremote <remote> storageclass=STANDARD_IA++### What version of git-annex are you using? On what operating system?++5.20150930 on Arch Linux++### Please provide any additional information below.++[[!format sh """++# Trying with RR+$ git annex enableremote amazon storageclass=REDUCED_REDUNDANCY+enableremote amazon (encryption update) (hybrid cipher with gpg key XXX) ok+(recording state in git...)++$ git annex info amazon                                        +remote: amazon+description: [amazon]+uuid: XXX+trust: semitrusted+cost: 250.0+type: S3+creds: embedded in git repository (gpg encrypted)+bucket: XXX+endpoint: s3.amazonaws.com+port: 80+storage class: ReducedRedundancy+# snip++# Trying with IA+$ git annex enableremote amazon storageclass=STANDARD_IA       +enableremote amazon (encryption update) (hybrid cipher with gpg key XXX) ok+(recording state in git...)++$ git annex info amazon                                 +remote: amazon+description: [amazon]+uuid: XXX+trust: semitrusted+cost: 250.0+type: S3+creds: embedded in git repository (gpg encrypted)+bucket: XXX+endpoint: s3.amazonaws.com+port: 80+storage class: Standard++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++I've had great experiences using git annex normally, but the special remotes tend+to get dicey, like here.
+ doc/bugs/Utility__47__libdiskfree.c_more_BSD_friendly.mdwn view
@@ -0,0 +1,81 @@+### Please describe the problem.++There exist more BSD systems than FreeBSD. Thus I propose diff for Utility/libdiskfree.c.++Based on pkgsrc patching guidelines https://www.netbsd.org/docs/pkgsrc/components.html#components.patches.guidelines+I tried to cook a diff even I'm not C developer.+++### What steps will reproduce the problem?++Add support for more OS for disk free check.++### What version of git-annex are you using? On what operating system?+git-annex-5.20150930+++### Please provide any additional information below.++The diff probably needs check, improvement...++[[!format sh """+--- libdiskfree.c.orig	Sun Oct  4 15:18:07 2015++++ libdiskfree.c	Sun Oct  4 15:23:23 2015+@@ -7,35 +7,30 @@+ + /* Include appropriate headers for the OS, and define what will be used to+  * check the free space. */+-#if defined(__APPLE__)+-# define _DARWIN_FEATURE_64_BIT_INODE 1+-# include <sys/param.h>+-# include <sys/mount.h>+-# define STATCALL statfs+-# define STATSTRUCT statfs64+-#else+-#if defined (__FreeBSD__)+-# include <sys/param.h>+-# include <sys/mount.h>+-# define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */+-# define STATSTRUCT statfs+-#else+-#if defined __ANDROID__+-# warning free space checking code not available for Android+-# define UNKNOWN+-#else+ #if defined (__linux__) || defined (__FreeBSD_kernel__)+ /* Linux or Debian kFreeBSD */+ /* This is a POSIX standard, so might also work elsewhere too. */+ # include <sys/statvfs.h>+ # define STATCALL statvfs+ # define STATSTRUCT statvfs+-#else+-# warning free space checking code not available for this OS++#endif++++#if defined __ANDROID__++# warning free space checking code not available for Android+ # define UNKNOWN+ #endif++++#if defined (HAVE_SYS_PARAM_H) && defined (HAVE_SYS_MOUNT_H)++#if defined(__APPLE__)++# define _DARWIN_FEATURE_64_BIT_INODE 1+ #endif+-#endif++# include <sys/param.h>++# include <sys/mount.h>++# define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */++# define STATSTRUCT statfs64++#else++# warning free space checking code not available for this OS++# define UNKNOWN+ #endif+ + #include <errno.h>++"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++Not tested this "feature" yet, I got another issue which blocks me for now.+
+ doc/bugs/concurrent_drop--from_presence_checking_failures.mdwn view
@@ -0,0 +1,398 @@+Concurrent dropping of a file has problems when drop --from is+used. (Also when the assistant or sync --content decided to drop from a+remote.)++> Now [[fixed|done]] --[[Joey]]++[[!toc]]++# refresher++First, let's remember how it works in the case where we're just dropping+from 2 repos concurrently. git-annex uses locking to detect and prevent+data loss:++<pre>+Two repos, each with a file:++A (has)+B (has)++A wants from drop from A         B wants to drop from B+A locks it                       B locks it+A checks if B has it             B checks if A has it+  (does, but locked, so fails)     (does, but locked, so fails)+A fails to drop it               B fails to drop it++The two processes are racing, so there are other orderings to+consider, for example:++A wants from drop from A        B wants to drop from B+A locks it                     +A checks if B has it (succeeds)+A drops it from A               B locks it +                                B checks if A has it (fails)+                                B fails to drop it++Which is also ok.++A wants from drop from A        B wants to drop from B+A locks it                     +A checks if B has it (succeeds)+                                B locks it           +                                B checks if A has it+                                  (does, but locked, so fails)+A drops it                      B fails to drop it++Yay, still ok.+</pre>++Locking works in those cases to prevent concurrent dropping of a file.++# the bug++But, when drop --from is used, the locking doesn't work:++<pre>+Two repos, each with a file:++A (has)+B (has)++A wants to drop from B                  B wants to drop from A+A checks to see if A has it (succeeds)  B checks to see if B has it (succeeds)+A tells B to drop it                    B tells A to drop it+B locks it, drops it                    A locks it, drops it++No more copies remain!+</pre>++Verified this one in the wild (adding an appropriate sleep to force the+race).++Best fix here seems to be for A to lock the content on A+as part of its check of numcopies, and keep it locked+while it's asking B to drop it. Then when B tells A to drop it,+it'll be locked and that'll fail (and vice-versa).++> Done, and verified the fix works in this situation.++# the bug part 2++<pre>+Three repos; C might be a special remote, so w/o its own locking:++A       C (has)+B (has)++A wants to drop from C         B wants to drop from B+                               B locks it+A checks if B has it           B checks if C has it (does)+ (does, but locked, so fails)  B drops it++Copy remains in C. But, what if the race goes the other way?++A wants to drop from C          B wants to drop from B+A checks if B has it (succeeds)+A drops it from C               B locks it+                                B checks if C has it (does not)++So ok, but then:++A wants to drop from C          B wants to drop from B+A checks if B has it (succeeds)+                                B locks it+                                B checks if C has it (does)+A drops it from C               B drops it from B++No more copies remain!+</pre>++To fix this, seems that A should not just check if B has it, but lock+the content on B and keep it locked while A is dropping from C.+This would prevent B dropping the content from itself while A is in the+process of dropping from C.++That would mean replacing the call to `git-annex-shell inannex`+with a new command that locks the content.++Note that this is analgous to the fix above; in both cases+the change is from checking if content is in a location, to locking it in+that location while performing a drop from another location.++> Done, and verified the fix works in this situation.++# the bug part 3 (where it gets really nasty)++<pre>+4 repos; C and D might be special remotes, so w/o their own locking:++A      C (has)+B      D (has)++B wants to drop from C        A wants to drop from D+B checks if D has it (does)   A checks if C has it (does)+B drops from C                A drops from D++No more copies remain!+</pre>++How do we get locking in this case?++Adding locking to C and D is not a general option, because special remotes+are dumb key/value stores; they may have no locking operations.++## a solution: remote locking++What could be done is, change from checking if the remote has content, to+trying to lock it there. If the remote doesn't support locking, it can't+be guaranteed to have a copy. Require N locked copies for a drop to+succeed.++So, drop --from would no longer be supported in these configurations.+To drop the content from C, B would have to --force the drop, or move the+content from C to B, and then drop it from B.++### impact when using assistant/sync --content++Need to consider whether this might cause currently working topologies+with the assistant/sync --content to no longer work. Eg, might content+pile up in a transfer remote?++> The assistant checks after any transfer of an object if it should drop+> it from anywhere. So, it gets/puts, and later drops.+> Similarly, for sync --content, it first gets, then puts, and finally drops.++> When dropping an object from remotes(s) + local, in `handleDropsFrom`,+> it drops from local first. So, this would cause content pile-up unless+> changed.+> +> Also, when numcopies > 1, a toplogy like+> `A(transfer) -- B(client) -- specials(backup)` would never be able to drop+> the file from A, because the specials don't support locking and it can't+> guarantee the content will remain on them.+> +> One solution might be to make sync --content/the assistant generate+> move operations, which can then ignore numcopies (like `move` does).+> So, move from A to B and then copy to the specials. +> +> Using moves does lead to a decrease in robustness. For example, in+> the topology `A(transfer) -- B(client) -- C (backup)`, with numcopies=2,+> and C intermittently connected, the current+> behavior with sync --content/assistant is for an object to reach B+> and then later C, and only then be removed from A.+> If moves were used, the object moves from A to B, and so there's only+> 1 copy instead of the 2 as before, in the interim until C gets connected.++## a solution: minimal remote locking++This avoids needing to special case moves, and has 2 parts.++### to drop from remote++Instead of requiring N locked copies of content when dropping,+require only 1 locked copy (either the local copy, or a git remote that+can be locked remotely). Check that content is on the other N-1+remotes w/o requiring locking (but use it if the remote supports locking).++Unlike using moves, this does not decrease robustness, most of the time;+barring the kind of race this bug is about, numcopies behaves as desired.+When there is a race, some of the non-locked copies might be removed,+dipping below numcopies, but the 1 locked copy remains, so the data is+never entirely lost.++Dipping below desired numcopies in an unusual race condition, and then+doing extra work later to recover may be good enough.++> Implemented, and I've now verified this solves the case above.+> Indeed, neither drop succeeds, because no copy can be locked.++### to drop from local repo++When dropping an object from the local repo, lock it for drop,+and then verify that N remotes have a copy +(without requiring locking on special remotes).++So, this is done exactly as git-annex already does it.++Like dropping from a remote, this can dip below numcopies in a race+condition involving special remotes. ++But, it's crucial that, despite the lack of locking of +content on special remotes, which may be the last copy,+the last copy never be removed in a race. Is this the case?++We can prove that the last copy is never removed +by considering shapes of networks.++1. Networks only connected by single special+   remotes, and not by git-git repo connections. Such networks are+   essentially a collection of disconnected smaller networks, each +   of the form `R--S`+2. Like 1, but with more special remotes. `S1--R--S2` etc.+3. More complicated (and less unusal) are networks with git-git+   repo connections, and no cycles.+   These can have arbitrary special remotes connected in too.+4. Finally, there can be a cycle of git-git connections.++The overall network may be larger and more complicated, but we need only+concern ourselves with the subset that has a particular object+or is directly connected to that subset; the rest is not relevant.++So, if we can prove local repo dropping is safe in each of these cases,+it follows it's safe for arbitrarily complicated networks.++Case 1:++<pre>+2 essentially disconnected networks, R1--S and R2--S++R1 (has)   S (has)+R1++R1 wants to drop its local copy     R2 wants to move from S+R1 locks its copy for drop          R2 copies from S+R1 checks that S has a copy         R2 locks its copy+R1 drops its local copy             R2 drops from S++R1 expected S to have the copy, and due to a race with R2,+S no longer had the copy it expected. But, this is not actually+a problem, because the copy moved to R2 and so still exists. ++So, this is ok!+</pre>++Case 2:++<pre>+2 essentially disconnected networks, S1--R1--S2 and S1--R2--S2++R1(has)        S1 (has)+R2(has)        S2 (has)++R1 wants to move from S1 to S2    R2 wants to move from S2 to S1+R1 locks its copy                 R2 locks its copy+R1 checks that S2 has a copy      R2 checks that S1 has a copy+R1 drops from S1                  R2 drops from S2++R1 and R2 end up each with a copy still, so this is ok,+despite S1 and S2 lacking a copy.++If R1/R2 had not had a local copy, they could not have done a remote drop.+</pre>++(Adding more special remotes shouldn't change how this works.)++Case 3:++<pre>+3 repos; B has A and C as remotes; A has C as remote; C is special remote.++A (has)      C (has)+B++B wants to drop from C        A wants to drop from A+B locks it on A+B drops from C                A locks it on A for drop+                                (fails; locked by B)+B drops from C                A keeps its copy++ok!++or, racing the other way++B wants to drop from C        A wants to drop from A+                              A locks it on A for drop+B locks it on A+   (fails; locked by A)+C keeps its copy              A drops its copy++ok!+</pre>++Case 4:++But, what if we have a cycle? The above case 3 also works if B and A are in a+cycle, but what about a larger cycle?++Well, a cycle necessarily involves only git repos, not special remotes.+Any special remote can't be part of a cycle, because a special remote+does not have remotes itself.++As the remotes in the cycle are not special remotes, locking is done+of content on remotes when dropping it from local or another remote.+This locking ensures that even with a cycle, we're ok. For example:++<pre>+4 repos; D is special remote w/o its own locking, and the rest are git+repos. A has remotes B,D; B has remotes C,D; C has remotes A,D++A (has) D+B (has)+C (has)++A wants to drop from A     B wants to drop from B     C wants to drop from C+A locks it on A for drop   B locks it on B for drop   C locks it on C for drop+A locks it on B            B locks it on C            C locks it on A+  (fails; locked by B)      (fails; locked by C)       (fails; locked by A)++Which is fine! But, check races..++A wants to drop from A     B wants to drop from B     C wants to drop from C+A locks it on A for drop                              C locks it on C for drop+A locks it on B (succeeds)                            C locks it on A+                           B locks it on B for drop       (fails; locked by A)+                               (fails; locked by A)+A drops                    B keeps                    C keeps++It can race other ways, but they all work out the same way essentially,+due to the locking.+</pre>++# the bug, with moves++`git annex move --from remote` is the same as a copy followed by drop --from,+so the same bug can occur then.++But, the implementation differs from Command.Drop, so will also+need some changes.++Command.Move.toPerform already locks local content for removal before+removing it, of course. So, that will interoperate fine with+concurrent drops/moves. Seems fine as-is.++Command.Move.fromPerform simply needs to lock the local content+in place before dropping it from the remote. This satisfies the need+for 1 locked copy when dropping from a remote, and so is sufficent to+fix the bug.++> done++# drop ordering++Consider the network: `T -- C --- (B1,B2)`++When numcopies is 2, a file could start on T, get copied to C, and then on+the B1 and B2. At which point, it can be dropped from C and T (if+unwanted).++Currently, the assistant and sync --content drop from the local repo 1st,+and then from remotes. Before the changes to fix this bug, that worked;+the content got removed from T and then from C, using the copies on B1 and+B2 as evidence. Now though, if B1 and B2 are special remotes, once the copy+is dropped from C, there is no locked copy available on (B1,B2), so the+subsequent drop from T fails.++Changing the drop order lets C lock its own copy in order to+drop from T. then the local drop from C proceeds successfully without locking,+as local drops don't need locking.++Course, there is a behavior change here.. Before, if B2 didn't exist,+the content would reach B2 and then be dropped from C, and then with only 2+copies, it could not also be dropped from T. If the drop order changes, the+content is instead dropped from T and left on C.++The new behavior seems better when T is a transfer remote, but perhaps not in+other cases.++> implemented
+ doc/bugs/direct_cripple_mode_crippled_my_other_non-crippled_repos.mdwn view
@@ -0,0 +1,299 @@+### Please describe the problem.++I am trying to setup a repo on my external music player, a Nokia N900+device. It appears as a `vfat` filesystem from the outside, so+git-annex see it as a "crippled filesystem" and sets up direct mode+appropriately.++The problem is that for some weird reason, the checkout somewhat fails+and i'm left with a basically emptied repo. The next `sync` commits+the removal of all files, and happily pushes that to the other repos,+pretty much irreversibly, unless i start fiddling with the git+history.++What's going on?++It seems the problem is because i setup my remote repo from scratch,+because doing a checkout fails, because the crippled filesystem+doesn't support files with:++* colons (`:`)+* question marks (`?`)+* backslashes (`\`)+* double quotes (`"`)+* stars (`*`)+* irregular encoding (i.e. non-UTF8)++I have found the following tools to be useful to cleanup the filesystem:++* [convmv](http://tracker.debian.org/convmv) can massively re-encode filenames and may also be able to fix all the issues above, but i didn't test that+* [rename](http://tracker.debian.org/rename) can massively rename files according to certain patterns, I have used:++    rename 's/\?//' *+    rename 's/://' *+    rename 's/\\//' *+    rename 's/"//' *+    rename 's/*//' *+    git add -A .++Similar issues:++* windows bugs:+  * [[bugs/Can__39__t_clone_on_Windows_because_some_filenames_have_a_colon_in_them/]]+  * [[bugs/Windows:_can__39__t_clone_repository/]]+* more general feature request:+  [[forum/Wishlist:_rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT/]]++The above issue specifically request that files with "special" characters be supported in vFAT or even Windows, but here is a different issue: I am worried about potential data loss and lack of anti-foot-shooting device in case a user adds an external hard drive or USB key that are widely formatted as vFAT and then triggers destruction of all files (after garbage collection).++### What steps will reproduce the problem?++<pre>+git init nokia-n900/repo+cd nokia-n900/repo+git annex init+git remote add /srv/mp3+git annex sync # sets up a synced/master branch with no files on the remote repo+cd /srv/mp3+git annex sync # commits the removal of all files+</pre>++I didn't expect that to fail: a test run here doesn't delete files on /srv/mp3...++### What version of git-annex are you using? On what operating system?++5.20141125, debian jessie.++### Please provide any additional information below.++Complete trace:++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++[986]anarcat@marcos:~130$ cd /media/anarcat/Nokia\ N900/.sounds/++[1026]anarcat@marcos:.sounds130$ git init mp3-test+Dépôt Git vide initialisé dans /media/anarcat/Nokia N900/.sounds/mp3-test/.git/+[1027]anarcat@marcos:.sounds$ cd mp3-test+[1028]anarcat@marcos:mp3-test$ git annex init+init+  Detected a filesystem without fifo support.++  Disabling ssh connection caching.++  Detected a crippled filesystem.++  Enabling direct mode.+ok+(Recording state in git...)+[1029]anarcat@marcos:mp3-test$ git remote add origin /srv/mp3+[1030]anarcat@marcos:mp3-test$ git annex sync+commit  ok+pull origin+warning: no common commits+remote: Décompte des objets: 721895, fait.+remote: Compression des objets: 100% (194286/194286), fait.+remote: Total 721895 (delta 565247), reused 660087 (delta 526635)+Réception d'objets: 100% (721895/721895), 53.76 MiB | 5.45 MiB/s, fait.+Résolution des deltas: 100% (565247/565247), fait.+Depuis /srv/mp3+ * [nouvelle branche] git-annex  -> origin/git-annex+ * [nouvelle branche] master     -> origin/master+ * [nouvelle branche] synced/git-annex -> origin/synced/git-annex+ * [nouvelle branche] synced/master -> origin/synced/master++error: unable to create file Dri/Dirty Rotten LP/04 -  Why?.mp3 (Argument invalide)+error: unable to create file Dri/Dirty Rotten LP/07 -  Who Am I?.mp3 (Argument invalide)+error: unable to create file Dysrhythmia/Barriers and Passages/10 - Will the Spirit Prevail?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 06 - How Long Has This Been Going On?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 10 - What Is There to Say?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 19 - Baby, What Else Can I Do?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/The Best of The Song Books/Ella Fitzgerald - The Best of The Song Books - 14 - Why Was I Born? (Jan 6, 1963 in L.A.).ogg (Argument invalide)+error: unable to create file Fantomas/Fantomas/01 - Book 1: Page 1.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/02 - Book 1: Page 2.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/03 - Book 1: Page 3.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/04 - Book 1: Page 4.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/05 - Book 1: Page 5.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/06 - Book 1: Page 6.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/07 - Book 1: Page 7.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/08 - Book 1: Page 8.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/09 - Book 1: Page 9.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/10 - Book 1: Page 10.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/11 - Book 1: Page 11.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/12 - Book 1: Page 12.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/13 - Book 1: Page 13.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/14 - Book 1: Page 14.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/15 - Book 1: Page 15.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/16 - Book 1: Page 16.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/17 - Book 1: Page 17.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/18 - Book 1: Page 18.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/19 - Book 1: Page 19.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/20 - Book 1: Page 20.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/21 - Book 1: Page 21.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/22 - Book 1: Page 22.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/23 - Book 1: Page 23.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/24 - Book 1: Page 24.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/25 - Book 1: Page 25.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/26 - Book 1: Page 26.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/27 - Book 1: Page 27.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/28 - Book 1: Page 28.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/29 - Book 1: Page 29.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/30 - Book 1: Page 30.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/11 - Henry: Portrait Of A Serial Ki.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/15 - Twin Peaks: Fire Walk With Me.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/Fant�mas - The Director's Cut.m3u (Argument invalide)+error: unable to create file Francois Perusse/Parodie "La Fureur".mp3 (Argument invalide)+error: unable to create file Frank zappa/Fillmore East, June 1971/Frank Zappa & The Mothers Of Invention - Fillmore East, June 1971 - 03 - What Kind Of Girl Do You Think We Are?.ogg (Argument invalide)+error: unable to create file Frank zappa/Fillmore East, June 1971/Frank Zappa & The Mothers Of Invention - Fillmore East, June 1971 - 07 - Do You Like My New Car?.ogg (Argument invalide)+fatal: cannot create directory at 'Frank zappa/Joe's Garage: Acts I, II & III': Argument invalide+failed+(merging origin/git-annex into git-annex...)+(Recording state in git...)+push origin+Décompte des objets: 6, fait.+Delta compression using up to 2 threads.+Compression des objets: 100% (5/5), fait.+Écriture des objets: 100% (6/6), 700 bytes | 0 bytes/s, fait.+Total 6 (delta 2), reused 1 (delta 0)+To /srv/mp3+   26fc58f..63cfaf8  git-annex -> synced/git-annex+   76ec411..8458b14  annex/direct/master -> synced/master+ok+git-annex: sync: 1 failed+[1052]anarcat@marcos:annex$ git annex sync+commit  ok+pull origin+ok+push origin+Everything up-to-date+ok+# End of transcript or log.+"""]]++Now on the remote repo, it destroys everything:++<pre>+$ cd /srv/mp3+$ git annex sync+[ backlog lost, because of the sheer number of files deleted ]+$ git diff --stat 91fda32 | tail -1+ 21923 files changed, 21923 deletions(-)+$ # 91fda32 is the last known good commit on the master branch there+</pre>++Boom! Doing the following restores some sanity:++<pre>+$ git reset --hard 91fda32+$ git branch -D synced/master+$ git annex sync+</pre>++On the direct repo, now sync doesn't destroy anything, but then again,+there are no files either. Eventually, after enough `sync` commands,+the destruction will return...++A clone also fails similarly, which is why i was trying with the+"clean init" approach:++<pre>+[1032]anarcat@marcos:.sounds$ git clone /srv/mp3 mp3-clone+Clonage dans 'mp3-clone'...+fait.+error: unable to create file Dri/Dirty Rotten LP/04 -  Why?.mp3 (Argument invalide)+error: unable to create file Dri/Dirty Rotten LP/07 -  Who Am I?.mp3 (Argument invalide)+error: unable to create file Dysrhythmia/Barriers and Passages/10 - Will the Spirit Prevail?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 06 - How Long Has This Been Going On?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 10 - What Is There to Say?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 19 - Baby, What Else Can I Do?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/The Best of The Song Books/Ella Fitzgerald - The Best of The Song Books - 14 - Why Was I Born? (Jan 6, 1963 in L.A.).ogg (Argument invalide)+Extraction des fichiers:  24% (5433/21923)+[...]+[1036]anarcat@marcos:.sounds130$ git clone /srv/mp3 mp3-clone+Clonage dans 'mp3-clone'...+fait.+error: unable to create file Dri/Dirty Rotten LP/04 -  Why?.mp3 (Argument invalide)+error: unable to create file Dri/Dirty Rotten LP/07 -  Who Am I?.mp3 (Argument invalide)+error: unable to create file Dysrhythmia/Barriers and Passages/10 - Will the Spirit Prevail?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 06 - How Long Has This Been Going On?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 10 - What Is There to Say?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/Pure Ella/Ella Fitzgerald - Pure Ella - 19 - Baby, What Else Can I Do?.ogg (Argument invalide)+error: unable to create file Ella Fitzgerald/The Best of The Song Books/Ella Fitzgerald - The Best of The Song Books - 14 - Why Was I Born? (Jan 6, 1963 in L.A.).ogg (Argument invalide)+error: unable to create file Fantomas/Fantomas/01 - Book 1: Page 1.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/02 - Book 1: Page 2.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/03 - Book 1: Page 3.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/04 - Book 1: Page 4.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/05 - Book 1: Page 5.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/06 - Book 1: Page 6.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/07 - Book 1: Page 7.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/08 - Book 1: Page 8.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/09 - Book 1: Page 9.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/10 - Book 1: Page 10.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/11 - Book 1: Page 11.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/12 - Book 1: Page 12.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/13 - Book 1: Page 13.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/14 - Book 1: Page 14.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/15 - Book 1: Page 15.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/16 - Book 1: Page 16.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/17 - Book 1: Page 17.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/18 - Book 1: Page 18.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/19 - Book 1: Page 19.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/20 - Book 1: Page 20.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/21 - Book 1: Page 21.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/22 - Book 1: Page 22.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/23 - Book 1: Page 23.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/24 - Book 1: Page 24.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/25 - Book 1: Page 25.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/26 - Book 1: Page 26.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/27 - Book 1: Page 27.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/28 - Book 1: Page 28.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/29 - Book 1: Page 29.mp3 (Argument invalide)+error: unable to create file Fantomas/Fantomas/30 - Book 1: Page 30.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/11 - Henry: Portrait Of A Serial Ki.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/15 - Twin Peaks: Fire Walk With Me.mp3 (Argument invalide)+error: unable to create file Fantomas/The Director's Cut/Fant�mas - The Director's Cut.m3u (Argument invalide)+error: unable to create file Francois Perusse/Parodie "La Fureur".mp3 (Argument invalide)+error: unable to create file Frank zappa/Fillmore East, June 1971/Frank Zappa & The Mothers Of Invention - Fillmore East, June 1971 - 03 - What Kind Of Girl Do You Think We Are?.ogg (Argument invalide)+error: unable to create file Frank zappa/Fillmore East, June 1971/Frank Zappa & The Mothers Of Invention - Fillmore East, June 1971 - 07 - Do You Like My New Car?.ogg (Argument invalide)+fatal: cannot create directory at 'Frank zappa/Joe's Garage: Acts I, II & III': Argument invalide+warning: Le clone a réussi, mais l'extraction a échoué.+Vous pouvez inspecter ce qui a été extrait avec 'git status'+et réessayer l'extraction avec 'git checkout -f HEAD'+</pre>++Besides, `clone` creates actually seems to create and transfer all the files and setup direct mode (!?), which takes up too much space on this external drive...++Interestingly, i have managed to clone the repo by cleaning up a lot of space and fixing the above errors. Interestingly, the git clone is only 2GB while the original repo is closer to 110GB. There's nevertheless a bunch of files checked out, and obviously, enabling git-annex on the repo gives the predictable:++<pre>+$ git annex sync++  Detected a filesystem without fifo support.++  Disabling ssh connection caching.++  Detected a crippled filesystem.++  Enabling direct mode.+git-annex: /media/anarcat/Nokia N900/.sounds/mp3-clone/.git/annex/objects/cec/e45/SHA256E-s3547512--6d0b48b144ba58cf649134c7b4d6597f4e0c5f319302f1c109d0967f22af607a.mp3: createDirectory: resource exhausted (No space left on device)+</pre>++... and leaves the repo in an inconsistent state again. Also note that the above took over 2 hours of wall clock time before failing.++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++Git-annex has been my bread and butter for a few months in the past+year. I absolutely love it and it generally makes my life much easier+when dealing with large files. Direct mode sometimes drives me nuts,+but it certainly is more the fault to the damn crippled filesystems+than git-annex's fault for sure. :)++Arguably, the above problems are partly due to me assuming that+git-annex will work well on crippled filesystems, regardless of my+dataset, which maybe an inaccurate assumption.++Thanks for all your hard work! --[[anarcat]]++> [[fixed|done]]
+ doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS.mdwn view
@@ -0,0 +1,58 @@+### Please describe the problem.++I'm trying to get git-annex running on a WD MyCloud EX2 NAS.++There are a number of issues, I imagine to overcome, a few of which I've already managed to overcome.+++### What steps will reproduce the problem?++I've managed to get it to work as special remote via rsync. But, it's nowhere near ideal. I was hoping to get working properly by installing git-annex locally on the machine.++FYI, git 2.6.1 is already installed on the NAS.++I can ssh into the NAS no problem. Only user sshd is allowed to ssh, but that's ok. Furthermore, the which command is missing, which prevented webapp lunched on my Mac from creating an rsync remote. Overcame that hurdle by "faking" it:+[[!format sh """+    > echo \#\!\/bin\/sh > /usr/bin/which+    > echo command \-v \$\@ >> /usr/bin/which+    > chmod +x /usr/bin/which+"""]]++I've downloaded the latest arm build. After untaring and running runshell successfully, I can't get through the next step, which is running git annex. It results in the following error:+[[!format sh """+    > git annex+    /mnt/HD/HD_a2/git-annex/shimmed/git-annex/git-annex: error while loading shared libraries: /mnt/HD/HD_a2/git-annex/shimmed/git-annex/git-annex: ELF load command alignment not page-aligned+    fatal: 'annex' appears to be a git command, but we were not able to execute it. Maybe git-annex is broken?+"""]]++same goes for+[[!format sh """+    > runshell git-annex-shell -c git annex:+    /mnt/HD/HD_a2/Nas_Prog/git-annex/shimmed/git-annex-shell/git-annex-shell: error while loading shared libraries: /mnt/HD/HD_a2/Nas_Prog/git-annex/shimmed/git-annex-shell/git-annex-shell: ELF load command alignment not page-aligned+"""]]++and pretty much any of the git-annex binaries.++Compiling from source is not an option.+Any ideas? Thanks.++### What version of git-annex are you using? On what operating system?++I'm trying the latest version of git-annex-standalone-armel.tar.gz downloaded 2015-10-10.++uname -a:  Linux WDMyCloudEX2 3.2.40 #4 Fri Jul 31 16:04:18 CST 2015 armv7l GNU/Linux+++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++Sure, runs on my Mac laptop
+ doc/bugs/git_annex_preferred_content_strange_behavior.mdwn view
@@ -0,0 +1,53 @@+### Please describe the problem.++With a version above 5.20141125, git annex wants all remotes to get all files+when one remote wants standard.++### What steps will reproduce the problem?++- Create repo "a" and "b"+- Add "b" as a remote of "a"+- Add a file in "a"+- In "a", try git annex copy --to b. Nothing happens: this is the excepted behaviour+- In "a", run git annex wanted here standard+- In "a", try git annex copy --to b. Now, it wants to copy the file to "b" without any obvious (to me) reason++It may be useful to emphasis the fact that git-annex wanted to copy a file to "b" while we did not change the preferred content of "b".++I tried with the version 5.20141125 and the problem does not occur. It occurs+though with the versions, 5.20150731-1, 5.20150916-1 and 5.20150930-g40fdbe9.++### What version of git-annex are you using? On what operating system?++    $ git annex version+    git-annex version: 5.20150930-g40fdbe9+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA Database+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external+    local repository version: 5+    supported repository version: 5+    upgrade supported from repository versions: 0 1 2 4++### Please provide any additional information below.++[[!format sh """+mkdir a+cd a+git init+git annex init+echo a > a+git annex add+git commit -m "first commit"+git clone . ../b+git remote add b file://$(pwd)/../b+git annex sync b+git annex copy --to b --auto # nothing happens, this is normal+git annex wanted here standard+git annex copy --to b --auto # now, it copies the file to b. Why ??+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)++Definitely! git-annex is a wonderful tool that I have been using every day to manage all my files for 2 years now. Thank you for this excellent software.++> [[fixed|done]] --[[Joey]]
+ doc/bugs/thread_blocked_indefinitely_in_an_MVar_operation_durin_fsck.mdwn view
@@ -0,0 +1,36 @@+### Please describe the problem.+I try to fsck my movies repo, but run in to a failure message. Running the same command on another repo on the same computer works (both being run with sudo and the same user.)++### What steps will reproduce the problem?+    $ sudo -u time_machine_carl git annex fsck --incremental-schedule 182d --time-limit 1h ++    sqlite worker thread crashed: SQLite3 returned ErrorCan'tOpen while attempting to perform prepare "SELECT null from fscked limit 1": unable to open database file+    git-annex: thread blocked indefinitely in an MVar operation++I guess I might just have created a fsck database using another user and this db is somehow locked?++### What version of git-annex are you using? On what operating system?++    $ git annex version+    git-annex version: 5.20150731-1+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA Database+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external+    +It runs on Raspbian Testing+    $ uname -a+    Linux pi 4.1.6+ #810 PREEMPT Tue Aug 18 15:19:58 BST 2015 armv6l GNU/Linux+### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)+Of course, The movie archive seems to be filling up nicely I just have some problems automating the checks.++> [[done]]
doc/bugs/windows_ssh_webapp_password_entry_broken.mdwn view
@@ -7,3 +7,5 @@ > I have a `winprocfix` branch that uses process-1.3 which has been > enhanced to allow fixing this. Merging is currently blocked on > <https://github.com/pcapriotti/optparse-applicative/issues/153> --[[Joey]]+> +> [[fixed|done]] --[[Joey]]
doc/design/encryption.mdwn view
@@ -26,7 +26,7 @@ Data is encrypted by GnuPG, using a symmetric cipher. The cipher is generated by GnuPG when the special remote is created. By default the best entropy pool is used, hence the generation may take a while; One-can use `initremote` with `highRandomQuality=false` or `--fast` options+can use `initremote` with the `--fast` option to speed up things, but at the expense of using random numbers of a lower quality. The generated cipher is then checked into your git repository, encrypted using one or more OpenPGP public keys. This scheme
doc/design/roadmap.mdwn view
@@ -2,17 +2,17 @@  * [[design/caching_database]] for metadata views, direct mode mappings * [[assistant/deltas]]-* [[assistant/gpgkeys]]-* [[assistant/telehash]]+* [[assistant/gpgkeys]] management for the assistant+* [[assistant/telehash]] or similar * [[design/requests_routing]]-* [[design/v6]]--## now--* Feb 2015 user-driven features and polishing, [[design/caching_database]] part 1+* [[design/v6]] repo format+* [[todo/deferred_update_mode]]+* [[use smudge and clean filters|todo/smudge]] like Git LFS  ## the rearview +* Mar-Oct 2015 busy busy busy, too busy to update!+* Feb 2015 user-driven features and polishing, [[design/caching_database]] part 1 * Jan 2015 Android 5, relative paths, workload [[tuning]] * Dec 2014 [[todo/extensible_addurl]], bittorrent special remote * Nov 2014 direct mode proxy, undo command, diffdriver
+ doc/devblog/day_320__porting_and_such.mdwn view
@@ -0,0 +1,13 @@+Lots of porting work ongoing recently:++* I've been working with [Goeke](http://www.goeke.net/) on building git-annex+  on Solaris/SmartOS. Who knows, this may lead to a binary distribution+  in some way, but to start with I got the disk free space code ported+  to Solaris, and have seen git-annex work there.+* Jirib has also been working on that same disk free code, porting it to+  OpenBSD. Hope to land an updated patch for that.+* Yury kindly updated the Windows autobuilder to a new Haskell Platform+  release, and I was able to land the `winprocfix` branch that fixes+  ssh password prompting in the webapp on Windows.+* The arm autobuilder is fixed and back in its colo, and should be making+  daily builds again.
+ doc/devblog/day_321__download_verification.mdwn view
@@ -0,0 +1,13 @@+While at the DerbyCon security conference, I got to thinking about+verifying objects that git-annex downloads from remotes. This can be+expensive for big files, so git-annex has never done it at download time,+instead deferring it to fsck time. But, that is a divergence from git,+which always verifies checksums of objects it receives. So, it violates+least surprise for git-annex to not verify checksums too. And this could+weaken security in some use cases.++So, today I changed that. Now whenever git-annex accepts an object into+.git/annex/objects, it first verifies its checksum and size. I did add a+setting to disable that and get back the old behavior: `git config+annex.verify false`, and there's also a per-remote setting if you want to+verify content from some remotes but not others.
+ doc/devblog/day_322-326__concurrent_drop_safety.mdwn view
@@ -0,0 +1,49 @@+Well, I've spent all week making `git annex drop --from` safe.++On Tuesday I got a sinking feeling in my stomach, as I realized that+there was hole in git-annex's armor to prevent concurrent drops from+violating numcopies or even losing the last copy of a file.+[[The bug involved an unlikely race condition|bugs/concurrent_drop--from_presence_checking_failures]],+and for all I know it's never happened in real life, but still this is not+good.++Since this is a potential data loss bug, expect a release pretty soon +with the fix. And, there are 2 things to keep in mind about the fix:++1. If a ssh remote is using an old version of git-annex, a drop may fail.+   Solution will be to just upgrade the git-annex on the remote to the+   fixed version.+2. When a file is present in several special remotes, but not in any+   accessible git repositories, dropping it from one of the special+   remotes will now fail, where before it was allowed.++   Instead, the file has to be moved from one of the special remotes to+   the git repository, and can then safely be dropped from the git repository.++   This is a worrysome behavior change, but unavoidable.++Solving this clearly called for more locking, to prevent concurrency+problems. But, at first I couldn't find a solution that would allow+dropping content that was only located on special remotes. I didn't want to+make special remotes need to involve locking; that would be a nightmare to+implement, and probably some existing special remotes don't have any+way to do locking anyway.++Happily, after thinking about it all through Wednesday, I found a solution,+that while imperfect (see above) is probably the best one feasible. If my+analysis is correct (and it seems so, although I'd like to write a more+formal proof than the ad-hoc one I have so far), no locking is needed on+special remotes, as long as the locking is done just right on the git repos+and remotes. While this is not able to guarantee that numcopies is always+preserved, it is able to guarantee that the last copy of a file is never+removed. And, numcopies *will* always be preserved except for when this+rare race condition occurs.++So, I've been implementing that all of yesterday and today. Getting it+right involves building up 4 different kinds of evidence, which can be+used to make sure that the last copy of a file can't possibly end up being+dropped, no matter what other concurrent drops could be happening.+I ended up with a very clean and robust implementation of this, and+a 2,000 line diff.++Whew!
+ doc/devblog/day_327__soaking.mdwn view
@@ -0,0 +1,16 @@+Feeling kind of ready to cut the next release of git-annex, but am giving+the recent large changes just a little time to soak in and make sure+they're ok.++Yesterday, changed the order that `git annex sync --content` and the+assistant do drops. When dropping from the local repo and also some+remotes, it now makes more sense to drop from the remotes first, and only+then the local repo. There are scenaries where that order lets content be+dropped from all the places that it should be, while the reverse order+doesn't.++Today, caught up on recent bug reports, including fixing a bad merge+commit that was made when git merge failed due to filenames not supported by+a crippled filesystem, and cleaning up a network transport warning that was+displayed incorrectly. Also developed a patch to the `aws` library to+support google nearline when creating buckets.
+ doc/forum/Help_fixing_S3_mistake.mdwn view
@@ -0,0 +1,7 @@+Hello, I'd like some help fixing a mistake I made setting up an S3 remote.++I created an S3 remote A, then at some later time accidentally created an S3 remote B with the same settings as A, and moved some files over to B.++I have fixed it by removing/marking as dead B, but I am missing some files now.++I believe that all of these files exist in A since it is the same bucket as B, but git annex doesn't know that they are in A.  Is there a way to somehow "reinject" or refresh git annex so it knows that the files are there?  I'm using chunking as well, so I don't know how to download all the chunks and use git annex's reinject command.
+ doc/forum/Need_some_help_installing_on_freeBSD.mdwn view
@@ -0,0 +1,48 @@+Hi there,++**I'm trying to install git-annex on my NAS which is freeNAS running freeBSD. +I've created a jail for it and installed via: pkg install hs-git-annex**++I've seen on this page: http://www.freshports.org/devel/hs-git-annex/ that there are Configuration Options. +I assume to use those I'd have to have compiled it myself from the ports like this: +To install the port: cd /usr/ports/devel/hs-git-annex/ && make install clean++So in this case of using the pkg, how can I check the way git-annex was compiled? +Anyone else ever used it on freeBSD and can give some input please? ++**Figured out how to see this info:**++git-annex version+git-annex version: 5.20150727+build flags: Assistant Webapp Webapp-secure Pairing S3 WebDAV Kqueue XMPP DNS Feeds Quvi TDFA Database+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external++**So apparently box.com isn't supported in this build?**++I installed via a web terminal, not via a normal ssh connection so when I came back the installation with all dependencies was finished but I couldn't scroll back to see all warning, only caught this last part of the screen; anyone got any idea if git-annex is supposed to run fine on FreeBSD 9.3-RELEASE-p26 amd64? ++**part of the text I caught:**+  IN_CLOSE_WRITE                                                               +                                                                                +Symbolic Link notifications:                                                    +   IN_DONT_FOLLOW                                                               +   IN_ATTRIB                                                                    +   IN_MOVE_SELF                                                                 +   IN_DELETE_SELF                                                               +                                                                                +Kernel patches to address the missing directory and symbolic link               +notifications are available from:                                               +                                                                                +https://github.com/dmatveev/libinotify-kqueue/tree/master/patches               +                                                                                +=============================================================================   +You might want to consider increasing the kern.maxfiles tunable if you plan     +to use this library for applications that need to monitor activity of a lot     +of files.                                                                       +                                                                                +If the default on your system is too low, add the following line to             +/boot/loader.conf, then reboot the system:                                      +                                                                                +    kern.maxfiles="25000"                                                       +=============================================================================
+ doc/forum/Trying_to_delete_an_S3_repo_fails.mdwn view
@@ -0,0 +1,3 @@+[2015-10-03 22:10:53.948377] TransferScanner: warning TransferScanner crashed: unknown response from git cat-file ("refs/heads/git-annex:706/2a9/SHA256E-s13840914--38cdcf384aee777000693f58f1302684bf18f577045c1340d7aeb72b6d6f2233.dmg.log.cnk missing","refs/heads/git-annex:b8f/4d2/SHA256E-s26301635--ce2f215486df83c6ada74a81679ff78f717404d0637bfd51ecbf0330c21db12b.dmg.log")++What exactly is this telling me?
+ doc/forum/external_special_remote_protocol_and_threads.mdwn view
@@ -0,0 +1,3 @@+As the sync command just got a -J option I have started to use threads to transfer new files to multiple remotes in parallel. A question that arises is if I still can assume that only one instance of the special remote will be started. I tend to store quite some state in the special remote program that I use to connect to a cloud server, and running two instances in parallel might run in to all sorts of fun behavior.++That said, I am of course aware that if I run multiple git annex commands I would run into these problems.
+ doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop.mdwn view
@@ -0,0 +1,16 @@+I installed git-annex on my freenas running freebsd according to the instructions on this site. +every now and then it'll pop up (when using the webapp) telling me git-annex has been updated, do I want to restart. +if I say yes, the webapp dies, I have to manually stop git annex assistant --stop and after a few minutes of using the webapp it again tells me it has been updated...++only saw this in the logs:+UpgradeWatcher: A new version of git-annex has been installed. +++Also, I connected 2 repos (on my NAS with freebsd and on my Macbook) but the NAS one can't connect to the MAcbook via SSH+ssh: connect to host 192.168.178.33 port 22: Connection refused ++How is this supposed to work? Obviously for now it shows its only syncing metadata with my MACbook: +syncing enabled (metadata only)++Also, both repositories are bare, direct but when a file got pulled from the NAS to the MACbook it was a symlink and not the file itself, so what's wrong here?+
+ doc/forum/git-annex_add_out_of_memory_error.mdwn view
@@ -0,0 +1,38 @@+Greetings all!++I have only been using git-annex for a few days, but I am loving it so far.++I have run into one problem:++At the top of a directory tree with some very large (GB+) blobs, I issue this command:++find . -size +5M  -exec git-annex add --debug {} \;  2>&1 | tee -a ./annex-add.log++When the command completes, I see a number of these types of errors in the log:++    [2015-10-05 21:14:13.502615] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]+    [2015-10-05 21:14:13.521586] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]+    [2015-10-05 21:14:13.533095] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--","./application/mozilla/firefox/linux/firefox-39.0/media/firefox-39.0.tar.bz2"]+    [2015-10-05 21:14:13.545739] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"]+    [2015-10-05 21:14:13.562727] process done ExitSuccess+    [2015-10-05 21:14:13.564103] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"]+    [2015-10-05 21:14:13.56616] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"]+    [2015-10-05 21:14:13.58032] process done ExitSuccess+    fatal: Out of memory, getdelim failed+    [2015-10-05 21:14:13.582109] process done ExitFailure 128+    ok+    (recording state in git...)+    git-annex: user error (git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] exited 128)++I've googled around and searched the forums/bug reports here without much success.+I have tried setting the queuesize to smaller values, but that does not seem to help.++I am using git-annex version: 5.20150930-g40fdbe9 on RHEL 6.++I am happy to help debug further...++Thanks,++JC++P.S. - Is it normal that some links in the .git dir are created without user write access?
+ doc/forum/git-like_git-annex_diff.mdwn view
@@ -0,0 +1,11 @@+It took me some trial and error to get `git-annex diffdriver` to work close to the way I wanted (basically, the same way `git diff` works in a normal git repository with `color.ui` set to `always` with `git config`, so I thought I'd put my notes here for others:++* Install `colordiff` (I couldn't figure out what `git`'s actual default diff program was or if it was possible to use it outside of `git diff`, so I went with this.+* Create a wrapper script to pass to `git-annex diffdriver`. `LANG=C` was to avoid a warning produced by `perl` when it runs `colordiff`. `exit 0` is needed because `colordiff`/`diff` has a non-zero exit status when the files it is passed are not identical and `git` expects the external diff program to exit with status 0:++        #!/usr/bin/env bash                                                             +        LANG=C colordiff -c "$1" "$2"                                                   +        exit 0++* Put the wrapper script somewhere on `PATH`, make it executable, and name it something like `git-annex-diff-wrapper`.+* Configure the repo to use the wrapper: `git config diff.external "git-annex diffdriver -- git-annex-diff-wrapper --"`
+ doc/forum/store_annex_in_different_place_on_hard_drive.mdwn view
@@ -0,0 +1,2 @@+I have the following setup on my work computer. The /home folder has limited space, but excellent backups, so I would like to keep my main repository there.+The /scratch/ folder has no backup, but has faster read/write speed, and more storage space. Is there a way to keep the main repo in /home, but somehow store the large annexed files on /scratch. I am open to making a remote repository, but the standard "git annex get" workflow will simply copy large files into the /home directory, which is what i want to avoid altogther. Can I just make a symlink from "/home/<username>/repo/.git/annex" to "/scratch/repodata" somehow? Thanks!
doc/git-annex-reinject.mdwn view
@@ -14,8 +14,8 @@ This can be useful if you have obtained the content of a file from elsewhere and want to put it in the local annex. -Automatically runs fsck on dest to check that the expected content was-provided.+Verifies that the src file's content matches with the content that the dest+file is expected to have, and refuses to reinject it otherwise.  Example:
doc/git-annex-setkey.mdwn view
@@ -11,8 +11,7 @@ This plumbing-level command makes the content of the specified key be set to the specified file. The file is moved into the annex. -No checking is done that the file contains the expected contents of the key.-So it's generally a better idea to use [[git-annex-reinject]](1) instead of+It's generally a better idea to use [[git-annex-reinject]](1) instead of this command.  # SEE ALSO
doc/git-annex-shell.mdwn view
@@ -43,6 +43,17 @@   Exits 100 if it's unable to tell (perhaps the key is in the process of   being removed from the annex). +* lockcontent directory key++  This locks a key's content in place in the annex, preventing it from+  being dropped.++  Once the content is successfully locked, outputs "OK". Then the content+  remains locked until a newline is received from the caller or the+  connection is broken.++  Exits nonzero if the content is not present, or could not be locked.+ * dropkey directory [key ...]    This drops the annexed data for the specified keys.
doc/git-annex.mdwn view
@@ -1026,6 +1026,12 @@   This both prevents git-annex sync from pushing changes, and prevents   storing or removing files from read-only remote. +* `remote.<name>.annex-verify`, `annex.verify`++  By default, git-annex will verify the checksums of objects downloaded+  from remotes. If you trust a remote and don't want the overhead+  of these checksums, you can set this to `false`.+ * `remote.<name>.annexUrl`    Can be used to specify a different url than the regular `remote.<name>.url`
− doc/news/version_5.20150812.mdwn 
@@ -1,40 +0,0 @@-git-annex 5.20150812 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Added support for SHA3 hashed keys (in 8 varieties), when git-annex is-     built using the cryptonite library.-   * metadata: Fix reversion introduced in 5.20150727 that caused recursive-     display of metadata to not work.-   * Windows: Fix bug that caused git-annex sync to fail due to missing-     environment variable.-   * Fix setting/setting/viewing metadata that contains unicode or other-     special characters, when in a non-unicode locale.-   * Simplify setup process for a ssh remote. Now it suffices to run git-     remote add, followed by git-annex sync. Now the remote is automatically-     initialized for use by git-annex, where before the git-annex branch had-     to manually be pushed before using git-annex sync. Note that this-     involved changes to git-annex-shell, so if the remote is using an old-     version, the manual push is still needed.-   * git-annex-shell: Don't let configlist auto-init repository when in-     readonly mode.-   * Perform a clean shutdown when --time-limit is reached.-     This includes running queued git commands, and cleanup actions normally-     run when a command is finished.-   * fsck: Commit incremental fsck database when --time-limit is reached.-     Previously, some of the last files fscked did not make it into the-     database when using --time-limit.-   * fsck: Commit incremental fsck database after every 1000 files-     fscked, or every 5 minutes, whichever comes first. Previously,-     commits were made every 1000 files fscked.-   * Linux standalone: Work around problem that prevented it from working-     properly if unpacked into a directory that contains ":" or ";" in its-     name.-   * proxy: Fix proxy git commit of non-annexed files in direct mode.-   * proxy: If a non-proxied git command, such as git revert-     would normally fail because of unstaged files in the work tree,-     make the proxied command fail the same way.-   * proxy: Fix removal of files deleted by the proxied command.-   * proxy: Fix behavior when run in subdirectory of git repo.-   * Improve Setup.hs file so that cabal copy --destdir works.-     Thanks, Magnus Therning.-   * Tighten dependency on optparse-applicative to 0.11.0.-   * Added back debian/cabal-wrapper, since it still seems needed after all."""]]
− doc/news/version_5.20150824.mdwn 
@@ -1,30 +0,0 @@-git-annex 5.20150824 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Sped up downloads of files from ssh remotes, reducing the-     non-data-transfer overhead 6x.-   * sync: Support --jobs-   * sync --content: Avoid unnecessary second pull from remotes when-     no file transfers are made.-   * External special remotes can now be built that can be used in readonly-     mode, where git-annex downloads content from the remote using regular-     http.-   * Added WHEREIS to external special remote protocol.-   * importfeed --relaxed: Avoid hitting the urls of items in the feed.-   * Fix reversion in init when ran as root, introduced in version 5.20150731.-   * Reorder declaration to fix build with yesod-core &gt; 1.4.13.-     Thanks, Michael Alan Dorman.-   * Fix building without quvi and without database.-     Thanks, Ben Boeckel.-   * Avoid building the assistant on the hurd, since an inotify equivalent-     is not yet implemented in git-annex for the hurd.-   * --debug log messages are now timestamped with fractional seconds.-   * --debug is passed along to git-annex-shell when git-annex is in debug mode.-   * Makefile: Pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc and on to-     ld, cc, and cpp.-   * As a result of the Makefile changes, the Debian package is built-     with various hardening options. Although their benefit to a largely-     haskell program is unknown."""]]--Note: The x86-64 bit linux standalone tarball shipped with this version had-a broken version of ssh that crashed on startup. The tarball has been-updated to fix this problem.
+ doc/news/version_5.20151019.mdwn view
@@ -0,0 +1,53 @@+git-annex 5.20151019 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * Fix a longstanding, but unlikely to occur bug, where dropping+     a file from a remote could race with other drops of the same file,+     and result in all copies of its content being lost.+   * git-annex-shell: Added lockcontent command, to prevent dropping of+     a key's content. This is necessary due to the above bugfix.+   * In some cases, the above bugfix changes what git-annex allows you to+     drop:+     - When a file is present in several special remotes,+       but not in any accessible git repositories, dropping it from one of+       the special remotes will now fail. Instead, the file has to be+       moved from one of the special remotes to the git repository, and can+       then safely be dropped from the git repository.+     - If a git remote has too old a version of git-annex-shell installed,+       git-annex won't trust it to hold onto a copy of a file when dropping+       that file from the local git repository.+   * Changed drop ordering when using git annex sync --content or the+     assistant, to drop from remotes first and from the local repo last.+     This works better with the behavior changes to drop in many cases.+   * Do verification of checksums of annex objects downloaded from remotes.+   * When annex objects are received into git repositories from other git+     repos, their checksums are verified then too.+   * To get the old, faster, behavior of not verifying checksums, set+     annex.verify=false, or remote.&lt;name&gt;.annex-verify=false.+   * setkey, rekey: These commands also now verify that the provided file+     matches the expected checksum of the key, unless annex.verify=false.+   * reinject: Already verified content; this can now be disabled by+     setting annex.verify=false.+   * sync, merge, assistant: When git merge failed for a reason other+     than a conflicted merge, such as a crippled filesystem not allowing+     particular characters in filenames, git-annex would make a merge commit+     that could omit such files or otherwise be bad. Fixed by aborting the+     whole merge process when git merge fails for any reason other than a+     merge conflict.+   * Allow building with S3 disabled again.+   * Ported disk free space checking code to work on Solaris.+   * Windows webapp: Fix support for entering password when setting+     up a ssh remote.+   * copy --auto was checking the wrong repo's preferred content.+     (--from was checking what --to should, and vice-versa.)+     Fixed this bug, which was introduced in version 5.20150727.+   * Avoid unncessary write to the location log when a file is unlocked+     and then added back with unchanged content.+   * S3: Fix support for using https.+   * Avoid displaying network transport warning when a ssh remote+     does not yet have an annex.uuid set.+   * Debian: Add torrent library to build-depends as it's packaged now,+     and stop recommending bittornado | bittorrent.+   * Debian: Remove build dependency on transformers library, as it is now+     included in ghc.+   * Debian: Remote menu file, since a desktop file is provided and+     lintian says there can be only one."""]]
+ doc/special_remotes/S3/comment_18_0dcb84048563fc9e45b66ca8f365f320._comment view
@@ -0,0 +1,7 @@+[[!comment format=mdwn+ username="darkfeline"+ subject="Sharing S3 bucket between repos"+ date="2015-10-09T01:16:25Z"+ content="""+What are the implications of sharing an S3 remote/bucket between two repositories with using fileprefix?  Can this be used to \"deduplicate\" files shared between repositories?+"""]]
+ doc/special_remotes/S3/comment_19_9d0d622b6202698f09eb06c097579fbb._comment view
@@ -0,0 +1,19 @@+[[!comment format=mdwn+ username="joey"+ subject="""comment 19"""+ date="2015-10-12T17:10:21Z"+ content="""+@darkfeline I suppose you're talking about two completely disjoint git+repositories, and not two clones of the same parent repo.++If you don't use fileprefix, and have the same file in two disjoint+repositories, git-annex will pick the same key for it in both cases, and+so you'll get deduplication, but only if you don't use different+fileprefixes.++And this will mostly work pretty well. The danger is, if you drop the file+from the S3 repo, because say, it's not used anymore in one repository,+then you're also removing it from the S3 repo as used by the other+repository. If that was the last copy of the file, that may not be what you+want.+"""]]
+ doc/special_remotes/comment_23_96ef232e13bc2dc102a667a06c856ee7._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="craig@6ddb6e2c94325e18a0d631a06e63fdc111ab1f12"+ nickname="craig"+ subject="Replicating my key for encrypted special remotes"+ date="2015-10-17T16:29:24Z"+ content="""+I'm using git annex assistant to auto backup my pictures off-site to glacier.  The files in glacier are encrypted.  However, if I lose my main machine, I've also lost the encryption key, which makes my off-site backup useless.  I figured I could fix this by creating a manual mode remote on a usb drive that I keep on my keychain.  I figured this would replicate the encryption key (as I might want to pull down files from the full backup glacier remote), but would not replicate the files themselves, as I have more pictures than space on the usb drive.++However, it seems the that the new remote is configured to only talk to my main machine and not glacier; the encryption key is not in the .git/ directory.  How do I ensure that I've got an off-site copy of the glacier encryption key?++Thanks,+Craig+"""]]
+ doc/special_remotes/comment_24_2c9eda62766c9d5000346a092fe5d0d8._comment view
@@ -0,0 +1,16 @@+[[!comment format=mdwn+ username="joey"+ subject="""comment 24"""+ date="2015-10-19T17:32:16Z"+ content="""+@craig, all of git-annex's information about a special remote is stored in+the git-annex branch in git, so any clone of the git repository is+sufficient to back that up. You can run `git annex enableremote`+in an clone to enable an existing special remote.++The only catch is that, if you have chosen to initremote a special remote+using a gpg key, with `encryption=shared keyid=whatever`, you'll of course+also need that gpg key to to use it. If you run `git annex info $myremote`+it will tell you amoung other things, any gpg keys that are used by that+remote.+"""]]
+ doc/tips/skydriveannex/comment_11_13692d8cb9ccf10ced26d90739385a6a._comment view
@@ -0,0 +1,47 @@+[[!comment format=mdwn+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"+ nickname="ovidiu"+ subject="comment 11"+ date="2015-10-03T19:18:24Z"+ content="""+shiny:skydriveannex ovi$ git annex initremote skydrive type=external externaltype=skydrive encryption=shared folder=gitannex+initremote skydrive (encryption setup) (shared cipher)  21:14:20 [skydriveannex-0.2.1] <module> : 'Starting'+ 21:14:20 [skydriveannex-0.2.1] login : 'EXCEPTION api.auth_get_token:AuthenticationError(u\"Insufficient authentication data provided (missing keys: [u'code'])\",)'+osascript: OpenScripting.framework - scripting addition \"/Library/ScriptingAdditions/AeroFSFinderExtension.osax\" cannot be used with the current OS because it has no OSAXHandlers entry in its Info.plist.+git-annex: You need to set OAUTH environment variables and folder and encryption parameters when running initremote.++the folder gitannex was created on my onedrive but that is all.++Log files:+[2015-10-03 21:12:43.322605] main: Syncing with onedrive + 21:12:50 [skydriveannex-0.2.1] <module> : 'Starting'+ 21:12:5++9  [ sYkoyud rnieveeda ntnoe xs-e0t. 2O.A1U]T Hl oegnivni r:o n'mEeXnCtE PvTaIrOiNa balpeis. aauntdh _fgoeltd_etro kaennd: PernoctroycpotliEornr opra(r4a0m0e,t e'r4s0 0f oCrl itehnet  rEermrootre:. +Bad Request')'+[2015-10-03 21:13:07.435901] main: Syncing with onedrive +++  You need to set OAUTH environment variables and folder and encryption parameters for the remote.+[2015-10-03 21:15:14.60821] main: Syncing with onedrive +++  You need to set OAUTH environment variables and folder and encryption parameters for the remote.+[2015-10-03 21:15:57.926083] main: starting assistant version 5.20150929-g7010007+(scanning...) [2015-10-03 21:15:58.506776] Watcher: Performing startup scan+(started...) gpg: Signature made Wed Sep 30 20:45:53 2015 CEST using DSA key ID 89C809CB+gpg: /var/folders/m6/zkd11n111m38ff37zbtgq0lr0000gp/T/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created+gpg: Good signature from \"git-annex distribution signing key (for Joey Hess) <id@joeyh.name>\"+gpg: WARNING: This key is not certified with a trusted signature!+gpg:          There is no indication that the signature belongs to the owner.+Primary key fingerprint: 4005 5C6A FD2D 526B 2961  E78F 5EE1 DBA7 89C8 09CB+ 21:16:01 [skydriveannex-0.2.1] <module> : 'Starting'+ 21:16:++01   [Ysokuy dnreievde atnon esxe-t0 .O2A.U1T]H  leongviinr o:n m'eEnXtC EvPaTrIiOaNb laepsi .aanudt hf_ogledte_rt oaknedn :ePnrcortyopctoiloEnr rpoarr(a4m0e0t,e r's4 0f0o rC ltiheen tr eEmrortoer.:+ Bad Request')'+[2015-10-03 21:16:34.655604] main: Syncing with onedrive +++  You need to set OAUTH environment variables and folder and encryption parameters for the remote.+"""]]
+ doc/todo/xmpp_removal/comment_1_457f98a4354ad6c17dcfb5eeefb4b11e._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="dxld@02c834b220f9ffc0410d37263aa29d9373cc455b"+ nickname="dxld"+ subject="Fully p2p alternative to XMPP"+ date="2015-10-01T17:22:44Z"+ content="""+It looks like no one else has suggested this yet so I guess I'll have to: [Tox](https://tox.chat/)++Tox is pretty easy to build on all platforms (GNU/Linux, Mac and WinDOS). All the protocol relevant bits are implemented as a single C library (libtoxcore). It supports bulk file transfers and handles all the NAT hole punching nastiness internally AFAIK.++Thoughts?++"""]]
doc/upgrades.mdwn view
@@ -2,11 +2,24 @@  # Software upgrades -Upgrading the code base of git-annex will be done differently depending on your [[install]] method. For most distribution-based packages, it is handled by the package management software.+Upgrading the code base of git-annex will be done differently depending on+your [[install]] method. For most distribution-based packages, it is+handled by the package management software. -For the standalone distribution, the [[git-annex-webapp]](1) will ask the user for confirmation when it detects a new version. Once that is confirmed, or if `annex.autoupgrade` is enabled (see the [[git-annex]](1) manpage) the assistant will start the upgrade. The upgrade process is fairly simple: the assistant will move the `git-annex.linux` directory out of the way and replace it with the new version, then re-execute itself. It therefore needs write access to the parent directory of the `git-annex.linux` directory.+For the standalone distribution, the [[git-annex-webapp]](1) will ask the+user for confirmation when it detects a new version. Once that is+confirmed, or if `annex.autoupgrade` is enabled (see the [[git-annex]](1)+manpage) the assistant will start the upgrade. The upgrade process is+fairly simple: the assistant will move the `git-annex.linux` directory out+of the way and replace it with the new version, then re-execute itself. It+therefore needs write access to the parent directory of the+`git-annex.linux` directory. -Note that "upgrading" from a distribution-based package to the [[install/Linux_standalone/]] version may cause weird problems, as an unexpected version of git-annex (e.g. the old one from packages) may be ran, see [[bugs/git-annex-shell_doesn__39__t_work_as_expected/]] for a full discussion.+Note that "upgrading" from a distribution-based package to the+[[install/Linux_standalone/]] version may cause weird problems, as an+unexpected version of git-annex (e.g. the old one from packages) may be+ran, see [[bugs/git-annex-shell_doesn__39__t_work_as_expected/]] for a full+discussion.  # Repository upgrades
doc/walkthrough/adding_a_remote.mdwn view
@@ -11,8 +11,11 @@ 	# git remote add usbdrive /media/usb/annex  This is all standard ad-hoc distributed git repository setup.-The only git-annex specific part is telling it the name-of the new repository created on the USB drive.++The only git-annex specific part is telling it a description+of the new repository created on the USB drive. This is optional, but+giving the repository a description helps when git-annex talks about it+later.  Notice that both repos are set up as remotes of one another. This lets either get annexed files from the other. You'll want to do that even
doc/walkthrough/creating_a_repository.mdwn view
@@ -1,6 +1,6 @@-This is very straightforward. Just tell it a description of the repository.+This is very straightforward.  	# mkdir ~/annex 	# cd ~/annex 	# git init-	# git annex init "my laptop"+	# git annex init
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20150930+Version: 5.20151019 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <id@joeyh.name>@@ -157,7 +157,8 @@     GHC-Options: -O2    if (os(windows))-    Build-Depends: Win32, Win32-extras, unix-compat (>= 0.4.1.3), setenv+    Build-Depends: Win32, Win32-extras, unix-compat (>= 0.4.1.3), setenv,+      process (>= 1.3.0.0)   else     Build-Depends: unix     -- Need to list these because they're generated from .hsc files.
man/git-annex-reinject.1 view
@@ -12,8 +12,8 @@ This can be useful if you have obtained the content of a file from elsewhere and want to put it in the local annex. .PP-Automatically runs fsck on dest to check that the expected content was-provided.+Verifies that the src file's content matches with the content that the dest+file is expected to have, and refuses to reinject it otherwise. .PP Example: .PP
man/git-annex-setkey.1 view
@@ -9,8 +9,7 @@ This plumbing\-level command makes the content of the specified key be set to the specified file. The file is moved into the annex. .PP-No checking is done that the file contains the expected contents of the key.-So it's generally a better idea to use git-annex\-reinject(1) instead of+It's generally a better idea to use git-annex\-reinject(1) instead of this command. .PP .SH SEE ALSO
man/git-annex-shell.1 view
@@ -38,6 +38,16 @@ Exits 100 if it's unable to tell (perhaps the key is in the process of being removed from the annex). .IP+.IP "lockcontent directory key"+This locks a key's content in place in the annex, preventing it from+being dropped.+.IP+Once the content is successfully locked, outputs "OK". Then the content+remains locked until a newline is received from the caller or the+connection is broken.+.IP+Exits nonzero if the content is not present, or could not be locked.+.IP .IP "dropkey directory [key ...]" This drops the annexed data for the specified keys. .IP
man/git-annex.1 view
@@ -883,6 +883,11 @@ This both prevents git-annex sync from pushing changes, and prevents storing or removing files from read\-only remote. .IP+.IP "\fBremote.<name>.annex\-verify\fP, \fBannex.verify\fP"+By default, git-annex will verify the checksums of objects downloaded+from remotes. If you trust a remote and don't want the overhead+of these checksums, you can set this to \fBfalse\fP.+.IP .IP "\fBremote.<name>.annexUrl\fP" Can be used to specify a different url than the regular \fBremote.<name>.url\fP for git-annex to use when talking with the remote. Similar to the \fBpushUrl\fP
stack.yaml view
@@ -27,5 +27,6 @@     webapp: true packages: - '.'-extra-deps: []+extra-deps:+- Win32-extras-0.2.0.1 resolver: lts-3.7
standalone/windows/build.sh view
@@ -38,6 +38,8 @@ # for haskell libraries to link them with the cygwin library. cabal update || true +cabal install warp-3.1.3.1+ cabal install --only-dependencies || true  # Detect when the last build was an incremental build and failed, @@ -62,16 +64,24 @@ # These are msys2 programs, from https://msys2.github.io/. # Since git for windows uses msys2, and includes its libraries, # these programs will work well with it.-if [ ! -e rsync.exe ] || [ "$(sha1sum rsync.exe)" != "85cb7a4d16d274fcf8069b39042965ad26abd6aa" ]; then-	rm -f rsync.exe || true-	withcyg wget https://downloads.kitenet.net/git-annex/windows/assets/rsync.exe-	withcyg chmod +x rsync.exe-fi-if [ ! -e wget.exe ] || [ "$(sha1sum wget.exe)" != "044380729200d5762965b10123a4f134806b01cf" ]; then-	rm -f wget.exe || true-	withcyg wget https://downloads.kitenet.net/git-annex/windows/assets/wget.exe-	withcyg chmod +x wget.exe-fi+getextra () {+	extrap="$1"+	extrasha="$2"+	curextrasha="$(withcyg sha1sum $extrap | sed 's/ .*//')"+	if [ ! -e "$extrap" ] || [ "$curextrasha" != "$extrasha" ]; then+		rm -f "$extrap" || true+		withcyg wget https://downloads.kitenet.net/git-annex/windows/assets/$extrap+		curextrasha="$(withcyg sha1sum $extrap | sed 's/ .*//')"+		if [ "$curextrasha" != "$extrasha" ]; then+			rm -f "$extrap"+			echo "CHECKSUM FAILURE" >&2+			exit 1+		fi+		withcyg chmod +x $extrap+	fi+}+getextra rsync.exe 85cb7a4d16d274fcf8069b39042965ad26abd6aa+getextra wget.exe 044380729200d5762965b10123a4f134806b01cf  # Build the installer cabal install nsis