packages feed

git-annex 4.20130827 → 4.20130909

raw patch · 163 files changed

+3289/−735 lines, 163 files

Files

Annex/Branch.hs view
@@ -1,6 +1,6 @@ {- management of the git-annex branch  -- - Copyright 2011-2012 Joey Hess <joey@kitenet.net>+ - Copyright 2011-2013 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -22,9 +22,12 @@ 	commit, 	files, 	withIndex,+	performTransitions, ) where  import qualified Data.ByteString.Lazy.Char8 as L+import qualified Data.Set as S+import qualified Data.Map as M  import Common.Annex import Annex.BranchState@@ -32,6 +35,7 @@ import qualified Git import qualified Git.Command import qualified Git.Ref+import qualified Git.Sha import qualified Git.Branch import qualified Git.UnionMerge import qualified Git.UpdateIndex@@ -42,6 +46,12 @@ import Annex.Perms import qualified Annex import Utility.Env+import Logs+import Logs.Transitions+import Logs.Trust.Pure+import Annex.ReplaceFile+import qualified Annex.Queue+import Annex.Branch.Transitions  {- Name of the branch that is used to store git-annex's information. -} name :: Git.Ref@@ -110,6 +120,9 @@  - later get staged, and might overwrite changes made during the merge.  - This is only done if some of the Refs do need to be merged.  -+ - Also handles performing any Transitions that have not yet been+ - performed, in either the local branch, or the Refs.+ -  - Returns True if any refs were merged in, False otherwise.  -} updateTo :: [(Git.Ref, Git.Branch)] -> Annex Bool@@ -117,7 +130,8 @@ 	-- ensure branch exists, and get its current ref 	branchref <- getBranch 	dirty <- journalDirty-	(refs, branches) <- unzip <$> filterM isnewer pairs+	ignoredrefs <- getIgnoredRefs+	(refs, branches) <- unzip <$> filterM (isnewer ignoredrefs) pairs 	if null refs 		{- Even when no refs need to be merged, the index 		 - may still be updated if the branch has gotten ahead @@ -132,7 +146,9 @@ 		else lockJournal $ go branchref dirty refs branches 	return $ not $ null refs   where-	isnewer (r, _) = inRepo $ Git.Branch.changed fullname r+	isnewer ignoredrefs (r, _)+		| S.member r ignoredrefs = return False+		| otherwise = inRepo $ Git.Branch.changed fullname r 	go branchref dirty refs branches = withIndex $ do 		cleanjournal <- if dirty then stageJournal else return noop 		let merge_desc = if null branches@@ -140,23 +156,26 @@ 			else "merging " ++ 				unwords (map Git.Ref.describe branches) ++  				" into " ++ show name+		localtransitions <- parseTransitionsStrictly "local"+			<$> getStale transitionsLog 		unless (null branches) $ do 			showSideAction merge_desc 			mergeIndex refs-		ff <- if dirty-			then return False-			else inRepo $ Git.Branch.fastForward fullname refs-		if ff-			then updateIndex branchref-			else commitBranch branchref merge_desc-				(nub $ fullname:refs)+		let commitrefs = nub $ fullname:refs+		unlessM (handleTransitions localtransitions commitrefs) $ do+			ff <- if dirty+				then return False+				else inRepo $ Git.Branch.fastForward fullname refs+			if ff+				then updateIndex branchref+				else commitBranch branchref merge_desc commitrefs 		liftIO cleanjournal  {- Gets the content of a file, which may be in the journal, or in the index  - (and committed to the branch).  -   - Updates the branch if necessary, to ensure the most up-to-date available- - content is available.+ - content is returned.  -  - Returns an empty string if the file doesn't exist yet. -} get :: FilePath -> Annex String@@ -175,7 +194,10 @@ get' file = go =<< getJournalFile file   where 	go (Just journalcontent) = return journalcontent-	go Nothing = withIndex $ L.unpack <$> catFile fullname file+	go Nothing = getRaw file+	+getRaw :: FilePath -> Annex String+getRaw file = withIndex $ L.unpack <$> catFile fullname file  {- Applies a function to modifiy the content of a file.  -@@ -225,7 +247,8 @@ 	committedref <- inRepo $ Git.Branch.commit message fullname parents 	setIndexSha committedref 	parentrefs <- commitparents <$> catObject committedref-	when (racedetected branchref parentrefs) $+	when (racedetected branchref parentrefs) $ do+		liftIO $ print ("race detected", branchref, parentrefs, "committing", (branchref, parents)) 		fixrace committedref parentrefs   where 	-- look for "parent ref" lines and return the refs@@ -253,14 +276,18 @@ files :: Annex [FilePath] files = do 	update-	withIndex $ do-		bfiles <- inRepo $ Git.Command.pipeNullSplitZombie-			[ Params "ls-tree --name-only -r -z"-			, Param $ show fullname-			]-		jfiles <- getJournalledFiles-		return $ jfiles ++ bfiles+	(++)+		<$> branchFiles+		<*> getJournalledFiles +{- Files in the branch, not including any from journalled changes,+ - and without updating the branch. -}+branchFiles :: Annex [FilePath]+branchFiles = withIndex $ inRepo $ Git.Command.pipeNullSplitZombie+	[ Params "ls-tree --name-only -r -z"+	, Param $ show fullname+	]+ {- Populates the branch's index file with the current branch contents.  -   - This is only done when the index doesn't yet exist, and the index @@ -361,3 +388,112 @@ 		sha <- hashFile h path 		streamer $ Git.UpdateIndex.updateIndexLine 			sha FileBlob (asTopFilePath $ fileJournal file)++{- This is run after the refs have been merged into the index,+ - but before the result is committed to the branch.+ - (Which is why it's passed the contents of the local branches's+ - transition log before that merge took place.)+ -+ - When the refs contain transitions that have not yet been done locally,+ - the transitions are performed on the index, and a new branch+ - is created from the result.+ -+ - When there are transitions recorded locally that have not been done+ - to the remote refs, the transitions are performed in the index,+ - and committed to the existing branch. In this case, the untransitioned+ - remote refs cannot be merged into the branch (since transitions+ - throw away history), so they are added to the list of refs to ignore,+ - to avoid re-merging content from them again.+ -}+handleTransitions :: Transitions -> [Git.Ref] -> Annex Bool+handleTransitions localts refs = do+	m <- M.fromList <$> mapM getreftransition refs+	let remotets = M.elems m+	if all (localts ==) remotets+		then return False+		else do+			let allts = combineTransitions (localts:remotets)+			let (transitionedrefs, untransitionedrefs) =+				partition (\r -> M.lookup r m == Just allts) refs+			performTransitions allts (localts /= allts) transitionedrefs+			ignoreRefs untransitionedrefs+			return True+  where+  	getreftransition ref = do+		ts <- parseTransitionsStrictly "remote" . L.unpack+			<$> catFile ref transitionsLog+		return (ref, ts)++ignoreRefs :: [Git.Ref] -> Annex ()+ignoreRefs rs = do+	old <- getIgnoredRefs+	let s = S.unions [old, S.fromList rs]+	f <- fromRepo gitAnnexIgnoredRefs+	replaceFile f $ \tmp -> liftIO $ writeFile tmp $+		unlines $ map show $ S.elems s++getIgnoredRefs :: Annex (S.Set Git.Ref)+getIgnoredRefs = S.fromList . mapMaybe Git.Sha.extractSha . lines <$> content+  where+  	content = do+		f <- fromRepo gitAnnexIgnoredRefs+		liftIO $ catchDefaultIO "" $ readFile f++{- Performs the specified transitions on the contents of the index file,+ - commits it to the branch, or creates a new branch. -}+performTransitions :: Transitions -> Bool -> [Ref] -> Annex ()+performTransitions ts neednewlocalbranch transitionedrefs = do+	-- For simplicity & speed, we're going to use the Annex.Queue to+	-- update the git-annex branch, while it usually holds changes+	-- for the head branch. Flush any such changes.+	Annex.Queue.flush+	withIndex $ do+		run $ mapMaybe getTransitionCalculator $ transitionList ts+		Annex.Queue.flush+		if neednewlocalbranch+			then do+				committedref <- inRepo $ Git.Branch.commit message fullname transitionedrefs+				setIndexSha committedref+			else do+				ref <- getBranch+				commitBranch ref message (nub $ fullname:transitionedrefs)+  where+  	message+		| neednewlocalbranch && null transitionedrefs = "new branch for transition " ++ tdesc+		| otherwise = "continuing transition " ++ tdesc+	tdesc = show $ map describeTransition $ transitionList ts++	{- The changes to make to the branch are calculated and applied to+	 - the branch directly, rather than going through the journal,+	 - which would be innefficient. (And the journal is not designed+	 - to hold changes to every file in the branch at once.)+	 -+	 - When a file in the branch is changed by transition code,+	 - that value is remembered and fed into the code for subsequent+	 - transitions.+	 -}+	run [] = noop+	run changers = do+		trustmap <- calcTrustMap <$> getRaw trustLog+		fs <- branchFiles+		hasher <- inRepo hashObjectStart+		forM_ fs $ \f -> do+			content <- getRaw f+			apply changers hasher f content trustmap+		liftIO $ hashObjectStop hasher+	apply [] _ _ _ _ = return ()+	apply (changer:rest) hasher file content trustmap =+		case changer file content trustmap of+			RemoveFile -> do+				Annex.Queue.addUpdateIndex+					=<< inRepo (Git.UpdateIndex.unstageFile file)+				-- File is deleted; can't run any other+				-- transitions on it.+				return ()+			ChangeFile content' -> do+				sha <- inRepo $ hashObject BlobObject content'+				Annex.Queue.addUpdateIndex $ Git.UpdateIndex.pureStreamer $+					Git.UpdateIndex.updateIndexLine sha FileBlob (asTopFilePath file)+				apply rest hasher file content' trustmap+			PreserveFile ->+				apply rest hasher file content trustmap
+ Annex/Branch/Transitions.hs view
@@ -0,0 +1,53 @@+{- git-annex branch transitions+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Annex.Branch.Transitions (+	FileTransition(..),+	getTransitionCalculator+) where++import Logs+import Logs.Transitions+import Logs.UUIDBased as UUIDBased+import Logs.Presence.Pure as Presence+import Types.TrustLevel+import Types.UUID++import qualified Data.Map as M++data FileTransition+	= ChangeFile String+	| RemoveFile+	| PreserveFile++type TransitionCalculator = FilePath -> String -> TrustMap -> FileTransition++getTransitionCalculator :: Transition -> Maybe TransitionCalculator+getTransitionCalculator ForgetGitHistory = Nothing+getTransitionCalculator ForgetDeadRemotes = Just dropDead++dropDead :: FilePath -> String -> TrustMap -> FileTransition+dropDead f content trustmap = case getLogVariety f of+	Just UUIDBasedLog -> ChangeFile $+		UUIDBased.showLog id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLog Just content+	Just (PresenceLog _) ->+		let newlog = Presence.compactLog $ dropDeadFromPresenceLog trustmap $ Presence.parseLog content+		in if null newlog+			then RemoveFile+			else ChangeFile $ Presence.showLog newlog+	Nothing -> PreserveFile++dropDeadFromUUIDBasedLog :: TrustMap -> UUIDBased.Log String -> UUIDBased.Log String+dropDeadFromUUIDBasedLog trustmap = M.filterWithKey $ notDead trustmap . const++{- Presence logs can contain UUIDs or other values. Any line that matches+ - a dead uuid is dropped; any other values are passed through. -}+dropDeadFromPresenceLog :: TrustMap -> [Presence.LogLine] -> [Presence.LogLine]+dropDeadFromPresenceLog trustmap = filter $ notDead trustmap (toUUID . Presence.info)++notDead :: TrustMap -> (v -> UUID) -> v -> Bool+notDead trustmap a v = M.findWithDefault SemiTrusted (a v) trustmap /= DeadTrusted
Annex/Content.hs view
@@ -279,6 +279,7 @@ 			then do 				updateInodeCache key src 				replaceFile f $ liftIO . moveFile src+				chmodContent f 				forM_ fs $ 					addContentWhenNotPresent key f 			else ifM (goodContent key f)@@ -498,6 +499,18 @@ 		addModes readModes 	go _ = modifyFileMode file $ 		removeModes writeModes .+		addModes [ownerReadMode]++{- Adjusts read mode of annexed file per core.sharedRepository setting. -}+chmodContent :: FilePath -> Annex ()+chmodContent file = unlessM crippledFileSystem $+	liftIO . go =<< fromRepo getSharedRepository+  where+	go GroupShared = modifyFileMode file $+		addModes [ownerReadMode, groupReadMode]+	go AllShared = modifyFileMode file $+		addModes readModes+	go _ = modifyFileMode file $ 		addModes [ownerReadMode]  {- Allows writing to an annexed file that freezeContent was called on
Annex/TaggedPush.hs view
@@ -13,13 +13,14 @@ import qualified Git import qualified Git.Ref import qualified Git.Command+import qualified Git.Branch import Utility.Base64  {- Converts a git branch into a branch that is tagged with a UUID, typically  - the UUID of the repo that will be pushing it, and possibly with other  - information.  -- - Pushing to branches on the remote that have out uuid in them is ugly,+ - Pushing to branches on the remote that have our uuid in them is ugly,  - but it reserves those branches for pushing by us, and so our pushes will  - never conflict with other pushes.  -@@ -50,7 +51,10 @@ taggedPush u info branch remote = Git.Command.runBool         [ Param "push"         , Param $ Remote.name remote-        , Param $ refspec Annex.Branch.name+	{- Using forcePush here is safe because we "own" the tagged branch+	 - we're pushing; it has no other writers. Ensures it is pushed+	 - even if it has been rewritten by a transition. -}+        , Param $ Git.Branch.forcePush $ refspec Annex.Branch.name         , Param $ refspec branch         ]   where
Annex/UUID.hs view
@@ -17,8 +17,11 @@ 	getUncachedUUID, 	prepUUID, 	genUUID,+	genUUIDInNameSpace,+	gCryptNameSpace, 	removeRepoUUID, 	storeUUID,+	setUUID, ) where  import Common.Annex@@ -27,7 +30,9 @@ import Config  import qualified Data.UUID as U+import qualified Data.UUID.V5 as U5 import System.Random+import Data.Bits.Utils  configkey :: ConfigKey configkey = annexConfig "uuid"@@ -36,6 +41,17 @@ genUUID :: IO UUID genUUID = UUID . show <$> (randomIO :: IO U.UUID) +{- Generates a UUID from a given string, using a namespace.+ - Given the same namespace, the same string will always result+ - in the same UUID. -}+genUUIDInNameSpace :: U.UUID -> String -> UUID+genUUIDInNameSpace namespace = UUID . show . U5.generateNamed namespace . s2w8++{- Namespace used for UUIDs derived from git-remote-gcrypt ids. -}+gCryptNameSpace :: U.UUID+gCryptNameSpace = U5.generateNamed U5.namespaceURL $+	s2w8 "http://git-annex.branchable.com/design/gcrypt/" + {- Get current repository's UUID. -} getUUID :: Annex UUID getUUID = getRepoUUID =<< gitRepo@@ -72,3 +88,9 @@  storeUUID :: ConfigKey -> UUID -> Annex () storeUUID configfield = setConfig configfield . fromUUID++{- Only sets the configkey in the Repo; does not change .git/config -}+setUUID :: Git.Repo -> UUID -> IO Git.Repo+setUUID r u = do+	let s = show configkey ++ "=" ++ fromUUID u+	Git.Config.store s r
Assistant/DaemonStatus.hs view
@@ -57,8 +57,7 @@  	return $ \dstatus -> dstatus 		{ syncRemotes = syncable-		, syncGitRemotes =-			filter (not . Remote.specialRemote) syncable+		, syncGitRemotes = filter Remote.syncableRemote syncable 		, syncDataRemotes = syncdata 		, syncingToCloudRemote = any iscloud syncdata 		}
Assistant/Install/AutoStart.o view

binary file changed (3428 → 5192 bytes)

Assistant/Install/Menu.o view

binary file changed (9360 → 15328 bytes)

Assistant/MakeRemote.hs view
@@ -68,8 +68,8 @@ 	go =<< Command.InitRemote.findExisting name   where   	go Nothing = setupSpecialRemote name Rsync.remote config-		=<< Command.InitRemote.generateNew name-	go (Just v) = setupSpecialRemote name Rsync.remote config v+		(Nothing, Command.InitRemote.newConfig name)+	go (Just (u, c)) = setupSpecialRemote name Rsync.remote config (Just u, c) 	config = M.fromList 		[ ("encryption", "shared") 		, ("rsyncurl", location)@@ -89,7 +89,7 @@ 		r <- Command.InitRemote.findExisting fullname 		case r of 			Nothing -> setupSpecialRemote fullname remotetype config-				=<< Command.InitRemote.generateNew fullname+				(Nothing, Command.InitRemote.newConfig fullname) 			Just _ -> go (n + 1)  {- Enables an existing special remote. -}@@ -98,15 +98,15 @@ 	r <- Command.InitRemote.findExisting name 	case r of 		Nothing -> error $ "Cannot find a special remote named " ++ name-		Just v -> setupSpecialRemote name remotetype config v+		Just (u, c) -> setupSpecialRemote name remotetype config (Just u, c) -setupSpecialRemote :: RemoteName -> RemoteType -> R.RemoteConfig -> (UUID, R.RemoteConfig) -> Annex RemoteName-setupSpecialRemote name remotetype config (u, c) = do+setupSpecialRemote :: RemoteName -> RemoteType -> R.RemoteConfig -> (Maybe UUID, R.RemoteConfig) -> Annex RemoteName+setupSpecialRemote name remotetype config (mu, c) = do 	{- Currently, only 'weak' ciphers can be generated from the 	 - assistant, because otherwise GnuPG may block once the entropy 	 - pool is drained, and as of now there's no way to tell the user 	 - to perform IO actions to refill the pool. -}-	c' <- R.setup remotetype u $+	(c', u) <- R.setup remotetype mu $ 		M.insert "highRandomQuality" "false" $ M.union config c 	describeUUID u name 	configSet u c'
Build/Configure.o view

binary file changed (54500 → 95744 bytes)

Build/DesktopFile.o view

binary file changed (14364 → 24176 bytes)

Build/InstallDesktopFile.o view

binary file changed (2536 → 4000 bytes)

Build/TestConfig.o view

binary file changed (33088 → 56512 bytes)

BuildFlags.hs view
@@ -48,7 +48,10 @@ #ifdef WITH_DNS 	, "DNS" #endif-#ifdef WITH_FEEDS+#ifdef WITH_FEED 	, "Feeds"+#endif+#ifdef WITH_QUVI+	, "Quvi" #endif 	]
CHANGELOG view
@@ -1,3 +1,42 @@+git-annex (4.20130909) unstable; urgency=low++  * initremote: Syntax change when setting up an encrypted special remote.+    Now use keyid=$KEYID rather than the old encryption=$KEYID+  * forget: New command, causes git-annex branch history to be forgotten+    in a way that will spread to other clones of the repository.+    (As long as they're running this version or newer of git-annex.)+  * forget --drop-dead: Completely removes mentions of repositories that+    have been marked as dead from the git-annex branch.+  * sync, assistant: Force push of the git-annex branch. Necessary+    to ensure it gets pushed to remotes after being rewritten by forget.+  * Added gcrypt support. This combines a fully encrypted git+    repository (using git-remote-gcrypt) with an encrypted git-annex special+    remote.+  * sync: Support syncing with gcrypt remotes.+  * importfeed: Also ignore transient problems with downloading content+    from feeds.+  * Honor core.sharedrepository when receiving and adding files in direct+    mode.+  * enableremote: gpg keys can be removed from those a remote encrypts+    to by passing "keyid-=$KEYID". keyid+= is also provided.+    (Thanks, guilhem for the patch.)+  * Added encryption=pubkey scheme, which encrypts to public keys directly+    rather than the hybrid approach. See documentation for advantages+    and disadvantages, but encryption=hybrid is the recommended scheme still.+    (Thanks, guilhem for the patch.)+  * Fix Feeds display in build flags.+  * Remind user when annex-ignore is set for some remotes, if unable to+    get or drop a file, possibly because it's on an ignored remote.+  * gpg: Force --no-textmode in case the user has it turned on in config.+  * webapp: Improve javascript's handling of longpolling connection+    failures, by reloading the current page in this case.+    Works around chromium behavior where ajax connections to urls+    that were already accessed are denied after navigating back to+    a previous page.+  * Allow building without quvi support.++ -- Joey Hess <joeyh@debian.org>  Mon, 09 Sep 2013 09:47:02 -0400+ git-annex (4.20130827) unstable; urgency=low    * Youtube support! (And 53 other video hosts). When quvi is installed,
@@ -14,10 +14,6 @@            2012 Joey Hess <joey@kitenet.net> License: GPL-3+ -Files: Utility/Gpg/Types.hs-Copyright: 2013 guilhem <guilhem@fripost.org>-License: GPL-3+- Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns standalone/android/icons/* Copyright: 2007 Henrik Nyh <http://henrik.nyh.se/>            2010 Joey Hess <joey@kitenet.net>
Command/AddUrl.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE CPP #-}+ module Command.AddUrl where  import Network.URI@@ -27,8 +29,10 @@ import Logs.Location import qualified Logs.Transfer as Transfer import Utility.Daemon (checkDaemon)+#ifdef WITH_QUVI import Annex.Quvi import qualified Utility.Quvi as Quvi+#endif  def :: [Command] def = [notBareRepo $ withOptions [fileOption, pathdepthOption, relaxedOption] $@@ -56,18 +60,25 @@   	(s', downloader) = getDownloader s 	bad = fromMaybe (error $ "bad url " ++ s') $ 		parseURI $ escapeURIString isUnescapedInURI s'-	badquvi = error $ "quvi does not know how to download url " ++ s' 	choosefile = flip fromMaybe optfile 	go url = case downloader of 		QuviDownloader -> usequvi-		DefaultDownloader -> ifM (liftIO $ Quvi.supported s')-			( usequvi-			, do-				pathmax <- liftIO $ fileNameLengthLimit "."-				let file = choosefile $ url2file url pathdepth pathmax-				showStart "addurl" file-				next $ perform relaxed s' file-			)+		DefaultDownloader -> +#ifdef WITH_QIVI+			ifM (liftIO $ Quvi.supported s')+				( usequvi+				, regulardownload url+				)+#else+			regulardownload url+#endif+	regulardownload url = do+		pathmax <- liftIO $ fileNameLengthLimit "."+		let file = choosefile $ url2file url pathdepth pathmax+		showStart "addurl" file+		next $ perform relaxed s' file+#ifdef WITH_QUVI+	badquvi = error $ "quvi does not know how to download url " ++ s' 	usequvi = do 		page <- fromMaybe badquvi 			<$> withQuviOptions Quvi.forceQuery [Quvi.quiet, Quvi.httponly] s'@@ -76,7 +87,11 @@ 			Quvi.pageTitle page ++ "." ++ Quvi.linkSuffix link 		showStart "addurl" file 		next $ performQuvi relaxed s' (Quvi.linkUrl link) file+#else+	usequvi = error "not built with quvi support"+#endif +#ifdef WITH_QUVI performQuvi :: Bool -> URLString -> URLString -> FilePath -> CommandPerform performQuvi relaxed pageurl videourl file = ifAnnexed file addurl geturl   where@@ -96,6 +111,7 @@ 					then next $ cleanup quviurl file key (Just tmp) 					else stop 			)+#endif  perform :: Bool -> URLString -> FilePath -> CommandPerform perform relaxed url file = ifAnnexed file addurl geturl
Command/EnableRemote.hs view
@@ -47,8 +47,8 @@  perform :: RemoteType -> UUID -> R.RemoteConfig -> CommandPerform perform t u c = do-	c' <- R.setup t u c-	next $ cleanup u c'+	(c', u') <- R.setup t (Just u) c+	next $ cleanup u' c'  cleanup :: UUID -> R.RemoteConfig -> CommandCleanup cleanup u c = do
+ Command/Forget.hs view
@@ -0,0 +1,52 @@+{- git-annex command+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Forget where++import Common.Annex+import Command+import qualified Annex.Branch as Branch+import Logs.Transitions+import qualified Annex+import qualified Option++import Data.Time.Clock.POSIX++def :: [Command]+def = [withOptions forgetOptions $ command "forget" paramNothing seek+		SectionMaintenance "prune git-annex branch history"]++forgetOptions :: [Option]+forgetOptions = [dropDeadOption]++dropDeadOption :: Option+dropDeadOption = Option.flag [] "drop-dead" "drop references to dead repositories"++seek :: [CommandSeek]+seek = [withFlag dropDeadOption $ \dropdead ->+	withNothing $ start dropdead]++start :: Bool -> CommandStart+start dropdead = do+	showStart "forget" "git-annex"+	now <- liftIO getPOSIXTime+	let basets = addTransition now ForgetGitHistory noTransitions+	let ts = if dropdead+		then addTransition now ForgetDeadRemotes basets+		else basets+	next $ perform ts =<< Annex.getState Annex.force++perform :: Transitions -> Bool -> CommandPerform+perform ts True = do+	recordTransitions Branch.change ts+	-- get branch committed before contining with the transition+	Branch.update+	void $ Branch.performTransitions ts True []+	next $ return True+perform _ False = do+	showLongNote "To forget git-annex branch history, you must specify --force. This deletes metadata!"+	stop
Command/ImportFeed.hs view
@@ -52,7 +52,9 @@ 		Just l | not (null l) -> do 			ok <- all id 				<$> mapM (downloadEnclosure relaxed cache) l-			next $ cleanup url ok+			unless ok $+				feedProblem url "problem downloading item"+			next $ cleanup url True 		_ -> do 			feedProblem url "bad feed content" 			next $ return True
Command/InitRemote.hs view
@@ -14,7 +14,6 @@ import qualified Remote import qualified Logs.Remote import qualified Types.Remote as R-import Annex.UUID import Logs.UUID import Logs.Trust @@ -34,18 +33,18 @@ 	( error $ "There is already a special remote named \"" ++ name ++ 		"\". (Use enableremote to enable an existing special remote.)" 	, do-		(u, c) <- generateNew name+		let c = newConfig name 		t <- findType config  		showStart "initremote" name-		next $ perform t u name $ M.union config c+		next $ perform t name $ M.union config c 	)   where 	config = Logs.Remote.keyValToConfig ws -perform :: RemoteType -> UUID -> String -> R.RemoteConfig -> CommandPerform-perform t u name c = do-	c' <- R.setup t u c+perform :: RemoteType -> String -> R.RemoteConfig -> CommandPerform+perform t name c = do+	(c', u) <- R.setup t Nothing c 	next $ cleanup u name c'  cleanup :: UUID -> String -> R.RemoteConfig -> CommandCleanup@@ -63,10 +62,8 @@ 		<$> Logs.Remote.readRemoteLog 	return $ headMaybe matches -generateNew :: String -> Annex (UUID, R.RemoteConfig)-generateNew name = do-	uuid <- liftIO genUUID-	return (uuid, M.singleton nameKey name)+newConfig :: String -> R.RemoteConfig+newConfig name = M.singleton nameKey name  findByName :: String ->  M.Map UUID R.RemoteConfig -> [(UUID, R.RemoteConfig)] findByName n = filter (matching . snd) . M.toList
Command/Log.hs view
@@ -17,7 +17,7 @@  import Common.Annex import Command-import qualified Logs.Location+import Logs import qualified Logs.Presence import Annex.CatFile import qualified Annex.Branch@@ -135,7 +135,7 @@ getLog key os = do 	top <- fromRepo Git.repoPath 	p <- liftIO $ relPathCwdToFile top-	let logfile = p </> Logs.Location.logFile key+	let logfile = p </> locationLogFile key 	inRepo $ pipeNullSplitZombie $ 		[ Params "log -z --pretty=format:%ct --raw --abbrev=40" 		, Param "--remove-empty"
Command/Sync.hs view
@@ -67,12 +67,12 @@ 		| otherwise = listed 	listed = do 		l <- catMaybes <$> mapM (Remote.byName . Just) rs-		let s = filter Remote.specialRemote l+		let s = filter (not . Remote.syncableRemote) l 		unless (null s) $ 			error $ "cannot sync special remotes: " ++ 				unwords (map Types.Remote.name s) 		return l-	available = filter (not . Remote.specialRemote)+	available = filter Remote.syncableRemote 		. filter (remoteAnnexSync . Types.Remote.gitconfig) 		<$> Remote.remoteList 	good = filterM $ Remote.Git.repoAvail . Types.Remote.repo@@ -167,29 +167,45 @@ 			showOutput 			inRepo $ pushBranch remote branch -{- If the remote is a bare git repository, it's best to push the branch- - directly to it. On the other hand, if it's not bare, pushing to the- - checked out branch will fail, and this is why we use the syncBranch.+{- Pushes a regular branch like master to a remote. Also pushes the git-annex+ - branch.  -+ - If the remote is a bare git repository, it's best to push the regular + - branch directly to it, so that cloning/pulling will get it.+ - On the other hand, if it's not bare, pushing to the checked out branch+ - will fail, and this is why we push to its syncBranch.+ -  - Git offers no way to tell if a remote is bare or not, so both methods  - are tried.  -  - The direct push is likely to spew an ugly error message, so stderr is- - elided. Since progress is output to stderr too, the sync push is done- - first, and actually sends the data. Then the direct push is tried,- - with stderr discarded, to update the branch ref on the remote.+ - elided. Since git progress display goes to stderr too, the sync push+ - is done first, and actually sends the data. Then the direct push is+ - tried, with stderr discarded, to update the branch ref on the remote.+ -+ - The sync push forces the update of the remote synced/git-annex branch.+ - This is necessary if a transition has rewritten the git-annex branch.+ - Normally any changes to the git-annex branch get pulled and merged before+ - this push, so this forcing is unlikely to overwrite new data pushed+ - in from another repository that is also syncing.+ -+ - But overwriting of data on synced/git-annex can happen, in a race.+ - The only difference caused by using a forced push in that case is that+ - the last repository to push wins the race, rather than the first to push.  -} pushBranch :: Remote -> Git.Ref -> Git.Repo -> IO Bool-pushBranch remote branch g = tryIO directpush `after` syncpush+pushBranch remote branch g = tryIO (directpush g) `after` syncpush g   where-	syncpush = Git.Command.runBool (pushparams (refspec branch)) g-	directpush = Git.Command.runQuiet (pushparams (show $ Git.Ref.base branch)) g-	pushparams b =+	syncpush = Git.Command.runBool $ pushparams+		[ Git.Branch.forcePush $ refspec Annex.Branch.name+		, refspec branch+		]+	directpush = Git.Command.runQuiet $ pushparams+		[show $ Git.Ref.base branch]+	pushparams branches = 		[ Param "push" 		, Param $ Remote.name remote-		, Param $ refspec Annex.Branch.name-		, Param b-		]+		] ++ map Param branches 	refspec b = concat  		[ show $ Git.Ref.base b 		,  ":"
Common.o view

binary file changed (622 → 906 bytes)

Config.hs view
@@ -25,6 +25,9 @@ getConfig :: ConfigKey -> String -> Annex String getConfig (ConfigKey key) def = fromRepo $ Git.Config.get key def +getConfigMaybe :: ConfigKey -> Annex (Maybe String)+getConfigMaybe (ConfigKey key) = fromRepo $ Git.Config.getMaybe key+ {- Changes a git config setting in both internal state and .git/config -} setConfig :: ConfigKey -> String -> Annex () setConfig (ConfigKey key) value = do
Config/Files.o view

binary file changed (15108 → 25568 bytes)

Creds.hs view
@@ -52,7 +52,7 @@ 		return c  	storeconfig creds key (Just cipher) = do-		s <- liftIO $ encrypt (GpgOpts []) cipher+		s <- liftIO $ encrypt [] cipher 			(feedBytes $ L.pack $ encodeCredPair creds) 			(readBytes $ return . L.unpack) 		return $ M.insert key (toB64 s) c
Crypto.hs view
@@ -8,6 +8,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE FlexibleInstances #-}+ module Crypto ( 	Cipher, 	KeyIds(..),@@ -22,9 +24,8 @@ 	feedBytes, 	readBytes, 	encrypt,-	decrypt,	-	GpgOpts(..),-	getGpgOpts,+	decrypt,+	getGpgEncParams,  	prop_HmacSha1WithCipher_sane ) where@@ -32,17 +33,18 @@ import qualified Data.ByteString.Lazy as L import Data.ByteString.Lazy.UTF8 (fromString) import Control.Applicative+import qualified Data.Map as M  import Common.Annex import qualified Utility.Gpg as Gpg-import Utility.Gpg.Types import Types.Key import Types.Crypto+import Types.Remote  {- The beginning of a Cipher is used for MAC'ing; the remainder is used- - as the GPG symmetric encryption passphrase. Note that the cipher- - itself is base-64 encoded, hence the string is longer than- - 'cipherSize': 683 characters, padded to 684.+ - as the GPG symmetric encryption passphrase when using the hybrid+ - scheme. Note that the cipher itself is base-64 encoded, hence the+ - string is longer than 'cipherSize': 683 characters, padded to 684.  -  - The 256 first characters that feed the MAC represent at best 192  - bytes of entropy.  However that's more than enough for both the@@ -62,59 +64,79 @@  cipherPassphrase :: Cipher -> String cipherPassphrase (Cipher c) = drop cipherBeginning c+cipherPassphrase (MacOnlyCipher _) = error "MAC-only cipher"  cipherMac :: Cipher -> String cipherMac (Cipher c) = take cipherBeginning c+cipherMac (MacOnlyCipher c) = c  {- Creates a new Cipher, encrypted to the specified key id. -}-genEncryptedCipher :: String -> Bool -> IO StorableCipher-genEncryptedCipher keyid highQuality = do+genEncryptedCipher :: String -> EncryptedCipherVariant -> Bool -> IO StorableCipher+genEncryptedCipher keyid variant highQuality = do 	ks <- Gpg.findPubKeys keyid-	random <- Gpg.genRandom highQuality cipherSize-	encryptCipher (Cipher random) ks+	random <- Gpg.genRandom highQuality size+	encryptCipher (mkCipher random) variant ks+  where+	(mkCipher, size) = case variant of+		Hybrid -> (Cipher, cipherSize) -- used for MAC + symmetric+		PubKey -> (MacOnlyCipher, cipherBeginning) -- only used for MAC  {- Creates a new, shared Cipher. -} genSharedCipher :: Bool -> IO StorableCipher genSharedCipher highQuality = 	SharedCipher <$> Gpg.genRandom highQuality cipherSize -{- Updates an existing Cipher, re-encrypting it to add a keyid. -}-updateEncryptedCipher :: String -> StorableCipher -> IO StorableCipher-updateEncryptedCipher _ (SharedCipher _) = undefined-updateEncryptedCipher keyid encipher@(EncryptedCipher _ ks) = do-	ks' <- Gpg.findPubKeys keyid+{- Updates an existing Cipher, re-encrypting it to add or remove keyids,+ - depending on whether the first component is True or False. -}+updateEncryptedCipher :: [(Bool, String)] -> StorableCipher -> IO StorableCipher+updateEncryptedCipher _ SharedCipher{} = undefined+updateEncryptedCipher [] encipher = return encipher+updateEncryptedCipher newkeys encipher@(EncryptedCipher _ variant (KeyIds ks)) = do+	dropKeys <- listKeyIds [ k | (False, k) <- newkeys ]+	forM_ dropKeys $ \k -> unless (k `elem` ks) $+		error $ "Key " ++ k ++ " was not present; cannot remove."+	addKeys <- listKeyIds [ k | (True, k) <- newkeys ]+	let ks' = (addKeys ++ ks) \\ dropKeys+	when (null ks') $+		error "Cannot remove the last key." 	cipher <- decryptCipher encipher-	encryptCipher cipher (merge ks ks')+	encryptCipher cipher variant $ KeyIds ks'   where-	merge (KeyIds a) (KeyIds b) = KeyIds $ a ++ b+	listKeyIds = mapM (Gpg.findPubKeys >=*> keyIds) >=*> concat  describeCipher :: StorableCipher -> String describeCipher (SharedCipher _) = "shared cipher"-describeCipher (EncryptedCipher _ (KeyIds ks)) =-	"with gpg " ++ keys ks ++ " " ++ unwords ks+describeCipher (EncryptedCipher _ variant (KeyIds ks)) =+	scheme ++ " with gpg " ++ keys ks ++ " " ++ unwords ks   where+	scheme = case variant of+		Hybrid -> "hybrid cipher"+		PubKey -> "pubkey crypto" 	keys [_] = "key" 	keys _ = "keys"  {- Encrypts a Cipher to the specified KeyIds. -}-encryptCipher :: Cipher -> KeyIds -> IO StorableCipher-encryptCipher (Cipher c) (KeyIds ks) = do+encryptCipher :: Cipher -> EncryptedCipherVariant -> KeyIds -> IO StorableCipher+encryptCipher c variant (KeyIds ks) = do 	-- gpg complains about duplicate recipient keyids 	let ks' = nub $ sort ks-	encipher <- Gpg.pipeStrict (Params "--encrypt" : recipients ks') c-	return $ EncryptedCipher encipher (KeyIds ks')+	let params = Gpg.pkEncTo ks' ++ Gpg.stdEncryptionParams False+	encipher <- Gpg.pipeStrict params cipher+	return $ EncryptedCipher encipher variant (KeyIds ks')   where-	recipients l = force_recipients :-		concatMap (\k -> [Param "--recipient", Param k]) l-	-- Force gpg to only encrypt to the specified-	-- recipients, not configured defaults.-	force_recipients = Params "--no-encrypt-to --no-default-recipient"+	cipher = case c of+		Cipher x -> x+		MacOnlyCipher x -> x  {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} decryptCipher :: StorableCipher -> IO Cipher decryptCipher (SharedCipher t) = return $ Cipher t-decryptCipher (EncryptedCipher t _) =-	Cipher <$> Gpg.pipeStrict [ Param "--decrypt" ] t+decryptCipher (EncryptedCipher t variant _) =+	mkCipher <$> Gpg.pipeStrict [ Param "--decrypt" ] t+  where+	mkCipher = case variant of+		Hybrid -> Cipher+		PubKey -> MacOnlyCipher  {- Generates an encrypted form of a Key. The encryption does not need to be  - reversable, nor does it need to be the same type of encryption used@@ -139,17 +161,25 @@ readBytes :: (L.ByteString -> IO a) -> Reader a readBytes a h = L.hGetContents h >>= a -{- Runs a Feeder action, that generates content that is symmetrically encrypted- - with the Cipher using the given GnuPG options, and then read by the Reader- - action. -}-encrypt :: GpgOpts -> Cipher -> Feeder -> Reader a -> IO a-encrypt opts = Gpg.feedRead ( Params "--symmetric --force-mdc" : toParams opts )-		. cipherPassphrase+{- Runs a Feeder action, that generates content that is symmetrically+ - encrypted with the Cipher (unless it is empty, in which case+ - public-key encryption is used) using the given gpg options, and then+ - read by the Reader action.  Note: For public-key encryption,+ - recipients MUST be included in 'params' (for instance using+ - 'getGpgEncParams'). -}+encrypt :: [CommandParam] -> Cipher -> Feeder -> Reader a -> IO a+encrypt params cipher = case cipher of+	Cipher{} -> Gpg.feedRead (params ++ Gpg.stdEncryptionParams True) $+			cipherPassphrase cipher+	MacOnlyCipher{} -> Gpg.pipeLazy $ params ++ Gpg.stdEncryptionParams False  {- Runs a Feeder action, that generates content that is decrypted with the- - Cipher, and read by the Reader action. -}+ - Cipher (or using a private key if the Cipher is empty), and read by the+ - Reader action. -} decrypt :: Cipher -> Feeder -> Reader a -> IO a-decrypt = Gpg.feedRead [Param "--decrypt"] . cipherPassphrase+decrypt cipher = case cipher of+	Cipher{} -> Gpg.feedRead [Param "--decrypt"] $ cipherPassphrase cipher+	MacOnlyCipher{} -> Gpg.pipeLazy [Param "--decrypt"]  macWithCipher :: Mac -> Cipher -> String -> String macWithCipher mac c = macWithCipher' mac (cipherMac c)@@ -161,3 +191,21 @@ prop_HmacSha1WithCipher_sane = known_good == macWithCipher' HmacSha1 "foo" "bar"   where 	known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51"++{- Return some options suitable for GnuPG encryption, symmetric or not. -}+class LensGpgEncParams a where getGpgEncParams :: a -> [CommandParam]++{- Extract the GnuPG options from a pair of a Remote Config and a Remote+ - Git Config. If the remote is configured to use public-key encryption,+ - look up the recipient keys and add them to the option list. -}+instance LensGpgEncParams (RemoteConfig, RemoteGitConfig) where+	getGpgEncParams (c,gc) = map Param (remoteAnnexGnupgOptions gc) ++ recipients+	  where+		recipients = case M.lookup "encryption" c of+			Just "pubkey" -> Gpg.pkEncTo $ maybe [] (split ",") $+						M.lookup "cipherkeys" c+			_ -> []++{- Extract the GnuPG options from a Remote. -}+instance LensGpgEncParams (RemoteA a) where+	getGpgEncParams r = getGpgEncParams (config r, gitconfig r)
Git/Branch.hs view
@@ -101,3 +101,7 @@ 	return sha   where 	ps = concatMap (\r -> ["-p", show r]) parentrefs++{- A leading + makes git-push force pushing a branch. -}+forcePush :: String -> String+forcePush b = "+" ++ b
+ Git/GCrypt.hs view
@@ -0,0 +1,78 @@+{- git-remote-gcrypt support+ -+ - https://github.com/blake2-ppc/git-remote-gcrypt+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Git.GCrypt where++import Common+import Git.Types+import Git.Construct+import qualified Git.Config as Config+import Utility.Gpg++urlPrefix :: String+urlPrefix = "gcrypt::"++isEncrypted :: Repo -> Bool+isEncrypted Repo { location = Url url } = urlPrefix `isPrefixOf` show url+isEncrypted _ = False++{- The first Repo is the git repository that has the second Repo+ - as one of its remotes.+ -+ - When the remote Repo uses gcrypt, returns the actual underlying+ - git repository that gcrypt is using to store its data. + -+ - Throws an exception if an url is invalid or the repo does not use+ - gcrypt.+ -}+encryptedRepo :: Repo -> Repo -> IO Repo+encryptedRepo baserepo = go+  where+  	go Repo { location = Url url }+		| urlPrefix `isPrefixOf` u =+			fromRemoteLocation (drop plen u) baserepo+		| otherwise = notencrypted+	  where+  		u = show url+		plen = length urlPrefix+	go _ = notencrypted+	notencrypted = error "not a gcrypt encrypted repository"++type RemoteName = String++{- gcrypt gives each encrypted repository a uique gcrypt-id,+ - which is stored in the repository (in encrypted form)+ - and cached in a per-remote gcrypt-id configuration setting. -}+remoteRepoId :: Repo -> Maybe RemoteName -> Maybe String+remoteRepoId = getRemoteConfig "gcrypt-id"++getRemoteConfig :: String -> Repo -> Maybe RemoteName -> Maybe String+getRemoteConfig field repo remotename = do+	n <- remotename+	Config.getMaybe (remoteConfigKey field n) repo++{- Gpg keys that the remote is encrypted for.+ - If empty, gcrypt uses --default-recipient-self -}+getParticiantList :: Maybe Repo -> Repo -> Maybe RemoteName -> KeyIds+getParticiantList globalconfigrepo repo remotename = KeyIds $ parse $ firstJust+	[ getRemoteConfig "gcrypt-participants" repo remotename+	, Config.getMaybe defaultkey repo+	, Config.getMaybe defaultkey =<< globalconfigrepo+	]+  where+	defaultkey = "gcrypt.participants"+  	parse (Just "simple") = []+	parse (Just l) = words l+	parse Nothing = []++remoteParticipantConfigKey :: RemoteName -> String+remoteParticipantConfigKey = remoteConfigKey "gcrypt-participants"++remoteConfigKey :: String -> RemoteName -> String+remoteConfigKey key remotename = "remote." ++ remotename ++ "." ++ key
Git/Version.o view

binary file changed (15924 → 26480 bytes)

GitAnnex.hs view
@@ -67,6 +67,7 @@ import qualified Command.Direct import qualified Command.Indirect import qualified Command.Upgrade+import qualified Command.Forget import qualified Command.Version import qualified Command.Help #ifdef WITH_ASSISTANT@@ -139,6 +140,7 @@ 	, Command.Direct.def 	, Command.Indirect.def 	, Command.Upgrade.def+	, Command.Forget.def 	, Command.Version.def 	, Command.Help.def #ifdef WITH_ASSISTANT
Locations.hs view
@@ -35,6 +35,7 @@ 	gitAnnexJournalLock, 	gitAnnexIndex, 	gitAnnexIndexLock,+	gitAnnexIgnoredRefs, 	gitAnnexPidFile, 	gitAnnexDaemonStatusFile, 	gitAnnexLogFile,@@ -224,6 +225,10 @@ {- Lock file for .git/annex/index. -} gitAnnexIndexLock :: Git.Repo -> FilePath gitAnnexIndexLock r = gitAnnexDir r </> "index.lck"++{- List of refs that should not be merged into the git-annex branch. -}+gitAnnexIgnoredRefs :: Git.Repo -> FilePath+gitAnnexIgnoredRefs r = gitAnnexDir r </> "ignoredrefs"  {- Pid file for daemon mode. -} gitAnnexPidFile :: Git.Repo -> FilePath
+ Logs.hs view
@@ -0,0 +1,110 @@+{- git-annex log file names+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs where++import Common.Annex+import Types.Key++data LogVariety = UUIDBasedLog | PresenceLog Key+	deriving (Show)++{- Converts a path from the git-annex branch into one of the varieties+ - of logs used by git-annex, if it's a known path. -}+getLogVariety :: FilePath -> Maybe LogVariety+getLogVariety f+	| f `elem` uuidBasedLogs = Just UUIDBasedLog+	| otherwise = PresenceLog <$> firstJust (presenceLogs f)++{- All the uuid-based logs stored in the git-annex branch. -}+uuidBasedLogs :: [FilePath]+uuidBasedLogs =+	[ uuidLog+	, remoteLog+	, trustLog+	, groupLog +	, preferredContentLog+	]++{- All the ways to get a key from a presence log file -}+presenceLogs :: FilePath -> [Maybe Key]+presenceLogs f =+	[ urlLogFileKey f+	, locationLogFileKey f+	]++uuidLog :: FilePath+uuidLog = "uuid.log"++remoteLog :: FilePath+remoteLog = "remote.log"++trustLog :: FilePath+trustLog = "trust.log"++groupLog :: FilePath+groupLog = "group.log"++preferredContentLog :: FilePath+preferredContentLog = "preferred-content.log"++{- The pathname of the location log file for a given key. -}+locationLogFile :: Key -> String+locationLogFile key = hashDirLower key ++ keyFile key ++ ".log"++{- Converts a pathname into a key if it's a location log. -}+locationLogFileKey :: FilePath -> Maybe Key+locationLogFileKey path+	| ["remote", "web"] `isPrefixOf` splitDirectories dir = Nothing+        | ext == ".log" = fileKey base+        | otherwise = Nothing+  where+	(dir, file) = splitFileName path+        (base, ext) = splitAt (length file - 4) file++{- The filename of the url log for a given key. -}+urlLogFile :: Key -> FilePath+urlLogFile key = hashDirLower key </> keyFile key ++ urlLogExt++{- Old versions stored the urls elsewhere. -}+oldurlLogs :: Key -> [FilePath]+oldurlLogs key =+	[ "remote/web" </> hashDirLower key </> key2file key ++ ".log"+	, "remote/web" </> hashDirLower key </> keyFile key ++ ".log"+	]++urlLogExt :: String+urlLogExt = ".log.web"++{- Converts a url log file into a key.+ - (Does not work on oldurlLogs.) -}+urlLogFileKey :: FilePath -> Maybe Key+urlLogFileKey path+	| ext == urlLogExt = fileKey base+	| otherwise = Nothing+  where+  	file = takeFileName path+	(base, ext) = splitAt (length file - extlen) file+	extlen = length urlLogExt++{- Does not work on oldurllogs. -}+isUrlLog :: FilePath -> Bool+isUrlLog file = urlLogExt `isSuffixOf` file++prop_logs_sane :: Key -> Bool+prop_logs_sane dummykey = all id+	[ isNothing (getLogVariety "unknown")+	, expect isUUIDBasedLog (getLogVariety uuidLog)+	, expect isPresenceLog (getLogVariety $ locationLogFile dummykey)+	, expect isPresenceLog (getLogVariety $ urlLogFile dummykey)+	]+  where+  	expect = maybe False+  	isUUIDBasedLog UUIDBasedLog = True+	isUUIDBasedLog _ = False+	isPresenceLog (PresenceLog k) = k == dummykey+	isPresenceLog _ = False
Logs/Group.hs view
@@ -21,15 +21,12 @@ import Data.Time.Clock.POSIX  import Common.Annex+import Logs import qualified Annex.Branch import qualified Annex import Logs.UUIDBased import Types.Group import Types.StandardGroups--{- Filename of group.log. -}-groupLog :: FilePath-groupLog = "group.log"  {- Returns the groups of a given repo UUID. -} lookupGroups :: UUID -> Annex (S.Set Group)
Logs/Location.hs view
@@ -20,36 +20,35 @@ 	loggedLocations, 	loggedKeys, 	loggedKeysFor,-	logFile,-	logFileKey ) where  import Common.Annex import qualified Annex.Branch+import Logs import Logs.Presence import Annex.UUID  {- Log a change in the presence of a key's value in current repository. -} logStatus :: Key -> LogStatus -> Annex ()-logStatus key status = do+logStatus key s = do 	u <- getUUID-	logChange key u status+	logChange key u s  {- Log a change in the presence of a key's value in a repository. -} logChange :: Key -> UUID -> LogStatus -> Annex ()-logChange key (UUID u) s = addLog (logFile key) =<< logNow s u+logChange key (UUID u) s = addLog (locationLogFile key) =<< logNow s u logChange _ NoUUID _ = noop  {- Returns a list of repository UUIDs that, according to the log, have  - the value of a key.  -} loggedLocations :: Key -> Annex [UUID]-loggedLocations key = map toUUID <$> (currentLog . logFile) key+loggedLocations key = map toUUID <$> (currentLog . locationLogFile) key  {- Finds all keys that have location log information.  - (There may be duplicate keys in the list.) -} loggedKeys :: Annex [Key]-loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files+loggedKeys = mapMaybe locationLogFileKey <$> Annex.Branch.files  {- Finds all keys that have location log information indicating  - they are present for the specified repository. -}@@ -62,15 +61,3 @@ 		us <- loggedLocations k 		let !there = u `elem` us 		return there--{- The filename of the log file for a given key. -}-logFile :: Key -> String-logFile key = hashDirLower key ++ keyFile key ++ ".log"--{- Converts a log filename into a key. -}-logFileKey :: FilePath -> Maybe Key-logFileKey file-	| ext == ".log" = fileKey base-	| otherwise = Nothing-  where-	(base, ext) = splitAt (length file - 4) file
Logs/PreferredContent.hs view
@@ -24,6 +24,7 @@ import Common.Annex import qualified Annex.Branch import qualified Annex+import Logs import Logs.UUIDBased import Limit import qualified Utility.Matcher@@ -34,10 +35,6 @@ import Logs.Group import Logs.Remote import Types.StandardGroups--{- Filename of preferred-content.log. -}-preferredContentLog :: FilePath-preferredContentLog = "preferred-content.log"  {- Changes the preferred content configuration of a remote. -} preferredContentSet :: UUID -> String -> Annex ()
Logs/Presence.hs view
@@ -12,37 +12,19 @@  -}  module Logs.Presence (-	LogStatus(..),-	LogLine(LogLine),+	module X, 	addLog, 	readLog,-	getLog,-	parseLog,-	showLog, 	logNow,-	compactLog,-	currentLog,-	prop_parse_show_log,+	currentLog ) where  import Data.Time.Clock.POSIX-import Data.Time-import System.Locale-import qualified Data.Map as M +import Logs.Presence.Pure as X import Common.Annex import qualified Annex.Branch-import Utility.QuickCheck -data LogLine = LogLine {-	date :: POSIXTime,-	status :: LogStatus,-	info :: String-} deriving (Eq, Show)--data LogStatus = InfoPresent | InfoMissing-	deriving (Eq, Show, Bounded, Enum)- addLog :: FilePath -> LogLine -> Annex () addLog file line = Annex.Branch.change file $ \s ->  	showLog $ compactLog (line : parseLog s)@@ -52,29 +34,6 @@ readLog :: FilePath -> Annex [LogLine] readLog = parseLog <$$> Annex.Branch.get -{- Parses a log file. Unparseable lines are ignored. -}-parseLog :: String -> [LogLine]-parseLog = mapMaybe parseline . lines-  where-	parseline l = LogLine-		<$> (utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" d)-		<*> parsestatus s-		<*> pure rest-	  where-		(d, pastd) = separate (== ' ') l-		(s, rest) = separate (== ' ') pastd-	parsestatus "1" = Just InfoPresent-	parsestatus "0" = Just InfoMissing-	parsestatus _ = Nothing--{- Generates a log file. -}-showLog :: [LogLine] -> String-showLog = unlines . map genline-  where-	genline (LogLine d s i) = unwords [show d, genstatus s, i]-	genstatus InfoPresent = "1"-	genstatus InfoMissing = "0"- {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> Annex LogLine logNow s i = do@@ -84,39 +43,3 @@ {- Reads a log and returns only the info that is still in effect. -} currentLog :: FilePath -> Annex [String] currentLog file = map info . filterPresent <$> readLog file--{- Given a log, returns only the info that is are still in effect. -}-getLog :: String -> [String]-getLog = map info . filterPresent . parseLog--{- Returns the info from LogLines that are in effect. -}-filterPresent :: [LogLine] -> [LogLine]-filterPresent = filter (\l -> InfoPresent == status l) . compactLog--{- Compacts a set of logs, returning a subset that contains the current- - status. -}-compactLog :: [LogLine] -> [LogLine]-compactLog = M.elems . foldr mapLog M.empty--type LogMap = M.Map String LogLine--{- Inserts a log into a map of logs, if the log has better (ie, newer)- - information than the other logs in the map -}-mapLog :: LogLine -> LogMap -> LogMap-mapLog l m-	| better = M.insert i l m-	| otherwise = m-  where-	better = maybe True newer $ M.lookup i m-	newer l' = date l' <= date l-	i = info l--instance Arbitrary LogLine where-	arbitrary = LogLine-		<$> arbitrary-		<*> elements [minBound..maxBound]-		<*> arbitrary `suchThat` ('\n' `notElem`)--prop_parse_show_log :: [LogLine] -> Bool-prop_parse_show_log l = parseLog (showLog l) == l-
+ Logs/Presence/Pure.hs view
@@ -0,0 +1,84 @@+{- git-annex presence log, pure operations+ -+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.Presence.Pure where++import Data.Time.Clock.POSIX+import Data.Time+import System.Locale+import qualified Data.Map as M++import Common.Annex+import Utility.QuickCheck++data LogLine = LogLine {+	date :: POSIXTime,+	status :: LogStatus,+	info :: String+} deriving (Eq, Show)++data LogStatus = InfoPresent | InfoMissing+	deriving (Eq, Show, Bounded, Enum)++{- Parses a log file. Unparseable lines are ignored. -}+parseLog :: String -> [LogLine]+parseLog = mapMaybe parseline . lines+  where+	parseline l = LogLine+		<$> (utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" d)+		<*> parsestatus s+		<*> pure rest+	  where+		(d, pastd) = separate (== ' ') l+		(s, rest) = separate (== ' ') pastd+	parsestatus "1" = Just InfoPresent+	parsestatus "0" = Just InfoMissing+	parsestatus _ = Nothing++{- Generates a log file. -}+showLog :: [LogLine] -> String+showLog = unlines . map genline+  where+	genline (LogLine d s i) = unwords [show d, genstatus s, i]+	genstatus InfoPresent = "1"+	genstatus InfoMissing = "0"++{- Given a log, returns only the info that is are still in effect. -}+getLog :: String -> [String]+getLog = map info . filterPresent . parseLog++{- Returns the info from LogLines that are in effect. -}+filterPresent :: [LogLine] -> [LogLine]+filterPresent = filter (\l -> InfoPresent == status l) . compactLog++{- Compacts a set of logs, returning a subset that contains the current+ - status. -}+compactLog :: [LogLine] -> [LogLine]+compactLog = M.elems . foldr mapLog M.empty++type LogMap = M.Map String LogLine++{- Inserts a log into a map of logs, if the log has better (ie, newer)+ - information than the other logs in the map -}+mapLog :: LogLine -> LogMap -> LogMap+mapLog l m+	| better = M.insert i l m+	| otherwise = m+  where+	better = maybe True newer $ M.lookup i m+	newer l' = date l' <= date l+	i = info l++instance Arbitrary LogLine where+	arbitrary = LogLine+		<$> arbitrary+		<*> elements [minBound..maxBound]+		<*> arbitrary `suchThat` ('\n' `notElem`)++prop_parse_show_log :: [LogLine] -> Bool+prop_parse_show_log l = parseLog (showLog l) == l+
Logs/Remote.hs view
@@ -25,11 +25,8 @@ import Common.Annex import qualified Annex.Branch import Types.Remote+import Logs import Logs.UUIDBased--{- Filename of remote.log. -}-remoteLog :: FilePath-remoteLog = "remote.log"  {- Adds or updates a remote's config in the log. -} configSet :: UUID -> RemoteConfig -> Annex ()
+ Logs/Transitions.hs view
@@ -0,0 +1,87 @@+{- git-annex transitions log+ -+ - This is used to record transitions that have been performed on the+ - git-annex branch, and when the transition was first started.+ -+ - We can quickly detect when the local branch has already had an transition+ - done that is listed in the remote branch by checking that the local+ - branch contains the same transition, with the same or newer start time.+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.Transitions where++import Data.Time.Clock.POSIX+import Data.Time+import System.Locale+import qualified Data.Set as S++import Common.Annex++transitionsLog :: FilePath+transitionsLog = "transitions.log"++data Transition+	= ForgetGitHistory+	| ForgetDeadRemotes+	deriving (Show, Ord, Eq, Read)++data TransitionLine = TransitionLine+	{ transitionStarted :: POSIXTime+	, transition :: Transition+	} deriving (Show, Ord, Eq)++type Transitions = S.Set TransitionLine++describeTransition :: Transition -> String+describeTransition ForgetGitHistory = "forget git history"+describeTransition ForgetDeadRemotes = "forget dead remotes"++noTransitions :: Transitions+noTransitions = S.empty++addTransition :: POSIXTime -> Transition -> Transitions -> Transitions+addTransition ts t = S.insert $ TransitionLine ts t++showTransitions :: Transitions -> String+showTransitions = unlines . map showTransitionLine . S.elems++{- If the log contains new transitions we don't support, returns Nothing. -}+parseTransitions :: String -> Maybe Transitions+parseTransitions = check . map parseTransitionLine . lines+  where+  	check l+		| all isJust l = Just $ S.fromList $ catMaybes l+		| otherwise = Nothing++parseTransitionsStrictly :: String -> String -> Transitions+parseTransitionsStrictly source = fromMaybe badsource . parseTransitions+  where+	badsource = error $ "unknown transitions listed in " ++ source ++ "; upgrade git-annex!"++showTransitionLine :: TransitionLine -> String+showTransitionLine (TransitionLine ts t) = unwords [show t, show ts]++parseTransitionLine :: String -> Maybe TransitionLine+parseTransitionLine s = TransitionLine <$> pdate ds <*> readish ts+  where+  	ws = words s+  	ts = Prelude.head ws+	ds = unwords $ Prelude.tail ws+	pdate = parseTime defaultTimeLocale "%s%Qs" >=*> utcTimeToPOSIXSeconds++combineTransitions :: [Transitions] -> Transitions+combineTransitions = S.unions++transitionList :: Transitions -> [Transition]+transitionList = map transition . S.elems++{- Typically ran with Annex.Branch.change, but we can't import Annex.Branch+ - here since it depends on this module. -}+recordTransitions :: (FilePath -> (String -> String) -> Annex ()) -> Transitions -> Annex ()+recordTransitions changer t = do+	changer transitionsLog $+		showTransitions . S.union t . parseTransitionsStrictly "local"
Logs/Trust.hs view
@@ -6,6 +6,7 @@  -}  module Logs.Trust (+	module X, 	trustLog, 	TrustLevel(..), 	trustGet,@@ -16,8 +17,6 @@ 	lookupTrust, 	trustMapLoad, 	trustMapRaw,-	-	prop_parse_show_TrustLog, ) where  import qualified Data.Map as M@@ -27,13 +26,11 @@ import Types.TrustLevel import qualified Annex.Branch import qualified Annex+import Logs import Logs.UUIDBased import Remote.List import qualified Types.Remote--{- Filename of trust.log. -}-trustLog :: FilePath-trustLog = "trust.log"+import Logs.Trust.Pure as X  {- Returns a list of UUIDs that the trustLog indicates have the  - specified trust level.@@ -97,26 +94,4 @@ {- Does not include forcetrust or git config values, just those from the  - log file. -} trustMapRaw :: Annex TrustMap-trustMapRaw = simpleMap . parseLog (Just . parseTrustLog)-	<$> Annex.Branch.get trustLog--{- The trust.log used to only list trusted repos, without a field for the- - trust status, which is why this defaults to Trusted. -}-parseTrustLog :: String -> TrustLevel-parseTrustLog s = maybe Trusted parse $ headMaybe $ words s-  where-	parse "1" = Trusted-	parse "0" = UnTrusted-	parse "X" = DeadTrusted-	parse _ = SemiTrusted--showTrustLog :: TrustLevel -> String-showTrustLog Trusted = "1"-showTrustLog UnTrusted = "0"-showTrustLog DeadTrusted = "X"-showTrustLog SemiTrusted = "?"--prop_parse_show_TrustLog :: Bool-prop_parse_show_TrustLog = all check [minBound .. maxBound]-  where-	check l = parseTrustLog (showTrustLog l) == l+trustMapRaw = calcTrustMap <$> Annex.Branch.get trustLog
+ Logs/Trust/Pure.hs view
@@ -0,0 +1,36 @@+{- git-annex trust log, pure operations+ -+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.Trust.Pure where++import Common.Annex+import Types.TrustLevel+import Logs.UUIDBased++calcTrustMap :: String -> TrustMap+calcTrustMap = simpleMap . parseLog (Just . parseTrustLog)++{- The trust.log used to only list trusted repos, without a field for the+ - trust status, which is why this defaults to Trusted. -}+parseTrustLog :: String -> TrustLevel+parseTrustLog s = maybe Trusted parse $ headMaybe $ words s+  where+	parse "1" = Trusted+	parse "0" = UnTrusted+	parse "X" = DeadTrusted+	parse _ = SemiTrusted++showTrustLog :: TrustLevel -> String+showTrustLog Trusted = "1"+showTrustLog UnTrusted = "0"+showTrustLog DeadTrusted = "X"+showTrustLog SemiTrusted = "?"++prop_parse_show_TrustLog :: Bool+prop_parse_show_TrustLog = all check [minBound .. maxBound]+  where+	check l = parseTrustLog (showTrustLog l) == l
Logs/UUID.hs view
@@ -28,12 +28,9 @@ import Common.Annex import qualified Annex import qualified Annex.Branch+import Logs import Logs.UUIDBased import qualified Annex.UUID--{- Filename of uuid.log. -}-uuidLog :: FilePath-uuidLog = "uuid.log"  {- Records a description for a uuid in the log. -} describeUUID :: UUID -> String -> Annex ()
Logs/Web.hs view
@@ -11,8 +11,6 @@ 	getUrls, 	setUrlPresent, 	setUrlMissing,-	urlLog,-	urlLogKey, 	knownUrls, 	Downloader(..), 	getDownloader,@@ -22,9 +20,9 @@ import qualified Data.ByteString.Lazy.Char8 as L  import Common.Annex+import Logs import Logs.Presence import Logs.Location-import Types.Key import qualified Annex.Branch import Annex.CatFile import qualified Git@@ -36,35 +34,9 @@ webUUID :: UUID webUUID = UUID "00000000-0000-0000-0000-000000000001" -urlLogExt :: String-urlLogExt = ".log.web"--urlLog :: Key -> FilePath-urlLog key = hashDirLower key </> keyFile key ++ urlLogExt--{- Converts a url log file into a key.- - (Does not work on oldurlLogs.) -}-urlLogKey :: FilePath -> Maybe Key-urlLogKey file-	| ext == urlLogExt = fileKey base-	| otherwise = Nothing-  where-  	(base, ext) = splitAt (length file - extlen) file-	extlen = length urlLogExt--isUrlLog :: FilePath -> Bool-isUrlLog file = urlLogExt `isSuffixOf` file--{- Used to store the urls elsewhere. -}-oldurlLogs :: Key -> [FilePath]-oldurlLogs key = -	[ "remote/web" </> hashDirLower key </> key2file key ++ ".log"-	, "remote/web" </> hashDirLower key </> keyFile key ++ ".log"-	]- {- Gets all urls that a key might be available from. -} getUrls :: Key -> Annex [URLString]-getUrls key = go $ urlLog key : oldurlLogs key+getUrls key = go $ urlLogFile key : oldurlLogs key   where 	go [] = return [] 	go (l:ls) = do@@ -77,13 +49,13 @@ setUrlPresent key url = do 	us <- getUrls key 	unless (url `elem` us) $ do-		addLog (urlLog key) =<< logNow InfoPresent url+		addLog (urlLogFile key) =<< logNow InfoPresent url 		-- update location log to indicate that the web has the key 		logChange key webUUID InfoPresent  setUrlMissing :: Key -> URLString -> Annex () setUrlMissing key url = do-	addLog (urlLog key) =<< logNow InfoMissing url+	addLog (urlLogFile key) =<< logNow InfoMissing url 	whenM (null <$> getUrls key) $ 		logChange key webUUID InfoMissing
Remote.hs view
@@ -19,7 +19,7 @@  	remoteTypes, 	remoteList,-	specialRemote,+	syncableRemote, 	remoteMap, 	uuidDescriptions, 	byName,@@ -231,6 +231,9 @@ 	ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted 	ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped 	showLongNote $ message ppuuidswanted ppuuidsskipped+	ignored <- filter (remoteAnnexIgnore . gitconfig) <$> remoteList+	unless (null ignored) $+		showLongNote $ "(Note that these git remotes have annex-ignore set: " ++ unwords (map name ignored) ++ ")"   where 	filteruuids l x = filter (`notElem` x) l 	message [] [] = nolocmsg
Remote/Bup.hs view
@@ -29,6 +29,7 @@ import Data.Digest.Pure.SHA import Utility.UserInfo import Annex.Content+import Annex.UUID import Utility.Metered  type BupRepo = String@@ -78,8 +79,10 @@   where 	buprepo = fromMaybe (error "missing buprepo") $ remoteAnnexBupRepo gc -bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig-bupSetup u c = do+bupSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+bupSetup mu c = do+	u <- maybe (liftIO genUUID) return mu+ 	-- verify configuration is sane 	let buprepo = fromMaybe (error "Specify buprepo=") $ 		M.lookup "buprepo" c@@ -96,7 +99,7 @@ 	-- persistant state, so it can vary between hosts. 	gitConfigSpecialRemote u c' "buprepo" buprepo -	return c'+	return (c', u)  bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] bupParams command buprepo params = @@ -133,7 +136,7 @@ 	sendAnnex k (rollback enck buprepo) $ \src -> do 		params <- bupSplitParams r buprepo enck [] 		liftIO $ catchBoolIO $-			encrypt (getGpgOpts r) cipher (feedFile src) $ \h ->+			encrypt (getGpgEncParams r) cipher (feedFile src) $ \h -> 				pipeBup params (Just h) Nothing  retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool
Remote/Directory.hs view
@@ -26,6 +26,7 @@ import Remote.Helper.Chunked import Crypto import Annex.Content+import Annex.UUID import Utility.Metered  remote :: RemoteType@@ -41,7 +42,7 @@ 	cst <- remoteCost gc cheapRemoteCost 	let chunksize = chunkSize c 	return $ encryptableRemote c-		(storeEncrypted dir (getGpgOpts gc) chunksize)+		(storeEncrypted dir (getGpgEncParams (c,gc)) chunksize) 		(retrieveEncrypted dir chunksize) 		Remote { 			uuid = u,@@ -65,8 +66,9 @@   where 	dir = fromMaybe (error "missing directory") $ remoteAnnexDirectory gc -directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig-directorySetup u c = do+directorySetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+directorySetup mu c = do+	u <- maybe (liftIO genUUID) return mu 	-- verify configuration is sane 	let dir = fromMaybe (error "Specify directory=") $ 		M.lookup "directory" c@@ -78,7 +80,7 @@ 	-- The directory is stored in git config, not in this remote's 	-- persistant state, so it can vary between hosts. 	gitConfigSpecialRemote u c' "directory" absdir-	return $ M.delete "directory" c'+	return (M.delete "directory" c', u)  {- Locations to try to access a given Key in the Directory.  - We try more than since we used to write to different hash directories. -}@@ -129,7 +131,7 @@ 					storeSplit meterupdate chunksize dests 						=<< L.readFile src -storeEncrypted :: FilePath -> GpgOpts -> ChunkSize -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool+storeEncrypted :: FilePath -> [CommandParam] -> ChunkSize -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool storeEncrypted d gpgOpts chunksize (cipher, enck) k p = sendAnnex k (void $ remove d enck) $ \src -> 	metered (Just p) k $ \meterupdate -> 		storeHelper d chunksize enck k $ \dests ->
+ Remote/GCrypt.hs view
@@ -0,0 +1,210 @@+{- git remotes encrypted using git-remote-gcrypt+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.GCrypt (remote, gen) where++import qualified Data.Map as M+import qualified Data.ByteString.Lazy as L++import Common.Annex+import Types.Remote+import Types.GitConfig+import Types.Crypto+import qualified Git+import qualified Git.Command+import qualified Git.Config+import qualified Git.GCrypt+import qualified Git.Types as Git ()+import qualified Annex.Branch+import qualified Annex.Content+import Config+import Config.Cost+import Remote.Helper.Git+import Remote.Helper.Encryptable+import Remote.Helper.Special+import Utility.Metered+import Crypto+import Annex.UUID+import Annex.Ssh+import qualified Remote.Rsync+import Utility.Rsync++remote :: RemoteType+remote = RemoteType {+	typename = "gcrypt",+	-- Remote.Git takes care of enumerating gcrypt remotes too,+	-- and will call our gen on them.+	enumerate = return [],+	generate = gen,+	setup = gCryptSetup+}++gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote+gen gcryptr u c gc = do+	g <- gitRepo+	-- get underlying git repo with real path, not gcrypt path+	r <- liftIO $ Git.GCrypt.encryptedRepo g gcryptr+	let r' = r { Git.remoteName = Git.remoteName gcryptr }+	-- read config of underlying repo if it's local+	r'' <- if Git.repoIsLocalUnknown r'+		then liftIO $ catchDefaultIO r' $ Git.Config.read r'+		else return r'+	gen' r'' u c gc++gen' :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote+gen' r u c gc = do+	cst <- remoteCost gc $+		if repoCheap r then nearlyCheapRemoteCost else expensiveRemoteCost+	(rsynctransport, rsyncurl) <- rsyncTransport r+	let rsyncopts = Remote.Rsync.genRsyncOpts c gc rsynctransport rsyncurl+	let this = Remote +		{ uuid = u+		, cost = cst+		, name = Git.repoDescribe r+		, storeKey = \_ _ _ -> noCrypto+		, retrieveKeyFile = \_ _ _ _ -> noCrypto+		, retrieveKeyFileCheap = \_ _ -> return False+		, removeKey = remove this rsyncopts+		, hasKey = checkPresent this rsyncopts+		, hasKeyCheap = repoCheap r+		, whereisKey = Nothing+		, config = M.empty+		, localpath = localpathCalc r+		, repo = r+		, gitconfig = gc { remoteGitConfig = Just $ extractGitConfig r }+		, readonly = Git.repoIsHttp r+		, globallyAvailable = globallyAvailableCalc r+		, remotetype = remote+	}+	return $ encryptableRemote c+		(store this rsyncopts)+		(retrieve this rsyncopts)+		this++rsyncTransport :: Git.Repo -> Annex ([CommandParam], String)+rsyncTransport r+	| "ssh://" `isPrefixOf` loc = sshtransport $ break (== '/') $ drop (length "ssh://") loc+	| "//:" `isInfixOf` loc = othertransport+	| ":" `isInfixOf` loc = sshtransport $ separate (== ':') loc+	| otherwise = othertransport+  where+  	loc = Git.repoLocation r+	sshtransport (host, path) = do+		opts <- sshCachingOptions (host, Nothing) []+		return (rsyncShell $ Param "ssh" : opts, host ++ ":" ++ path)+	othertransport = return ([], loc)++noCrypto :: Annex a+noCrypto = error "cannot use gcrypt remote without encryption enabled"++unsupportedUrl :: Annex a+unsupportedUrl = error "using non-ssh remote repo url with gcrypt is not supported"++gCryptSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+gCryptSetup mu c = go $ M.lookup "gitrepo" c+  where+	remotename = fromJust (M.lookup "name" c)+  	go Nothing = error "Specify gitrepo="+	go (Just gitrepo) = do+		c' <- encryptionSetup c+		inRepo $ Git.Command.run +			[ Params "remote add"+			, Param remotename+			, Param $ Git.GCrypt.urlPrefix ++ gitrepo+			]++		{- Configure gcrypt to use the same list of keyids that+		 - were passed to initremote, unless shared encryption+		 - was used. -}+		case extractCipher c' of+			Nothing -> noCrypto+			Just (EncryptedCipher _ _ (KeyIds { keyIds = ks})) ->+				setConfig (ConfigKey $ Git.GCrypt.remoteParticipantConfigKey remotename) (unwords ks)+			_ -> noop++		{- Run a git fetch and a push to the git repo in order to get+		 - its gcrypt-id set up, so that later git annex commands+		 - will use the remote as a ggcrypt remote. The fetch is+		 - needed if the repo already exists; the push is needed+		 - if the repo has not yet been initialized by gcrypt. -}+		void $ inRepo $ Git.Command.runBool+			[ Param "fetch"+			, Param remotename+			]+		void $ inRepo $ Git.Command.runBool+			[ Param "push"+			, Param remotename+			, Param $ show $ Annex.Branch.fullname+			]+		g <- inRepo Git.Config.reRead+		case Git.GCrypt.remoteRepoId g (Just remotename) of+			Nothing -> error "unable to determine gcrypt-id of remote"+			Just v -> do+				let u = genUUIDInNameSpace gCryptNameSpace v+				if Just u == mu || mu == Nothing+					then do+						gitConfigSpecialRemote u c' "gcrypt" "true"+						return (c', u)+					else error "uuid mismatch"++store :: Remote -> Remote.Rsync.RsyncOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool+store r rsyncopts (cipher, enck) k p+	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $+		sendwith $ \meterupdate h -> do+			createDirectoryIfMissing True $ parentDir dest+			readBytes (meteredWriteFile meterupdate dest) h+			return True+	| Git.repoIsSsh (repo r) = Remote.Rsync.storeEncrypted rsyncopts gpgopts (cipher, enck) k p+	| otherwise = unsupportedUrl+  where+  	gpgopts = getGpgEncParams r+	dest = gCryptLocation r enck+  	sendwith a = metered (Just p) k $ \meterupdate ->+		Annex.Content.sendAnnex k noop $ \src ->+			liftIO $ catchBoolIO $+				encrypt gpgopts cipher (feedFile src) (a meterupdate)++retrieve :: Remote -> Remote.Rsync.RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> MeterUpdate -> Annex Bool+retrieve r rsyncopts (cipher, enck) k d p+	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do+		retrievewith $ L.readFile src+		return True+	| Git.repoIsSsh (repo r) = Remote.Rsync.retrieveEncrypted rsyncopts (cipher, enck) k d p+	| otherwise = unsupportedUrl+  where+	src = gCryptLocation r enck+	retrievewith a = metered (Just p) k $ \meterupdate -> liftIO $+		a >>= \b -> +			decrypt cipher (feedBytes b)+				(readBytes $ meteredWriteFile meterupdate d)++remove :: Remote -> Remote.Rsync.RsyncOpts -> Key -> Annex Bool+remove r rsyncopts k+	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do+		liftIO $ removeDirectoryRecursive (parentDir dest)+		return True+	| Git.repoIsSsh (repo r) = Remote.Rsync.remove rsyncopts k+	| otherwise = unsupportedUrl+  where+	dest = gCryptLocation r k++checkPresent :: Remote -> Remote.Rsync.RsyncOpts -> Key -> Annex (Either String Bool)+checkPresent r rsyncopts k+	| not $ Git.repoIsUrl (repo r) =+		guardUsable (repo r) unknown $+			liftIO $ catchDefaultIO unknown $+				Right <$> doesFileExist (gCryptLocation r k)+	| Git.repoIsSsh (repo r) = Remote.Rsync.checkPresent (repo r) rsyncopts k+	| otherwise = unsupportedUrl+  where+	unknown = Left $ "unable to check " ++ Git.repoDescribe (repo r) ++ show (repo r)++{- Annexed objects are stored directly under the top of the gcrypt repo+ - (not in annex/objects), and are hashed using lower-case directories for max+ - portability. -}+gCryptLocation :: Remote -> Key -> FilePath+gCryptLocation r key = Git.repoLocation (repo r) </> keyPath key hashDirLower
Remote/Git.hs view
@@ -13,9 +13,6 @@ 	repoAvail, ) where -import qualified Data.Map as M-import Control.Exception.Extensible- import Common.Annex import Utility.Rsync import Remote.Helper.Ssh@@ -26,6 +23,7 @@ import qualified Git.Config import qualified Git.Construct import qualified Git.Command+import qualified Git.GCrypt import qualified Annex import Logs.Presence import Logs.Transfer@@ -46,10 +44,14 @@ #ifndef mingw32_HOST_OS import Utility.CopyFile #endif+import Remote.Helper.Git+import qualified Remote.GCrypt  import Control.Concurrent import Control.Concurrent.MSampleVar import System.Process (std_in, std_err)+import qualified Data.Map as M+import Control.Exception.Extensible  remote :: RemoteType remote = RemoteType {@@ -90,11 +92,10 @@ 		(False, _, NoUUID) -> tryGitConfigRead r 		_ -> return r -repoCheap :: Git.Repo -> Bool-repoCheap = not . Git.repoIsUrl- gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote-gen r u _ gc = go <$> remoteCost gc defcst+gen r u c gc+	| Git.GCrypt.isEncrypted r = Remote.GCrypt.gen r u c gc+	| otherwise = go <$> remoteCost gc defcst   where 	defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost 	go cst = new@@ -111,14 +112,12 @@ 			, hasKeyCheap = repoCheap r 			, whereisKey = Nothing 			, config = M.empty-			, localpath = if Git.repoIsLocal r || Git.repoIsLocalUnknown r-				then Just $ Git.repoPath r-				else Nothing+			, localpath = localpathCalc r 			, repo = r 			, gitconfig = gc 				{ remoteGitConfig = Just $ extractGitConfig r } 			, readonly = Git.repoIsHttp r-			, globallyAvailable = not $ Git.repoIsLocal r || Git.repoIsLocalUnknown r+			, globallyAvailable = globallyAvailableCalc r 			, remotetype = remote 			} @@ -126,17 +125,18 @@ repoAvail :: Git.Repo -> Annex Bool repoAvail r  	| Git.repoIsHttp r = return True+	| Git.GCrypt.isEncrypted r = do+		g <- gitRepo+		liftIO $ do+			er <- Git.GCrypt.encryptedRepo g r+			if Git.repoIsLocal er || Git.repoIsLocalUnknown er+				then catchBoolIO $+					void (Git.Config.read er) >> return True+				else return True 	| Git.repoIsUrl r = return True 	| Git.repoIsLocalUnknown r = return False 	| otherwise = liftIO $ catchBoolIO $ onLocal r $ return True -{- Avoids performing an action on a local repository that's not usable.- - Does not check that the repository is still available on disk. -}-guardUsable :: Git.Repo -> a -> Annex a -> Annex a-guardUsable r onerr a-	| Git.repoIsLocalUnknown r = return onerr-	| otherwise = a- {- Tries to read the config for a specified remote, updates state, and  - returns the updated repo. -} tryGitConfigRead :: Git.Repo -> Annex Git.Repo@@ -152,6 +152,7 @@ 	| Git.repoIsHttp r = do 		headers <- getHttpHeaders 		store $ geturlconfig headers+	| Git.GCrypt.isEncrypted r = handlegcrypt =<< getConfigMaybe (remoteConfig r "uuid") 	| Git.repoIsUrl r = return r 	| otherwise = store $ safely $ onLocal r $ do  		ensureInitialized@@ -219,6 +220,15 @@ 			let k = "remote." ++ n ++ ".annex-ignore" 			warning $ "Remote " ++ n ++ " " ++ msg ++ "; setting " ++ k 			inRepo $ Git.Command.run [Param "config", Param k, Param "true"]+		+	handlegcrypt Nothing = return r+	handlegcrypt (Just _cacheduuid) = do+		-- Generate UUID from the gcrypt-id+		g <- gitRepo+		case Git.GCrypt.remoteRepoId g (Git.remoteName r) of+			Nothing -> return r+			Just v -> store $ liftIO $ setUUID r $+				genUUIDInNameSpace gCryptNameSpace v  {- Checks if a given remote has the content for a key inAnnex.  - If the remote cannot be accessed, or if it cannot determine@@ -254,17 +264,6 @@ 	unknown = Left $ "unable to check " ++ Git.repoDescribe r 	showchecking = showAction $ "checking " ++ Git.repoDescribe r -{- Runs an action on a local repository inexpensively, by making an annex- - monad using that repository. -}-onLocal :: Git.Repo -> Annex a -> IO a-onLocal r a = do-	s <- Annex.new r-	Annex.eval s $ do-		-- No need to update the branch; its data is not used-		-- for anything onLocal is used to do.-		Annex.BranchState.disableUpdate-		a- keyUrls :: Git.Repo -> Key -> [String] keyUrls r key = map tourl locs   where@@ -408,15 +407,16 @@ 							(\d -> rsyncOrCopyFile params object d p) 			) -rsyncHelper :: Maybe MeterUpdate -> [CommandParam] -> Annex Bool-rsyncHelper callback params = do-	showOutput -- make way for progress bar-	ifM (liftIO $ (maybe rsync rsyncProgress callback) params)-		( return True-		, do-			showLongNote "rsync failed -- run git annex again to resume file transfer"-			return False-		)+{- Runs an action on a local repository inexpensively, by making an annex+ - monad using that repository. -}+onLocal :: Git.Repo -> Annex a -> IO a+onLocal r a = do+	s <- Annex.new r+	Annex.eval s $ do+		-- No need to update the branch; its data is not used+		-- for anything onLocal is used to do.+		Annex.BranchState.disableUpdate+		a  {- Copys a file with rsync unless both locations are on the same  - filesystem. Then cp could be faster. -}@@ -448,6 +448,16 @@ #endif 	dorsync = rsyncHelper (Just p) $ 		rsyncparams ++ [File src, File dest]++rsyncHelper :: Maybe MeterUpdate -> [CommandParam] -> Annex Bool+rsyncHelper callback params = do+	showOutput -- make way for progress bar+	ifM (liftIO $ (maybe rsync rsyncProgress callback) params)+		( return True+		, do+			showLongNote "rsync failed -- run git annex again to resume file transfer"+			return False+		)  {- Generates rsync parameters that ssh to the remote and asks it  - to either receive or send the key's content. -}
Remote/Glacier.hs view
@@ -25,6 +25,7 @@ import Utility.Metered import qualified Annex import Annex.Content+import Annex.UUID  import System.Process @@ -67,13 +68,18 @@ 			remotetype = remote 		} -glacierSetup :: UUID -> RemoteConfig -> Annex RemoteConfig-glacierSetup u c = do+glacierSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+glacierSetup mu c = do+	u <- maybe (liftIO genUUID) return mu+	glacierSetup' u c+glacierSetup' :: UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+glacierSetup' u c = do 	c' <- encryptionSetup c 	let fullconfig = c' `M.union` defaults 	genVault fullconfig u 	gitConfigSpecialRemote u fullconfig "glacier" "true"-	setRemoteCredPair fullconfig (AWS.creds u)+	c'' <- setRemoteCredPair fullconfig (AWS.creds u)+	return (c'', u)   where 	remotename = fromJust (M.lookup "name" c) 	defvault = remotename ++ "-" ++ fromUUID u@@ -95,7 +101,7 @@ storeEncrypted r (cipher, enck) k p = sendAnnex k (void $ remove r enck) $ \src -> do 	metered (Just p) k $ \meterupdate -> 		storeHelper r enck $ \h ->-			encrypt (getGpgOpts r) cipher (feedFile src)+			encrypt (getGpgEncParams r) cipher (feedFile src) 				(readBytes $ meteredWrite meterupdate h)  retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool
Remote/Helper/Encryptable.hs view
@@ -23,30 +23,52 @@  - updated to be accessible to an additional encryption key. Or the user  - could opt to use a shared cipher, which is stored unencrypted. -} encryptionSetup :: RemoteConfig -> Annex RemoteConfig-encryptionSetup c = case (M.lookup "encryption" c, extractCipher c) of-	(Nothing, Nothing) -> error "Specify encryption=key or encryption=none or encryption=shared"-	(Just "none", Nothing) -> return c-	(Nothing, Just _) -> return c-	(Just "shared", Just (SharedCipher _)) -> return c-	(Just "none", Just _) -> cannotchange-	(Just "shared", Just (EncryptedCipher _ _)) -> cannotchange-	(Just _, Just (SharedCipher _)) -> cannotchange-	(Just "shared", Nothing) -> use "encryption setup" . genSharedCipher-		=<< highRandomQuality-	(Just keyid, Nothing) -> use "encryption setup" . genEncryptedCipher keyid-		=<< highRandomQuality-	(Just keyid, Just v) -> use "encryption update" $ updateEncryptedCipher keyid v+encryptionSetup c = maybe genCipher updateCipher $ extractCipher c   where-	cannotchange = error "Cannot change encryption type of existing remote."+	-- The type of encryption+	encryption = M.lookup "encryption" c+	-- Generate a new cipher, depending on the chosen encryption scheme+	genCipher = case encryption of+		_ | M.member "cipher" c || M.member "cipherkeys" c -> cannotchange+		Just "none" -> return c+		Just "shared" -> use "encryption setup" . genSharedCipher+			=<< highRandomQuality+		-- hybrid encryption is the default when a keyid is+                -- specified but no encryption+		_ | maybe (M.member "keyid" c) (== "hybrid") encryption ->+			use "encryption setup" . genEncryptedCipher key Hybrid+				=<< highRandomQuality+		Just "pubkey" -> use "encryption setup" . genEncryptedCipher key PubKey+				=<< highRandomQuality+		_ -> error $ "Specify " ++ intercalate " or "+			(map ("encryption=" ++)+				["none","shared","hybrid","pubkey"])+			++ "."+	key = fromMaybe (error "Specifiy keyid=...") $ M.lookup "keyid" c+	newkeys = maybe [] (\k -> [(True,k)]) (M.lookup "keyid+" c) +++		maybe [] (\k -> [(False,k)]) (M.lookup "keyid-" c)+	cannotchange = error "Cannot set encryption type of existing remotes."+	-- Update an existing cipher if possible.+	updateCipher v = case v of+		SharedCipher _ | maybe True (== "shared") encryption -> return c'+		EncryptedCipher _ variant _+			| maybe True (== if variant == Hybrid then "hybrid" else "pubkey") encryption ->+				use "encryption update" $ updateEncryptedCipher newkeys v+		_ -> cannotchange 	use m a = do 		showNote m 		cipher <- liftIO a 		showNote $ describeCipher cipher-		return $ M.delete "encryption" $ M.delete "highRandomQuality" $-				storeCipher c cipher+		return $ storeCipher c' cipher 	highRandomQuality =  		(&&) (maybe True ( /= "false") $ M.lookup "highRandomQuality" c) 			<$> fmap not (Annex.getState Annex.fast)+	c' = foldr M.delete c+                -- git-annex used to remove 'encryption' as well, since+                -- it was redundant; we now need to keep it for+                -- public-key incryption, hence we leave it on newer+                -- remotes (while being backward-compatible).+		[ "keyid", "keyid+", "keyid-", "highRandomQuality" ]  {- Modifies a Remote to support encryption.  -@@ -113,25 +135,30 @@  {- Gets encryption Cipher, and encrypted version of Key. -} cipherKey :: RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))-cipherKey c k = maybe Nothing make <$> remoteCipher c+cipherKey c k = fmap make <$> remoteCipher c   where-	make ciphertext = Just (ciphertext, encryptKey mac ciphertext k)+	make ciphertext = (ciphertext, encryptKey mac ciphertext k) 	mac = fromMaybe defaultMac $ M.lookup "mac" c >>= readMac  {- Stores an StorableCipher in a remote's configuration. -} storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig storeCipher c (SharedCipher t) = M.insert "cipher" (toB64 t) c-storeCipher c (EncryptedCipher t ks) = +storeCipher c (EncryptedCipher t _ ks) = 	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c   where 	showkeys (KeyIds l) = intercalate "," l  {- Extracts an StorableCipher from a remote's configuration. -} extractCipher :: RemoteConfig -> Maybe StorableCipher-extractCipher c = -	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of-		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)-		(Just t, Nothing) -> Just $ SharedCipher (fromB64 t)-		_ -> Nothing+extractCipher c = case (M.lookup "cipher" c,+			M.lookup "cipherkeys" c,+			M.lookup "encryption" c) of+	(Just t, Just ks, encryption) | maybe True (== "hybrid") encryption ->+		Just $ EncryptedCipher (fromB64 t) Hybrid (readkeys ks)+	(Just t, Just ks, Just "pubkey") ->+		Just $ EncryptedCipher (fromB64 t) PubKey (readkeys ks)+	(Just t, Nothing, encryption) | maybe True (== "shared") encryption ->+		Just $ SharedCipher (fromB64 t)+	_ -> Nothing   where 	readkeys = KeyIds . split ","
+ Remote/Helper/Git.hs view
@@ -0,0 +1,30 @@+{- Utilities for git remotes.+ -+ - Copyright 2011-2012 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Helper.Git where++import Common.Annex+import qualified Git++repoCheap :: Git.Repo -> Bool+repoCheap = not . Git.repoIsUrl++localpathCalc :: Git.Repo -> Maybe FilePath+localpathCalc r = if globallyAvailableCalc r+	then Nothing+	else Just $ Git.repoPath r++globallyAvailableCalc :: Git.Repo -> Bool+globallyAvailableCalc r = not $+	Git.repoIsLocal r || Git.repoIsLocalUnknown r++{- Avoids performing an action on a local repository that's not usable.+ - Does not check that the repository is still available on disk. -}+guardUsable :: Git.Repo -> a -> Annex a -> Annex a+guardUsable r onerr a+	| Git.repoIsLocalUnknown r = return onerr+	| otherwise = a
Remote/Hook.hs view
@@ -18,6 +18,7 @@ import Config import Config.Cost import Annex.Content+import Annex.UUID import Remote.Helper.Special import Remote.Helper.Encryptable import Crypto@@ -38,7 +39,7 @@ gen r u c gc = do 	cst <- remoteCost gc expensiveRemoteCost 	return $ encryptableRemote c-		(storeEncrypted hooktype $ getGpgOpts gc)+		(storeEncrypted hooktype $ getGpgEncParams (c,gc)) 		(retrieveEncrypted hooktype) 		Remote { 			uuid = u,@@ -62,13 +63,14 @@   where 	hooktype = fromMaybe (error "missing hooktype") $ remoteAnnexHookType gc	 -hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig-hookSetup u c = do+hookSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+hookSetup mu c = do+	u <- maybe (liftIO genUUID) return mu 	let hooktype = fromMaybe (error "Specify hooktype=") $ 		M.lookup "hooktype" c 	c' <- encryptionSetup c 	gitConfigSpecialRemote u c' "hooktype" hooktype-	return c'+	return (c', u)  hookEnv :: Action -> Key -> Maybe FilePath -> IO (Maybe [(String, String)]) hookEnv action k f = Just <$> mergeenv (fileenv f ++ keyenv)@@ -118,7 +120,7 @@ store h k _f _p = sendAnnex k (void $ remove h k) $ \src -> 	runHook h "store" k (Just src) $ return True -storeEncrypted :: HookName -> GpgOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool+storeEncrypted :: HookName -> [CommandParam] -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool storeEncrypted h gpgOpts (cipher, enck) k _p = withTmp enck $ \tmp -> 	sendAnnex k (void $ remove h enck) $ \src -> do 		liftIO $ encrypt gpgOpts cipher (feedFile src) $
Remote/List.hs view
@@ -22,6 +22,7 @@ import qualified Git.Config  import qualified Remote.Git+import qualified Remote.GCrypt #ifdef WITH_S3 import qualified Remote.S3 #endif@@ -38,6 +39,7 @@ remoteTypes :: [RemoteType] remoteTypes = 	[ Remote.Git.remote+	, Remote.GCrypt.remote #ifdef WITH_S3 	, Remote.S3.remote #endif@@ -98,6 +100,7 @@ 			Remote.Git.configRead r 		| otherwise = return r -{- Checks if a remote is a special remote -}-specialRemote :: Remote -> Bool-specialRemote r = remotetype r /= Remote.Git.remote+{- Checks if a remote is syncable using git. -}+syncableRemote :: Remote -> Bool+syncableRemote r = remotetype r `elem`+	[ Remote.Git.remote, Remote.GCrypt.remote ]
Remote/Rsync.hs view
@@ -7,7 +7,16 @@  {-# LANGUAGE CPP #-} -module Remote.Rsync (remote) where+module Remote.Rsync (+	remote,+	storeEncrypted,+	retrieveEncrypted,+	remove,+	checkPresent,+	withRsyncScratchDir,+	genRsyncOpts,+	RsyncOpts+) where  import qualified Data.ByteString.Lazy as L import qualified Data.Map as M@@ -23,6 +32,7 @@ import Config import Config.Cost import Annex.Content+import Annex.UUID import Annex.Ssh import Remote.Helper.Special import Remote.Helper.Encryptable@@ -51,11 +61,12 @@ gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote gen r u c gc = do 	cst <- remoteCost gc expensiveRemoteCost-	(transport, url) <- rsyncTransport-	let o = RsyncOpts url (transport ++ opts) escape-	    islocal = rsyncUrlIsPath $ rsyncUrl o+	(transport, url) <- rsyncTransport gc $+		fromMaybe (error "missing rsyncurl") $ remoteAnnexRsyncUrl gc+	let o = genRsyncOpts c gc transport url+	let islocal = rsyncUrlIsPath $ rsyncUrl o 	return $ encryptableRemote c-		(storeEncrypted o $ getGpgOpts gc)+		(storeEncrypted o $ getGpgEncParams (c,gc)) 		(retrieveEncrypted o) 		Remote 			{ uuid = u@@ -78,6 +89,9 @@ 			, globallyAvailable = not $ islocal 			, remotetype = remote 			}++genRsyncOpts :: RemoteConfig -> RemoteGitConfig -> [CommandParam] -> RsyncUrl -> RsyncOpts+genRsyncOpts c gc transport url = RsyncOpts url (transport ++ opts) escape   where 	opts = map Param $ filter safe $ remoteAnnexRsyncOptions gc 	escape = M.lookup "shellescape" c /= Just "no"@@ -88,31 +102,34 @@ 		| opt == "--delete" = False 		| opt == "--delete-excluded" = False 		| otherwise = True-	rawurl = fromMaybe (error "missing rsyncurl") $ remoteAnnexRsyncUrl gc++rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex ([CommandParam], RsyncUrl)+rsyncTransport gc rawurl+	| rsyncUrlIsShell rawurl =+		(\rsh -> return (rsyncShell rsh, resturl)) =<<+		case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of+			"ssh":sshopts -> do+				let (port, sshopts') = sshReadPort sshopts+				    host = takeWhile (/=':') resturl+				-- Connection caching+				(Param "ssh":) <$> sshCachingOptions+					(host, port)+					(map Param $ loginopt ++ sshopts')+			"rsh":rshopts -> return $ map Param $ "rsh" :+				loginopt ++ rshopts+			rsh -> error $ "Unknown Rsync transport: "+				++ unwords rsh+	| otherwise = return ([], rawurl)+  where 	(login,resturl) = case separate (=='@') rawurl of-			(h, "") -> (Nothing, h)-			(l, h)  -> (Just l, h)+		(h, "") -> (Nothing, h)+		(l, h)  -> (Just l, h) 	loginopt = maybe [] (\l -> ["-l",l]) login-	fromNull as xs | null xs = as-		       | otherwise = xs-	rsyncTransport = if rsyncUrlIsShell rawurl-		then (\rsh -> return (rsyncShell rsh, resturl)) =<<-			case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of-				"ssh":sshopts -> do-					let (port, sshopts') = sshReadPort sshopts-					    host = takeWhile (/=':') resturl-					-- Connection caching-					(Param "ssh":) <$> sshCachingOptions-						(host, port)-						(map Param $ loginopt ++ sshopts')-				"rsh":rshopts -> return $ map Param $ "rsh" :-					loginopt ++ rshopts-				rsh -> error $ "Unknown Rsync transport: "-					++ unwords rsh-		else return ([], rawurl)+	fromNull as xs = if null xs then as else xs -rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig-rsyncSetup u c = do+rsyncSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+rsyncSetup mu c = do+	u <- maybe (liftIO genUUID) return mu 	-- verify configuration is sane 	let url = fromMaybe (error "Specify rsyncurl=") $ 		M.lookup "rsyncurl" c@@ -121,7 +138,7 @@ 	-- The rsyncurl is stored in git config, not only in this remote's 	-- persistant state, so it can vary between hosts. 	gitConfigSpecialRemote u c' "rsyncurl" url-	return c'+	return (c', u)  rsyncEscape :: RsyncOpts -> String -> String rsyncEscape o s@@ -137,7 +154,7 @@ store :: RsyncOpts -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool store o k _f p = sendAnnex k (void $ remove o k) $ rsyncSend o p k False -storeEncrypted :: RsyncOpts -> GpgOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool+storeEncrypted :: RsyncOpts -> [CommandParam] -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool storeEncrypted o gpgOpts (cipher, enck) k p = withTmp enck $ \tmp -> 	sendAnnex k (void $ remove o enck) $ \src -> do 		liftIO $ encrypt gpgOpts cipher (feedFile src) $
Remote/S3.hs view
@@ -30,6 +30,7 @@ import Creds import Utility.Metered import Annex.Content+import Annex.UUID import Logs.Web  type Bucket = String@@ -70,8 +71,12 @@ 			remotetype = remote 		} -s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig-s3Setup u c = if isIA c then archiveorg else defaulthost+s3Setup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+s3Setup mu c = do+	u <- maybe (liftIO genUUID) return mu+	s3Setup' u c+s3Setup' :: UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+s3Setup' u c = if isIA c then archiveorg else defaulthost   where 	remotename = fromJust (M.lookup "name" c) 	defbucket = remotename ++ "-" ++ fromUUID u@@ -85,7 +90,8 @@ 		 	use fullconfig = do 		gitConfigSpecialRemote u fullconfig "s3" "true"-		setRemoteCredPair fullconfig (AWS.creds u)+		c' <- setRemoteCredPair fullconfig (AWS.creds u)+		return (c', u)  	defaulthost = do 		c' <- encryptionSetup c@@ -129,7 +135,7 @@ 	-- To get file size of the encrypted content, have to use a temp file. 	-- (An alternative would be chunking to to a constant size.) 	withTmp enck $ \tmp -> sendAnnex k (void $ remove' r enck) $ \src -> do-		liftIO $ encrypt (getGpgOpts r) cipher (feedFile src) $+		liftIO $ encrypt (getGpgEncParams r) cipher (feedFile src) $ 			readBytes $ L.writeFile tmp 		s3Bool =<< storeHelper (conn, bucket) r enck p tmp
Remote/Web.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE CPP #-}+ module Remote.Web (remote) where  import Common.Annex@@ -18,8 +20,10 @@ import Types.Key import Utility.Metered import qualified Utility.Url as Url+#ifdef WITH_QUVI import Annex.Quvi import qualified Utility.Quvi as Quvi+#endif  import qualified Data.Map as M @@ -72,8 +76,14 @@ 		untilTrue urls $ \u -> do 			let (u', downloader) = getDownloader u 			case downloader of-				QuviDownloader -> flip downloadUrl dest-					=<< withQuviOptions Quvi.queryLinks [Quvi.httponly, Quvi.quiet] u'+				QuviDownloader -> do+#ifdef WITH_QUVI+					flip downloadUrl dest+						=<< withQuviOptions Quvi.queryLinks [Quvi.httponly, Quvi.quiet] u'+#else+					warning "quvi support needed for this url"+					return False+#endif 				DefaultDownloader -> downloadUrl [u'] dest  downloadKeyCheap :: Key -> FilePath -> Annex Bool@@ -94,14 +104,25 @@ 	us <- getUrls key 	if null us 		then return $ Right False-		else return . Right =<< checkKey' key us-checkKey' :: Key -> [URLString] -> Annex Bool-checkKey' key us = untilTrue us $ \u -> do+		else return =<< checkKey' key us+checkKey' :: Key -> [URLString] -> Annex (Either String Bool)+checkKey' key us = firsthit us (Right False) $ \u -> do 	let (u', downloader) = getDownloader u 	showAction $ "checking " ++ u' 	case downloader of 		QuviDownloader ->-			withQuviOptions Quvi.check [Quvi.httponly, Quvi.quiet] u'+#ifdef WITH_QUVI+			Right <$> withQuviOptions Quvi.check [Quvi.httponly, Quvi.quiet] u'+#else+			return $ Left "quvi support needed for this url"+#endif 		DefaultDownloader -> do 			headers <- getHttpHeaders-			liftIO $ Url.check u' headers (keySize key)+			liftIO $ Right <$> Url.check u' headers (keySize key)+  where+  	firsthit [] miss _ = return miss+	firsthit (u:rest) _ a = do+		r <- a u+		case r of+			Right _ -> return r+			Left _ -> firsthit rest r a
Remote/WebDAV.hs view
@@ -32,6 +32,7 @@ import Creds import Utility.Metered import Annex.Content+import Annex.UUID  type DavUrl = String type DavUser = B8.ByteString@@ -73,15 +74,17 @@ 			remotetype = remote 		} -webdavSetup :: UUID -> RemoteConfig -> Annex RemoteConfig-webdavSetup u c = do+webdavSetup :: Maybe UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+webdavSetup mu c = do+	u <- maybe (liftIO genUUID) return mu 	let url = fromMaybe (error "Specify url=") $ 		M.lookup "url" c 	c' <- encryptionSetup c 	creds <- getCreds c' u 	testDav url creds 	gitConfigSpecialRemote u c' "webdav" "true"-	setRemoteCredPair c' (davCreds u)+	c'' <- setRemoteCredPair c' (davCreds u)+	return (c'', u)  store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool store r k _f p = metered (Just p) k $ \meterupdate ->@@ -94,7 +97,7 @@ storeEncrypted r (cipher, enck) k p = metered (Just p) k $ \meterupdate -> 	davAction r False $ \(baseurl, user, pass) -> 		sendAnnex k (void $ remove r enck) $ \src ->-			liftIO $ encrypt (getGpgOpts r) cipher+			liftIO $ encrypt (getGpgEncParams r) cipher 				(streamMeteredFile src meterupdate) $ 				readBytes $ storeHelper r enck baseurl user pass
Setup.o view

binary file changed (10440 → 16352 bytes)

Test.hs view
@@ -33,6 +33,7 @@ import qualified Types.Backend import qualified Types.TrustLevel import qualified Types+import qualified Logs import qualified Logs.UUIDBased import qualified Logs.Trust import qualified Logs.Remote@@ -55,11 +56,13 @@ import qualified Utility.Misc import qualified Utility.InodeCache import qualified Utility.Env-import qualified Utility.Gpg import qualified Utility.Matcher import qualified Utility.Exception #ifndef mingw32_HOST_OS import qualified GitAnnex+import qualified Remote.Helper.Encryptable+import qualified Types.Crypto+import qualified Utility.Gpg #endif  type TestEnv = M.Map String String@@ -115,6 +118,7 @@ 	, check "prop_idempotent_key_encode" Types.Key.prop_idempotent_key_encode 	, check "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape 	, check "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword+	, check "prop_logs_sane" Logs.prop_logs_sane 	, check "prop_idempotent_configEscape" Logs.Remote.prop_idempotent_configEscape 	, check "prop_parse_show_Config" Logs.Remote.prop_parse_show_Config 	, check "prop_parentDir_basics" Utility.Path.prop_parentDir_basics@@ -872,18 +876,21 @@  -- gpg is not a build dependency, so only test when it's available test_crypto :: TestEnv -> Test-test_crypto env = "git-annex crypto" ~: intmpclonerepo env $ whenM (Utility.Path.inPath Utility.Gpg.gpgcmd) $ do #ifndef mingw32_HOST_OS+test_crypto env = "git-annex crypto" ~: TestList $ flip map ["shared","hybrid","pubkey"] $+    \scheme -> TestCase $ intmpclonerepo env $ whenM (Utility.Path.inPath Utility.Gpg.gpgcmd) $ do 	Utility.Gpg.testTestHarness @? "test harness self-test failed" 	Utility.Gpg.testHarness $ do 		createDirectory "dir"-		let a cmd = git_annex env cmd+		let a cmd = git_annex env cmd $ 			[ "foo" 			, "type=directory"-			, "encryption=" ++ Utility.Gpg.testKeyId+			, "encryption=" ++ scheme 			, "directory=dir" 			, "highRandomQuality=false"-			]+			] ++ if scheme `elem` ["hybrid","pubkey"]+				then ["keyid=" ++ Utility.Gpg.testKeyId]+				else [] 		a "initremote" @? "initremote failed" 		not <$> a "initremote" @? "initremote failed to fail when run twice in a row" 		a "enableremote" @? "enableremote failed"@@ -891,6 +898,16 @@ 		git_annex env "get" [annexedfile] @? "get of file failed" 		annexed_present annexedfile 		git_annex env "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"+		(c,k) <- annexeval $ do+			uuid <- Remote.nameToUUID "foo"+			rs <- Logs.Remote.readRemoteLog+			Just (k,_) <- Backend.lookupFile annexedfile+			return (fromJust $ M.lookup uuid rs, k)+		let key = if scheme `elem` ["hybrid","pubkey"]+				then Just $ Utility.Gpg.KeyIds [Utility.Gpg.testKeyId]+				else Nothing+		testEncryptedRemote scheme key c [k] @? "invalid crypto setup"+ 		annexed_present annexedfile 		git_annex env "drop" [annexedfile, "--numcopies=2"] @? "drop failed" 		annexed_notpresent annexedfile@@ -898,8 +915,35 @@ 		annexed_present annexedfile 		not <$> git_annex env "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" 		annexed_present annexedfile+  where+	{- Ensure the configuration complies with the encryption scheme, and+	 - that all keys are encrypted properly for the given directory remote. -}+	testEncryptedRemote scheme ks c keys = case Remote.Helper.Encryptable.extractCipher c of+		Just cip@Crypto.SharedCipher{} | scheme == "shared" && isNothing ks ->+			checkKeys cip Nothing+		Just cip@(Crypto.EncryptedCipher encipher v ks')+			| checkScheme v && keysMatch ks' ->+				checkKeys cip (Just v) <&&> checkCipher encipher ks'+		_ -> return False+	  where+		keysMatch (Utility.Gpg.KeyIds ks') =+			maybe False (\(Utility.Gpg.KeyIds ks2) ->+					sort (nub ks2) == sort (nub ks')) ks+		checkCipher encipher = Utility.Gpg.checkEncryptionStream encipher . Just+		checkScheme Types.Crypto.Hybrid = scheme == "hybrid"+		checkScheme Types.Crypto.PubKey = scheme == "pubkey"+		checkKeys cip mvariant = do+			cipher <- Crypto.decryptCipher cip+			files <- filterM doesFileExist $+				map ("dir" </>) $ concatMap (key2files cipher) keys+			return (not $ null files) <&&> allM (checkFile mvariant) files+		checkFile mvariant filename =+			Utility.Gpg.checkEncryptionFile filename $+				if mvariant == Just Types.Crypto.PubKey then ks else Nothing+		key2files cipher = Locations.keyPaths .+			Crypto.encryptKey Types.Crypto.HmacSha1 cipher #else-		putStrLn "gpg testing not implemented on Windows"+test_crypto _env = "git-annex crypto" ~: putStrLn "gpg testing not implemented on Windows" #endif  -- This is equivilant to running git-annex, but it's all run in-process
Types/Crypto.hs view
@@ -8,6 +8,7 @@ module Types.Crypto ( 	Cipher(..), 	StorableCipher(..),+	EncryptedCipherVariant(..), 	KeyIds(..), 	Mac(..), 	readMac,@@ -22,10 +23,13 @@ import Utility.Gpg (KeyIds(..))  -- XXX ideally, this would be a locked memory region-newtype Cipher = Cipher String+data Cipher = Cipher String | MacOnlyCipher String -data StorableCipher = EncryptedCipher String KeyIds | SharedCipher String+data StorableCipher = EncryptedCipher String EncryptedCipherVariant KeyIds+		| SharedCipher String 	deriving (Ord, Eq)+data EncryptedCipherVariant = Hybrid | PubKey+	deriving (Ord, Eq)  {- File names are (client-side) MAC'ed on special remotes.  - The chosen MAC algorithm needs to be same for all files stored on the@@ -66,4 +70,4 @@ 	HmacSha384 -> showDigest $* hmacSha384 	HmacSha512 -> showDigest $* hmacSha512   where-  ($*) g f x y = g $ f x y+	($*) g f x y = g $ f x y
Types/Remote.hs view
@@ -31,7 +31,7 @@ 	-- generates a remote of this type 	generate :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> a (RemoteA a), 	-- initializes or changes a remote-	setup :: UUID -> RemoteConfig -> a RemoteConfig+	setup :: Maybe UUID -> RemoteConfig -> a (RemoteConfig, UUID) }  instance Eq (RemoteTypeA a) where
Upgrade/V2.hs view
@@ -12,9 +12,9 @@ import qualified Git.Command import qualified Git.Ref import qualified Annex.Branch-import Logs.Location import Annex.Content import Utility.Tmp+import Logs  olddir :: Git.Repo -> FilePath olddir g@@ -47,7 +47,7 @@  	e <- liftIO $ doesDirectoryExist old 	when e $ do-		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs+		mapM_ (\(k, f) -> inject f $ locationLogFile k) =<< locationLogs 		mapM_ (\f -> inject f f) =<< logFiles old  	saveState False@@ -73,7 +73,7 @@   where 	tryDirContents d = catchDefaultIO [] $ dirContents d 	islogfile f = maybe Nothing (\k -> Just (k, f)) $-			logFileKey $ takeFileName f+			locationLogFileKey f  inject :: FilePath -> FilePath -> Annex () inject source dest = do
Utility/Applicative.o view

binary file changed (1308 → 1936 bytes)

Utility/Directory.o view

binary file changed (15500 → 25096 bytes)

Utility/Env.o view

binary file changed (3664 → 5736 bytes)

Utility/Exception.o view

binary file changed (8648 → 14008 bytes)

Utility/ExternalSHA.o view

binary file changed (11664 → 18448 bytes)

Utility/FileSystemEncoding.o view

binary file changed (7068 → 11440 bytes)

Utility/FreeDesktop.o view

binary file changed (23532 → 39336 bytes)

Utility/Gpg.hs view
@@ -24,7 +24,7 @@ import Utility.Tmp #endif -newtype KeyIds = KeyIds [String]+newtype KeyIds = KeyIds { keyIds :: [String] } 	deriving (Ord, Eq)  {- If a specific gpg command was found at configure time, use it.@@ -32,6 +32,10 @@ gpgcmd :: FilePath gpgcmd = fromMaybe "gpg" SysConfig.gpg +-- Generate an argument list to asymetrically encrypt to the given recipients.+pkEncTo :: [String] -> [CommandParam]+pkEncTo = concatMap (\r -> [Param "--recipient", Param r])+ stdParams :: [CommandParam] -> IO [String] stdParams params = do #ifndef mingw32_HOST_OS@@ -48,9 +52,23 @@ 	return $ defaults ++ toCommand params #endif   where-	-- be quiet, even about checking the trustdb+	-- Be quiet, even about checking the trustdb. 	defaults = ["--quiet", "--trust-model", "always"] +{- Usual options for symmetric / public-key encryption. -}+stdEncryptionParams :: Bool -> [CommandParam]+stdEncryptionParams symmetric =+	[ enc symmetric+	, Param "--force-mdc"+	, Param "--no-textmode"+	]+  where+	enc True = Param "--symmetric"+	-- Force gpg to only encrypt to the specified recipients, not+	-- configured defaults. Recipients are assumed to be specified in+	-- elsewhere.+	enc False = Params "--encrypt --no-encrypt-to --no-default-recipient"+ {- Runs gpg with some params and returns its stdout, strictly. -} readStrict :: [CommandParam] -> IO String readStrict params = do@@ -71,10 +89,11 @@ 		hClose to 		hGetContentsStrict from -{- Runs gpg with some parameters. First sends it a passphrase via- - --passphrase-fd. Then runs a feeder action that is passed a handle and- - should write to it all the data to input to gpg. Finally, runs- - a reader action that is passed a handle to gpg's output. +{- Runs gpg with some parameters. First sends it a passphrase (unless it+ - is empty) via '--passphrase-fd'. Then runs a feeder action that is+ - passed a handle and should write to it all the data to input to gpg.+ - Finally, runs a reader action that is passed a handle to gpg's+ - output.  -  - Runs gpg in batch mode; this is necessary to avoid gpg 2.x prompting for  - the passphrase.@@ -92,20 +111,23 @@ 		hClose toh 	let Fd pfd = frompipe 	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]--	params' <- stdParams $ [Param "--batch"] ++ passphrasefd ++ params-	closeFd frompipe `after` go params'+	closeFd frompipe `after` go (passphrasefd ++ params) #else 	-- store the passphrase in a temp file for gpg 	withTmpFile "gpg" $ \tmpfile h -> do 		hPutStr h passphrase 		hClose h 		let passphrasefile = [Param "--passphrase-file", File tmpfile]-		params' <- stdParams $ [Param "--batch"] ++ passphrasefile ++ params-		go params'+		go $ passphrasefile ++ params #endif   where-	go params' = withBothHandles createProcessSuccess (proc gpgcmd params')+	go params' = pipeLazy params' feeder reader++{- Like feedRead, but without passphrase. -}+pipeLazy :: [CommandParam] -> (Handle -> IO ()) -> (Handle -> IO a) -> IO a+pipeLazy params feeder reader = do+	params' <- stdParams $ Param "--batch" : params+	withBothHandles createProcessSuccess (proc gpgcmd params') 		$ \(to, from) -> do 			void $ forkIO $ do 				feeder to@@ -259,4 +281,42 @@ testTestHarness = do 	keys <- testHarness $ findPubKeys testKeyId 	return $ KeyIds [testKeyId] == keys+#endif++#ifndef mingw32_HOST_OS+checkEncryptionFile :: FilePath -> Maybe KeyIds -> IO Bool+checkEncryptionFile filename keys =+	checkGpgPackets keys =<< readStrict params+  where+	params = [Params "--list-packets --list-only", File filename]++checkEncryptionStream :: String -> Maybe KeyIds -> IO Bool+checkEncryptionStream stream keys =+	checkGpgPackets keys =<< pipeStrict params stream+  where+	params = [Params "--list-packets --list-only"]++{- Parses an OpenPGP packet list, and checks whether data is+ - symmetrically encrypted (keys is Nothing), or encrypted to some+ - public key(s).+ - /!\ The key needs to be in the keyring! -}+checkGpgPackets :: Maybe KeyIds -> String -> IO Bool+checkGpgPackets keys str = do+	let (asym,sym) = partition (pubkeyEncPacket `isPrefixOf`) $+			filter (\l' -> pubkeyEncPacket `isPrefixOf` l' ||+				symkeyEncPacket `isPrefixOf` l') $+			takeWhile (/= ":encrypted data packet:") $+			lines str+	case (keys,asym,sym) of+		(Nothing, [], [_]) -> return True+		(Just (KeyIds ks), ls, []) -> do+			-- Find the master key associated with the+			-- encryption subkey.+			ks' <- concat <$> mapM (findPubKeys >=*> keyIds)+					[ k | k:"keyid":_ <- map (reverse . words) ls ]+			return $ sort (nub ks) == sort (nub ks')+		_ -> return False+  where+	pubkeyEncPacket = ":pubkey enc packet: "+	symkeyEncPacket = ":symkey enc packet: " #endif
− Utility/Gpg/Types.hs 
@@ -1,30 +0,0 @@-{- gpg data types- -- - Copyright 2013 guilhem <guilhem@fripost.org>- -- - Licensed under the GNU GPL version 3 or higher.- -}--module Utility.Gpg.Types where--import Utility.SafeCommand-import Types.GitConfig-import Types.Remote--{- GnuPG options. -}-type GpgOpt = String-newtype GpgOpts = GpgOpts [GpgOpt]--toParams :: GpgOpts -> [CommandParam]-toParams (GpgOpts opts) = map Param opts--class LensGpgOpts a where-	getGpgOpts :: a -> GpgOpts--{- Extract the GnuPG options from a Remote Git Config. -}-instance LensGpgOpts RemoteGitConfig where-	getGpgOpts = GpgOpts . remoteAnnexGnupgOptions--{- Extract the GnuPG options from a Remote. -}-instance LensGpgOpts (RemoteA a) where-	getGpgOpts = getGpgOpts . gitconfig
Utility/Misc.hs view
@@ -91,6 +91,12 @@ 			go (replacement:acc) vs (drop (length val) s) 		| otherwise = go acc rest s +{- First item in the list that is not Nothing. -}+firstJust :: Eq a => [Maybe a] -> Maybe a+firstJust ms = case dropWhile (== Nothing) ms of+	[] -> Nothing+	(md:_) -> md+ {- Given two orderings, returns the second if the first is EQ and returns  - the first otherwise.  -
Utility/Misc.o view

binary file changed (19232 → 33792 bytes)

Utility/Monad.o view

binary file changed (9320 → 14976 bytes)

Utility/OSX.o view

binary file changed (8808 → 14632 bytes)

Utility/PartialPrelude.o view

binary file changed (5456 → 8648 bytes)

Utility/Path.o view

binary file changed (32700 → 56016 bytes)

Utility/Process.o view

binary file changed (43324 → 71704 bytes)

Utility/SafeCommand.o view

binary file changed (29280 → 49128 bytes)

Utility/Tmp.o view

binary file changed (13404 → 22000 bytes)

Utility/UserInfo.o view

binary file changed (6280 → 10264 bytes)

debian/changelog view
@@ -1,3 +1,42 @@+git-annex (4.20130909) unstable; urgency=low++  * initremote: Syntax change when setting up an encrypted special remote.+    Now use keyid=$KEYID rather than the old encryption=$KEYID+  * forget: New command, causes git-annex branch history to be forgotten+    in a way that will spread to other clones of the repository.+    (As long as they're running this version or newer of git-annex.)+  * forget --drop-dead: Completely removes mentions of repositories that+    have been marked as dead from the git-annex branch.+  * sync, assistant: Force push of the git-annex branch. Necessary+    to ensure it gets pushed to remotes after being rewritten by forget.+  * Added gcrypt support. This combines a fully encrypted git+    repository (using git-remote-gcrypt) with an encrypted git-annex special+    remote.+  * sync: Support syncing with gcrypt remotes.+  * importfeed: Also ignore transient problems with downloading content+    from feeds.+  * Honor core.sharedrepository when receiving and adding files in direct+    mode.+  * enableremote: gpg keys can be removed from those a remote encrypts+    to by passing "keyid-=$KEYID". keyid+= is also provided.+    (Thanks, guilhem for the patch.)+  * Added encryption=pubkey scheme, which encrypts to public keys directly+    rather than the hybrid approach. See documentation for advantages+    and disadvantages, but encryption=hybrid is the recommended scheme still.+    (Thanks, guilhem for the patch.)+  * Fix Feeds display in build flags.+  * Remind user when annex-ignore is set for some remotes, if unable to+    get or drop a file, possibly because it's on an ignored remote.+  * gpg: Force --no-textmode in case the user has it turned on in config.+  * webapp: Improve javascript's handling of longpolling connection+    failures, by reloading the current page in this case.+    Works around chromium behavior where ajax connections to urls+    that were already accessed are denied after navigating back to+    a previous page.+  * Allow building without quvi support.++ -- Joey Hess <joeyh@debian.org>  Mon, 09 Sep 2013 09:47:02 -0400+ git-annex (4.20130827) unstable; urgency=low    * Youtube support! (And 53 other video hosts). When quvi is installed,
debian/control view
@@ -72,7 +72,7 @@ 	wget, 	curl, 	openssh-client (>= 1:5.6p1)-Recommends: lsof, gnupg, bind9-host, ssh-askpass, quvi+Recommends: lsof, gnupg, bind9-host, ssh-askpass, quvi, git-remote-gcrypt Suggests: graphviz, bup, libnss-mdns Description: manage files with git, without checking their contents into git  git-annex allows managing files with git, without checking the file
debian/copyright view
@@ -14,10 +14,6 @@            2012 Joey Hess <joey@kitenet.net> License: GPL-3+ -Files: Utility/Gpg/Types.hs-Copyright: 2013 guilhem <guilhem@fripost.org>-License: GPL-3+- Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns standalone/android/icons/* Copyright: 2007 Henrik Nyh <http://henrik.nyh.se/>            2010 Joey Hess <joey@kitenet.net>
doc/bugs/400_mode_leakage.mdwn view
@@ -15,3 +15,11 @@ individual files is not wise.)  --[[Joey]]++> Revisiting this, git-annex already honors core.sharedrepository settings,+> so I just needed to set it to `world` to allow everyone to read.+> +> There was a code path in direct mode where that didn't work; fixed that.+> +> [[done]]+> --[[Joey]]
+ doc/bugs/Could_not_read_from_remote_repository.mdwn view
@@ -0,0 +1,24 @@+### Please describe the problem.++I've been using git-annex for a few weeks now, and everything was working fine until today. Now, when git-annex goes to sync my files, it will hang for a while, and then display a red box on the left side, saying `! Synced with adam.liter`. So, basically, it doesn't really sync. The logs say that it cannot read from the remote repository, which is set up on box.com and was working just fine until today.++I've tried reconfiguring my jabber account on all my devices with git-annex as well as deleting and remaking the box.com repository, and none of these steps have solved the problem.++### What steps will reproduce the problem?++Making changes to any file located in the repository.++### What version of git-annex are you using? On what operating system?++Mac OS X 10.8.4 and git-annex version 4.20130801-gc88bbc4.++### Please provide any additional information below.++[[!format sh """+[2013-09-09 01:33:03 EDT] XMPPSendPack: Syncing with adam.liter +Already up-to-date.+fatal: Could not read from remote repository.++Please make sure you have the correct access rights+and the repository exists.+"""]]
+ doc/bugs/Feature_request:___34__quvi__34___flag.mdwn view
@@ -0,0 +1,14 @@+### Please describe the problem.+git-annex v4.20130827 can't be built on ARM. Technically it's vector that can't be built due to a lack of Template Haskell compilers for this architecture. Vector is a dependency of aeson, which is a dependency of git-annex, which therefore fails to compile.++The only functionality that relies on aeson is, to my knowledge, quvi. Thus my feature request: If you were to introduce a flag to switch quvi support on or off, ARM users like me could circumvent the aeson dependency at build time. In this case we weren't stuck with 4.20130815 (the latest version to not depend on aeson) and could use current and future versions of git-annex. I would appreciate it.+++### What steps will reproduce the problem?+See above.+++### What version of git-annex are you using? On what operating system?+I'm running Raspbian Wheezy on a Raspberry Pi. The git-annex version to be built is 4.20130827. ++> [[done]] --[[Joey]]
+ doc/bugs/GPG_can__39__t_handle_some_files.mdwn view
@@ -0,0 +1,23 @@+### Please describe the problem.++It looks like GPG is being used in text mode, or at least isn't overriding the GPG config.++### What steps will reproduce the problem?++Have a binary file with long lines, and attempt to copy it into git-annex.++This will happen:++    $ git-annex copy 09\ Into\ The\ Dissonance.mp3 -t rsync.net_annex+    copy 09 Into The Dissonance.mp3 (gpg) (checking rsync.net_annex...) (to rsync.net_annex...) gpg: can't handle text lines longer than 19995 characters+    failed+    git-annex: copy: 1 failed++A workaround is to remove "textmode" from your gpg.conf, but git-annex should force this.++### What version of git-annex are you using? On what operating system?++7ae625363bcb6e1fc8b3733c1d7814aca05a2368 on Ubuntu 13.04 x86_64++> The sheer number of ways gpg offers of shooting yourself in the foot..+> Ok [[done]] --[[Joey]]
doc/bugs/Problem_when_dropping_unused_files.mdwn view
@@ -15,3 +15,7 @@ ### What version of git-annex are you using? On what operating system?  git-annex version: 4.20130815, OS X 10.8.4++> [[done]]; no indication this is anything other than a corrupt git+> repository, which can be caused by system crash, disk data loss, +> cosmic rays, etc. This is why we keep backups... --[[Joey]]
doc/bugs/Using_a_revoked_GPG_key.mdwn view
@@ -29,3 +29,6 @@ failed git-annex: enableremote: 1 failed """]]++> [[done]]; can now use: `git annex enableremote foo keyid-=REVOKEDKEY+> keyid+=NEWKEY` to remove it, and add a new key. --[[Joey]]
+ doc/bugs/Windows_and_Linux_in_direct_mode_confuses_git.mdwn view
@@ -0,0 +1,379 @@+### Please describe the problem.+I try to share some files between a windows machine and a linux machine in direct mode, with a remote repository between the two. I can't seem to reproduce the problem in indirect mode...++I add a file on the windows machine then copy it to the remote repository. I then get it on the linux machine and edit it there. If I `git annex sync` and copy the file's data to the remote repository, a `git annex sync` on windows will fail like this:++[[!format sh """+$ git annex sync+commit+ok+pull origin+remote: Counting objects: 41, done.+remote: Compressing objects: 100% (24/24), done.+remote: Total 31 (delta 9), reused 0 (delta 0)+Unpacking objects: 100% (31/31), done.+From ssh://192.168.1.29:/home/raz/work/test+   b23cb44..9073cc3  git-annex  -> origin/git-annex+   7d4f0e2..87438fc  master     -> origin/master+   57a661c..b33b025  synced/git-annex -> origin/synced/git-annex+   7d4f0e2..87438fc  synced/master -> origin/synced/master+ok+(merging origin/git-annex origin/synced/git-annex into git-annex...)+(Recording state in git...)+push origin+Counting objects: 17, done.+Delta compression using up to 8 threads.+Compressing objects: 100% (4/4), done.+Writing objects: 100% (5/5), 565 bytes | 0 bytes/s, done.+Total 5 (delta 2), reused 0 (delta 0)+To ssh://192.168.1.29:/home/raz/work/test.git+   b33b025..dcfaf23  git-annex -> synced/git-annex+ ! [rejected]        master -> synced/master (non-fast-forward)+error: failed to push some refs to 'ssh://192.168.1.29:/home/raz/work/test.git'+hint: Updates were rejected because a pushed branch tip is behind its remote+hint: counterpart. Check out this branch and merge the remote changes+hint: (e.g. 'git pull') before pushing again.+hint: See the 'Note about fast-forwards' in 'git push --help' for details.+failed+git-annex: sync: 1 failed+"""]]++A `git log` tells me the master branch is not properly merged so I assume the pull failed silently (I paste the log in the transcript).++I can then try to manually fix the windows working copy and sometimes it kind of work, but sometimes I have a hard time recovering and I am better off cloning a new version and moving data's around...++### What steps will reproduce the problem?+Create a repository on a windows.++Add a file.++`git clone --bare` the repository and put it on a remote machine.++Clone the remote repository to a linux machine.++Get the file's data.++Edit the file on linux.++Commit it and copy the data's to the remote machine.++`git clone sync` on the windows machine --> Error.++### What version of git-annex are you using? On what operating system?+On Windows:+[[!format sh """+git-annex version: 4.20130827-g4f18612+build flags: Pairing Testsuite S3 WebDAV DNS+local repository version: 4+default repository version: 3+supported repository versions: 3 4+upgrade supported from repository versions: 2+"""]]++On Linux+[[!format sh """+git-annex version: 4.20130815+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP+local repository version: 4+default repository version: 3+supported repository versions: 3 4+upgrade supported from repository versions: 0 1 2+"""]]++### Please provide any additional information below.+[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++### On Windows++$ mkdir test_windows++$ cd test_windows/++$ git init+Initialized empty Git repository in c:/Users/raz/test_windows/.git/++$ git annex init windows+init windows+  Detected a crippled filesystem.++  Enabling direct mode.++  Detected a filesystem without fifo support.++  Disabling ssh connection caching.+ok+(Recording state in git...)++$ echo "test 1" > test.txt++$ git annex add test.txt+add test.txt (checksum...) ok+(Recording state in git...)++$ git annex sync+commit+ok+git-annex: no branch is checked out++$ cd ..++$ git clone --bare test_windows/ test.git+Cloning into bare repository 'test.git'...+done.++$ scp -r test.git 192.168.1.29:/home/raz/work/+config                                        100%  183     0.2KB/s   00:00+description                                   100%   73     0.1KB/s   00:00+HEAD                                          100%   23     0.0KB/s   00:00+applypatch-msg.sample                         100%  452     0.4KB/s   00:00+commit-msg.sample                             100%  896     0.9KB/s   00:00+post-commit.sample                            100%  160     0.2KB/s   00:00+post-receive.sample                           100%  552     0.5KB/s   00:00+post-update.sample                            100%  189     0.2KB/s   00:00+pre-applypatch.sample                         100%  398     0.4KB/s   00:00+pre-commit.sample                             100% 1704     1.7KB/s   00:00+pre-push.sample                               100% 1348     1.3KB/s   00:00+pre-rebase.sample                             100% 4951     4.8KB/s   00:00+prepare-commit-msg.sample                     100% 1239     1.2KB/s   00:00+update.sample                                 100% 3611     3.5KB/s   00:00+exclude                                       100%  240     0.2KB/s   00:00+825dc642cb6eb9a060e54bf8d69288fbee4904        100%   15     0.0KB/s   00:00+a31b7a55380d79248658d3fffff6d2dc41a726        100%  116     0.1KB/s   00:00+dcabbf728abe62b6e2bcc06b8306eb3aa9a497        100%  176     0.2KB/s   00:00+332ecbfe923879df51a7a3f9bb86ebdfb64273        100%   45     0.0KB/s   00:00+4f0e20d38d6dc757340a8c569270b5a857fc67        100%  160     0.2KB/s   00:00+62956bcf45c63b24a637b22a742db0d9458248        100%   86     0.1KB/s   00:00+7ec519eb1b257a85ae8f9373d0835dea9ddc04        100%   82     0.1KB/s   00:00+1d1559a050b4a2df2e653b1e7ac15bdf7d56b1        100%   51     0.1KB/s   00:00+b74dbedfac4d6a0a90580cff2088a5d61f6675        100%  130     0.1KB/s   00:00+0ca042f439ef997f04a25a449aa1f539b93cf4        100%   53     0.1KB/s   00:00+bbce5e51e5a90ffd26900de3546fc4f9704181        100%  151     0.2KB/s   00:00+18bd66316298fcc5c06af9865c30a2fe4d2476        100%  176     0.2KB/s   00:00+edf8b5cc207e16ca0173785a2ba569c5d4208c        100%   71     0.1KB/s   00:00+packed-refs                                   100%  160     0.2KB/s   00:00++$ cd test_windows/++$ git remote add origin ssh://192.168.1.29:/home/raz/work/test.git++$ git annex copy --to origin+copy test.txt (checking origin...) (to origin...)+test.txt+           7 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 79 bytes  received 31 bytes  220.00 bytes/sec+total size is 7  speedup is 0.06+ok+(Recording state in git...)++$ git annex sync+commit+ok+pull origin+remote: Counting objects: 11, done.+remote: Compressing objects: 100% (5/5), done.+remote: Total 6 (delta 1), reused 0 (delta 0)+Unpacking objects: 100% (6/6), done.+From ssh://192.168.1.29:/home/raz/work/test+ * [new branch]      git-annex  -> origin/git-annex+ * [new branch]      master     -> origin/master+ok+(merging origin/git-annex into git-annex...)+(Recording state in git...)+push origin+Counting objects: 18, done.+Delta compression using up to 8 threads.+Compressing objects: 100% (8/8), done.+Writing objects: 100% (10/10), 874 bytes | 0 bytes/s, done.+Total 10 (delta 4), reused 0 (delta 0)+To ssh://192.168.1.29:/home/raz/work/test.git+ * [new branch]      git-annex -> synced/git-annex+ * [new branch]      master -> synced/master+ok++### On Linux++$ git clone test.git test_linux+Cloning into 'test_linux'...+done.++$ cd test_linux ++$ git annex init linux+init linux ok+(Recording state in git...)++$ git annex sync+(merging origin/git-annex origin/synced/git-annex into git-annex...)+(Recording state in git...)+commit  +ok+pull origin +ok+push origin +Counting objects: 11, done.+Delta compression using up to 4 threads.+Compressing objects: 100% (6/6), done.+Writing objects: 100% (8/8), 775 bytes | 0 bytes/s, done.+Total 8 (delta 3), reused 0 (delta 0)+To /home/raz/work/test.git+   0b4f175..41fba1d  git-annex -> synced/git-annex+ok++$ git annex get+get test.txt (from origin...) ok+(Recording state in git...)++$ git annex direct+commit  +# On branch master+nothing to commit, working directory clean+ok+direct test.txt ok+direct  ok++$ git annex sync+commit  +ok+pull origin +ok+push origin +Counting objects: 9, done.+Delta compression using up to 4 threads.+Compressing objects: 100% (4/4), done.+Writing objects: 100% (5/5), 492 bytes | 0 bytes/s, done.+Total 5 (delta 1), reused 0 (delta 0)+To /home/raz/work/test.git+   41fba1d..57a661c  git-annex -> synced/git-annex+ok++$ cat test.txt  +test 1++$ echo "test 2" > test.txt++$ git annex sync+add test.txt (checksum...) ok+(Recording state in git...)+commit  +(Recording state in git...)+ok+pull origin +ok+push origin +Counting objects: 22, done.+Delta compression using up to 4 threads.+Compressing objects: 100% (12/12), done.+Writing objects: 100% (16/16), 1.40 KiB | 0 bytes/s, done.+Total 16 (delta 4), reused 0 (delta 0)+To /home/raz/work/test.git+   57a661c..4883cad  git-annex -> synced/git-annex+   7d4f0e2..87438fc  master -> synced/master+ok++$ git annex copy --to origin+copy test.txt (to origin...) ok+(Recording state in git...)++$ git annex sync            +commit  +ok+pull origin +remote: Counting objects: 6, done.+remote: Compressing objects: 100% (4/4), done.+remote: Total 5 (delta 0), reused 0 (delta 0)+Unpacking objects: 100% (5/5), done.+From /home/raz/work/test+   b23cb44..9073cc3  git-annex  -> origin/git-annex+ok+(merging origin/git-annex into git-annex...)+(Recording state in git...)+push origin +Counting objects: 18, done.+Delta compression using up to 4 threads.+Compressing objects: 100% (8/8), done.+Writing objects: 100% (10/10), 924 bytes | 0 bytes/s, done.+Total 10 (delta 3), reused 0 (delta 0)+To /home/raz/work/test.git+   4883cad..b33b025  git-annex -> synced/git-annex+ok++$ +++### On Windows++$ git annex sync+commit+ok+pull origin+remote: Counting objects: 41, done.+remote: Compressing objects: 100% (24/24), done.+remote: Total 31 (delta 9), reused 0 (delta 0)+Unpacking objects: 100% (31/31), done.+From ssh://192.168.1.29:/home/raz/work/test+   b23cb44..9073cc3  git-annex  -> origin/git-annex+   7d4f0e2..87438fc  master     -> origin/master+   57a661c..b33b025  synced/git-annex -> origin/synced/git-annex+   7d4f0e2..87438fc  synced/master -> origin/synced/master+ok+(merging origin/git-annex origin/synced/git-annex into git-annex...)+(Recording state in git...)+push origin+Counting objects: 17, done.+Delta compression using up to 8 threads.+Compressing objects: 100% (4/4), done.+Writing objects: 100% (5/5), 565 bytes | 0 bytes/s, done.+Total 5 (delta 2), reused 0 (delta 0)+To ssh://192.168.1.29:/home/raz/work/test.git+   b33b025..dcfaf23  git-annex -> synced/git-annex+ ! [rejected]        master -> synced/master (non-fast-forward)+error: failed to push some refs to 'ssh://192.168.1.29:/home/raz/work/test.git'+hint: Updates were rejected because a pushed branch tip is behind its remote+hint: counterpart. Check out this branch and merge the remote changes+hint: (e.g. 'git pull') before pushing again.+hint: See the 'Note about fast-forwards' in 'git push --help' for details.+failed+git-annex: sync: 1 failed++$ git log --graph --oneline --decorate --all+* 87438fc (origin/synced/master, origin/master) git-annex automatic sync+* 7d4f0e2 (HEAD, synced/master, master) git-annex automatic sync+*-.   dcfaf23 (origin/synced/git-annex, git-annex) merging origin/git-annex orig+|\ \+| | *   b33b025 merging origin/git-annex into git-annex+| | |\+| | |/+| |/|+| * | 9073cc3 (origin/git-annex) update+| | * 73518bc update+| | * 4883cad update+| | * 1b447f5 update+| |/+|/|+* | 57a661c update+| |+|  \+*-. \   41fba1d merging origin/git-annex origin/synced/git-annex into git-annex+|\ \ \+| | |/+| |/|+| | *   0b4f175 merging origin/git-annex into git-annex+| | |\+| | |/+| |/|+| * | b23cb44 update+| | * b5755a2 update+| |/+| * 6adcabb update+| * df18bd6 update+| * dcbbce5 branch created+* 0d138eb update+* fe6bccc branch created++$++# End of transcript or log.+"""]]
+ doc/bugs/clicking_back_in_the_web_browser_crashes.mdwn view
@@ -0,0 +1,23 @@+### Please describe the problem.++When I click on the "back" button in my web browser, I get the following error message:++    git-annex has shut down+    +    You can now close this browser window.++### What steps will reproduce the problem?++ 1. git annex webapp+ 2. click on "configuration" (or whatever)+ 3. click on the back button of your web browser++### What version of git-annex are you using? On what operating system?++4.20130815~bpo70+1 on Debian Wheezy, with Chromium Version 29.0.1547.57 Debian 7.1 (217859)++### Please provide any additional information below.++Click "forward" actually brings us back to sanity, and the webapp doesn't actually *crash*. -- [[anarcat]]++> I have applied the page reload fix/hack. [[done]] --[[Joey]]
doc/bugs/git-annex_quit_unexpectedly___40__macosx__41__.mdwn view
@@ -352,3 +352,7 @@  # End of transcript or log. """]]++> This got fixed in haskell gnutls 0.1.5. I have updated the Mtn Lion+> autobuilder and it will be in tonight's build. [[done]]+> --[[Joey]]
doc/bugs/git_annex_add_error_with_Andrew_File_System.mdwn view
@@ -24,3 +24,5 @@  It seems to me that AFS doesn't support hard links between different directories.++> [[done]] per comments --[[Joey]]
+ doc/bugs/merge_causes_out_of_memory_on_large_repos.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.++`git annex merge` goes out of memory on a large repo.+++### What steps will reproduce the problem?++    $ git annex merge+    merge git-annex git-annex: out of memory (requested 1048576 bytes)++### What version of git-annex are you using? On what operating system?++    git-annex version: 4.20130827+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS++On debian wheezy i386++### Please provide any additional information below.++The repository contains a lot of files:++    $ cd .git/objects+    $ find . -type f | wc -l +    1091548+
+ doc/bugs/rename:_permission_denied__44___after_direct_mode_switch.mdwn view
@@ -0,0 +1,77 @@+### Please describe the problem.++On Mac OS X, I tried to switch a repository to direct mode, but there was a+problem in the middle of the switch (permission denied) and the switch+aborted, leaving the repository in a half switched state.++I tried different manipulations, one of which was a checkout (oops), switch+back to indirect, then direct again, and now I have the repository in direct+mode except one file which caused the permission denied error.++### What steps will reproduce the problem?++Do not know exactly why this file is special. I still have the repository, and+each time I try to get this file, it fails with the same error message.++### What version of git-annex are you using? On what operating system?++On Umba, git-annex version: 4.20130723, on Mac OS X 10.6.8.++### Please provide any additional information below.++Umba is the Mac OS X, camaar and riva are Debian machines.++[[!format sh """+Umba$ git annex version+git-annex version: 4.20130723+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS+Umba$++Umba$ git annex get --from riva --not --in here+get 2013-07-31/2013-07-31_180411.jpg (from riva...) +Password: +SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+     2819887 100%  943.08kB/s    0:00:02 (xfer#1, to-check=0/1)++sent 42 bytes  received 2820397 bytes  433913.69 bytes/sec+total size is 2819887  speedup is 1.00+failed+git-annex: get: 1 failed+Umba$ find . -name SHA256-s2819887-\*+./.git/annex/objects/wq/3j/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+./.git/annex/objects/wq/3j/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c.cache+./.git/annex/objects/wq/3j/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c.map+./.git/annex/transfer/failed/download/13fd5d5a-ed97-11e2-9178-574d3b1c0618/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+./.git/annex/transfer/failed/download/95443f2e-ed96-11e2-9d3f-8ffa5b1aae7a/SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+Umba$ git annex fsck+fsck 2013-07-31/2013-07-31_180411.jpg ok+(Recording state in git...)+Umba$ git annex drop 2013-07-31/2013-07-31_180411.jpg +Umba$ git annex get --from riva --not --in here+get 2013-07-31/2013-07-31_180411.jpg (from riva...) +Password: +SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+     2819887 100%  949.58kB/s    0:00:02 (xfer#1, to-check=0/1)++sent 42 bytes  received 2820397 bytes  512807.09 bytes/sec+total size is 2819887  speedup is 1.00+failed+git-annex: get: 1 failed+Umba$++camaar% git annex copy --to umba --not --in umba+copy 2013-07-31/2013-07-31_180411.jpg (checking umba...) (to umba...)+SHA256-s2819887--987f9811d7b5c7a287a74b7adbb852be4d18eeda61c3507f4e08c534d2356f4c+     2819887 100%    4.19MB/s    0:00:00 (xfer#1, to-check=0/1)+git-annex: //Users/nicolas/Pictures/Petites Boutes/.git/annex/tmp/2013-07-31_18041141700.jpg: rename: permission denied (Operation not permitted)+git-annex-shell: recvkey: 1 failed++sent 2820393 bytes  received 42 bytes  1128174.00 bytes/sec+total size is 2819887  speedup is 1.00+rsync error: syntax or usage error (code 1) at main.c(1070) [sender=3.0.9]+  +  rsync failed -- run git annex again to resume file transfer+failed+git-annex: copy: 1 failed+camaar%+"""]]
+ doc/bugs/webapp_usability:_put_the_notices_on_the_right.mdwn view
@@ -0,0 +1,15 @@+### Please describe the problem.++The colored notices on the left of the screen are a little off. When they disappear, it also leaves a huge gap on the left side.++### What steps will reproduce the problem?++Click the `X` on all the messages on the left.++### What version of git-annex are you using? On what operating system?++4.20130815~bpo70+1 on Debian Wheezy, with Chromium Version 29.0.1547.57 Debian 7.1 (217859)++### Please provide any additional information below.++This is admittedly a very mild problem, but from a usability perspective, it would be less confusing to show those on the right. When they were gone, I thought git-annex was broken somewhat... -- [[anarcat]]
doc/design.mdwn view
@@ -3,4 +3,6 @@  See [[encryption]] for design of encryption elements. +See [[roadmap]] for planning and scheduling of new stuff.+ See [[assistant]] for the design site for the git-annex [[/assistant]].
doc/design/assistant.mdwn view
@@ -1,11 +1,11 @@-This is the design site for the git-annex [[/assistant]].+These are the design pages for the git-annex [[/assistant]].  Parts of the design is still being fleshed out, still many ideas and use cases to add. Feel free to chip in with comments! --[[Joey]] -## roadmap+See [[roadmap]] for current plans. -kickstarter year+## initial development kickstarter year overview (2012-2013)  * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]] * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]@@ -21,22 +21,6 @@ * Month 11: [[!traillink Windows]] porting, finishing touches * Month 12: [presentation at SELF2013](http://www.southeastlinuxfest.org/), finishing touches -[sustaining git-annex development](https://campaign.joeyh.name/) year-(starts September 2013).--* Month 1 [[!traillink encrypted_git_remotes]]-* Month 2 [[!traillink disaster_recovery]]-* Month 3 user-driven features and polishing-* Month 4 improve special remote interface & git-annex enhancement contest-* Month 5 [[!traillink xmpp_security]]-* Month 6 Windows assistant and webapp-* Month 7 user-driven features and polishing-* Month 8 [[!traillink gpgkeys]] [[!traillink sshpassword]]-* Month 9 get [[Android]] out of beta-* Month 10 user-driven features and polishing-* Month 11 [[!traillink chunks]] [[!traillink deltas]]-* Month 12 user-driven features and polishing- ## porting  * [[OSX]] port is in fairly good shape, but still has some room for improvement@@ -57,5 +41,5 @@  ## blog -I'm blogging about my progress in the [[blog]] on a semi-daily basis.+I'm blogging about my progress in the [[devblog]] on a semi-daily basis. Follow along!
doc/design/assistant/blog.mdwn view
@@ -1,10 +1,11 @@-Work on the git-annex assistant is [crowdfunded](https://campaign.joeyh.name/).-I'll be blogging about my progress here on a semi-daily basis.+This blog was updated on a semi-daily basis by Joey during the year of work+concentrating on the git-annex assistant that was funded by his kickstarter+campaign. -[[!sidebar content="""-[[!calendar type="month" pages="page(design/assistant/blog/*)"]]-[[!calendar type="month" month="-1" pages="page(design/assistant/blog/*)"]]-[[!calendar type="month" month="-2" pages="page(design/assistant/blog/*)"]]-"""]]+Post-kickstarter work will instead appear on the [[devblog]]. However,+this page's RSS feed will continue to work, so you don't have to migrate+your RSS reader. -[[!inline pages="page(design/assistant/blog/*)" show=0]]+[[!inline pages="page(devblog/*)" show=-1 feedshow=7]]++[[!inline pages="page(design/assistant/blog/*)" show=0 feeds=no]]
doc/design/assistant/polls/Android_default_directory.mdwn view
@@ -4,4 +4,4 @@ want the first time they run it, but to save typing on android, anything that gets enough votes will be included in a list of choices as well. -[[!poll open=yes expandable=yes 55 "/sdcard/annex" 5 "Whole /sdcard" 4 "DCIM directory (photos and videos only)" 1 "Same as for regular git-annex. ~/annex/"]]+[[!poll open=yes expandable=yes 58 "/sdcard/annex" 5 "Whole /sdcard" 4 "DCIM directory (photos and videos only)" 1 "Same as for regular git-annex. ~/annex/"]]
doc/design/assistant/polls/prioritizing_special_remotes.mdwn view
@@ -6,7 +6,7 @@ Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 16 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 9 "Box.com (done)" 71 "My phone (or MP3 player)" 21 "Tahoe-LAFS" 10 "OpenStack SWIFT" 31 "Google Drive"]]+[[!poll open=yes 16 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 9 "Box.com (done)" 71 "My phone (or MP3 player)" 23 "Tahoe-LAFS" 10 "OpenStack SWIFT" 31 "Google Drive"]]  This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they
doc/design/encryption.mdwn view
@@ -103,14 +103,17 @@  ## risks -A risk of this scheme is that, once the symmetric cipher has been obtained, it-allows full access to all the encrypted content. This scheme does not allow-revoking a given gpg key access to the cipher, since anyone with such a key-could have already decrypted the cipher and stored a copy. +A risk of this scheme is that, once the symmetric cipher has been+obtained, it allows full access to all the encrypted content. Indeed+anyone owning a key that used to be granted access could already have+decrypted the cipher and stored a copy. While it is in possible to+remove a key with `keyid-=`, it is designed for a+[[completely_different_purpose|/encryption]] and does not actually revoke+access.  If git-annex stores the decrypted symmetric cipher in memory, then there is a risk that it could be intercepted from there by an attacker. Gpg-amelorates these type of risks by using locked memory. For git-annex, note+ameliorates these type of risks by using locked memory. For git-annex, note that an attacker with local machine access can tell at least all the filenames and metadata of files stored in the encrypted remote anyway, and can access whatever content is stored locally.
+ doc/design/gcrypt.mdwn view
@@ -0,0 +1,8 @@+To integrate with git-remote-gcrypt, a key thing is to have a way to map+from the gcrypt-id of an encrypted repository to a git-annex repository+uuid.++To do this, we'll make a v5 UUID, feeding in the gcrypt-id.+The namespace used is itself a v5 UUID, generated using the URL+namespace and the URL of this page at the time this scheme was+developed: "http://git-annex.branchable.com/design/gcrypt/"
+ doc/design/roadmap.mdwn view
@@ -0,0 +1,18 @@+## roadmap++Now in the +[sustaining git-annex development](https://campaign.joeyh.name/) year+(starting September 2013).++* Month 1 [[!traillink assistant/encrypted_git_remotes]]+* Month 2 [[!traillink assistant/disaster_recovery]]+* Month 3 user-driven features and polishing+* Month 4 improve special remote interface & git-annex enhancement contest+* Month 5 [[!traillink assistant/xmpp_security]]+* Month 6 Windows assistant and webapp+* Month 7 user-driven features and polishing+* Month 8 [[!traillink assistant/gpgkeys]] [[!traillink assistant/sshpassword]]+* Month 9 get [[assistant/Android]] out of beta+* Month 10 user-driven features and polishing+* Month 11 [[!traillink assistant/chunks]] [[!traillink assistant/deltas]]+* Month 12 user-driven features and polishing
+ doc/devblog.mdwn view
@@ -0,0 +1,10 @@+Work on git-annex is [crowdfunded](https://campaign.joeyh.name/).+Joey blogs about his progress here on a semi-daily basis.++[[!sidebar content="""+[[!calendar type="month" pages="page(devblog/*)"]]+[[!calendar type="month" month="-1" pages="page(devblog/*)"]]+[[!calendar type="month" month="-2" pages="page(devblog/*)"]]+"""]]++[[!inline pages="page(devblog/*)" show=0]]
+ doc/devblog/day_-1__drop_dead.mdwn view
@@ -0,0 +1,5 @@+Implemented `git annex forget --drop-dead`, which is finally a way to+remove all references to old repositories that you've marked as dead.++I've still not merged in the `forget` branch, because I developed this+while slightly ill, and have not tested it very well yet.
+ doc/devblog/day_-3__.mdwn view
@@ -0,0 +1,29 @@+John Millikin came through and fixed that haskell-gnutls segfault+on OSX that I developed a reproducible test case for the other day.+It's a bit hard to test, since the bug doesn't always happen, but the+fix is already deployed for Mountain Lion autobuilder.++However, I then found another way to make haskell-gnutls segfault, more+reliably on OSX, and even sometimes on Linux. Just entering the wrong XMPP+password in the assistant can trigger this crash. Hopefully John will work+his magic again.++---++Meanwhile, I fixed the sync-after-forget problem. Now sync always forces+its push of the git-annex branch (as does the assistant). I considered but+rejected having sync do the kind of uuid-tagged branch push that the+assistant sometimes falls back to if it's failing to do a normal sync. It's+ugly, but worse, it wouldn't work in the workflow where multiple clients+are syncing to a central bare repository, because they'd not pull down the+hidden uuid-tagged branches, and without the assistant running on the+repository, nothing would ever merge their data into the git-annex branch.+Forcing the push of synced/git-annex was easy, once I satisfied myself+that it was always ok to do so.++Also factored out a module that knows about all the different log files+stored on the git-annex branch, which is all the support infrastructure+that will be needed to make `git annex forget --drop-dead` work. Since this+is basically a routing module, perhaps I'll get around to making it use+a nice bidirectional routing library like+[Zwaluw](http://hackage.haskell.org/package/Zwaluw) one day.
+ doc/devblog/day_-4__forgetting.mdwn view
@@ -0,0 +1,80 @@+Yesterday I spent making a release, and shopping for a new laptop, since+this one is dying. (Soon I'll be able to compile git-annex fast-ish! Yay!)+And thinking about [[todo/wishlist:_dropping_git-annex_history]].++Today, I added the `git annex forget` command. It's currently been lightly+tested, seems to work, and is living in the `forget` branch until I gain+confidence with it. It should be perfectly safe to use, even if it's buggy,+because you can use `git reflog git-annex` to pull out and revert to an old+version of your git-annex branch. So if you're been wanting this feature,+please beta test!++----++I actually implemented something more generic than just forgetting git+history. There's now a whole mechanism for git-annex doing distributed+transitions of whatever sort is needed.++There were several subtleties involved in distributed transitions:++First is how to tell when a given transition has already been done on a+branch. At first I was thinking that the transition log should include the+sha of the first commit on the old branch that got rewritten. However, that+would mean that after a single transition had been done, every git-annex+branch merge would need to look up the first commit of the current branch,+to see if it's done the transition yet. That's slow! Instead, transitions+are logged with a timestamp, and as long as a branch contains a transition+with the same timestamp, it's been done.++A really tricky problem is what to do if the local repository has+transitioned, but a remote has not, and changes keep being made to the+remote. What it does so far is incorporate the changes from the remote into+the index, and re-run the transition code over the whole thing to yeild a+single new commit. This might not be very efficient (once I write the more+full-featured transition code), but it lets the local repo keep up with+what's going on in the remote, without directly merging with it (which+would revert the transition). And once the remote repository has its+git-annex upgraded to one that knows about transitions, it will finish up+the transition on its side automatically, and the two branches will once+again merge.++Related to the previous problem, we don't want to keep trying to merge+from a remote branch when it's not yet transitioned. So a blacklist is+used, of untransitioned commits that have already been integrated.++One really subtle thing is that when the user does a transition more+complicated than `git annex forget`, like the `git annex forget --dead`+that I need to implement to forget dead remotes, they're not just telling+git-annex to forget whatever dead remotes it knows right now. They're+actually telling git-annex to perform the transition one time on every+existing clone of the repository, at some point in the future. Repositories+with unfinished transitions could hang around for years, and at some future+point when git-annex runs in the repository again, it would merge in the+current state of the world, and re-do the transition. So you might tell it+to forget dead remotes today, and then the very repository you ran that in+later becomes dead, and a long-slumbering repo wakes up and forgets about+the repo that started the whole process! I hope users don't find this+massively confusing, but that's how the implementation works right now.++----++I think I have at least two more days of work to do to finish up this+feature.++* I still need to add some extra features like forgetting about dead remotes,+  and forgetting about keys that are no longer present on any remote.++* After `git annex forget`, `git annex sync`+  will fail to push the synced/annex branch to remotes, since the branch+  is no longer a fast-forward of the old one. I will probably fix this by+  making `git annex sync` do a fallback push of a unique branch in this case,+  like the assistant already does. Although I may need to adjust that code+  to handle this case, too..++* For some reason the automatic transitioning code triggers+  a "(recovery from race)" commit. This is certianly a bug somewhere,+  because you can't have a race with only 1 participant.++----++Today's work was sponsored by Richard Hartmann.
+ doc/devblog/day_1__inauspicious_beginning.mdwn view
@@ -0,0 +1,11 @@+I try hard to keep this devblog about git-annex development and not me.+However, it is a shame that what I wanted to be the beginning of my first+real month of work funded by the new campaign has been marred by my home's+internet connection being taken out by a lightning strike, and by illness.+Nearly back on my feet after that, and waiting for my new laptop to+finally get here.++Today's work: Finished up the `git annex forget` feature and merged it in.+Fixed the bug that was causing the commit race detection code to+incorrectly fire on the commit made by the transition code. Few other bits+and pieces.
+ doc/devblog/day_2__new_laptop.mdwn view
@@ -0,0 +1,8 @@+Now I can build git-annex twice as fast! And a typical incremental build is+down to 10 seconds, from 51 seconds.++Spent a productive evening working with Guilhem to get his encryption+patches reviewed and merged. Now there is a way to remove revoked gpg keys,+and there is a new encryption scheme available that uses public key+encryption by default rather than git-annex's usual approach. That's not+for everyone, but it is a good option to have available.
+ doc/devblog/day_3__gcrypt_uuids.mdwn view
@@ -0,0 +1,63 @@+Started work on [gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt)+support.++The first question is, should git-annex leave it up to gcrypt to transport+the data to the encrypted repository on a push/pull? gcrypt hooks into git+nicely to make that just work. However, if I go this route, it limits+the places the encrypted git repositores can be stored to regular git+remotes (and rsync). The alternative is to somehow use gcrypt to+generate/consume the data, but use the git-annex special remotes to store+individual files. Which would allow for a git repo stored on S3, etc.+For now, I am going with the simple option, but I have not ruled out+trying to make the latter work. It seems it would need changes to gcrypt+though.++Next question: Given a remote that uses gcrypt, how do I determine the+annex.uuid of that repository. I found a nice solutuon to this. gcrypt has+its own gcrypt-id, and I convert it to a UUID in a +[[reproducible, and even standards-compliant way|design/gcrypt]]. So+the same encrypted remote will automatically get the same annex.uuid+wherever it's used. Nice. Does mean that git-annex cannot find a uuid+until `git pull` or `git push` has been used, to let gcrypt get the+gcrypt-id. Implemented that.++The next step is actually making git-annex store data on gcrypt remotes.+And it needs to store it encrypted of course. It seems best to avoid+needing a `git annex initremote` for these gcrypt remotes, and just have+git-annex automatically encrypt data stored on them. But I don't+know. Without initializing them like a special remote is, I'm limited to+using the gpg keys that gcrypt is configured to encrypt to, and cannot use+the regular git-annex hybrid encryption scheme. Also, I need to generate+and store a nonce anyway to HMAC ecrypt keys. (Or modify gcrypt+to put enough entropy in gcrypt-id that I can use it?) ++Another concern I have is that gcrypt's own encryption scheme is simply+to use a list of public keys to encrypt to. It would be nicer if the+full set of git-annex encryption schemes could be used. Then the webapp+could use shared encryption to avoid needing to make the user set up a gpg+key, or hybrid encryption could be used to add keys later, etc.++But I see why gcrypt works the way it does. Otherwise, you can't make an+encrypted repo with a friend set as one of the particpants and have them be+able to git clone it. Both hybrid and shared encryption store a secret+inside the repo, which is not accessible if it's encrypted using that+secret. There are use cases where not being able to blindly clone a gcrypt+repo would be ok. For example, you use the assistant to pair with a friend+and then set up an encrypted repo in the cloud for both of you to use.++Anyway, for now, I will need to deal with+setting up gpg keys etc in the assistant. I don't want to tackle+full [[design/assistant/gpgkeys]] yet. Instead, I think I will start by+adding some simple stuff to the assistant:++* When adding a USB drive, offer to encrypt the repository on the drive+  so that only you can see it.+* When adding a ssh remote make a similar offer.+* Add a UI to add an arbitrary git remote with encryption.+  Let the user paste in the url to an empty remote they have,+  which could be to eg github. (In most cases this won't be used for+  annexed content..)+* When the user has no gpg key, prompt to set one up. (Securely!)+* Maybe have an interface to add another gpg key that can access the gcrypt+  repo. Note that this will need to re-encrypt and re-push the whole+  git history.
+ doc/devblog/day_4__unexpected_windows_day.mdwn view
@@ -0,0 +1,10 @@+Woke up with a pretty solid plan for gcrypt. It will be structured as a+separate special remote, so `initremote` will be needed, with a gitrepo=+parameter (unless the remote already exists). git-annex will then set up+the git remote, including pushing to it (needed to get a gcrypt-id).++Didn't feel up to implementing that today. Instead I expectedly spent+the day doing mostly Windows work, including setting up a VM on my new+laptop for development. Including a ssh server in Windows, so I can+script local builds and tests on Windows without ever having to+touch the desktop. Much better!
+ doc/devblog/day_5__gcrypt_special_remote_part_1.mdwn view
@@ -0,0 +1,7 @@+About half way done with a gcrypt special remote. I can initremote it (the+hard part to get working), and can send files to it. Can't yet get files+back, or remove files, and only local repositories work so far, but this is+enough to know it's going to be pretty nice!++Did find one issue in gcrypt that I may need to develop a patch for:+<https://github.com/blake2-ppc/git-remote-gcrypt/issues/3>
+ doc/devblog/day_6__gcrypt_fully_working.mdwn view
@@ -0,0 +1,8 @@+gcrpyt is fully working now. *Most* of the examples in+[[tips/fully_encrypted_git_repositories_with_gcrypt]] should work.++A few known problems:++* `git annex sync` refuses to sync with gcrypt remotes. some url parsing issue.+* Swapping two drives with gcrypt repositories on the same mount point doesn't work yet.+* http urls are not supported
+ doc/devblog/moving_blogs.mdwn view
@@ -0,0 +1,5 @@+I've started a new page for my devblog, since I'm not focusing extensively+on the assistant and so keeping the blog [[here|design/assistant/blog]]+increasingly felt wrong. Also, my new year of+[crowdfunded development](https://campaign.joeyh.name)+formally starts in September, so a new blog seemed good.
+ doc/devblog/moving_blogs/comment_1_6caa7e67461a6ea5de8155ae9cf75fab._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://pnijjar.livejournal.com/"+ ip="99.236.22.229"+ subject="comment 1"+ date="2013-08-31T00:05:16Z"+ content="""+Do we need to update our RSS feeds? I appear to be getting your devblog posts in my old feed, but I do not know whether that will continue working.+"""]]
+ doc/devblog/moving_blogs/comment_2_e3e2048fc2397b87a2f29c9fe49394cb._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"+ nickname="develop"+ subject="comment 2"+ date="2013-08-31T10:03:04Z"+ content="""+The old RSS feed will continue working.++So sit back, relax, and enjoy the show.+"""]]
doc/encryption.mdwn view
@@ -6,50 +6,94 @@ file content is sent to an untrusted party who does not have access to the git repository. -Such an encrypted remote uses strong GPG encryption on the contents of files,-as well as HMAC hashing of the filenames. The size of the encrypted files,-and access patterns of the data, should be the only clues to what is-stored in such a remote.+Such an encrypted remote uses strong ([[symmetric|design/encryption]] or+asymmetric) encryption on the contents of files, as well as HMAC hashing+of the filenames. The size of the encrypted files, and access patterns+of the data, should be the only clues to what is stored in such a+remote.  You should decide whether to use encryption with a special remote before any data is stored in it. So, `git annex initremote` requires you to specify "encryption=none" when first setting up a remote in order-to disable encryption.+to disable encryption. To use encryption, you run +run `git-annex initremote` in one of these ways: -If you want to use encryption, run `git annex initremote` with-"encryption=USERID". The value will be passed to `gpg` to find encryption keys.-Typically, you will say "encryption=2512E3C7" to use a specific gpg key.-Or, you might say "encryption=joey@kitenet.net" to search for matching keys.+* `git annex initremote newremote type=... encryption=hybrid keyid=KEYID ...`+* `git annex initremote newremote type=... encryption=shared`+* `git annex initremote newremote type=... encryption=pubkey keyid=KEYID ...` -The default MAC algorithm to be applied on the filenames is HMACSHA1. A-stronger one, for instance HMACSHA512, one can be chosen upon creation-of the special remote with the option `mac=HMACSHA512`. The available-MAC algorithms are HMACSHA1, HMACSHA224, HMACSHA256, HMACSHA384, and-HMACSHA512. Note that it is not possible to change algorithm for a-non-empty remote.+## hybrid encryption keys -The [[encryption_design|design/encryption]] allows additional encryption keys-to be added on to a special remote later. Once a key is added, it is able-to access content that has already been stored in the special remote.-To add a new key, just run `git annex enableremote` specifying the-new encryption key:+The [[hybrid_key_design|design/encryption]] allows additional+encryption keys to be added on to a special remote later. Due to this+flexability, it is the default and recommended encryption scheme.+ +	git annex initremote newremote type=... [encryption=hybrid] keyid=KEYID ... -	git annex enableremote myremote encryption=788A3F4C+Here the KEYID(s) are passed to `gpg` to find encryption keys. +Typically, you will say "keyid=2512E3C7" to use a specific gpg key.+Or, you might say "keyid=joey@kitenet.net" to search for matching keys. -Note that once a key has been given access to a remote, it's not-possible to revoke that access, short of deleting the remote. See-[[encryption_design|design/encryption]] for other security risks-associated with encryption.+To add a new key and allow it to access all the content that is stored+in the encrypted special remote, just run `git annex+enableremote` specifying the new encryption key: -## shared cipher mode+	git annex enableremote myremote keyid+=788A3F4C +While a key can later be removed from the list, note that+that will **not** necessarily prevent the owner of the key+from accessing data on the remote (which is by design impossible to prevent,+short of deleting the remote). In fact the only sound use of `keyid-=` is+probably to replace a revoked key:++	git annex enableremote myremote keyid-=2512E3C7 keyid+=788A3F4C++See also [[encryption_design|design/encryption]] for other security+risks associated with encryption.++## shared encryption key+ Alternatively, you can configure git-annex to use a shared cipher to encrypt data stored in a remote. This shared cipher is stored,-**unencrypted** in the git repository. So it's shared amoung every-clone of the git repository. The advantage is you don't need to set up gpg-keys. The disadvantage is that this is **insecure** unless you-trust every clone of the git repository with access to the encrypted data-stored in the special remote.+**unencrypted** in the git repository. So it's shared among every+clone of the git repository. +	+	git annex initremote newremote type=... encryption=shared -To use shared encryption, specify "encryption=shared" when first setting-up a special remote.+The advantage is you don't need to set up gpg keys. The disadvantage is+that this is **insecure** unless you trust every clone of the git+repository with access to the encrypted data stored in the special remote.++## regular public key encryption++This alternative simply encrypts the files in the special remotes to one or+more public keys. It might be considered more secure due to its simplicity+and since it's exactly the way everyone else uses gpg.++	git annex initremote newremote type=.... encryption=pubkey keyid=KEYID ...++A disavantage is that is not easy to later add additional public keys+to the special remote. While the `enableremote` parameters `keyid+=` and+`keyid-=` can be used, they have **no effect** on files that are already+present on the remote. Probably the only use for these parameters is +to replace a revoked key:++	git annex enableremote myremote keyid-=2512E3C7 keyid+=788A3F4C++But even in this case, since the files are not re-encrypted, the revoked+key has to be kept around to be able to decrypt those files.+(Of course, if the reason for revocation is+that the key has been compromised, it is **insecure** to leave files+encrypted using that old key, and the user should re-encrypt everything.)++(Because filenames are MAC'ed, a cipher still needs to be+generated (and encrypted to the given key IDs).)++## MAC algorithm++The default MAC algorithm to be applied on the filenames is HMACSHA1. A+stronger one, for instance HMACSHA512, one can be chosen upon creation+of the special remote with the option `mac=HMACSHA512`. The available+MAC algorithms are HMACSHA1, HMACSHA224, HMACSHA256, HMACSHA384, and+HMACSHA512. Note that it is not possible to change algorithm for a+non-empty remote.
doc/footer/column_a.mdwn view
@@ -2,6 +2,6 @@  [[!inline pages="news/* and !*/Discussion" archive=yes show=2 feeds=no]] -### [[Dev blog|design/assistant/blog]]+### [[devblog|devblog]] -[[!inline pages="design/assistant/blog/* and !*/Discussion" archive=yes show=5 feeds=no]]+[[!inline pages="devblog/* and !*/Discussion" archive=yes show=5 feeds=no]]
+ doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days.mdwn view
@@ -0,0 +1,72 @@+I've been experiencing problems with Box.com for a few days now and I don't know what's causing them. Is anyone else experiencing anything similar?++I paste the log.++    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "c9e1d5421e78924c21e3d68e84f80a8d1f64f9488020107ca0eeee0c4f10e763.py", keyBackendName = "SHA256E", keySize = Just 1891, keyMtime = Nothing}}+    [2013-09-02 12:27:26 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kant.xml Nothing : expensive scan found missing object+    [2013-09-02 12:27:26 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing+    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing+    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing+    +    +    100%          0.0 B/s 0s[2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Just 437+    ResponseTimeout+    ResponseTimeout+    +                            +    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "dd3cc45d91430c6f7d68eb807f4ac1561cd0297b11a2de77b5fe66017d125798.py", keyBackendName = "SHA256E", keySize = Just 437, keyMtime = Nothing}}+    [2013-09-02 12:27:44 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kgp.dtd Nothing : expensive scan found missing object+    [2013-09-02 12:27:44 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing+    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing+    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing+    +    +    100%          0.0 B/s 0s[2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Just 2861+    ResponseTimeout+    ResponseTimeout+    +                            +    [2013-09-02 12:28:02 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "d6b7940ac68768a8e37e72f248e2d94f19fb0d47062084d9baf0ec08cebbf692.py", keyBackendName = "SHA256E", keySize = Just 2861, keyMtime = Nothing}}+    [2013-09-02 12:28:02 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kgp.py Nothing : expensive scan found missing object+    [2013-09-02 12:28:02 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing+    [2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing+    [2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing+    +    +    100%          0.0 B/s 0s[2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Just 1090+    ResponseTimeout+    ResponseTimeout+    +                            +    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "f1492b80d05b96cc7cf2904d461c99d430fa86a4eb1d99f1b155c9147ff4420f.py", keyBackendName = "SHA256E", keySize = Just 1090, keyMtime = Nothing}}+    [2013-09-02 12:28:21 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/russiansample.xml Nothing : expensive scan found missing object+    [2013-09-02 12:28:21 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing+    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing+    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing+    +    +    100%          0.0 B/s 0s[2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Just 4864+    ResponseTimeout+    ResponseTimeout+    +    +    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "b577eaf8b6ddbf9fef866c455cae248aec3b22e3f2e91aa2b75ece90f1801689.py", keyBackendName = "SHA256E", keySize = Just 4864, keyMtime = Nothing}}+    [2013-09-02 12:28:40 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/stderr.py Nothing : expensive scan found missing object+    [2013-09-02 12:28:40 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing+    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing+    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing+    +    +    100%          0.0 B/s 0s[2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Just 4449+    ResponseTimeout+    ResponseTimeout+    +                            +    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "c5e5d9b1bee2710c7ed05270a363d3e93270b0fb6779c4c8d59ace06c11db684.py", keyBackendName = "SHA256E", keySize = Just 4449, keyMtime = Nothing}}+    [2013-09-02 12:28:58 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/stdout.py Nothing : expensive scan found missing object+    [2013-09-02 12:28:58 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing+    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing+    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing+++    100%          0.0 B/s 0s[2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Just 532
+ doc/forum/Git_annex_assistant_on_EC2.mdwn view
@@ -0,0 +1,5 @@+With Amazon EC2, it is possible to take a snapshot of the virtual machine and then bring up multiple instances based on the snapshot. ++If I configure git annex assistant to sync a folder to S3, will it work correctly if I spin up multiple instances, all identical git annex assistant configurations?++My use case is to allow users of my forums to upload attachments with their posts. The attachments are stored locally on disk, but I'm hoping that git annex assistant can help synchroise the attachments across all forum instances (I usually run only 1 instance, but their could be as many as 3 under heavy load).
+ doc/forum/Git_annex_on_Windows.mdwn view
@@ -0,0 +1,11 @@+Hi,+I want to download some files in git and git annex in Windows OS.+I have installed msysgit for git-scm in D:\Git directory,+Now I download git-annex assistant for Windows, and install it+in D:\Git\cmd directory.+But when I run the command "git annex" in git-bash, it says it can not+found the "git annex" command.+My question is that, how should I use git annex in Windows to download files?+For example, git annex get <filename>.+Doea anyone has any idear to help me figure it out?+Thanks in advance!
+ doc/forum/Help_Windows_walkthrough.mdwn view
@@ -0,0 +1,177 @@+Hello,++i am trying to run the walkthrough on Windows 7. When i try to get the contents of a file, i only get a some git annex text string and not the real file. Both repositories are on the same ntfs filesystem. ++C:\tmp>git annex version  +git-annex version: 4.20130827-g4f18612  +build flags: Pairing Testsuite S3 WebDAV DNS  +local repository version: 4  +default repository version: 3  +supported repository versions: 3 4  +upgrade supported from repository versions: 2  +  +C:\tmp\server>git --version  +git version 1.8.3.msysgit.0  +   ++# walkthrough.bat+    +    doskey /history > commands.log+    mkdir laptop+    cd laptop+    git init+    git annex init laptop+    cd ..+    +    git clone laptop server+    cd server+    git annex init server+    git remote add laptop c:\tmp\laptop+    +    cd ..\laptop+    git remote add server c:\tmp\server+    copy ..\1.pdf .+    git annex add 1.pdf+    git commit -m add+    dir +    +    cd ..\server+    dir+    git fetch laptop+    git merge laptop/master+    git annex get 1.pdf+    dir+    type 1.pdf+    ++# walkthrough.log++    C:\tmp>walkthrough.bat+    +    C:\tmp>doskey /history  1>commands.log+    +    C:\tmp>mkdir laptop+    +    C:\tmp>cd laptop+    +    C:\tmp\laptop>git init+    Initialized empty Git repository in C:/tmp/laptop/.git/+    +    C:\tmp\laptop>git annex init laptop+    init laptop+      Detected a crippled filesystem.+    +      Enabling direct mode.+    +      Detected a filesystem without fifo support.+    +      Disabling ssh connection caching.+    ok+    (Recording state in git...)+    +    C:\tmp\laptop>cd ..+    +    C:\tmp>git clone laptop server+    Cloning into 'server'...+    done.+    warning: remote HEAD refers to nonexistent ref, unable to checkout.+    +    +    C:\tmp>cd server+    +    C:\tmp\server>git annex init server+    init server+      Detected a crippled filesystem.+    +      Enabling direct mode.+    +      Detected a filesystem without fifo support.+    +      Disabling ssh connection caching.+    ok+    (Recording state in git...)+    +    C:\tmp\server>git remote add laptop c:\tmp\laptop+    +    C:\tmp\server>cd ..\laptop+    +    C:\tmp\laptop>git remote add server c:\tmp\server+    +    C:\tmp\laptop>copy ..\1.pdf .+            1 file(s) copied.+    +    C:\tmp\laptop>git annex add 1.pdf+    add 1.pdf (checksum...) ok+    (Recording state in git...)+    +    C:\tmp\laptop>git commit -m add+    [master (root-commit) 7ad1514] add+     1 file changed, 1 insertion(+)+     create mode 120000 1.pdf+    +    C:\tmp\laptop>dir+     Volume in drive C has no label.+     Volume Serial Number is x+    +     Directory of C:\tmp\laptop+    +    09/01/2013  11:03 AM    <DIR>          .+    09/01/2013  11:03 AM    <DIR>          ..+    08/30/2013  12:43 PM            37,500 1.pdf+                   1 File(s)         37,500 bytes+                   2 Dir(s)   7,698,817,024 bytes free+    +    C:\tmp\laptop>cd ..\server+    +    C:\tmp\server>dir+     Volume in drive C has no label.+     Volume Serial Number is x+    +     Directory of C:\tmp\server+    +    09/01/2013  11:03 AM    <DIR>          .+    09/01/2013  11:03 AM    <DIR>          ..+                   0 File(s)              0 bytes+                   2 Dir(s)   7,698,817,024 bytes free+    +    C:\tmp\server>git fetch laptop+    remote: Counting objects: 9, done.+    remote: Compressing objects: 100% (6/6), done.+    remote: Total 8 (delta 1), reused 0 (delta 0)+    Unpacking objects: 100% (8/8), done.+    From c:\tmp\laptop+     * [new branch]      git-annex  -> laptop/git-annex+     * [new branch]      master     -> laptop/master+    +    C:\tmp\server>git merge laptop/master+    +    C:\tmp\server>git annex get 1.pdf+    get 1.pdf (merging laptop/git-annex origin/git-annex into git-annex...)+    (Recording state in git...)+    (from laptop...)+    1.pdf+           37500 100%    4.51MB/s    0:00:00 (xfer#1, to-check=0/1)+    +    sent 37573 bytes  received 31 bytes  75208.00 bytes/sec+    total size is 37500  speedup is 1.00+    ok+    (Recording state in git...)+    +    C:\tmp\server>dir+     Volume in drive C has no label.+     Volume Serial Number is x+    +     Directory of C:\tmp\server+    +    09/01/2013  11:03 AM    <DIR>          .+    09/01/2013  11:03 AM    <DIR>          ..+    09/01/2013  11:03 AM               194 1.pdf+                   1 File(s)            194 bytes+                   2 Dir(s)   7,698,767,872 bytes free+    +    C:\tmp\server>type 1.pdf+    .git/annex/objects/kM/0q/SHA256E-s37500--32d8190c7e189d45f48245a100e4cc981ea1bbc+    02ac8bfa6188db73e41ce06f3.pdf/SHA256E-s37500--32d8190c7e189d45f48245a100e4cc981e+    a1bbc02ac8bfa6188db73e41ce06f3.pdfC:\tmp\server>+    C:\tmp\server>+
+ doc/forum/Manual_webapp_behaviour_on_ARM.mdwn view
@@ -0,0 +1,15 @@+Hello,++I have a Samsung ARM Chromebook running Ubuntu with crouton (chroot in chromeos). I want to use git-annex, and add the chromebook as a second "device" to my already set up git-annex repo on my main pc. That repo is one folder set up with the assistant, and it has one encrypted ssh(rsync) remote.++I've read that the webapp is not available on ARM, and I've also found a topic with a simple script to replicate git-annex-assistant steps: http://git-annex.branchable.com/forum/Running_assistant_steps_manually/. So, I've used the `cabal` steps to install git-annex manually, which is working.++I want to use git-annex in a dropbox-like fashion, folder and file sync between these two devices (and maybe later on another encrypted ssh remote). However I'm in need of some guidance regarding the command line usage and such.++My questions are:++- How do I "add" the chromebook repo to the existing git-annex setup?+- The machines are not on at the same time. One is my main workstation and the chromebook is my "mobile friend". The encrypted ssh remote is on a VPS, that is always on.+  - How do I configure the syncing between the main machine, the chromebook and the encrypted ssh remote on the vps? If possible, bi-directional.++If it is possible, both explanation and command line examples would help a lot.
+ doc/forum/Pruning_out_unwanted_Git_objects.mdwn view
@@ -0,0 +1,3 @@+I have a backups repository with a few files in it, that at one point had some huge filesets erroneously added to it.  As a result, even though there are only 23,334 annexed files, the number of non-dangling Git objects in the repository comes to 593,584.++Normally I would use `git filter-branch` to clear out the deadwood in situations like this, since it is a completely private repository.  What I'm wondering is, is any such thing possible with git-annex, or is the best option just to start over, copy all the files into the new repository, and then `git-add` them all?
+ doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__.mdwn view
@@ -0,0 +1,6 @@+Hi,++When using git-annex assistant, is there a possibility to share only certain files or directories from a repository (similar to what Dropbox can do), or do we have to share whole repositories so far?+Could this be a nice wishlist item, or is there a strong reason against adding this feature to git-annex?++Thanks a lot!
doc/git-annex.mdwn view
@@ -307,19 +307,30 @@   types of special remotes need different configuration values. The   command will prompt for parameters as needed. -  All special remotes support encryption. You must either specify-  encryption=none to disable encryption, or use encryption=keyid-  (or encryption=emailaddress) to specify a gpg key that can access-  the encrypted special remote.+  All special remotes support encryption. You can either specify+  `encryption=none` to disable encryption, or specify +  `encryption=hybrid keyid=$keyid ...` to specify a gpg key id (or an email+  address accociated with a key. -  Note that with encryption enabled, a gpg key is created. This requires-  sufficient entropy. If initremote seems to hang or take a long time-  while generating the key, you may want to ctrl-c it and re-run with --fast,-  which causes it to use a lower-quality source of randomness.+  There are actually three schemes that can be used for management of the+  encryption keys. When using the encryption=hybrid scheme, additional+  gpg keys can be given access to the encrypted special remote easily+  (without re-encrypting everything). When using encryption=shared,+  a shared key is generated and stored in the git repository, allowing+  anyone who can clone the git repository to access it. Finally, when using+  encryption=pubkey, content in the special remote is directly encrypted+  to the specified gpg keys, and additional ones cannot easily be given+  access. +  Note that with encryption enabled, a cryptographic key is created.+  This requires sufficient entropy. If initremote seems to hang or take+  a long time while generating the key, you may want to ctrl-c it and+  re-run with --fast, which causes it to use a lower-quality source of+  randomness.+   Example Amazon S3 remote: -	git annex initremote mys3 type=S3 encryption=me@example.com datacenter=EU+	git annex initremote mys3 type=S3 encryption=hybrid keyid=me@example.com datacenter=EU  * enableremote name [param=value ...] @@ -335,12 +346,29 @@   For example, the directory special remote requires a directory= parameter.    This command can also be used to modify the configuration of an existing-  special remote, by specifying new values for parameters that were originally-  set when using initremote. For example, to add a new gpg key to the keys-  that can access an encrypted remote:+  special remote, by specifying new values for parameters that were+  originally set when using initremote. (However, some settings such as+  the as the encryption scheme cannot be changed once a special remote+  has been created.) -	git annex enableremote mys3 encryption=friend@example.com+  The gpg keys that an encrypted special remote is encrypted to can be+  changed using the keyid+= and keyid-= parameters. These respectively+  add and remove keys from the list. However, note that removing a key+  does NOT necessarily prevent the key's owner from accessing data+  in the encrypted special remote+  (which is by design impossible, short of deleting the remote). +  One use-case of keyid-= is to replace a revoked key with+  a new key:++	git annex enableremote mys3 keyid-=revokedkey keyid+=newkey++  Also, note that for encrypted special remotes using plain public-key+  encryption (encryption=pubkey), adding or removing a key has NO effect+  on files that have already been copied to the remote. Hence using+  keyid+= and keyid-= with such remotes should be used with care, and+  make little sense except in cases like the revoked key example above.+ * trust [repository ...]    Records that a repository is trusted to not unexpectedly lose@@ -478,6 +506,23 @@ * upgrade    Upgrades the repository to current layout.++* forget++  Causes the git-annex branch to be rewritten, throwing away historical+  data about past locations of files. The resulting branch will use less+  space, but `git annex log` will not be able to show where+  files used to be located.+  +  To also prune references to repositories that have been marked as dead,+  specify --drop-dead.++  When this rewritten branch is merged into other clones of+  the repository, git-annex will automatically perform the same rewriting+  to their local git-annex branches. So the forgetfulness will automatically+  propigate out from its starting point until all repositories running+  git-annex have forgotten their old history. (You may need to force+  git to push the branch to any git repositories not running git-annex.  # QUERY COMMANDS
+ doc/install/Android/comment_9_f3d289b78d6bdb3cc65689495a8439a5._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="http://openlifechallenge.wordpress.com/"+ nickname="O"+ subject="Add to F-Droid"+ date="2013-09-08T09:56:38Z"+ content="""+Hi,+Really enjoy this app for Android! Super simple to use but it would be great if you could fix a folder navigator option when choosing location for device repo, right now it does not work. ++Could you also add it to the F-Droid repository?+"""]]
doc/install/Windows.mdwn view
@@ -26,7 +26,8 @@  To build git-annex from source on Windows, you need to install the Haskell Platform, Mingw, and Cygwin. Use Cygwin to install-gcc, rsync, git, wget, ssh, and gpg.+gcc, rsync, git, wget, ssh, and gnupg. To build the git-annex installer,+you also need to install the NulSoft installer system.  There is a shell script `standalone/windows/build.sh` that can be used to build git-annex. Note that this shell script cannot be run
+ doc/install/cabal/comment_18_1efa0c7a963ec452fc6336fbe4964f6e._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="RaspberryPie"+ ip="96.47.226.20"+ subject="git-annex assistant for the Raspberry Pi"+ date="2013-09-04T03:58:37Z"+ content="""+It took a while and a few tries, but I finally built the git-annex binary including the assistant on a Raspberry Pi. The build comes without the flags webapp, webdav, and dbus as these rely on a Template Haskell compiler that hasn't been ported to Arm architecture yet. ++I put the binary up on Github in case anyone's interested: <https://github.com/tradloff/git-annex-RPi>+"""]]
− doc/news/version_4.20130709.mdwn 
@@ -1,35 +0,0 @@-git-annex 4.20130709 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * --all: New switch that makes git-annex operate on all data stored-     in the git annex, including old versions of files. Supported by-     fsck, get, move, copy.-   * --unused: New switch that makes git-annex operate on all data found-     by the last run of git annex unused. Supported by fsck, get, move, copy.-   * get, move, copy: Can now be run in a bare repository,-     like fsck already could. --all is enabled automatically in this case.-   * merge: Now also merges synced/master or similar branches, which-     makes it useful to put in a post-receive hook to make a repository-     automatically update its working copy when git annex sync or the assistant-     sync with it.-   * webapp: Fix ssh setup with nonstandard port, broken in last release.-   * init: Detect systems on which git commit fails due to not being able to-     determine the FQDN, and put in a workaround so committing to the git-annex-     branch works.-   * addurl --pathdepth: Fix failure when the pathdepth specified is deeper-     than the urls's path.-   * Windows: Look for .exe extension when searching for a command in path.-   * Pass -f to curl when downloading a file with it, so it propigates failure.-   * Windows: Fix url to object when using a http remote.-   * webapp: Fix authorized\_keys line added when setting up a rsync remote-     on a server that also supports git-annex, to not force running-     git-annex-shell.-   * OSX Mountain Lion: Fixed gpg bundled in dmg to not fail due to a missing-     gpg-agent.-   * Android: gpg is built without --enable-minimal, so it interoperates-     better with other gpg builds that may default to using other algorithms-     for encryption.-   * dropunused, addunused: Complain when asked to operate on a number that-     does not correspond to any unused key.-   * fsck: Don't claim to fix direct mode when run on a symlink whose content-     is not present.-   * Make --numcopies override annex.numcopies set in .gitattributes."""]]
− doc/news/version_4.20130723.mdwn 
@@ -1,36 +0,0 @@-git-annex 4.20130723 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Fix data loss bug when adding an (uncompressed) tarball of a-     git-annex repository, or other file that begins with something-     that can be mistaken for a git-annex link. Closes: #[717456](http://bugs.debian.org/717456)-   * New improved version of the git-annex logo, contributed by-     John Lawrence.-   * Rsync.net have committed to support git-annex and offer a special-     discounted rate for git-annex users. Updated the webapp to reflect this.-     http://www.rsync.net/products/git-annex-pricing.html-   * Install XDG desktop icon files.-   * Support unannex and uninit in direct mode.-   * Support import in direct mode.-   * webapp: Better display of added files.-   * fix: Preserve the original mtime of fixed symlinks.-   * uninit: Preserve .git/annex/objects at the end, if it still-     has content, so that old versions of files and deleted files-     are not deleted. Print a message with some suggested actions.-   * When a transfer is already being run by another process,-     proceed on to the next file, rather than dying.-   * Fix checking when content is present in a non-bare repository-     accessed via http.-   * Display byte sizes with more precision.-   * watcher: Fixed a crash that could occur when a directory was renamed-     or deleted before it could be scanned.-   * watcher: Partially worked around a bug in hinotify, no longer crashes-     if hinotify cannot process a directory (but can't detect changes in it)-   * directory special remote: Fix checking that there is enough disk space-     to hold an object, was broken when using encryption.-   * webapp: Differentiate between creating a new S3/Glacier/WebDav remote,-     and initializing an existing remote. When creating a new remote, avoid-     conflicts with other existing (or deleted) remotes with the same name.-   * When an XMPP server has SRV records, try them, but don't then fall-     back to the regular host if they all fail.-   * For long hostnames, use a hash of the hostname to generate the socket-     file for ssh connection caching."""]]
+ doc/news/version_4.20130909.mdwn view
@@ -0,0 +1,36 @@+git-annex 4.20130909 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * initremote: Syntax change when setting up an encrypted special remote.+     Now use keyid=$KEYID rather than the old encryption=$KEYID+   * forget: New command, causes git-annex branch history to be forgotten+     in a way that will spread to other clones of the repository.+     (As long as they're running this version or newer of git-annex.)+   * forget --drop-dead: Completely removes mentions of repositories that+     have been marked as dead from the git-annex branch.+   * sync, assistant: Force push of the git-annex branch. Necessary+     to ensure it gets pushed to remotes after being rewritten by forget.+   * Added gcrypt support. This combines a fully encrypted git+     repository (using git-remote-gcrypt) with an encrypted git-annex special+     remote.+   * sync: Support syncing with gcrypt remotes.+   * importfeed: Also ignore transient problems with downloading content+     from feeds.+   * Honor core.sharedrepository when receiving and adding files in direct+     mode.+   * enableremote: gpg keys can be removed from those a remote encrypts+     to by passing "keyid-=$KEYID". keyid+= is also provided.+     (Thanks, guilhem for the patch.)+   * Added encryption=pubkey scheme, which encrypts to public keys directly+     rather than the hybrid approach. See documentation for advantages+     and disadvantages, but encryption=hybrid is the recommended scheme still.+     (Thanks, guilhem for the patch.)+   * Fix Feeds display in build flags.+   * Remind user when annex-ignore is set for some remotes, if unable to+     get or drop a file, possibly because it's on an ignored remote.+   * gpg: Force --no-textmode in case the user has it turned on in config.+   * webapp: Improve javascript's handling of longpolling connection+     failures, by reloading the current page in this case.+     Works around chromium behavior where ajax connections to urls+     that were already accessed are denied after navigating back to+     a previous page.+   * Allow building without quvi support."""]]
doc/special_remotes.mdwn view
@@ -10,6 +10,7 @@ * [[S3]] (Amazon S3, and other compatible services) * [[Amazon_Glacier|glacier]] * [[bup]]+* [[gcrypt]] (encrypted git repositories!) * [[directory]] * [[rsync]] * [[webdav]]
doc/special_remotes/S3.mdwn view
@@ -15,13 +15,10 @@ A number of parameters can be passed to `git annex initremote` to configure the S3 remote. -* `encryption` - Required. Either "none" to disable encryption (not recommended),-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to access the encrypted data (use with caution).+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `embedcreds` - Optional. Set to "yes" embed the login credentials inside   the git repository, which allows other clones to also access them. This is
doc/special_remotes/bup.mdwn view
@@ -19,14 +19,10 @@  These parameters can be passed to `git annex initremote` to configure bup: -* `encryption` - Required. Either "none" to disable encryption of content-  stored in bup (ssh will still be used to transport it securely),-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to access the encrypted data (use with caution).+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `buprepo` - Required. This is passed to `bup` as the `--remote`   to use to store data. To create the repository,`bup init` will be run.
doc/special_remotes/directory.mdwn view
@@ -14,13 +14,10 @@ These parameters can be passed to `git annex initremote` to configure the remote: -* `encryption` - Required. Either "none" to disable encryption,-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to decrypt the encrypted data.+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `chunksize` - Avoid storing files larger than the specified size in the   directory. For use on directories on mount points that have file size
+ doc/special_remotes/gcrypt.mdwn view
@@ -0,0 +1,43 @@+[git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt/)+adds support for encrypted remotes to git. The git-annex gcrypt special+remote allows git-annex to also store its files in such repositories.+Naturally, git-annex encrypts the files it stores too, so everything+stored on the remote is encrypted.++See [[tips/fully_encrypted_git_repositories_with_gcrypt]] for some examples+of using gcrypt.++## configuration++These parameters can be passed to `git annex initremote` to configure+gcrypt:++* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]].++* `keyid` - Specifies the gpg key to use for encryption of both the files+  git-annex stores in the repository, as well as to encrypt the git+  repository itself. May be repeated when multiple participants+  should have access to the repository.++* `gitrepo` - Required. The path or url to the git repository+  for gcrypt to use. This repository should be either empty, or an existing+  gcrypt repositry.++* `shellescape` - See [[rsync]] for the details of this option.++## notes++For git-annex to store files in a repository on a remote server, you need+shell access, and `rsync` must be installed. ++While you can use git-remote-gcrypt with servers like github, git-annex+can't store files on them. In such a case, you can just use+git-remote-gcrypt directly.++If you use encryption=hybrid, you can add more gpg keys that can access+the files git-annex stored in the gcrypt repository. However, due to the+way git-remote-gcrypt encrypts the git repository, you will need to somehow+force it to re-push everything again, so that the encrypted repository can+be decrypted by the added keys. Probably this can be done by setting+`GCRYPT_FULL_REPACK` and doing a forced push of branches.
doc/special_remotes/glacier.mdwn view
@@ -21,13 +21,10 @@ A number of parameters can be passed to `git annex initremote` to configure the Glacier remote. -* `encryption` - Required. Either "none" to disable encryption (not recommended),-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to access the encrypted data (use with caution).+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `embedcreds` - Optional. Set to "yes" embed the login credentials inside   the git repository, which allows other clones to also access them. This is
doc/special_remotes/hook.mdwn view
@@ -25,16 +25,13 @@  These parameters can be passed to `git annex initremote`: -* `encryption` - Required. Either "none" to disable encryption,-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to access the encrypted data.--  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].- * `hooktype` - Required. This specifies a collection of hooks to use for   this remote.++* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]].++* `keyid` - Specifies the gpg key to use for [[encryption]].  ## hooks
doc/special_remotes/rsync.mdwn view
@@ -2,26 +2,22 @@  Setup example: -	# git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net+	# git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync keyid=joey@kitenet.net 	# git annex describe myrsync "rsync server"  Or for using rsync over SSH -	# git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=joey@kitenet.net+	# git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync keyid=joey@kitenet.net 	# git annex describe myrsync "rsync server"  ## configuration  These parameters can be passed to `git annex initremote` to configure rsync: -* `encryption` - Required. Either "none" to disable encryption of content-  stored on the remote rsync server,-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to decrypt the encrypted data.+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `rsyncurl` - Required. This is the url or `hostname:/directory` to    pass to rsync to tell it where to store content.
doc/special_remotes/webdav.mdwn view
@@ -10,13 +10,10 @@ A number of parameters can be passed to `git annex initremote` to configure the webdav remote. -* `encryption` - Required. Either "none" to disable encryption (not recommended),-  or a value that can be looked up (using gpg -k) to find a gpg encryption-  key that will be given access to the remote, or "shared" which allows-  every clone of the repository to access the encrypted data (use with caution).+* `encryption` - One of "none", "hybrid", "shared", or "pubkey".+  See [[encryption]]. -  Note that additional gpg keys can be given access to a remote by-  running enableremote with the new key id. See [[encryption]].+* `keyid` - Specifies the gpg key to use for [[encryption]].  * `embedcreds` - Optional. Set to "yes" embed the login credentials inside   the git repository, which allows other clones to also access them. This is@@ -42,4 +39,4 @@  Setup example: -	# WEBDAV_USERNAME=joey@kitenet.net WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://www.box.com/dav/git-annex chunksize=75mb encryption=joey@kitenet.net+	# WEBDAV_USERNAME=joey@kitenet.net WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://www.box.com/dav/git-annex chunksize=75mb keyid=joey@kitenet.net
doc/tips/assume-unstaged.mdwn view
@@ -11,7 +11,7 @@  	git config core.ignoreStat true -When git mv and git rm are used, those changes *do* get noticed, even+When `git mv` and `git rm` are used, those changes *do* get noticed, even on assume-unchanged files. When new files are added, eg by `git annex add`, they are also noticed.
+ doc/tips/assume-unstaged/comment_2_5b589f37cfc03bf7be33a51826cc4dba._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnxlx1UrzVhdy6_gFjzmF42x6QXxBUxg00"+ nickname="Jakukyo"+ subject="comment 2"+ date="2013-09-05T12:14:42Z"+ content="""+> There are two gotchas...++So just always run `git annex add` after editing a file+and `git update-index --really-refresh` after migrating+backend?++"""]]
+ doc/tips/fully_encrypted_git_repositories_with_gcrypt.mdwn view
@@ -0,0 +1,104 @@+[git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt/)+adds support for encrypted remotes to git. The git-annex +[[gcrypt special remote|special_remotes/gcrypt]] allows git-annex to+also store its files in such repositories. Naturally, git-annex encrypts+the files it stores too, so everything stored on the remote is encrypted.++Here are some ways you can use this awesome stuff..++[[!toc ]]++## prerequisites++* Install+[git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt/)+* Install git-annex version 4.20130909 or newer.++## encrypted backup drive++Let's make a USB drive into an encrypted backup repository. It will contain+both the full contents of your git repository, and all the files you+instruct git-annex to store on it, and everything will be encrypted so that+only you can see it.++First, you need to set up a gpg key. You might consider generating a+special purpose key just for this use case, since you may end up wanting to+put the key on multiple machines that you would not trust with your+main gpg key. You need to tell git-annex the keyid of the key.++	git init --bare /mnt/encryptedbackup+	git annex initremote encryptedbackup type=gcrypt gitrepo=/mnt/encryptedbackup keyid=$mykey+	git annex sync encryptedbackup+	git annex copy --to encryptedbackup ...++Note that if you lose your gpg key, it will be *impossible* to get the+data out of your encrypted backup. You need to find a secure way to store a+backup of your gpg key. Printing it out and storing it in a safe deposit box,+for example.++You can actually specifiy keyid= as many times as you like to allow any one+of a set of gpg keys to access this repository. So you could add a friend's+key, or another gpg key you have.++To restore from the backup, just plug the drive into any machine that has+the gpg key used to encrypt it, and then:++	git clone gcrypt::/mnt/encryptedbackup restored+	cd restored+	git annex enableremote encryptedbackup gitrepo=/mnt/encryptedbackup+	git annex get --from encryptedbackup++## encrypted git-annex repository on a ssh server++If you have a ssh server that has git-annex and rsync installed, you can+set up an encrypted repository there. Works just like the encrypted drive+except without the cable.++First, on the server, run:++	git init --bare encryptedrepo++Now, in your existing git-annex repository:++	git annex initremote encryptedrepo type=gcrypt gitrepo=ssh://my.server/home/me/encryptedrepo keyid=$mykey+	git annex sync encryptedrepo+	git annex copy --to encryptedrepo ...++If you're going to be sharing this repository with others, be sure to also+include their keyids, by specifying keyid= repeatedly.++Now that the repo is set up, anyone who has access to it and has one of the keys+used to encrypt it can check it out:++	git clone gcrypt::ssh://my.server/home/me/encryptedrepo myrepo+	cd myrepo+	git annex enableremote encryptedrepo gitrepo=ssh://my.server/home/me/encryptedrepo+	git annex get --from encryptedrepo++## private encrypted git remote on hosting site++You can use gcrypt to store your git repository in encrypted form on any+hosting site that supports git. Only you can decrypt its contents.+Using it this way, git-annex does not store large files on the hosting site; it's+only used to store your git repository itself.++	git remote add encrypted gcrypt::ssh://hostingsite/myrepo.git+	git config git push encrypted master git-annex++Now you can carry on using git-annex with your new repository. For example,+`git annex sync` will sync with it.++To check out the repository from the hosting site, use the same gcrypt::+url you used when setting it up:++	git clone gcrypt::ssh://hostingsite/myrepo.git++## multiuser encrypted git remote on hosting site++Suppose two users want to share an encrypted git remote. Both of you+need to set up the remote, and configure gcrypt to encrypt it so that both+of you can see it.+	+	git remote add sharedencrypted gcrypt::ssh://hostingsite/myrepo.git+	git config remote.sharedencrypted.gcryt-participants "$mykey $friendkey"+	git config git push sharedencrypted master git-annex
doc/tips/setup_a_public_repository_on_a_web_site.mdwn view
@@ -9,18 +9,20 @@ 1. Set up a web site. I used Apache, and configured it to follow symlinks.    `Options FollowSymLinks` 2. Put some files on the website. Make sure it works.-4. `git init; git annex init`-3. We want users to be able to clone the git repository over http, because+3. `git init; git annex init`+4. `git config core.sharedrepository world` (Makes sure files+   are always added with permissions that allow everyone to read them.)+5. We want users to be able to clone the git repository over http, because    git-annex can download files from it over http as well. For this to    work, `git update-server-info` needs to get run after commits. The    git `post-update` hook will take care of this, you just need to enable    the hook. `chmod +x .git/hooks/post-update`-5. `git annex add; git commit -m added`-6. Make sure users can still download files from the site directly.-7. Instruct advanced users to clone a http url that ends with the "/.git/"+6. `git annex add; git commit -m added`+7. Make sure users can still download files from the site directly.+8. Instruct advanced users to clone a http url that ends with the "/.git/"    directory. For example, for downloads.kitenet.net, the clone url    is `https://downloads.kitenet.net/.git/`-8. Set up a git `post-receive` hook to update the repository's working tree+9. Set up a git `post-receive` hook to update the repository's working tree    when changes are pushed to it. See below for details.  When users clone over http, and run git-annex, it will
doc/tips/using_Amazon_Glacier.mdwn view
@@ -16,7 +16,7 @@  Next, create the Glacier remote. -	# git annex initremote glacier type=glacier encryption=2512E3C7+	# git annex initremote glacier type=glacier keyid=2512E3C7 	initremote glacier (encryption setup with gpg key C910D9222512E3C7) (gpg) ok  The configuration for the Glacier remote is stored in git. So to make another
doc/tips/using_Amazon_S3.mdwn view
@@ -14,7 +14,7 @@  Next, create the S3 remote, and describe it.  -	# git annex initremote cloud type=S3 encryption=2512E3C7+	# git annex initremote cloud type=S3 keyid=2512E3C7 	initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) (gpg) ok 	# git annex describe cloud "at Amazon's US datacenter" 	describe cloud ok
doc/tips/using_box.com_as_a_special_remote.mdwn view
@@ -5,7 +5,7 @@ git-annex can use Box as a [[special remote|special_remotes]]. Recent versions of git-annex make this very easy to set up: -	WEBDAV_USERNAME=you@example.com WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://www.box.com/dav/git-annex chunksize=75mb encryption=you@example.com+	WEBDAV_USERNAME=you@example.com WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://www.box.com/dav/git-annex chunksize=75mb encryption=shared  Note the use of chunksize; Box has a 100 mb maximum file size, and this breaks up large files into chunks before that limit is reached.
+ doc/todo/Wishlist:_additional_environment_variables_for_hooks.mdwn view
@@ -0,0 +1,14 @@+It would be nice if a couple of additional environment variables to be set for hook uses.++In particular:++    GIT_ANNEX_DIRECT=`git config annex.direct`++and++    GIT_TOP_LEVEL=`git rev-parse --show-toplevel`+++I've made some changes to flickrannex to allow the sub-directories above the uploaded image to be added as tags.  This change has been merged into trunk: [[https://github.com/TobiasTheViking/flickrannex]]++What I needed was both the environment variables mentioned above.  One is set as part of the annex-hook and the other I guestimate from the file path.  If it was set in git-annex it would be much cleaner (and accurate).  So...I think this info would be useful for other hook.
+ doc/todo/checksum_verification_on_transfer.mdwn view
@@ -0,0 +1,7 @@+Since most file transfers, particularly to/from encrypted special remotes involve git-annex streaming through the contents of the file anyway, it should be possible to add a verification of the checksum nearly for free. The main thing needed is probably a faster haskell checksum library than Data.Digest.Pure.Sha, which is probably slow enough to be annoying.++I have not verified if an upload could be aborted before sending the data to the remote if a checksum failure is detected. It may be dependent on the individual special remote implementations. Some probably stream the encrypted data directly out the wire, while others need to set up a temp file to run a command on. It would certianly be possible to at least make the upload abort and fail if a bad checksum was detected.++Doing the same for downloads is less useful, because the data is there locally to be fscked. The real advantage would be doing the check for uploads, to ensure that hard-to-detect corrupted files don't reach special remotes.++--[[Joey]]
doc/todo/faster_gnupg_cipher.mdwn view
@@ -1,1 +1,9 @@-Apparently newer gnupg has support for hardware-accelerated AES-NI. It would be good to have an option to use that. I also wonder if using the same symmetric key for many files presents a security issues (and whether using GPG keys directly would be more secure).+Apparently newer gnupg has support for hardware-accelerated AES-NI. It+would be good to have an option to use that. I also wonder if using the+same symmetric key for many files presents a security issues (and whether+using GPG keys directly would be more secure).++> [[done]]; you can now use encryption=pubkey when setting up a special+> remote to use pure public keys without the hybrid symmetric key scheme.+> Which you choose is up to you. Also,  annex.gnupg-options can configure+> the ciphers used. --[[Joey]]
doc/todo/wishlist:_dropping_git-annex_history.mdwn view
@@ -24,3 +24,5 @@ --[[Joey]]   See also : [[forum/safely_dropping_git-annex_history]]++> [[done]] --[[Joey]]
+ doc/users/anarcat.mdwn view
@@ -0,0 +1,3 @@+Miaou.++See <http://anarc.at/>.
doc/walkthrough/getting_file_content.mdwn view
@@ -6,7 +6,7 @@ USB drive.  	# cd /media/usb/annex-	# git fetch laptop; git merge laptop/master+	# git annex sync laptop 	# git annex get . 	get my_cool_big_file (from laptop...) ok 	get iso/debian.iso (from laptop...) ok
doc/walkthrough/syncing.mdwn view
@@ -1,11 +1,9 @@-Notice that in the [[previous example|getting_file_content]], you had to-git fetch and merge from laptop first. This lets git-annex know what has-changed in laptop, and so it knows about the files present there and can+Notice that in the [[previous example|getting_file_content]], `git annex+sync` was used. This lets git-annex know what has changed in the other+repositories like the laptop, and so it knows about the files present there and can get them. -If you have a lot of repositories to keep in sync, manually fetching and -merging from them can become tedious. To automate it there is a handy-sync command, which also even commits your changes for you.+Let's look at what the sync command does in more detail:  	# cd /media/usb/annex 	# git annex sync
git-annex.1 view
@@ -282,19 +282,30 @@ types of special remotes need different configuration values. The command will prompt for parameters as needed. .IP-All special remotes support encryption. You must either specify-encryption=none to disable encryption, or use encryption=keyid-(or encryption=emailaddress) to specify a gpg key that can access-the encrypted special remote.+All special remotes support encryption. You can either specify+encryption=none to disable encryption, or specify +encryption=hybrid keyid=$keyid ... to specify a gpg key id (or an email+address accociated with a key. .IP-Note that with encryption enabled, a gpg key is created. This requires-sufficient entropy. If initremote seems to hang or take a long time-while generating the key, you may want to ctrl\-c it and re\-run with \-\-fast,-which causes it to use a lower\-quality source of randomness.+There are actually three schemes that can be used for management of the+encryption keys. When using the encryption=hybrid scheme, additional+gpg keys can be given access to the encrypted special remote easily+(without re\-encrypting everything). When using encryption=shared,+a shared key is generated and stored in the git repository, allowing+anyone who can clone the git repository to access it. Finally, when using+encryption=pubkey, content in the special remote is directly encrypted+to the specified gpg keys, and additional ones cannot easily be given+access. .IP+Note that with encryption enabled, a cryptographic key is created.+This requires sufficient entropy. If initremote seems to hang or take+a long time while generating the key, you may want to ctrl\-c it and+re\-run with \-\-fast, which causes it to use a lower\-quality source of+randomness.+.IP Example Amazon S3 remote: .IP- git annex initremote mys3 type=S3 encryption=me@example.com datacenter=EU+ git annex initremote mys3 type=S3 encryption=hybrid keyid=me@example.com datacenter=EU .IP .IP "enableremote name [param=value ...]" Enables use of an existing special remote in the current repository,@@ -309,12 +320,29 @@ For example, the directory special remote requires a directory= parameter. .IP This command can also be used to modify the configuration of an existing-special remote, by specifying new values for parameters that were originally-set when using initremote. For example, to add a new gpg key to the keys-that can access an encrypted remote:+special remote, by specifying new values for parameters that were+originally set when using initremote. (However, some settings such as+the as the encryption scheme cannot be changed once a special remote+has been created.) .IP- git annex enableremote mys3 encryption=friend@example.com+The gpg keys that an encrypted special remote is encrypted to can be+changed using the keyid+= and keyid\-= parameters. These respectively+add and remove keys from the list. However, note that removing a key+does NOT necessarily prevent the key's owner from accessing data+in the encrypted special remote+(which is by design impossible, short of deleting the remote). .IP+One use\-case of keyid\-= is to replace a revoked key with+a new key:+.IP+ git annex enableremote mys3 keyid\-=revokedkey keyid+=newkey+.IP+Also, note that for encrypted special remotes using plain public\-key+encryption (encryption=pubkey), adding or removing a key has NO effect+on files that have already been copied to the remote. Hence using+keyid+= and keyid\-= with such remotes should be used with care, and+make little sense except in cases like the revoked key example above.+.IP .IP "trust [repository ...]" Records that a repository is trusted to not unexpectedly lose content. Use with care.@@ -436,6 +464,22 @@ .IP .IP "upgrade" Upgrades the repository to current layout.+.IP+.IP "forget"+Causes the git\-annex branch to be rewritten, throwing away historical+data about past locations of files. The resulting branch will use less+space, but git annex log will not be able to show where+files used to be located.+.IP+To also prune references to repositories that have been marked as dead,+specify \-\-drop\-dead.+.IP+When this rewritten branch is merged into other clones of+the repository, git\-annex will automatically perform the same rewriting+to their local git\-annex branches. So the forgetfulness will automatically+propigate out from its starting point until all repositories running+git\-annex have forgotten their old history. (You may need to force+git to push the branch to any git repositories not running git\-annex. .IP .SH QUERY COMMANDS .IP "version"
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 4.20130827+Version: 4.20130909 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <joey@kitenet.net>@@ -68,6 +68,9 @@ Flag Feed   Description: Enable podcast feed support +Flag Quvi+  Description: Enable use of quvi to download videos+ Executable git-annex   Main-Is: git-annex.hs   Build-Depends: MissingH, hslogger, directory, filepath,@@ -76,7 +79,7 @@    extensible-exceptions, dataenc, SHA, process, json,    base (>= 4.5 && < 4.8), monad-control, MonadCatchIO-transformers,    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,-   SafeSemaphore, uuid, random, dlist, unix-compat, aeson+   SafeSemaphore, uuid, random, dlist, unix-compat   -- Need to list these because they're generated from .hsc files.   Other-Modules: Utility.Touch Utility.Mounts   Include-Dirs: Utility@@ -142,7 +145,7 @@      yesod, yesod-default, yesod-static, yesod-form, yesod-core,      case-insensitive, http-types, transformers, wai, wai-logger, warp,      blaze-builder, crypto-api, hamlet, clientsession,-     template-haskell, data-default+     template-haskell, data-default, aeson     CPP-Options: -DWITH_WEBAPP    if flag(Pairing)@@ -160,6 +163,10 @@   if flag(Feed)     Build-Depends: feed     CPP-Options: -DWITH_FEED+  +  if flag(Quvi)+    Build-Depends: aeson+    CPP-Options: -DWITH_QUVI  source-repository head   type: git
templates/notifications/longpolling.julius view
@@ -3,7 +3,7 @@ 		var f = function() { 			longpoll(url, #{ident} 				, function() { setTimeout(f, #{delay}); }-				, function() { webapp_disconnected(); }+				, function() { window.location.reload(true); } 			);	 		}; 		setTimeout(f, #{startdelay});
− templates/page.julius 
@@ -1,17 +0,0 @@-connfailed =-  '<div id="modal" class="modal fade">' +-  '  <div class="modal-header">' +-  '    <h3>git-annex has shut down</h3>' +-  '  </div>' +-  '  <div class="modal-body">' +-  '    You can now close this browser window.' +-  '  </div>' +-  '</div>' ;--function webapp_disconnected () {-	$('#modal').replaceWith(connfailed);-	$('#modal').modal('show');--	// ideal, but blocked by many browsers-	window.close();-}