packages feed

git-annex 5.20141125 → 5.20141203

raw patch · 94 files changed

+1946/−365 lines, 94 filesdep +awsdep +conduitdep +conduit-extradep −hS3

Dependencies added: aws, conduit, conduit-extra, resourcet

Dependencies removed: hS3

Files

Annex/Direct.hs view
@@ -225,10 +225,18 @@   where 	canff = maybe (return False) (\o -> inRepo $ Git.Branch.fastForwardable o branch) old -{- Cleans up after a direct mode merge. The merge must have been staged- - in the index. Uses diff-index to compare the staged changes with- - the tree before the merge, and applies those changes to the work tree.+mergeDirectCleanup :: FilePath -> Git.Ref -> Annex ()+mergeDirectCleanup d oldref = do+	updateWorkTree d oldref+	liftIO $ removeDirectoryRecursive d++{- Updates the direct mode work tree to reflect the changes staged in the+ - index by a git command, that was run in a temporary work tree.  -+ - Uses diff-index to compare the staged changes with provided ref+ - which should be the tree before the merge, and applies those+ - changes to the work tree.+ -  - There are really only two types of changes: An old item can be deleted,  - or a new item added. Two passes are made, first deleting and then  - adding. This is to handle cases where eg, a file is deleted and a@@ -236,8 +244,8 @@  - order, but we cannot add the directory until the file with the  - same name is removed.)  -}-mergeDirectCleanup :: FilePath -> Git.Ref -> Annex ()-mergeDirectCleanup d oldref = do+updateWorkTree :: FilePath -> Git.Ref -> Annex ()+updateWorkTree d oldref = do 	(items, cleanup) <- inRepo $ DiffTree.diffIndex oldref 	makeabs <- flip fromTopFilePath <$> gitRepo 	let fsitems = zip (map (makeabs . DiffTree.file) items) items@@ -246,7 +254,6 @@ 	forM_ fsitems $ 		go makeabs DiffTree.dstsha DiffTree.dstmode movein movein_raw 	void $ liftIO cleanup-	liftIO $ removeDirectoryRecursive d   where 	go makeabs getsha getmode a araw (f, item) 		| getsha item == nullSha = noop
Annex/Init.hs view
@@ -21,8 +21,6 @@ import qualified Git import qualified Git.LsFiles import qualified Git.Config-import qualified Git.Construct-import qualified Git.Types as Git import qualified Git.Objects import qualified Annex.Branch import Logs.UUID@@ -41,12 +39,7 @@ import Utility.FileMode #endif import Annex.Hook-import Git.Hook (hookFile) import Upgrade-import Annex.Content-import Logs.Location--import System.Log.Logger  genDescription :: Maybe String -> Annex String genDescription (Just d) = return d
Annex/Link.hs view
@@ -35,13 +35,17 @@  - content.  -} getAnnexLinkTarget :: FilePath -> Annex (Maybe LinkTarget)-getAnnexLinkTarget file = ifM (coreSymlinks <$> Annex.getGitConfig)-	( check readSymbolicLink $+getAnnexLinkTarget f = getAnnexLinkTarget' f+	=<< (coreSymlinks <$> Annex.getGitConfig)++{- Pass False to force looking inside file. -}+getAnnexLinkTarget' :: FilePath -> Bool -> Annex (Maybe LinkTarget)+getAnnexLinkTarget' file coresymlinks = if coresymlinks+	then check readSymbolicLink $ 		return Nothing-	, check readSymbolicLink $+	else check readSymbolicLink $ 		check probefilecontent $ 			return Nothing-	)   where 	check getlinktarget fallback = do 		v <- liftIO $ catchMaybeIO $ getlinktarget file
Assistant/Install.hs view
@@ -92,18 +92,20 @@ installFileManagerHooks :: FilePath -> IO () #ifdef linux_HOST_OS installFileManagerHooks program = do+	let actions = ["get", "drop", "undo"]+ 	-- Gnome 	nautilusScriptdir <- (\d -> d </> "nautilus" </> "scripts") <$> userDataDir 	createDirectoryIfMissing True nautilusScriptdir-	genNautilusScript nautilusScriptdir "get"-	genNautilusScript nautilusScriptdir "drop"+	forM_ actions $+		genNautilusScript nautilusScriptdir  	-- KDE 	home <- myHomeDir 	let kdeServiceMenusdir = home </> ".kde" </> "share" </> "kde4" </> "services" </> "ServiceMenus" 	createDirectoryIfMissing True kdeServiceMenusdir 	writeFile (kdeServiceMenusdir </> "git-annex.desktop")-		(kdeDesktopFile ["get", "drop"])+		(kdeDesktopFile actions)   where 	genNautilusScript scriptdir action = 		installscript (scriptdir </> scriptname action) $ unlines
Build/BundledPrograms.hs view
@@ -20,7 +20,7 @@ bundledPrograms = catMaybes 	[ Nothing #ifndef mingw32_HOST_OS-	-- git is not included in the windows bundle+	-- git is not included in the windows bundle; msysgit is used 	, Just "git" 	-- Not strictly needed in PATH by git-annex, but called 	-- by git when it sshes to a remote.@@ -36,7 +36,9 @@ 	, Just "rsync" #ifndef darwin_HOST_OS 	-- OS X has ssh installed by default.-	-- (Linux probably, but not guaranteed.)+	-- Linux probably has ssh, but not guaranteed.+	-- On Windows, msysgit provides ssh, but not in PATH, +	-- so we ship our own. 	, Just "ssh" 	, Just "ssh-keygen" #endif
Build/DistributionUpdate.hs view
@@ -33,7 +33,7 @@ autobuilds =  	(map linuxarch ["i386", "amd64", "armel"]) ++ 	(map androidversion ["4.0", "4.3"]) ++-	[ (autobuild "x86_64-apple-mavericks/git-annex.dmg", "git-annex/OSX/current/10.9_Mavericks/git-annex.dmg")+	[ (autobuild "x86_64-apple-yosemite/git-annex.dmg", "git-annex/OSX/current/10.10_Yosemite/git-annex.dmg") 	, (autobuild "windows/git-annex-installer.exe", "git-annex/windows/current/git-annex-installer.exe") 	]   where@@ -93,6 +93,7 @@  makeinfos :: [(FilePath, Version)] -> Annex () makeinfos updated = do+	mapM_ (\f -> inRepo $ runBool [Param "annex", Param "add", File f]) (map fst updated) 	version <- liftIO getChangelogVersion 	void $ inRepo $ runBool  		[ Param "commit"
Build/NullSoftInstaller.hs view
@@ -123,7 +123,6 @@ 		, StartOptions "SW_SHOWNORMAL"
 		, IconFile "$INSTDIR/cmd/git-annex.exe"
 		, IconIndex 2
-		, KeyboardShortcut "ALT|CONTROL|a"
 		, Description "git-annex webapp"
 		]
 	createShortcut autoStartItem
@@ -134,12 +133,9 @@ 		, IconIndex 2
 		, Description "git-annex autostart"
 		]
-	section "bins" [] $ do
-		setOutPath "$INSTDIR\\bin"
-		mapM_ addfile extrabins
 	section "cmd" [] $ do
 		setOutPath "$INSTDIR\\cmd"
-		addfile gitannex
+		mapM_ addfile (gitannex:extrabins)
 	section "meta" [] $ do
 		setOutPath "$INSTDIR"
 		addfile license
@@ -148,8 +144,7 @@ 	uninstall $ do
 		delete [RebootOK] $ startMenuItem
 		delete [RebootOK] $ autoStartItem
-		removefilesFrom "$INSTDIR/bin" extrabins
-		removefilesFrom "$INSTDIR/cmd" [gitannex]
+		removefilesFrom "$INSTDIR/cmd" (gitannex:extrabins)
 		removefilesFrom "$INSTDIR" $
 			launchers ++
 			[ license
CHANGELOG view
@@ -1,3 +1,34 @@+git-annex (5.20141203) unstable; urgency=medium++  * proxy: New command for direct mode repositories, allows bypassing+    the direct mode guard in a safe way to do all sorts of things+    including git revert, git mv, git checkout ...+  * undo: New command to undo the most recent change to a file+    or to the contents of a directory.+  * Add undo action to nautilus and konqueror integration.+  * diffdriver: New git-annex command, to make git external diff drivers+    work with annexed files.+  * pre-commit: Block partial commit of unlocked annexed file, since+    that left a typechange staged in index due to some infelicity of git's+    handling of partial commits.+  * Work around behavior change in lsof 4.88's -F output format.+  * S3: Switched to using the haskell aws library.+  * S3: No longer buffers entire files in memory when uploading without+    chunking.+  * S3: When built with a new enough version of the haskell aws library,+    supports doing multipart uploads, in order to store extremely large+    files in S3 when not using chunking.+  * Don't show "(gpg)" when decrypting the remote encryption cipher,+    since this could be taken to read that's the only time git-annex+    runs gpg, which is not the case.+  * Debian package is now maintained by Gergely Nagy.+  * Windows: Remove Alt+A keyboard shortcut, which turns out to have scope+    outside the menus.+  * Windows: Install ssh and other bundled programs to Git/cmd,+    instead of Git/bin, since the latter is not in the default msysgit PATH.++ -- Joey Hess <id@joeyh.name>  Wed, 03 Dec 2014 15:16:52 -0400+ git-annex (5.20141125) unstable; urgency=medium    * Remove fixup code for bad bare repositories created by
CmdLine/GitAnnex.hs view
@@ -83,6 +83,9 @@ import qualified Command.Indirect import qualified Command.Upgrade import qualified Command.Forget+import qualified Command.Proxy+import qualified Command.DiffDriver+import qualified Command.Undo import qualified Command.Version import qualified Command.Help #ifdef WITH_ASSISTANT@@ -175,6 +178,9 @@ 	, Command.Indirect.cmd 	, Command.Upgrade.cmd 	, Command.Forget.cmd+	, Command.Proxy.cmd+	, Command.DiffDriver.cmd+	, Command.Undo.cmd 	, Command.Version.cmd 	, Command.Help.cmd #ifdef WITH_ASSISTANT
CmdLine/Seek.hs view
@@ -107,9 +107,11 @@ withFilesUnlocked' typechanged a params = seekActions $ 	prepFiltered a unlockedfiles   where-	check f = liftIO (notSymlink f) <&&> -		(isJust <$> catKeyFile f <||> isJust <$> catKeyFileHEAD f)-	unlockedfiles = filterM check =<< seekHelper typechanged params+	unlockedfiles = filterM isUnlocked =<< seekHelper typechanged params++isUnlocked :: FilePath -> Annex Bool+isUnlocked f = liftIO (notSymlink f) <&&> +	(isJust <$> catKeyFile f <||> isJust <$> catKeyFileHEAD f)  {- Finds files that may be modified. -} withFilesMaybeModified :: (FilePath -> CommandStart) -> CommandSeek
+ Command/DiffDriver.hs view
@@ -0,0 +1,102 @@+{- git-annex command+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.DiffDriver where++import Common.Annex+import Command+import Annex.Content+import Annex.Link+import Git.Types++cmd :: [Command]+cmd = [dontCheck repoExists $+	command "diffdriver" ("[-- cmd --opts]") seek+		SectionPlumbing "external git diff driver shim"]++seek :: CommandSeek+seek = withWords start++start :: [String] -> CommandStart+start opts = do+	let (req, differ) = parseReq opts+	void $ liftIO . exitBool =<< liftIO . differ =<< fixupReq req+	stop++data Req +	= Req+		{ rPath :: FilePath+		, rOldFile :: FilePath+		, rOldHex :: String+		, rOldMode :: String+		, rNewFile :: FilePath+		, rNewHex :: String+		, rNewMode ::String+		}+	| UnmergedReq+		{ rPath :: FilePath+		}++type Differ = Req -> IO Bool++serializeReq :: Req -> [CommandParam]+serializeReq req@(UnmergedReq {}) = [Param $ rPath req]+serializeReq req@(Req {}) = map Param+	[ rPath req+	, rOldFile req+	, rOldHex req+	, rOldMode req+	, rNewFile req+	, rNewHex req+	, rNewMode req+	]++parseReq :: [String] -> (Req, Differ)+parseReq opts = case separate (== "--") opts of+	(c:ps, l) -> (mk l, externalDiffer c ps)+	([],_) -> badopts+  where+	mk (path:old_file:old_hex:old_mode:new_file:new_hex:new_mode:[]) =+		Req+			{ rPath = path+			, rOldFile = old_file+			, rOldHex = old_hex+			, rOldMode = old_mode+			, rNewFile = new_file+			, rNewHex = new_hex+			, rNewMode = new_mode+			}+	mk (unmergedpath:[]) = UnmergedReq { rPath = unmergedpath }+	mk _ = badopts++	badopts = error $ "Unexpected input: " ++ unwords opts++{- Check if either file is a symlink to a git-annex object,+ - which git-diff will leave as a normal file containing the link text.+ - Adjust the Req to instead point to the actual location of the annexed+ - object (which may or may not exist). -}+fixupReq :: Req -> Annex Req+fixupReq req@(UnmergedReq {}) = return req+fixupReq req@(Req {}) = +	check rOldFile rOldMode (\r f -> r { rOldFile = f }) req+		>>= check rNewFile rNewMode (\r f -> r { rNewFile = f })+  where+	check getfile getmode setfile r = case readBlobType (getmode r) of+		Just SymlinkBlob -> do+			v <- getAnnexLinkTarget' (getfile r) False+			case fileKey . takeFileName =<< v of+				Nothing -> return r+				Just k -> setfile r <$>+					withObjectLoc k+						-- indirect mode+						return +						-- direct mode+						(return . Prelude.head)+		_ -> return r++externalDiffer :: String -> [String] -> Differ+externalDiffer c ps = \req -> boolSystem c (map Param ps ++ serializeReq req )
Command/PreCommit.hs view
@@ -23,6 +23,8 @@ import Logs.MetaData import Types.View import Types.MetaData+import qualified Git.Index as Git+import qualified Git.LsFiles as Git  import qualified Data.Set as S @@ -37,10 +39,18 @@ 		withWords startDirect ps 		runAnnexHook preCommitAnnexHook 	, do-		-- fix symlinks to files being committed-		withFilesToBeCommitted (whenAnnexed Command.Fix.start) ps-		-- inject unlocked files into the annex-		withFilesUnlockedToBeCommitted startIndirect ps+		ifM (liftIO Git.haveFalseIndex)+			( do+				(fs, cleanup) <- inRepo $ Git.typeChangedStaged ps+				whenM (anyM isUnlocked fs) $+					error "Cannot make a partial commit with unlocked annexed files. You should `git annex add` the files you want to commit, and then run git commit."+				void $ liftIO cleanup+			, do+				-- fix symlinks to files being committed+				withFilesToBeCommitted (whenAnnexed Command.Fix.start) ps+				-- inject unlocked files into the annex+				withFilesUnlockedToBeCommitted startIndirect ps+			) 		runAnnexHook preCommitAnnexHook 		-- committing changes to a view updates metadata 		mv <- currentView
+ Command/Proxy.hs view
@@ -0,0 +1,46 @@+{- git-annex command+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Proxy where++import Common.Annex+import Command+import Config+import Utility.Tmp+import Utility.Env+import Annex.Direct+import qualified Git.Branch+import qualified Git.Sha++cmd :: [Command]+cmd = [notBareRepo $+	command "proxy" ("-- git command") seek+		SectionPlumbing "safely bypass direct mode guard"]++seek :: CommandSeek+seek = withWords start++start :: [String] -> CommandStart+start [] = error "Did not specify command to run."+start (c:ps) = liftIO . exitWith =<< ifM isDirect+	( do+		g <- gitRepo+		withTmpDirIn (gitAnnexTmpMiscDir g) "proxy" go+	, liftIO $ safeSystem c (map Param ps)+	)+  where+	go tmp = do+		oldref <- fromMaybe Git.Sha.emptyTree+			<$> inRepo Git.Branch.currentSha+		exitcode <- liftIO $ proxy tmp+		mergeDirectCleanup tmp oldref+		return exitcode+	proxy tmp = do+		usetmp <- Just . addEntry "GIT_WORK_TREE" tmp  <$> getEnvironment+		unlessM (boolSystemEnv "git" [Param "checkout", Param "--", Param "."] usetmp) $+			error "Failed to set up proxy work tree."+		safeSystemEnv c (map Param ps) usetmp
+ Command/Undo.hs view
@@ -0,0 +1,84 @@+{- git-annex command+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Undo where++import Common.Annex+import Command+import Config+import Annex.Direct+import Annex.CatFile+import Git.DiffTree+import Git.FilePath+import Git.UpdateIndex+import Git.Sha+import qualified Git.LsFiles as LsFiles+import qualified Git.Command as Git+import qualified Git.Branch+import qualified Command.Sync++cmd :: [Command]+cmd = [notBareRepo $+	command "undo" paramPaths seek+		SectionCommon "undo last change to a file or directory"]++seek :: CommandSeek+seek ps = do+	-- Safety first; avoid any undo that would touch files that are not+	-- in the index.+	(fs, cleanup) <- inRepo $ LsFiles.notInRepo False ps+	unless (null fs) $+		error $ "Cannot undo changes to files that are not checked into git: " ++ unwords fs+	void $ liftIO $ cleanup++	-- Committing staged changes before undo allows later+	-- undoing the undo. It would be nicer to only commit staged+	-- changes to the specified files, rather than all staged changes,+	-- but that is difficult to do; a partial git-commit can't be done +	-- in direct mode.+	void $ Command.Sync.commitStaged Git.Branch.ManualCommit+		"commit before undo"+	+	withStrings start ps++start :: FilePath -> CommandStart+start p = do+	showStart "undo" p+	next $ perform p++perform :: FilePath -> CommandPerform+perform p = do+	g <- gitRepo++	-- Get the reversed diff that needs to be applied to undo.+	(diff, cleanup) <- inRepo $+		diffLog [Param "-R", Param "--", Param p]+	top <- inRepo $ toTopFilePath p+	let diff' = filter (`isDiffOf` top) diff+	liftIO $ streamUpdateIndex g (map stageDiffTreeItem diff')++	-- Take two passes through the diff, first doing any removals,+	-- and then any adds. This order is necessary to handle eg, removing+	-- a directory and replacing it with a file.+	let (removals, adds) = partition (\di -> dstsha di == nullSha) diff'+	let mkrel di = liftIO $ relPathCwdToFile $ fromTopFilePath (file di) g++	forM_ removals $ \di -> do+		f <- mkrel di+		whenM isDirect $+			maybe noop (`removeDirect` f)+				=<< catKey (srcsha di) (srcmode di)+		liftIO $ nukeFile f++	forM_ adds $ \di -> do+		f <- mkrel di+		inRepo $ Git.run [Param "checkout", Param "--", File f]+		whenM isDirect $+			maybe noop (`toDirect` f)+				=<< catKey (dstsha di) (dstmode di)++	next $ liftIO cleanup
Git/Branch.hs view
@@ -43,6 +43,9 @@ 		| null l = Nothing 		| otherwise = Just $ Git.Ref l +currentSha :: Repo -> IO (Maybe Git.Sha)+currentSha r = maybe (pure Nothing) (`Git.Ref.sha` r) =<< current r+ {- Checks if the second branch has any commits not present on the first  - branch. -} changed :: Branch -> Branch -> Repo -> IO Bool
Git/DiffTree.hs view
@@ -7,10 +7,12 @@  module Git.DiffTree ( 	DiffTreeItem(..),+	isDiffOf, 	diffTree, 	diffTreeRecursive, 	diffIndex, 	diffWorkTree,+	diffLog, ) where  import Numeric@@ -33,6 +35,13 @@ 	, file :: TopFilePath 	} deriving Show +{- Checks if the DiffTreeItem modifies a file with a given name+ - or under a directory by that name. -}+isDiffOf :: DiffTreeItem -> TopFilePath -> Bool+isDiffOf diff f = case getTopFilePath f of+	"" -> True -- top of repo contains all+	d -> d `dirContains` getTopFilePath (file diff)+ {- Diffs two tree Refs. -} diffTree :: Ref -> Ref -> Repo -> IO ([DiffTreeItem], IO Bool) diffTree src dst = getdiff (Param "diff-tree")@@ -66,16 +75,23 @@ 		, return ([], return True) 		) +{- Runs git log in --raw mode to get the changes that were made in+ - a particular commit. The output format is adjusted to be the same+ - as diff-tree --raw._-}+diffLog :: [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool)+diffLog params = getdiff (Param "log")+	(Param "-n1" : Param "--abbrev=40" : Param "--pretty=format:" : params)+ getdiff :: CommandParam -> [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) getdiff command params repo = do 	(diff, cleanup) <- pipeNullSplit ps repo-	return (parseDiffTree diff, cleanup)+	return (parseDiffRaw diff, cleanup)   where 	ps = command : Params "-z --raw --no-renames -l0" : params -{- Parses diff-tree output. -}-parseDiffTree :: [String] -> [DiffTreeItem]-parseDiffTree l = go l []+{- Parses --raw output used by diff-tree and git-log. -}+parseDiffRaw :: [String] -> [DiffTreeItem]+parseDiffRaw l = go l []   where 	go [] c = c 	go (info:f:rest) c = go rest (mk info f : c)
Git/Index.hs view
@@ -11,6 +11,9 @@ import Git import Utility.Env +indexEnv :: String+indexEnv = "GIT_INDEX_FILE"+ {- Forces git to use the specified index file.  -  - Returns an action that will reset back to the default@@ -25,7 +28,7 @@ 	return $ reset res   where 	var = "GIT_INDEX_FILE"-	reset (Just v) = setEnv var v True+	reset (Just v) = setEnv indexEnv v True 	reset _ = unsetEnv var  indexFile :: Repo -> FilePath@@ -34,3 +37,19 @@ {- Git locks the index by creating this file. -} indexFileLock :: Repo -> FilePath indexFileLock r = indexFile r ++ ".lock"++{- When the pre-commit hook is run, and git commit has been run with+ - a file or files specified to commit, rather than committing the staged+ - index, git provides the pre-commit hook with a "false index file".+ -+ - Changes made to this index will influence the commit, but won't+ - affect the real index file.+ -+ - This detects when we're in this situation, using a heuristic, which+ - might be broken by changes to git. Any use of this should have a test+ - case to make sure it works.+ -}+haveFalseIndex :: IO Bool+haveFalseIndex = maybe (False) check <$> getEnv indexEnv+  where+	check f = "next-index" `isPrefixOf` takeFileName f
Git/UpdateIndex.hs view
@@ -19,7 +19,8 @@ 	updateIndexLine, 	stageFile, 	unstageFile,-	stageSymlink+	stageSymlink,+	stageDiffTreeItem, ) where  import Common@@ -28,6 +29,7 @@ import Git.Command import Git.FilePath import Git.Sha+import qualified Git.DiffTree as Diff  {- Streamers are passed a callback and should feed it lines in the form  - read by update-index, and generated by ls-tree. -}@@ -95,8 +97,11 @@ unstageFile :: FilePath -> Repo -> IO Streamer unstageFile file repo = do 	p <- toTopFilePath file repo-	return $ pureStreamer $ "0 " ++ fromRef nullSha ++ "\t" ++ indexPath p+	return $ unstageFile' p +unstageFile' :: TopFilePath -> Streamer+unstageFile' p = pureStreamer $ "0 " ++ fromRef nullSha ++ "\t" ++ indexPath p+ {- A streamer that adds a symlink to the index. -} stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer stageSymlink file sha repo = do@@ -105,6 +110,12 @@ 		<*> pure SymlinkBlob 		<*> toTopFilePath file repo 	return $ pureStreamer line++{- A streamer that applies a DiffTreeItem to the index. -}+stageDiffTreeItem :: Diff.DiffTreeItem -> Streamer+stageDiffTreeItem d = case toBlobType (Diff.dstmode d) of+	Nothing -> unstageFile' (Diff.file d)+	Just t -> pureStreamer $ updateIndexLine (Diff.dstsha d) t (Diff.file d)  indexPath :: TopFilePath -> InternalGitPath indexPath = toInternalGitPath . getTopFilePath
README view
@@ -3,4 +3,4 @@ dealing with files larger than git can currently easily handle, whether due to limitations in memory, checksumming time, or disk space. -For documentation, see doc/ or <http://git-annex.branchable.com/>+For documentation, see doc/ or <https://git-annex.branchable.com/>
Remote/Helper/AWS.hs view
@@ -1,6 +1,6 @@ {- Amazon Web Services common infrastructure.  -- - Copyright 2011,2012 Joey Hess <joey@kitenet.net>+ - Copyright 2011-2014 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -12,8 +12,14 @@ import Common.Annex import Creds +import qualified Aws+import qualified Aws.S3 as S3 import qualified Data.Map as M+import qualified Data.ByteString as B+import qualified Data.Text as T+import Data.Text.Encoding (encodeUtf8) import Data.Text (Text)+import Data.IORef  creds :: UUID -> CredPairStorage creds u = CredPairStorage@@ -22,6 +28,13 @@ 	, credPairRemoteKey = Just "s3creds" 	} +genCredentials :: CredPair -> IO Aws.Credentials+genCredentials (keyid, secret) = Aws.Credentials+	<$> pure (encodeUtf8 (T.pack keyid))+	<*> pure (encodeUtf8 (T.pack secret))+	<*> newIORef []+	<*> pure Nothing+ data Service = S3 | Glacier 	deriving (Eq) @@ -33,9 +46,10 @@ defaultRegion :: Service -> Region defaultRegion = snd . Prelude.head . regionInfo -{- S3 and Glacier use different names for some regions. Ie, "us-east-1"- - cannot be used with S3, while "US" cannot be used with Glacier. Dunno why.- - Also, Glacier is not yet available in all regions. -}+data ServiceRegion = BothRegion Region | S3Region Region | GlacierRegion Region++{- The "US" and "EU" names are used as location constraints when creating a+ - S3 bucket. -} regionInfo :: Service -> [(Text, Region)] regionInfo service = map (\(t, r) -> (t, fromServiceRegion r)) $  	filter (matchingService . snd) $@@ -45,9 +59,7 @@ 		[ ("US East (N. Virginia)", [S3Region "US", GlacierRegion "us-east-1"]) 		, ("US West (Oregon)", [BothRegion "us-west-2"]) 		, ("US West (N. California)", [BothRegion "us-west-1"])-		-- Requires AWS4-HMAC-SHA256 which S3 library does not-		-- currently support.-		-- , ("EU (Frankfurt)", [BothRegion "eu-central-1"])+		, ("EU (Frankfurt)", [BothRegion "eu-central-1"]) 		, ("EU (Ireland)", [S3Region "EU", GlacierRegion "eu-west-1"]) 		, ("Asia Pacific (Singapore)", [S3Region "ap-southeast-1"]) 		, ("Asia Pacific (Tokyo)", [BothRegion "ap-northeast-1"])@@ -63,4 +75,14 @@ 	matchingService (S3Region _) = service == S3 	matchingService (GlacierRegion _) = service == Glacier -data ServiceRegion = BothRegion Region | S3Region Region | GlacierRegion Region+s3HostName :: Region -> B.ByteString+s3HostName "US" = "s3.amazonaws.com"+s3HostName "EU" = "s3-eu-west-1.amazonaws.com"+s3HostName r = encodeUtf8 $ T.concat ["s3-", r, ".amazonaws.com"]++s3DefaultHost :: String+s3DefaultHost = "s3.amazonaws.com"++mkLocationConstraint :: Region -> S3.LocationConstraint+mkLocationConstraint "US" = S3.locationUsClassic+mkLocationConstraint r = r
Remote/Helper/Encryptable.hs view
@@ -109,7 +109,6 @@ 		case M.lookup encipher cache of 			Just cipher -> return $ Just (cipher, encipher) 			Nothing -> do-				showNote "gpg" 				cipher <- liftIO $ decryptCipher encipher 				Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache }) 				return $ Just (cipher, encipher)
Remote/Helper/Http.hs view
@@ -5,13 +5,15 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE BangPatterns #-}+ module Remote.Helper.Http where  import Common.Annex import Types.StoreRetrieve import Utility.Metered import Remote.Helper.Special-import Network.HTTP.Client (RequestBody(..), Response, responseStatus, responseBody, BodyReader)+import Network.HTTP.Client (RequestBody(..), Response, responseStatus, responseBody, BodyReader, NeedsPopper) import Network.HTTP.Types  import qualified Data.ByteString.Lazy as L@@ -24,17 +26,45 @@ -- Implemented as a fileStorer, so that the content can be streamed -- from the file in constant space. httpStorer :: (Key -> RequestBody -> Annex Bool) -> Storer-httpStorer a = fileStorer $ \k f m -> do-	size <- liftIO $ (fromIntegral . fileSize <$> getFileStatus f :: IO Integer)-	let streamer sink = withMeteredFile f m $ \b -> do-		mvar <- newMVar $ L.toChunks b-		let getnextchunk = modifyMVar mvar $ pure . pop-		sink getnextchunk-	let body = RequestBodyStream (fromInteger size) streamer-	a k body+httpStorer a = fileStorer $ \k f m -> a k =<< liftIO (httpBodyStorer f m)++-- Reads the file and generates a streaming request body, that will update+-- the meter as it's sent.+httpBodyStorer :: FilePath -> MeterUpdate -> IO RequestBody+httpBodyStorer src m = do+	size <- fromIntegral . fileSize <$> getFileStatus src :: IO Integer+	let streamer sink = withMeteredFile src m $ \b -> byteStringPopper b sink+	return $ RequestBodyStream (fromInteger size) streamer++byteStringPopper :: L.ByteString -> NeedsPopper () -> IO ()+byteStringPopper b sink = do+	mvar <- newMVar $ L.toChunks b+	let getnextchunk = modifyMVar mvar $ \v ->+		case v of+			[] -> return ([], S.empty)+			(c:cs) -> return (cs, c)+	sink getnextchunk++{- Makes a Popper that streams a given number of chunks of a given+ - size from the handle, updating the meter as the chunks are read. -}+handlePopper :: Integer -> Int -> MeterUpdate -> Handle -> NeedsPopper () -> IO ()+handlePopper numchunks chunksize meterupdate h sink = do+	mvar <- newMVar zeroBytesProcessed+	let getnextchunk = do+		sent <- takeMVar mvar+		if sent >= target+			then do+				putMVar mvar sent+				return S.empty+			else do+				b <- S.hGet h chunksize+				let !sent' = addBytesProcessed sent chunksize+				putMVar mvar sent'+                                meterupdate sent'+				return b+	sink getnextchunk   where-	pop [] = ([], S.empty)-	pop (c:cs) = (cs, c)+	target = toBytesProcessed (numchunks * fromIntegral chunksize)  -- Reads the http body and stores it to the specified file, updating the -- meter as it goes.
Remote/S3.hs view
@@ -1,22 +1,31 @@ {- S3 remotes  -- - Copyright 2011-2013 Joey Hess <joey@kitenet.net>+ - Copyright 2011-2014 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE CPP #-}+ module Remote.S3 (remote, iaHost, configIA, iaItemUrl) where -import Network.AWS.AWSConnection-import Network.AWS.S3Object hiding (getStorageClass)-import Network.AWS.S3Bucket hiding (size)-import Network.AWS.AWSResult+import qualified Aws as AWS+import qualified Aws.Core as AWS+import qualified Aws.S3 as S3 import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.ByteString.Lazy as L+import qualified Data.ByteString as S import qualified Data.Map as M import Data.Char import Network.Socket (HostName)+import Network.HTTP.Conduit (Manager, newManager, closeManager)+import Network.HTTP.Client (defaultManagerSettings, managerResponseTimeout, responseStatus, responseBody, RequestBody(..))+import Network.HTTP.Types+import Control.Monad.Trans.Resource+import Control.Monad.Catch+import Data.Conduit  import Common.Annex import Types.Remote@@ -25,13 +34,15 @@ import Config import Config.Cost import Remote.Helper.Special+import Remote.Helper.Http import qualified Remote.Helper.AWS as AWS import Creds-import Utility.Metered import Annex.UUID import Logs.Web+import Utility.Metered+import Utility.DataUnits -type Bucket = String+type BucketName = String  remote :: RemoteType remote = RemoteType {@@ -42,13 +53,16 @@ }  gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote)-gen r u c gc = new <$> remoteCost gc expensiveRemoteCost+gen r u c gc = do+	cst <- remoteCost gc expensiveRemoteCost+	info <- extractS3Info c+	return $ new cst info   where-	new cst = Just $ specialRemote c-		(prepareStore this)-		(prepareRetrieve this)-		(simplyPrepare $ remove this c)-		(simplyPrepare $ checkKey this)+	new cst info = Just $ specialRemote c+		(prepareS3 this info $ store this)+		(prepareS3 this info retrieve)+		(prepareS3 this info remove)+		(prepareS3 this info $ checkKey this) 		this 	  where 		this = Remote {@@ -73,10 +87,11 @@ 			remotetype = remote, 			mkUnavailable = gen r u (M.insert "host" "!dne!" c) gc, 			getInfo = includeCredsInfo c (AWS.creds u) $ catMaybes-				[ Just ("bucket", fromMaybe "unknown" (getBucket c))+				[ Just ("bucket", fromMaybe "unknown" (getBucketName c)) 				, if configIA c-					then Just ("internet archive item", iaItemUrl $ fromMaybe "unknown" $ getBucket c)+					then Just ("internet archive item", iaItemUrl $ fromMaybe "unknown" $ getBucketName c) 					else Nothing+				, Just ("partsize", maybe "unlimited" (roughSize storageUnits False) (getPartSize c)) 				] 		} @@ -92,8 +107,8 @@ 	defaults = M.fromList 		[ ("datacenter", T.unpack $ AWS.defaultRegion AWS.S3) 		, ("storageclass", "STANDARD")-		, ("host", defaultAmazonS3Host)-		, ("port", show defaultAmazonS3Port)+		, ("host", AWS.s3DefaultHost)+		, ("port", "80") 		, ("bucket", defbucket) 		] 		@@ -113,239 +128,378 @@ 		c' <- setRemoteCredPair noEncryptionUsed c (AWS.creds u) mcreds 		-- Ensure user enters a valid bucket name, since 		-- this determines the name of the archive.org item.-		let bucket = replace " " "-" $ map toLower $+		let validbucket = replace " " "-" $ map toLower $ 			fromMaybe (error "specify bucket=") $-				getBucket c'+				getBucketName c' 		let archiveconfig = -			-- hS3 does not pass through x-archive-* headers+			-- IA acdepts x-amz-* as an alias for x-archive-* 			M.mapKeys (replace "x-archive-" "x-amz-") $ 			-- encryption does not make sense here 			M.insert "encryption" "none" $-			M.insert "bucket" bucket $+			M.insert "bucket" validbucket $ 			M.union c' $ 			-- special constraints on key names-			M.insert "mungekeys" "ia" $-			-- bucket created only when files are uploaded-			M.insert "x-amz-auto-make-bucket" "1" defaults-		writeUUIDFile archiveconfig u+			M.insert "mungekeys" "ia" defaults+		info <- extractS3Info archiveconfig+		withS3Handle archiveconfig u info $+			writeUUIDFile archiveconfig u 		use archiveconfig -prepareStore :: Remote -> Preparer Storer-prepareStore r = resourcePrepare (const $ s3Action r False) $ \(conn, bucket) -> -	fileStorer $ \k src p -> do-		ok <- s3Bool =<< liftIO (store (conn, bucket) r k p src)+-- Sets up a http connection manager for S3 encdpoint, which allows+-- http connections to be reused across calls to the helper.+prepareS3 :: Remote -> S3Info -> (S3Handle -> helper) -> Preparer helper+prepareS3 r info = resourcePrepare $ const $+	withS3Handle (config r) (uuid r) info -		-- Store public URL to item in Internet Archive.-		when (ok && configIA (config r) && not (isChunkKey k)) $-			setUrlPresent k (iaKeyUrl r k)+store :: Remote -> S3Handle -> Storer+store r h = fileStorer $ \k f p -> do+	case partSize (hinfo h) of+		Just partsz | partsz > 0 -> do+			fsz <- fromIntegral . fileSize <$> liftIO (getFileStatus f)+			if fsz > partsz+				then multipartupload fsz partsz k f p+				else singlepartupload k f p+		_ -> singlepartupload k f p	+	-- Store public URL to item in Internet Archive.+	when (isIA (hinfo h) && not (isChunkKey k)) $+		setUrlPresent k (iaKeyUrl r k)+	return True+  where+	singlepartupload k f p = do+		rbody <- liftIO $ httpBodyStorer f p+		void $ sendS3Handle h $ putObject h (bucketObject (hinfo h) k) rbody+	multipartupload fsz partsz k f p = do+#if MIN_VERSION_aws(0,10,6)+		let info = hinfo h+		let object = bucketObject info k -		return ok+		let startreq = (S3.postInitiateMultipartUpload (bucket info) object)+				{ S3.imuStorageClass = Just (storageClass info)+				, S3.imuMetadata = metaHeaders info+				, S3.imuAutoMakeBucket = isIA info+				, S3.imuExpires = Nothing -- TODO set some reasonable expiry+				}+		uploadid <- S3.imurUploadId <$> sendS3Handle h startreq -store :: (AWSConnection, Bucket) -> Remote -> Key -> MeterUpdate -> FilePath -> IO (AWSResult ())-store (conn, bucket) r k p file = do-	size <- (fromIntegral . fileSize <$> getFileStatus file) :: IO Integer-	withMeteredFile file p $ \content -> do-		-- size is provided to S3 so the whole content-		-- does not need to be buffered to calculate it-		let object = S3Object-			bucket (bucketFile r k) ""-			(("Content-Length", show size) : getXheaders (config r))-			content-		sendObject conn $-			setStorageClass (getStorageClass $ config r) object+		-- The actual part size will be a even multiple of the+		-- 32k chunk size that hGetUntilMetered uses.+		let partsz' = (partsz `div` toInteger defaultChunkSize) * toInteger defaultChunkSize -prepareRetrieve :: Remote -> Preparer Retriever-prepareRetrieve r = resourcePrepare (const $ s3Action r False) $ \(conn, bucket) ->-	byteRetriever $ \k sink ->-		liftIO (getObject conn $ bucketKey r bucket k)-			>>= either s3Error (sink . obj_data)+		-- Send parts of the file, taking care to stream each part+		-- w/o buffering in memory, since the parts can be large.+		etags <- bracketIO (openBinaryFile f ReadMode) hClose $ \fh -> do+			let sendparts meter etags partnum = do+				pos <- liftIO $ hTell fh+				if pos >= fsz+					then return (reverse etags)+					else do+						-- Calculate size of part that will+						-- be read.+						let sz = if fsz - pos < partsz'+							then fsz - pos+							else partsz'+						let p' = offsetMeterUpdate p (toBytesProcessed pos)+						let numchunks = ceiling (fromIntegral sz / fromIntegral defaultChunkSize :: Double)+						let popper = handlePopper numchunks defaultChunkSize p' fh+						let req = S3.uploadPart (bucket info) object partnum uploadid $+							 RequestBodyStream (fromIntegral sz) popper+						S3.UploadPartResponse _ etag <- sendS3Handle h req+						sendparts (offsetMeterUpdate meter (toBytesProcessed sz)) (etag:etags) (partnum + 1)+			sendparts p [] 1 +		void $ sendS3Handle h $ S3.postCompleteMultipartUpload+			(bucket info) object uploadid (zip [1..] etags)+#else+		warning $ "Cannot do multipart upload (partsize " ++ show partsz ++ ") of large file (" ++ show fsz ++ "); built with too old a version of the aws library."+		singlepartupload k f p+#endif++{- Implemented as a fileRetriever, that uses conduit to stream the chunks+ - out to the file. Would be better to implement a byteRetriever, but+ - that is difficult. -}+retrieve :: S3Handle -> Retriever+retrieve h = fileRetriever $ \f k p -> liftIO $ runResourceT $ do+	(fr, fh) <- allocate (openFile f WriteMode) hClose+	let req = S3.getObject (bucket info) (bucketObject info k)+	S3.GetObjectResponse { S3.gorResponse = rsp } <- sendS3Handle' h req+	responseBody rsp $$+- sinkprogressfile fh p zeroBytesProcessed+	release fr+  where+	info = hinfo h+	sinkprogressfile fh meterupdate sofar = do+		mbs <- await+		case mbs of+			Nothing -> return ()+			Just bs -> do+				let sofar' = addBytesProcessed sofar (S.length bs)+				liftIO $ do+					void $ meterupdate sofar'+					S.hPut fh bs+				sinkprogressfile fh meterupdate sofar'+ retrieveCheap :: Key -> FilePath -> Annex Bool retrieveCheap _ _ = return False  {- Internet Archive doesn't easily allow removing content.  - While it may remove the file, there are generally other files  - derived from it that it does not remove. -}-remove :: Remote -> RemoteConfig -> Remover-remove r c k-	| configIA c = do+remove :: S3Handle -> Remover+remove h k+	| isIA info = do 		warning "Cannot remove content from the Internet Archive" 		return False-	| otherwise = remove' r k--remove' :: Remote -> Key -> Annex Bool-remove' r k = s3Action r False $ \(conn, bucket) ->-	s3Bool =<< liftIO (deleteObject conn $ bucketKey r bucket k)--checkKey :: Remote -> CheckPresent-checkKey r k = s3Action r noconn $ \(conn, bucket) -> do-	showAction $ "checking " ++ name r-	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k-	case res of-		Right _ -> return True-		Left (AWSError _ _) -> return False-		Left e -> s3Error e+	| otherwise = do+		res <- tryNonAsync $ sendS3Handle h $+			S3.DeleteObject (bucketObject info k) (bucket info)+		return $ either (const False) (const True) res   where-	noconn = error "S3 not configured"-			-s3Warning :: ReqError -> Annex Bool-s3Warning e = do-	warning $ prettyReqError e-	return False--s3Error :: ReqError -> a-s3Error e = error $ prettyReqError e--s3Bool :: AWSResult () -> Annex Bool-s3Bool (Right _) = return True-s3Bool (Left e) = s3Warning e--s3Action :: Remote -> a -> ((AWSConnection, Bucket) -> Annex a) -> Annex a-s3Action r noconn action = do-	let bucket = M.lookup "bucket" $ config r-	conn <- s3Connection (config r) (uuid r)-	case (bucket, conn) of-		(Just b, Just c) -> action (c, b)-		_ -> return noconn+	info = hinfo h -bucketFile :: Remote -> Key -> FilePath-bucketFile r = munge . key2file+checkKey :: Remote -> S3Handle -> CheckPresent+checkKey r h k = do+	showAction $ "checking " ++ name r+#if MIN_VERSION_aws(0,10,0)+	rsp <- go+	return (isJust $ S3.horMetadata rsp)+#else+	catchMissingException $ do+		void go+		return True+#endif   where-	munge s = case M.lookup "mungekeys" c of-		Just "ia" -> iaMunge $ filePrefix c ++ s-		_ -> filePrefix c ++ s-	c = config r--filePrefix :: RemoteConfig -> String-filePrefix = M.findWithDefault "" "fileprefix"--bucketKey :: Remote -> Bucket -> Key -> S3Object-bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty+	go = sendS3Handle h $+		S3.headObject (bucket (hinfo h)) (bucketObject (hinfo h) k) -{- Internet Archive limits filenames to a subset of ascii,- - with no whitespace. Other characters are xml entity- - encoded. -}-iaMunge :: String -> String-iaMunge = (>>= munge)-  where-	munge c-		| isAsciiUpper c || isAsciiLower c || isNumber c = [c]-		| c `elem` "_-.\"" = [c]-		| isSpace c = []-		| otherwise = "&" ++ show (ord c) ++ ";"+#if ! MIN_VERSION_aws(0,10,0)+	{- Catch exception headObject returns when an object is not present+	 - in the bucket, and returns False. All other exceptions indicate a+	 - check error and are let through. -}+	catchMissingException :: Annex Bool -> Annex Bool+	catchMissingException a = catchJust missing a (const $ return False)+	  where+		missing :: AWS.HeaderException -> Maybe ()+		missing e+			| AWS.headerErrorMessage e == "ETag missing" = Just ()+			| otherwise = Nothing+#endif  {- Generate the bucket if it does not already exist, including creating the  - UUID file within the bucket.  -- - To check if the bucket exists, ask for its location. However, some ACLs- - can allow read/write to buckets, but not querying location, so first- - check if the UUID file already exists and we can skip doing anything.+ - Some ACLs can allow read/write to buckets, but not querying them,+ - so first check if the UUID file already exists and we can skip doing+ - anything.  -} genBucket :: RemoteConfig -> UUID -> Annex () genBucket c u = do-	conn <- s3ConnectionRequired c u 	showAction "checking bucket"-	unlessM ((== Right True) <$> checkUUIDFile c u conn) $ do-		loc <- liftIO $ getBucketLocation conn bucket-		case loc of-			Right _ -> writeUUIDFile c u-			Left err@(NetworkError _) -> s3Error err-			Left (AWSError _ _) -> do-				showAction $ "creating bucket in " ++ datacenter-				res <- liftIO $ createBucketIn conn bucket datacenter-				case res of-					Right _ -> writeUUIDFile c u-					Left err -> s3Error err+	info <- extractS3Info c+	withS3Handle c u info $ \h ->+		go h =<< checkUUIDFile c u h   where-	bucket = fromJust $ getBucket c+	go _ (Right True) = noop+	go h _ = do+		v <- tryNonAsync $ sendS3Handle h (S3.getBucket $ bucket $ hinfo h)+		case v of+			Right _ -> noop+			Left _ -> do+				showAction $ "creating bucket in " ++ datacenter+				void $ sendS3Handle h $+					S3.PutBucket (bucket $ hinfo h) Nothing $+						AWS.mkLocationConstraint $+							T.pack datacenter+		writeUUIDFile c u h+	 	datacenter = fromJust $ M.lookup "datacenter" c  {- Writes the UUID to an annex-uuid file within the bucket.  -  - If the file already exists in the bucket, it must match.  -- - Note that IA items do not get created by createBucketIn. - - Rather, they are created the first time a file is stored in them.- - So this also takes care of that.+ - Note that IA buckets can only created by having a file+ - stored in them. So this also takes care of that.  -}-writeUUIDFile :: RemoteConfig -> UUID -> Annex ()-writeUUIDFile c u = do-	conn <- s3ConnectionRequired c u-	v <- checkUUIDFile  c u conn+writeUUIDFile :: RemoteConfig -> UUID -> S3Handle -> Annex ()+writeUUIDFile c u h = do+	v <- checkUUIDFile c u h 	case v of-		Left e -> error e-		Right True -> return ()-		Right False -> do-			let object = setStorageClass (getStorageClass c) (mkobject uuidb)-			either s3Error return =<< liftIO (sendObject conn object)+		Right True -> noop+		_ -> void $ sendS3Handle h mkobject   where-	file = uuidFile c+	file = T.pack $ uuidFile c 	uuidb = L.fromChunks [T.encodeUtf8 $ T.pack $ fromUUID u]-	bucket = fromJust $ getBucket c -	mkobject = S3Object bucket file "" (getXheaders c)+	mkobject = putObject h file (RequestBodyLBS uuidb) -{- Checks if the UUID file exists in the bucket and has the specified UUID already. -}-checkUUIDFile :: RemoteConfig -> UUID -> AWSConnection -> Annex (Either String Bool)-checkUUIDFile c u conn = check <$> liftIO (tryNonAsync $ getObject conn $ mkobject L.empty)+{- Checks if the UUID file exists in the bucket+ - and has the specified UUID already. -}+checkUUIDFile :: RemoteConfig -> UUID -> S3Handle -> Annex (Either SomeException Bool)+checkUUIDFile c u h = tryNonAsync $ check <$> get   where-	check (Right (Right o))-		| obj_data o == uuidb = Right True-		| otherwise = Left $ "This bucket is already in use by a different S3 special remote, with UUID: " ++ show (obj_data o)-	check _ = Right False+	get = liftIO +		. runResourceT +		. either (pure . Left) (Right <$$> AWS.loadToMemory)+		=<< tryS3 (sendS3Handle h (S3.getObject (bucket (hinfo h)) file))+	check (Right (S3.GetObjectMemoryResponse _meta rsp)) =+		responseStatus rsp == ok200 && responseBody rsp == uuidb+	check (Left _S3Error) = False +	file = T.pack $ uuidFile c 	uuidb = L.fromChunks [T.encodeUtf8 $ T.pack $ fromUUID u]-	bucket = fromJust $ getBucket c-	file = uuidFile c -	mkobject = S3Object bucket file "" (getXheaders c)- uuidFile :: RemoteConfig -> FilePath-uuidFile c = filePrefix c ++ "annex-uuid"+uuidFile c = getFilePrefix c ++ "annex-uuid" -s3ConnectionRequired :: RemoteConfig -> UUID -> Annex AWSConnection-s3ConnectionRequired c u =-	maybe (error "Cannot connect to S3") return =<< s3Connection c u+putObject :: S3Handle -> T.Text -> RequestBody -> S3.PutObject+putObject h file rbody = (S3.putObject (bucket info) file rbody)+	{ S3.poStorageClass = Just (storageClass info)+	, S3.poMetadata = metaHeaders info+	, S3.poAutoMakeBucket = isIA info+	}+  where+	info = hinfo h -s3Connection :: RemoteConfig -> UUID -> Annex (Maybe AWSConnection)-s3Connection c u = go =<< getRemoteCredPairFor "S3" c (AWS.creds u)+data S3Handle = S3Handle +	{ hmanager :: Manager+	, hawscfg :: AWS.Configuration+	, hs3cfg :: S3.S3Configuration AWS.NormalQuery+	, hinfo :: S3Info+	}++{- Sends a request to S3 and gets back the response.+ - + - Note that pureAws's use of ResourceT is bypassed here;+ - the response should be fully processed while the S3Handle+ - is still open, eg within a call to withS3Handle.+ -}+sendS3Handle+	:: (AWS.Transaction req res, AWS.ServiceConfiguration req ~ S3.S3Configuration)+	=> S3Handle +	-> req+	-> Annex res+sendS3Handle h r = liftIO $ runResourceT $ sendS3Handle' h r++sendS3Handle'+	:: (AWS.Transaction r a, AWS.ServiceConfiguration r ~ S3.S3Configuration)+	=> S3Handle+	-> r+	-> ResourceT IO a+sendS3Handle' h = AWS.pureAws (hawscfg h) (hs3cfg h) (hmanager h)++withS3Handle :: RemoteConfig -> UUID -> S3Info -> (S3Handle -> Annex a) -> Annex a+withS3Handle c u info a = do+	creds <- getRemoteCredPairFor "S3" c (AWS.creds u)+	awscreds <- liftIO $ AWS.genCredentials $ fromMaybe nocreds creds+	let awscfg = AWS.Configuration AWS.Timestamp awscreds (AWS.defaultLog AWS.Error)+	bracketIO (newManager httpcfg) closeManager $ \mgr -> +		a $ S3Handle mgr awscfg s3cfg info   where-	go Nothing = return Nothing-	go (Just (ak, sk)) = return $ Just $ AWSConnection host port ak sk+	s3cfg = s3Configuration c+	httpcfg = defaultManagerSettings+		{ managerResponseTimeout = Nothing }+	nocreds = error "Cannot use S3 without credentials configured" +s3Configuration :: RemoteConfig -> S3.S3Configuration AWS.NormalQuery+s3Configuration c = (S3.s3 proto endpoint False) { S3.s3Port = port }+  where+	proto+		| port == 443 = AWS.HTTPS+		| otherwise = AWS.HTTP 	host = fromJust $ M.lookup "host" c+	datacenter = fromJust $ M.lookup "datacenter" c+	-- When the default S3 host is configured, connect directly to+	-- the S3 endpoint for the configured datacenter.+	-- When another host is configured, it's used as-is.+	endpoint+		| host == AWS.s3DefaultHost = AWS.s3HostName $ T.pack datacenter+		| otherwise = T.encodeUtf8 $ T.pack host 	port = let s = fromJust $ M.lookup "port" c in 		case reads s of 		[(p, _)] -> p 		_ -> error $ "bad S3 port value: " ++ s -getBucket :: RemoteConfig -> Maybe Bucket-getBucket = M.lookup "bucket"+tryS3 :: Annex a -> Annex (Either S3.S3Error a)+tryS3 a = (Right <$> a) `catch` (pure . Left) -getStorageClass :: RemoteConfig -> StorageClass-getStorageClass c = case fromJust $ M.lookup "storageclass" c of-	"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY-	_ -> STANDARD-	-getXheaders :: RemoteConfig -> [(String, String)]-getXheaders = filter isxheader . M.assocs+data S3Info = S3Info+	{ bucket :: S3.Bucket+	, storageClass :: S3.StorageClass+	, bucketObject :: Key -> T.Text+	, metaHeaders :: [(T.Text, T.Text)]+	, partSize :: Maybe Integer+	, isIA :: Bool+	}++extractS3Info :: RemoteConfig -> Annex S3Info+extractS3Info c = do+	b <- maybe+		(error "S3 bucket not configured")+		(return . T.pack)+		(getBucketName c)+	return $ S3Info+		{ bucket = b+		, storageClass = getStorageClass c+		, bucketObject = T.pack . getBucketObject c+		, metaHeaders = getMetaHeaders c+		, partSize = getPartSize c+		, isIA = configIA c+		}++getBucketName :: RemoteConfig -> Maybe BucketName+getBucketName = M.lookup "bucket"++getStorageClass :: RemoteConfig -> S3.StorageClass+getStorageClass c = case M.lookup "storageclass" c of+	Just "REDUCED_REDUNDANCY" -> S3.ReducedRedundancy+	_ -> S3.Standard++getPartSize :: RemoteConfig -> Maybe Integer+getPartSize c = readSize dataUnits =<< M.lookup "partsize" c++getMetaHeaders :: RemoteConfig -> [(T.Text, T.Text)]+getMetaHeaders = map munge . filter ismetaheader . M.assocs   where-	isxheader (h, _) = "x-amz-" `isPrefixOf` h+	ismetaheader (h, _) = metaprefix `isPrefixOf` h+	metaprefix = "x-amz-meta-"+	metaprefixlen = length metaprefix+	munge (k, v) = (T.pack $ drop metaprefixlen k, T.pack v) +getFilePrefix :: RemoteConfig -> String+getFilePrefix = M.findWithDefault "" "fileprefix"++getBucketObject :: RemoteConfig -> Key -> FilePath+getBucketObject c = munge . key2file+  where+	munge s = case M.lookup "mungekeys" c of+		Just "ia" -> iaMunge $ getFilePrefix c ++ s+		_ -> getFilePrefix c ++ s++{- Internet Archive limits filenames to a subset of ascii,+ - with no whitespace. Other characters are xml entity+ - encoded. -}+iaMunge :: String -> String+iaMunge = (>>= munge)+  where+	munge c+		| isAsciiUpper c || isAsciiLower c || isNumber c = [c]+		| c `elem` "_-.\"" = [c]+		| isSpace c = []+		| otherwise = "&" ++ show (ord c) ++ ";"++configIA :: RemoteConfig -> Bool+configIA = maybe False isIAHost . M.lookup "host"+ {- Hostname to use for archive.org S3. -} iaHost :: HostName iaHost = "s3.us.archive.org" -configIA :: RemoteConfig -> Bool-configIA c = maybe False isIAHost (M.lookup "host" c)- isIAHost :: HostName -> Bool isIAHost h = ".archive.org" `isSuffixOf` map toLower h -iaItemUrl :: Bucket -> URLString-iaItemUrl bucket = "http://archive.org/details/" ++ bucket+iaItemUrl :: BucketName -> URLString+iaItemUrl b = "http://archive.org/details/" ++ b  iaKeyUrl :: Remote -> Key -> URLString-iaKeyUrl r k = "http://archive.org/download/" ++ bucket ++ "/" ++ bucketFile r k+iaKeyUrl r k = "http://archive.org/download/" ++ b ++ "/" ++ getBucketObject (config r) k   where-	bucket = fromMaybe "" $ getBucket $ config r+	b = fromMaybe "" $ getBucketName $ config r
Test.hs view
@@ -194,6 +194,7 @@ 	, check "lock" test_lock 	, check "edit (no pre-commit)" test_edit 	, check "edit (pre-commit)" test_edit_precommit+	, check "partial commit" test_partial_commit 	, check "fix" test_fix 	, check "trust" test_trust 	, check "fsck (basics)" test_fsck_basic@@ -501,6 +502,14 @@ 	c <- readFile annexedfile 	assertEqual "content of modified file" c (changedcontent annexedfile) 	not <$> git_annex testenv "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of modified file"++test_partial_commit :: TestEnv -> Assertion+test_partial_commit testenv = intmpclonerepoInDirect testenv $ do+	git_annex testenv "get" [annexedfile] @? "get of file failed"+	annexed_present annexedfile+	git_annex testenv "unlock" [annexedfile] @? "unlock failed"+	not <$> boolSystem "git" [Params "commit -q -m test", File annexedfile]+		@? "partial commit of unlocked file not blocked by pre-commit hook"  test_fix :: TestEnv -> Assertion test_fix testenv = intmpclonerepoInDirect testenv $ do
Utility/Lsof.hs view
@@ -93,11 +93,15 @@ 		_ -> parsefail  	parsefiles c [] = (c, [])-	parsefiles c (l:ls) = case splitnull l of-		['a':mode, 'n':file, ""] ->-			parsefiles ((file, parsemode mode):c) ls-		(('p':_):_) -> (c, l:ls)-		_ -> parsefail+	parsefiles c (l:ls) = parsefiles' c (splitnull l) l ls++	parsefiles' c ['a':mode, 'n':file, ""] _ ls =+		parsefiles ((file, parsemode mode):c) ls+	parsefiles' c (('p':_):_) l ls = (c, l:ls)+	-- Some buggy versions of lsof emit a f field+	-- that was not requested, so ignore it.+	parsefiles' c (('f':_):rest) l ls = parsefiles' c rest l ls+	parsefiles' _ _ _ _ = parsefail  	parsemode ('r':_) = OpenReadOnly 	parsemode ('w':_) = OpenWriteOnly
Utility/Process.hs view
@@ -13,6 +13,7 @@ 	CreateProcess(..), 	StdHandle(..), 	readProcess,+	readProcess', 	readProcessEnv, 	writeReadProcessEnv, 	forceSuccessProcess,@@ -66,16 +67,18 @@ readProcess cmd args = readProcessEnv cmd args Nothing  readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String-readProcessEnv cmd args environ =-	withHandle StdoutHandle createProcessSuccess p $ \h -> do-		output  <- hGetContentsStrict h-		hClose h-		return output+readProcessEnv cmd args environ = readProcess' p   where 	p = (proc cmd args) 		{ std_out = CreatePipe 		, env = environ 		}++readProcess' :: CreateProcess -> IO String+readProcess' p = withHandle StdoutHandle createProcessSuccess p $ \h -> do+	output  <- hGetContentsStrict h+	hClose h+	return output  {- Runs an action to write to a process on its stdin,   - returns its output, and also allows specifying the environment.
Utility/Tmp.hs view
@@ -24,8 +24,8 @@ {- Runs an action like writeFile, writing to a temp file first and  - then moving it into place. The temp file is stored in the same  - directory as the final file to avoid cross-device renames. -}-viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()-viaTmp a file content = bracket setup cleanup use+viaTmp :: (MonadMask m, MonadIO m) => (FilePath -> String -> m ()) -> FilePath -> String -> m ()+viaTmp a file content = bracketIO setup cleanup use   where 	(dir, base) = splitFileName file 	template = base ++ ".tmp"@@ -36,9 +36,9 @@ 		_ <- tryIO $ hClose h 		tryIO $ removeFile tmpfile 	use (tmpfile, h) = do-		hClose h+		liftIO $ hClose h 		a tmpfile content-		rename tmpfile file+		liftIO $ rename tmpfile file  {- Runs an action with a tmp file located in the system's tmp directory  - (or in "." if there is none) then removes the file. -}@@ -61,15 +61,15 @@ {- Runs an action with a tmp directory located within the system's tmp  - directory (or within "." if there is none), then removes the tmp  - directory and all its contents. -}-withTmpDir :: Template -> (FilePath -> IO a) -> IO a+withTmpDir :: (MonadMask m, MonadIO m) => Template -> (FilePath -> m a) -> m a withTmpDir template a = do-	tmpdir <- catchDefaultIO "." getTemporaryDirectory+	tmpdir <- liftIO $ catchDefaultIO "." getTemporaryDirectory 	withTmpDirIn tmpdir template a  {- Runs an action with a tmp directory located within a specified directory,  - then removes the tmp directory and all its contents. -}-withTmpDirIn :: FilePath -> Template -> (FilePath -> IO a) -> IO a-withTmpDirIn tmpdir template = bracket create remove+withTmpDirIn :: (MonadMask m, MonadIO m) => FilePath -> Template -> (FilePath -> m a) -> m a+withTmpDirIn tmpdir template = bracketIO create remove   where 	remove d = whenM (doesDirectoryExist d) $ do #if mingw32_HOST_OS
debian/changelog view
@@ -1,3 +1,34 @@+git-annex (5.20141203) unstable; urgency=medium++  * proxy: New command for direct mode repositories, allows bypassing+    the direct mode guard in a safe way to do all sorts of things+    including git revert, git mv, git checkout ...+  * undo: New command to undo the most recent change to a file+    or to the contents of a directory.+  * Add undo action to nautilus and konqueror integration.+  * diffdriver: New git-annex command, to make git external diff drivers+    work with annexed files.+  * pre-commit: Block partial commit of unlocked annexed file, since+    that left a typechange staged in index due to some infelicity of git's+    handling of partial commits.+  * Work around behavior change in lsof 4.88's -F output format.+  * S3: Switched to using the haskell aws library.+  * S3: No longer buffers entire files in memory when uploading without+    chunking.+  * S3: When built with a new enough version of the haskell aws library,+    supports doing multipart uploads, in order to store extremely large+    files in S3 when not using chunking.+  * Don't show "(gpg)" when decrypting the remote encryption cipher,+    since this could be taken to read that's the only time git-annex+    runs gpg, which is not the case.+  * Debian package is now maintained by Gergely Nagy.+  * Windows: Remove Alt+A keyboard shortcut, which turns out to have scope+    outside the menus.+  * Windows: Install ssh and other bundled programs to Git/cmd,+    instead of Git/bin, since the latter is not in the default msysgit PATH.++ -- Joey Hess <id@joeyh.name>  Wed, 03 Dec 2014 15:16:52 -0400+ git-annex (5.20141125) unstable; urgency=medium    * Remove fixup code for bad bare repositories created by
debian/control view
@@ -14,7 +14,9 @@ 	libghc-cryptohash-dev, 	libghc-dataenc-dev, 	libghc-utf8-string-dev,-	libghc-hs3-dev (>= 0.5.6),+	libghc-aws-dev (>= 0.9.2),+	libghc-conduit-dev,+	libghc-resourcet-dev, 	libghc-dav-dev (>= 1.0) [amd64 i386 kfreebsd-amd64 kfreebsd-i386 powerpc], 	libghc-quickcheck2-dev, 	libghc-monad-control-dev (>= 0.3),@@ -75,7 +77,7 @@ 	openssh-client, 	git-remote-gcrypt (>= 0.20130908-6), 	llvm-3.4 [armel armhf],-Maintainer: Joey Hess <joeyh@debian.org>+Maintainer: Gergely Nagy <algernon@madhouse-project.org> Standards-Version: 3.9.5 Vcs-Git: git://git.kitenet.net/git-annex Homepage: http://git-annex.branchable.com/
+ doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t.mdwn view
@@ -0,0 +1,37 @@+### Please describe the problem.++git-annex assistant starts to drop files in remote repos, even when they are set to manual.++### What steps will reproduce the problem?+++Create 3 repos:++* A -- standard, archive+* m1 -- standard, manual+* m2 -- standard, manual++System has 3 files annexed: file1, file2, file3. Repo "A" has all three files, m1 has none, m2 has file3.++So, while in m1:++    m1 $ git annex find --want-drop --in m2+    file3++file3 shouln't be dropped from m2. There is no reason to do this. m2 is set as manual, and it shouldn't be touched in any case.++Now, let's get this file in m1:++    m1 $ git annex get file3+    get file3 (from m2...) ok+    (Recording state in git...)+    m1 $ git annex find --want-drop --in m2++So when 'file3' is present in local repo, it's not going to be dropped from m2.++I guess that rule 'present' works in local repo context while 'drop' acts on remote files.+++### What version of git-annex are you using? On what operating system?++I'm using latest version in Debian Jessie (5.20141024) on amd64 and armhf. I've also reproduced the bug with manually compiled 5.20141105-g8d8b248 on amd64.
+ doc/bugs/Default_startup_command.mdwn view
@@ -0,0 +1,29 @@+[[!meta title="Android Default startup command"]]++### Please describe the problem.+On a CM11 4.4.4 install, the startup will fail with the default startup command "git annex webapp".+The error message shown is:  +```+Falling back to hardcoded app location; cannot find expected files in /data/app-lib+git annex webapp+```++However, using "git-annex webapp", git-annex will run fine.++### What steps will reproduce the problem?++Launch the beta or nightly build for 4.3 and 4.4, wait for error message++### What version of git-annex are you using? On what operating system?++Version "current" as of 2014-11-15 and also version "nightly build" as of 2014-11-15 on Android 4.4.4 CM11, device: Oppo Find 7a++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]
+ doc/bugs/Fails_to_add_files_with_a_newline_in_their_name___59____41__.mdwn view
@@ -0,0 +1,71 @@+### Please describe the problem.++I know that it is actually a problem with my script and that some might treat it as a feature ;)  but -- apparently I can have such filenames, git also manages to add it no problem (see below)++### What steps will reproduce the problem?+++### What version of git-annex are you using? On what operating system?++5.20141024~bpo70+1++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+$> cd 1000genomes/+total 352+  4 20131219.populations.tsv?         4 CHANGELOG?                    4 phase1/                           4 README.crams?              4 README.sequence_data?+  4 20131219.superpopulations.tsv?  124 changelog_details/            4 pilot_data/                       4 README.ebi_aspera_info?    4 release/+  4 alignment.index?                  0 complete_genomics_indices/    4 README.alignment_data?            4 README.ftp_structure?      4 sequence.index?+  4 alignment_indices/              128 data/                         4 README.analysis_history?          4 README.pilot_data?        16 sequence_indices/+  4 analysis.sequence.index?          4 exome.alignment.index?        4 README.complete_genomics_data?    4 README.populations?        4 technical/+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#9 !5130 [0].....................................:Wed Nov 26 11:40:13:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+$> git annex add  20131219.*+add 20131219.populations.tsv+ git-annex: unknown response from git cat-file ("HEAD:./20131219.populations.tsv missing","HEAD:./20131219.populations.tsv\n")+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#10 !5131 [0].....................................:Wed Nov 26 11:40:36:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+*$> git annex add  '20131219.populations.tsv+'+add 20131219.populations.tsv+ git-annex: unknown response from git cat-file ("HEAD:./20131219.populations.tsv missing","HEAD:./20131219.populations.tsv\n")+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#11 !5132 [0].....................................:Wed Nov 26 11:41:07:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+$> ls -ld '20131219.populations.tsv+'+4 -r-------- 2 yoh yoh 91 Nov 26 10:44 20131219.populations.tsv?+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#12 !5133 [0].....................................:Wed Nov 26 11:41:15:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+$> acpolicy git-annex+git-annex:+  Installed: 5.20141024~bpo70+1+  Candidate: 5.20141024~bpo70+1+  Version table:+ *** 5.20141024~bpo70+1 0+        100 http://debproxy/debian/ wheezy-backports/main amd64 Packages+        100 /var/lib/dpkg/status+     3.20120629 0+        500 http://debproxy/debian/ wheezy/main amd64 Packages+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#13 !5134 [0].....................................:Wed Nov 26 11:41:31:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+*$> git add  '20131219.populations.tsv+'+(git)/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124:1000genomes[master]+#14 !5135 [0].....................................:Wed Nov 26 11:42:58:.+head2:/backup/users/yoh/tmp/nobackup_1000genome-ls.20141124/1000genomes+$> +++# End of transcript or log.+"""]]++> [[dup|done]] of+> [[git_annex_import_fails_on_filenames_with_newlines_in_them]] --[[Joey]]
+ doc/bugs/Git_annex_for_Windows_keyboard_shortcut.mdwn view
@@ -0,0 +1,22 @@+### Please describe the problem.+After git annex been installed, I cannot any longer (with no hacks) use my right ALT+A shortcut creating special char.+After installing git annex for Windows it overtook this shortcut and redirects my browser to a weird localhost server with a weird port.++### What steps will reproduce the problem?+Installing git annex for Windows  and tapping rAlt+A. :)++### What version of git-annex are you using? On what operating system?+The newest version, Windows 8.++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++> Seems I misunderstood, I thought that would only apply in the start menu.+> Removed the shortcut. [[done]] --[[Joey]]
doc/bugs/Internal_server_error_while_adding_remote_server_on_Windows.mdwn view
@@ -27,3 +27,7 @@  # End of transcript or log. """]]++> [[fixed|done]]; git-annex's ssh is now installed to Git/cmd, +> so it will be in the PATH as long as the instructions are followed to let+> msysgit add git to the PATH. --[[Joey]]
doc/bugs/S3_memory_leaks.mdwn view
@@ -2,8 +2,12 @@  Sending a file to S3 causes a slow memory increase toward the file size. +> This is fixed, now that it uses aws. --[[Joey]]+ Copying the file back from S3 causes a slow memory increase toward the file size.++> [[fixed|done]] too! --[[Joey]]  The author of hS3 is aware of the problem, and working on it. I think I have identified the root cause of the buffering; it's done by hS3 so it can
+ doc/bugs/S3_memory_leaks/comment_5_1b6d13852aeb37c5c29baf100bd62789._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"+ nickname="Justin"+ subject="comment 5"+ date="2014-11-30T22:18:56Z"+ content="""+This is getting in my way, but I'm concerned about using a mostly-untested branch (and also, compiling for ARM sounds really unpleasant).  Can you elaborate on the risks here?  Can I perhaps donate developer time to get this merged into mainline?+"""]]
+ doc/bugs/S3_memory_leaks/comment_6_71ffb88f548969fd4d45eac4656380c0._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="joey"+ subject="""comment 6"""+ date="2014-12-02T17:10:42Z"+ content="""+The s3-aws branch is well tested now.++I am blocked from merging it by some dependency issues, in particular +<http://hackage.haskell.org/package/dbus> needs to have a small dependency+fix. I have emailed its author a second time now to try to get that done.+"""]]
doc/bugs/S3_upload_not_using_multipart.mdwn view
@@ -52,3 +52,11 @@ 	upgrade supported from repository versions: 0 1 2  [[!tag confirmed]]++> [[fixed|done]] This is now supported, when git-annex is built with a new+> enough version of the aws library. You need to configure the remote to+> use an appropriate value for multipart, eg:+> +> git annex enableremote cloud multipart=1GiB+> +> --[[Joey]]
+ doc/bugs/Unclear_output_during_git-annex_copy.mdwn view
@@ -0,0 +1,17 @@+I noticed the following output during a `git-annex sync --content` run+which I found a little disconcerting:++	copy file.idx copy file.idx (gpg) (checking s3...) (to s3...) +	ok+	copy file.pack copy file.pack (checking s3...) (to s3...) +	ok+	(Recording state in git...)++It's confusing that seemingly only the first file got encrypted (gpg).++Fortunately, it appears that actually all files were encrypted (only GPGHMAC file names in remote location),+so this is hopefully just a cosmetic issue.++git-annex version: 5.20141024-g613f396++> removed confusing output, [[done]] --[[Joey]]
+ doc/bugs/Uninstalling_removes_libcrypto.dll.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.+Just feedback to you:+After I uninstalled git-annex (Windows), traditional git was not able to push changes to an origin server via ssh.+Instead I got an error "libcrypto.dll is missing".+Installing git once again after uninstalling git-annex solved the problem.++### What steps will reproduce the problem?+Installing and uninstalling git-annex.++### What version of git-annex are you using? On what operating system?+File "git-annex-installer.exe" downloaded 2014-11-09.+Windows 8.++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++> [[fixed|done]]; git-annex no longer includes its own ssh, and uses the+> one from msysgit. --[[Joey]]
+ doc/bugs/annex.autocommit_seems_ignored_for_new_files.mdwn view
@@ -0,0 +1,39 @@+### Please describe the problem.++When `annex.autocommit` is set to `false`, the assistant continues to commit to the repository, at least in indirect mode (direct mode not tested). I'd like it to not even `git annex add`, but it should at least not `git commit`.++### What steps will reproduce the problem?++I started the assistant with `git annex assistant`. Then I also opened the webapp with `git annex website`.++Next, I ran `git config --bool --local --replace-all annex.autocommit false`. Supposedly that'll stop the assistant from committing things.++Then I did a `echo "hello world" > test`.++The webapp showed the file syncing, and `ls -l test` confirmed it'd been added the annex. `git status` confirmed it'd been committed as well.++I did a `git rm test`, and then a `git status`... and it turns out the annex committed that as well.++NOTE: My repository is in indirect mode.++I tried `--global` instead of `--local` for the config, and that didn't make a difference.+++### What version of git-annex are you using? On what operating system?++5.20141125 on Debian testing, from the package. (And, thank you Joey, for all your work on Debian. You'll be missed.)++### Please provide any additional information below.++> The assistant does not notice changes to the +> git configuration that are made while it's already running.+> +> I have verified that setting `git config annex.autocommit true`+> and then starting the assistant behaves as documented.+>+> So will asking the webapp to restart.+> +> Retitling bug report appropriately. --[[Joey]]++[[!meta title="git-annex assistant could watch .git/config and react to config changes on the fly"]]+[[!tag confirmed]]
+ doc/bugs/assistant_windows_does_not_start.mdwn view
@@ -0,0 +1,35 @@+### Please describe the problem.++webapp does not start. ++Reports the following error:++Script: c:\Programm Files(x86)\Git\git-annex-webapp.vbs+Line: 2+Character: 1+Error: System can not fine the specified File+Code: 80070002+Source: null+ ++### What steps will reproduce the problem?++1. Downloaded the installer. +2. Run the installer+3. In folder c:\Programm Files(x86)\Git doubleclick on git-annex-webapp+4. Content vbs: +    Set objshell=CreateObject("Wscript.Shell")+    objShell.Run("git-annex webapp"), 0, False+4. +### What version of git-annex are you using? On what operating system?+Version 2014-11-05 14:38 +Server  Windows Server 2008 R2+### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]
+ doc/bugs/fsck_of_special_remotes_incorrectly_reports_ok.mdwn view
@@ -0,0 +1,35 @@+### Please describe the problem.++I am trying to verify that my files are where they should after having updated my remotes to use chunking. I do get the following two error messages, but then git-annex reports that everything is Ok. I find it slightly scary and thus wonder if the fsck command in all cases do a conservative enough check. All other files (except two other ones where it noticed that they where in fact not on the remote do not report any errors.)++When running the fsck command for all files I got the following slightly worring message for one file.++    fsck Ekonomi/2014-06-02_Elavräkning.pdf (checking box.com5...) (checking box.com5...) (checking box.com5...) +    79%         23.2KB/s 26s+      Error_Packet "partial packet: expecting 16448 bytes, got: 891"+    ok+                      +Reading the message it seems that the file was not correctly downloaded, but still deemed Ok enough by git-annex. ++Further while writing this report I get the following message from a remote  using the external protocol:+ +    fsck Deklaration/archive/2012/DE/Steuerklärung Formular.pdf +    93%        221.9KB/s 12s+      /Users/carlmod/Annex/.git/annex/tmp/GPGHMACSHA1--0e7e4951e2650ac636bc3fc17c673f1f2dfb3071: openBinaryFile: does not exist (No such file or directory)+    ok                      ++Source code for this remote is not yet published online, as i would want to build it slightly more reliable before releasing it. It however passes the remote test suite without any problem. +### What steps will reproduce the problem?++Not really sure, but rerunning the commands show that the openBinaryFile happens here for this particular file. ++### What version of git-annex are you using? On what operating system?++    $ git annex version+    git-annex version: 5.20141125+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external++built using Homebrew (i.e. in a cabal sandbox) on OS X Yosemite 10.10.1+
+ doc/bugs/git_annex_sync_--content_may_copy_then_drop_a_file_to_a_remote.mdwn view
@@ -0,0 +1,33 @@+### Please describe the problem.++When using `git annex sync --content` with two archive with different cost, git annex will upload my file to the first archive, then to the second one, then it will drop it from the first one.++### What steps will reproduce the problem?++I've set both remote to "trusted", both in the "archive" group, both with "standard" preferred content, the remote named "hubic2" as an annex-cost of 200.0, and the remote named "hubic3" as an annex-cost of 175.0++### What version of git-annex are you using? On what operating system?+I'm using git-annex 5.20141125 on Debian Sid (upgraded today on November 7 2014).++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++$ git annex list | head+here+|hubic2+||hubic3+|||+X__ Avatars/archer.jpg+$ git annex sync --content+commit  ok+copy Avatars/archer.jpg copy Avatars/archer.jpg (to hubic3...)+ok+copy Avatars/archer.jpg copy Avatars/archer.jpg (to hubic2...)+ok+drop hubic3 Avatars/archer.jpg ok++# End of transcript or log.+"""]]
+ doc/bugs/local_pair_fails_if_non-ascii_characters_present_on_annex_path.mdwn view
@@ -0,0 +1,16 @@+### Please describe the problem.++When the annex directory has a non-ascii character (like a tilde) on its path, local pairing fails and a couple of instances of the following message appear on the log++"illegal control characters in pairing message; ignoring"++### What steps will reproduce the problem?++* make a new annex repo named on /home/someone/Vídeos/blah/ (notice the i with a tilde) on two computers+* try to pair them++### What version of git-annex are you using? On what operating system?++git-annex version: 5.20141016-g26b38fd on Arch Linux++git-annex version: 5.20140717 on Ubuntu 14.10
+ doc/bugs/main_repo_not_available_on_downloads.kitenet.net.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.++The [downloads.kitenet.net](http://downloads.kitenet.net/) site is really a git annex repository, yet it is not possible to "git annex get" the files from there.++I regularly have to do something like this to workaround that issue:++    git annex addurl --file git-annex-standalone-amd64.tar.gz https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz++Shouldn't this be done automatically?++### What steps will reproduce the problem?++    git clone https://downloads.kitenet.net/+    cd downloads.kitenet.net/+    git annex get git-annex/linux/++### What version of git-annex are you using? On what operating system?++5.20141024 on Debian Jessie.++### Please provide any additional information below.++Thanks! -- [[anarcat]]++> [[done]]; apparently some pebak. --[[Joey]]
doc/bugs/modified_permissions_persist_after_unlock__44___commit.mdwn view
@@ -38,3 +38,6 @@  [[!tag confirmed]] [[!meta title="git commit of unlocked file leaves typechange staged in index"]]++> [[fixed|done]]; git-annex pre-commit will block such commits from being+> made. Git devs didn't seem to have a way to improve its behavior. --[[Joey]]
doc/bugs/new_AWS_region___40__eu-central-1__41__.mdwn view
@@ -6,3 +6,5 @@ * Region: eu-central-1  This should be added to the "Adding an Amazon S3 repository" page in the Datacenter dropdown of the webapp.++> [[fixed|done]] --[[Joey]]
+ doc/bugs/proxy_command_fails_on_Windows.mdwn view
@@ -0,0 +1,36 @@+### Please describe the problem.++The new ''git annex proxy'' command fails every call in the 2014-11-28 daily Windows build.++### What steps will reproduce the problem?++1) Install daily Windows build via https://downloads.kitenet.net/git-annex/autobuild/windows/.++2) Run, in git bash:++    $ git annex proxy -- pull+    git-annex: pull: createProcess: does not exist (No such file or directory)+    failed+    git-annex: proxy: 1 failed++### What version of git-annex are you using? On what operating system?++Daily Windows build 5.20141128-g70f997e on Windows 7 x64++### Please provide any additional information below.++    $ git annex version+    git-annex version: 5.20141128-g70f997e+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feed+    s Quvi TDFA CryptoHash+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SH+    A256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar ho+    ok external+    local repository version: 5+    supported repository version: 5+    upgrade supported from repository versions: 2 3 4++> Seems the man page was mis-read, or possibly I've fixed it,+> since it certianly says you need to specify "-- git cmd", not+> just "-- cmd". [[done]] --[[Joey]]
+ doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config.mdwn view
@@ -0,0 +1,26 @@+### Please describe the problem.++I am using a ssh-host configured in .ssh/config in a corporate-environment like this:++[[!format sh """+  Host home+        Hostname 1.2.3.4+        Port 443+        ProxyCommand corkscrew proxy.fqdn 8080 %h %p ~/.authfile+        # http://www.agroman.net/corkscrew/+"""]]++I now want to be able to add this host as a remote repository, but don't have the ability to add it. A tunnel to localhost would be possible but then I would have to setup this portforwarding for git-annex separately every time it is started.++### What steps will reproduce the problem?++Add a remote host that is only reachable through a proxy.++### What version of git-annex are you using? On what operating system?++gentoo+git-annex-5.20141125 {+assistant +cryptohash +dbus +desktop-notify +dns +feed +inotify +pairing +production +quvi +s3 +tahoe +tdfa +testsuite +webapp +webapp-secure +webdav +xmpp}++++Is it possible to use this predefined host, or add the possibility to configure git-annex like above?
doc/builds.mdwn view
@@ -15,8 +15,8 @@ <h2>Android</h2> <iframe width=1024 height=20em scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/android/4.3/build-version"> </iframe>-<h2>OSX Mavericks</h2>-<iframe width=1024 height=20em scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/build-version">+<h2>OSX Yosemite</h2>+<iframe width=1024 height=20em scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/build-version"> </iframe> <h2>Windows</h2> <iframe width=1024 height=20em scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/windows/build-version">@@ -37,8 +37,8 @@ <h2>Android</h2> <iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/android/"> </iframe>-<h2>OSX Mavericks</h2>-<iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/">+<h2>OSX Yosemite</h2>+<iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/"> </iframe> <h2><a href="https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/">Windows</a></h2> <a href="https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/">here</a>
doc/design/assistant/polls/Android_default_directory.mdwn view
@@ -4,4 +4,4 @@ want the first time they run it, but to save typing on android, anything that gets enough votes will be included in a list of choices as well. -[[!poll open=yes expandable=yes 69 "/sdcard/annex" 6 "Whole /sdcard" 7 "DCIM directory (photos and videos only)" 2 "Same as for regular git-annex. ~/annex/"]]+[[!poll open=yes expandable=yes 71 "/sdcard/annex" 6 "Whole /sdcard" 7 "DCIM directory (photos and videos only)" 2 "Same as for regular git-annex. ~/annex/"]]
+ doc/devblog/day_232__OSX_Yosemite.mdwn view
@@ -0,0 +1,6 @@+The OSX autobuilder has been updated to OSX 10.10 Yosemite. The +[resulting build](https://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg)+might also work on 10.9 Mavericks too, and I'd appreciate help testing that.++Went ahead and fixed the [[partial commit problem|bugs/modified_permissions_persist_after_unlock__44___commit]]+by making the pre-commit hook detect and block problimatic partial commits.
+ doc/devblog/day_233__direct_mode_proxy.mdwn view
@@ -0,0 +1,29 @@+Ever since the direct mode guard was added a year ago, direct mode has been+a lot safer to use, but very limited in the git commands that could be run+in a direct mode repository.++The worst limitation was that there was no way to `git revert` unwanted+changes. But also, there was no way to check out different branches,+or run commands like `git mv`.++Today I made `git annex proxy`, which allows doing all of those things,+and more. [documentation here](http://git-annex.branchable.com/direct_mode/#index5h2)++It's so flexible that I'm not sure where the boundries lie yet, but it+seems it will work for any git command that updates both the work tree and+the index. Some git commands only update one or the other and not both and+won't work with the proxy. As an advanced user tool, I think this is a+great solution. I still want to make a simpler+[[undo command|todo/direct_mode_undo]] that can nicely integrate into file+managers.++The implementation of `git annex proxy` is+[quite simple](http://source.git-annex.branchable.com/?p=source.git;a=blob;f=Command/Proxy.h), because it+reuses all the complicated work tree update code that was already written+for `git annex merge`.++----++And here's the lede I buried: I've gotten two years of funding to work+on git-annex part-time!+[Details in my personal blog](http://joeyh.name/blog/entry/continuing_to_be_pleasantly_surprised/).
+ doc/devblog/day_234__undo_undo.mdwn view
@@ -0,0 +1,6 @@+Built the `git annex undo` command. This is intended to be a simple+interface for users who have changed one file, and want to undo the change+without the complexities of `git revert` or `git annex proxy`. It's simple+enough that I added undo as an action in the file manager integration.++And yes, you can undo an undo. :)
+ doc/devblog/day_235__thanksgiving_backlog.mdwn view
@@ -0,0 +1,18 @@+Back from the holiday, catching up on traffic. Backlog stands at 113+messages.++Here's a nice tip that Giovanni added:+[[tips/publishing_your_files_to_the_public]] (using a public S3 bucket)++Just before going on break, I added a new feature that I didn't mention+here. `git annex diffdriver` integrates git-annex with git's external diff+driver support. So if you have a smart diff program that can diff, say,+genome sequences, or cat videos, or something in some useful way, it+can be hooked up to `git diff` and will be able to see the content of+annexed files.++Also today, I spent a couple hours today updating the license file included+in the standalone git-annex builds to include the licenses of all the+haskell libraries git-annex depends on. Which I had for some reason not+thought to include before, despite them getting built into the git-annex+binary.
doc/direct_mode.mdwn view
@@ -3,9 +3,14 @@ `.git/annex/objects/`. Direct mode gets rid of the symlinks.  The advantage of direct mode is that you can access files directly,-including modifying them. The disadvantage is that most regular git-commands cannot be used in a direct mode repository.+including modifying them. The disadvantage is that many regular git+commands cannot be used in a direct mode repository, since they don't+understand how to update its working tree. +[[!toc]]++## enabling (and disabling) direct mode+ Normally, git-annex repositories start off in indirect mode. With some exceptions: @@ -15,8 +20,6 @@   into direct mode. * Windows always uses direct mode. -## enabling (and disabling) direct mode- Any repository can be converted to use direct mode at any time, and if you decide not to use it, you can convert back to indirect mode just as easily. Also, you can have one clone of a repository using direct mode, and another@@ -82,11 +85,29 @@ example, you can run `git log` on files, run `git push`, `git fetch`, `git config`, `git remote add` etc. +## proxing git commands in direct mode++For those times when you really need to run a command like `git revert+HEAD` in a direct mode repository, git-annex has the ability to proxy+the command to work in direct mode.++For example:++	git annex proxy -- git revert HEAD++	git annex proxy -- git checkout HEAD^^++	git annex proxy -- git mv mydir newname++This works by setting up a temporary work tree, letting the git+command run on that work tree, and then updating the real work+tree to reflect any changes staged or committed by the git command,+with appropriate handling of the direct mode files.+ ## forcing git to use the work tree in direct mode  This is for experts only. You can lose data doing this, or check enormous files directly into your git repository, and it's your fault if you do!-Also, there should be no good reason to need to do this, ever.  Ok, with the warnings out of the way, all you need to do to make any git command access the work tree in direct mode is pass it
+ doc/forum/Bare_repo_on_USB_drive_not_providing_files.mdwn view
@@ -0,0 +1,17 @@+Dear all,+     In the first place I will explain what I want to achieve and then the way I figured out to do it. I have a lot of scientific simulations which I need to be traceable (cross check experiment version with code version, etc). I run them mostly on my ubuntu box (14.10, ext4)  and store them on an indirect repo. From time to time, I find useful to work on them on my laptop (OSX 10.10, low disk space, hfs+). +My idea was to full sync the main repo with a usb drive, and then at home get only the files I need to work from the USB drive.+I faced a huge interoperability problem regarding the filesystem format. I tried using ext4, hfs+, fat32 and exfat. Regarding performance and stability, exfat is the only usable option between linux and osx (using hfs+ got the repo corrupted many times while writing from linux). However that forces me to use either a direct mode repo on the USB drive or a bare one. ++Scenario 1: Direct mode on USB drive +While this worked in another case, in this case when setting up the repo with the assistant it deleted all symlinks from the main repo. I wasn't able to correctly do this and finally gave up. +Can anyone help me setting up this scenario? It will be like the following: ++* ubuntu box (ext4): main repo, indirect mode, all files+* usb drive (exfat): sync repo, direct mode, all files (having the files available is a plus)+* osx laptop (hpfs+): partial repo, indirect mode, some files+++Scenario 2: Bare repo on USB drive +In this case the bare repo gets synced, with all data (2.6Gb). However when I clone it on the laptop, I get the  message that files are not available, it says that they are only on main repo. This puzzles me because files seem to be on the bare repo.+What's wrong in this case? How can I debug this?
+ doc/forum/Recovery_after_freeze_while_importing_files.mdwn view
@@ -0,0 +1,3 @@+My PC froze while git-annex was in the process of importing a file. It was in the final stage of updating git status when the freeze occurred. git status shows the imported file under "changes to be committed" as "new file."++I don't know how git-annex works under the hood, so I don't know how the freeze might've affected the repo. I don't want to lose my files. What do you recommend is the best way of recovering from this and making sure there are no errors?
+ doc/forum/Shared_Encryption_does_not_Work_as_Expected.mdwn view
@@ -0,0 +1,11 @@+I recently started to set up a share that would be synced between multiple machines.  Since I control all of them I thought it might be cool to try using the suggested "shared encryption".  This seems to have been a mistake.++My project folder is now a big mess of symlinks to read-only files.  ++I can fix this with a "cp -RH" and get my data back and I can muddle through fixing the permissions but this leads me to two conclusions:++1.  I did something wrong but I don't know what.  Should I have created the ssh remote first and the local repo second?  ++2.  This feature should either be hidden or the UI should be much more explicit about how to use it properly.  This cost me quite a bit of time and frustration.++I really like git-annex so I would appreciate any comments or suggestions.
+ doc/forum/Standards_usage_pattern__63__.mdwn view
@@ -0,0 +1,8 @@+Hi,++I have few devices: 2x android, 2x windows, 2x linux and I would like to share at least one folder on all of them.+None of the devices have public IPs and some regularly change location (work laptops). Because the XMPP support is missing on windows I couldn't find a way to synchronize the repositories (the keys) or did I miss something in the documentation?+How can I archieve "seamless" synchronization when I start the webapp on the devices?++Thank you,+Karol
+ doc/forum/Update_on_remote_repo.mdwn view
@@ -0,0 +1,6 @@+Hello,++using the git-annex assistant I can activate auto update on the client machines. What about the server that serves as transfer repository? It needs to have git-annex installed. Is there any possibility of auto update there? What is the recommended procedure?++Thanks,+Florian
+ doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks.mdwn view
@@ -0,0 +1,27 @@+Dear all,+        I had (snif!) and indirect repo with 2.7 of pdf files with filenames were manually added by me with author and title, and took years of work. I had them in a git annex repo from quite some time.  +Now I'm trying to sync them to an usb drive. Main repo lives in an ubuntu box with ext4 filesystem. I failed many times to do this, both on fat and exfat external drives (direct).  +As I was failing, I tried to to a test on the same local filesystem. So I have my main repo in /home/juan/papers and the other in /tmp/papers.  +While trying to sync them using the assistant, i noticed that most of the folders and symlinks from the main repo just dissapeared. +I still have them in .git/annex/objects but I don't know how to recover the links. +I must say I'm an everyday git user, and big fan of git-annex but I'm trying to use it for six months without luck. I think I will quit after recovering the data, if possible.++EDIT 1+-------+After digging around a bit, I found that I had the keys, and could find out the original filename, for example:++    git log --stat -S 1279+    commit a6362df13d728037875320a457182acb073d08ea+    Author: Juan Cardelino <x@gmail.com>+    Date:   Fri Nov 21 11:25:31 2014 -0200++     2013_august/feature_matching_by_searching_maximum_clique_on_high_order_association.pdf | 1 ++     LevelSets/fedkiw/stanford2002-06.pdf                                                   | 1 ++     new/nfa/00106994.pdf                                                                   | 1 ++     3 files changed, 3 insertions(+)++How can I recover them?++EDIT 2+-------+Seems that I overwritten my repo with a new repo. Because git log only shows 4 entries from today. Is it possible that the webapp has made that?
+ doc/forum/metadata_limits.mdwn view
@@ -0,0 +1,10 @@+Hello,++I was wondering what `The metadata values can contain absolutely anything you like -- but you're recommended to keep it simple and reasonably short.` listed on the [[metadata]] page really means?+++Here is my use case (which is part of a larger solution)...++I've been storing my photos in git annex for a while (years perhaps), I (or actually the wife) tried to find one and couldn't, so I'm thinking I need to add metadata (Tags and captions, etc) to each picture. I was looking for ways to do so. I was actually considering using gthumb or shotwell or similar to create the tags and somehow pull that out and associate it with the file (either with a plugin or program) and then sort out how to distribute the metadata with the pics. The git annex metadata seems like a reasonable solution for keeping metadata with the pictures. The question is in this case, how long can the text in say a caption tag before git-annex has problems? I could probably do something like caption-<seq> to handle really long captions, but then captions are short by definition, so perhaps I should say description instead. And for this particular use case, I am aware this doesn't solve the problem of actually creating the data, just trying to sort out if get annex metadata to the right solution to tie the metadata to a pic, err file.++
+ doc/forum/possible_gpg_issue.mdwn view
@@ -0,0 +1,28 @@+Hi,++I'm having an issue and not sure if it's to do with git-annex or something external. I'm on Ubuntu 14.04, and have been using git-annex for several months without this issue, but it suddenly started (possibly) after several packages were updated via apt-get upgrade.++We have two remotes configured, one is a local (LAN) smb share, the other is on Amazon S3. We're using shared encryption on the S3 remote, and no encryption on the smb remote.++The problem that started happening recently is when copying to the smb remote. There is no problem copying from the remote, and no problem reading or writing to the drive outside of git-annex. However, copying to the remote fails after it seems to copy most or all of the file and then hang on a gpg step. Again, there is no encryption on this remote. The other S3 remote with shared encryption has no issues. Other devices on the LAN, all OS X, have no issues writing to the remote.++I don't have enough info to necessarily claim this is a bug in git-annex, but I'm not sure what to poke at next to try to figure it out. Any help or advice would be greatly appreciated.++Below is the debug output from a failed git annex copy command.++    cw@ubuntu$ git annex copy annexedfile --to smbremote --debug+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","show-ref","git-annex"]+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","show-ref","--hash","refs/heads/git-annex"]+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","log","refs/heads/git-annex..aa8813d486939544701359dc28fa7b0916917961","--oneline","-n1"]+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","log","refs/heads/git-annex..097d5b482d6856ce22814a0c2c5eee43e3e030e4","--oneline","-n1"]+    [2014-11-07 15:35:13 PST] chat: git ["--git-dir=/repobase/.git","--work-tree=/repobase","cat-file","--batch"]+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","ls-files","--cached","-z","--","annexedfile"]+    copy annexedfile (gpg) (to smbremote...) +    [2014-11-07 15:35:13 PST] chat: gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","11","--symmetric","--force-mdc","--no-textmode"]+    95%           0.0 B/s 0s/mnt/annex/tmp/GPGHMACSHA1--a097a9b653d1facbe7d37d0e8f9f580261d9adef/GPGHMACSHA1--a097a9b653d1facbe7d37d0e8f9f580261d9adef: hClose: does not exist (Host is down)+    failed                  +    git-annex: copy: 1 failed+    cw@ubuntu$++Thanks,+cw
+ doc/forum/repair_stuck_on_ls-tree_command.mdwn view
@@ -0,0 +1,54 @@+Hi,++I have run the git annex repair command on one of my repositories and it has been running for 38h hours now. I have monitored the process and its sons with htop and it seems that git annex repair is stuck.++It repeatedly launches the following command: ++    git --git-dir=/home/vincent/photos2/.git --work-tree=/home/vincent/photos2 ls-tree --full-tree -z -r -- 5fe5193e079631c0ceac0688ae0a6c1636491b61++I have tried to execute it by hand and it produces a lot of output. I have redirected it into some file in order to count the content produced by this command.++    vincent@berlioz:~/photos2/.git$ git --git-dir=/home/vincent/photos2/.git --work-tree=/home/vincent/photos2 ls-tree --full-tree -z -r -- 5fe5193e079631c0ceac0688ae0a6c1636491b61 > /tmp/ls-tree+    vincent@berlioz:/tmp$ wc ls-tree +           0   232525 11611220 ls-tree++Could this cause some troubles to git-annex ? Maybe my repository is just too big and it demands some time to deal with the whole thing?++Before doing the git annex repair command I have tried to delete a distant annex (hosted on a raspberry pi, running an arm raspbian) from this one, using the git annex webapp website. Since it was kind of stuck I asked webapp to shutdown. I ran git annex repair for some time. It looked like stuck. I stopped it, then tried git fsck. It ran well. I tried git annex fsck, it took all my RAM and swap. The process was eating 14 Giga bytes of memory. I tried git gc and the same memory problem occured. I ran git annex repair again and it has been running for 38 hours now. I am lost. Please help.++    vincent@berlioz:~$ LANG=C apt-cache policy git-annex+    git-annex:+      Installed: 5.20141125+      Candidate: 5.20141125+      Package pin: 5.20141125+      Version table:+     *** 5.20141125 950+            400 http://ftp.fr.debian.org/debian/ testing/main amd64 Packages+             50 http://ftp.fr.debian.org/debian/ unstable/main amd64 Packages+            100 /var/lib/dpkg/status+         3.20120629 950+            900 http://ftp.fr.debian.org/debian/ wheezy/main amd64 Packages++    vincent@berlioz:~/photos2$ du -sh .+    152G	.++Info gathered from another client annex:++    vincent@tigrou:/mnt/mars/images/photos2$ git annex info+    repository mode: indirect+    trusted repositories: 0+    semitrusted repositories: 11+        8<------------skipped repositories list------------->8+    untrusted repositories: 1+    	a357bb9e-7a9d-4112-b74c-13707e2c7f85 -- vincent@rhett:~/photos [gateway]+    transfers in progress: none+    available local disk space: 945.91 gigabytes (+1 megabyte reserved)+    bad keys size: 368 bytes (clean up with git-annex unused)+    local annex keys: 76070+    local annex size: 383.91 gigabytes+    annexed files in working tree: 59117+    size of annexed files in working tree: 319.73 gigabytes+    bloom filter size: 16 mebibytes (15.2% full)+    backend usage: +            SHA256E: 101902+            SHA256: 33285
@@ -0,0 +1,5 @@+Dear all,+   I have a repository in direct mode which still has some symlinks pointing to .git/annex/objects. Is there a way to turn them back into regular files. I don't recall why this happen, I guess I started on indirect mode and after that I converted it to direct mode.+Thanks in advance.+Best regards,+              Juan
+ doc/forum/weird_view_file_naming.mdwn view
@@ -0,0 +1,13 @@+Very quick one, if anyone has to share experience with views (i would find them useful to get partial checkout views, as long as I can find a special setup to preserve the tree/filenames hyerarchy): ++by using (filtering) views I am getting file names changed to include the (orignal) dir name,+for instance:++2008/pippo.doc++after a git annex view 2008/=*, comes out as pippo_%2008%.doc+is this avoidable ?++(i've been trying to follow metadata walkthrough etc. but i always get the dirname in the filename between percents...)++btw: version is 5.20141105
doc/git-annex.mdwn view
@@ -282,6 +282,22 @@   are on a video hosting site, and the video is downloaded. This allows   importing e.g., youtube playlists. +* `undo [filename|directory] ...`++  When passed a filename, undoes the last change that was made to that+  file.++  When passed a directory, undoes the last change that was made to the+  contents of that directory.++  Running undo a second time will undo the undo, returning the working+  tree to the same state it had before. In order for undoing an undo of+  staged changes, any staged changes are first committed by the+  undo command.++  Note that this does not undo get/drop of a file's content; it only+  operates on the file tree committed to git.+ * `watch`    Watches for changes to files in the current directory and its subdirectories,@@ -499,9 +515,9 @@    As part of the switch to direct mode, any changed files will be committed. -  Note that git commands that operate on the work tree are often unsafe to-  use in direct mode repositories, and can result in data loss or other-  bad behavior.+  Note that git commands that operate on the work tree will refuse to+  run in direct mode repositories. Use `git annex proxy` to safely run such+  commands.  * `indirect` @@ -944,6 +960,25 @@   Most MATCHING OPTIONS can be used with findref, to limit the files it   finds. However, the --include and --exclude options will not work. +* `proxy -- git cmd [options]`++  Only useful in a direct mode repository, this runs the specified git+  command with a temporary work tree, and updates the working tree to+  reflect any changes staged or committed by the git command.+  +  For example, to revert the most recent change that was committed+  to the repository:++    git annex proxy -- git revert HEAD++  To check out a past version of the repository:++    git annex proxy -- git checkout HEAD^^++  To rename a directory:++    git annex proxy -- git mv mydir newname+ * `resolvemerge`    Resolves a conflicted merge, by adding both conflicting versions of the@@ -953,6 +988,21 @@   Note that only merge conflicts that involve an annexed file are resolved.   Merge conflicts between two files that are not annexed will not be   automatically resolved.++* `diffdriver`++  This is an external git diff driver shim. Normally, when using `git diff`+  with an external git driver, the symlinks to annexed files are not set up+  right, so the external git driver cannot read them in order to perform+  smart diffing of their contents. This command works around the problem,+  by passing the fixed up files to the real external diff driver.++  To use, just configure git to use "git-annex diffdriver -- cmd params --"+  as the external diff command, where cmd is the real external diff+  command you want to use, and params are any extra parameters to pass+  to it. Note the trailing "--", which is required.++  For example, set `GIT_EXTERNAL_DIFF=git-annex diffdriver -- j-c-diff --`  * `remotedaemon`
doc/install/Docker.mdwn view
@@ -13,6 +13,10 @@ Since the Android autobuilder container can take quite a lot of work to get built, it is published. `docker pull joeyh/git-annex-android-builder` +So's the armel autobuilder container. +`docker pull joeyh/git-annex-armel-builder`, and its companion container+`docker pull joeyh/git-annex-armel-builder-companion`+ # building autobuilder containers using Propellor  The Docker containers are built using@@ -22,10 +26,9 @@ [[!format haskell """ import qualified Propellor.Property.SiteSpecific.GitAnnexBuilder as GitAnnexBuilder -	, host hostname@"your.machine.net" = Just $ props+	, host "your.machine.net" 	        & Docker.configured-		& Docker.docked container hostname "amd64-git-annex-builder"-	, GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h"+		& Docker.docked (GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h") """]]  This will autobuild every hour at :15, and the autobuilt image will be
doc/install/OSX.mdwn view
@@ -3,7 +3,7 @@ [[!img /assistant/osx-app.png align=right link=/assistant]] For easy installation, use the prebuilt app bundle. -* 10.9 Mavericks: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.9_Mavericks/git-annex.dmg)+* 10.10 Yosemite / 10.9 Mavericks: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg) * 10.8.2 Mountain Lion: [git-annex.dmg.bz2](https://downloads.kitenet.net/git-annex/OSX/current/10.8.2_Mountain_Lion/git-annex.dmg.bz2) **warning: not being updated any longer** * 10.7.5 Lion: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.7.5_Lion/git-annex.dmg) **warning: not being updated any longer** @@ -20,10 +20,11 @@  ## autobuilds -[[Joey]] autobuilds the app for Mavericks, thanks to Kevin McKenzie-for hosting the autobuilder.+[[Joey]] autobuilds the app for 10.10 Yosemite+(also reported to work on 10.9 Mavericks).+Thanks to Kevin McKenzie for hosting the autobuilder. -* [autobuild of git-annex.dmg](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/git-annex.dmg) ([build logs](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/))+* [autobuild of git-annex.dmg](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/git-annex.dmg) ([build logs](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/))  ## using Homebrew
doc/install/Windows.mdwn view
@@ -36,5 +36,5 @@ 	cd gitannex 	build -(To build the git-annex installer, you also need to install the NulSoft+(To build the git-annex installer, you also need to install the NullSoft installer system.)
+ doc/internals/lockdown/comment_1_caa8b378e8c35177083e6cc2f6937180._comment view
@@ -0,0 +1,15 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnLM6CIUo2loSFNTLPDdBgAEoz9ehP2hz8"+ nickname="Duncan"+ subject="Having the write bit not set can cause problems with automated usage (e.g. build systems)"+ date="2014-12-02T23:16:57Z"+ content="""+I'm using a git-annex to store build artefacts on a remote bare repo. Some of these artefacts are used in subsequent builds, which clone the artefacts repo, and use 'git annex get' to retrieve the artefacts of interest. ++Unfortunately, I've had to add a little kludge along the following lines to my build script fragment:++    git annex get ${file}+    find .git/annex/objects -type d -exec chmod +w {} \;++This is necessary because I need to ensure that the cloned git repo is able to be deleted at all times (I'm using yocto/openembedded and it may want to delete the clone for a variety of reasons).+"""]]
doc/metadata.mdwn view
@@ -21,7 +21,7 @@  The field names are limited to alphanumerics (and `[_-.]`), and are case insensitive. The metadata values can contain absolutely anything you-like -- but you're recommended to keep it simple and reasonably short.+like -- but you're recommended to keep it simple and [[reasonably short|forum/metadata_limits]]  Here are some metadata fields that git-annex has special support for:
− doc/news/version_5.20140926.mdwn 
@@ -1,5 +0,0 @@-git-annex 5.20140926 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Depend on new enough git for --no-gpg-sign to work. Closes: #[762446](http://bugs.debian.org/762446)-   * Work around failure to build on mips by using cabal, not Setup,-     to build in debian/rules."""]]
− doc/news/version_5.20140927.mdwn 
@@ -1,6 +0,0 @@-git-annex 5.20140927 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * Really depend (not just build-depend) on new enough git for --no-gpg-sign-     to work. Closes: #[763057](http://bugs.debian.org/763057)-   * Add temporary workaround for bug #763078 which broke building on armel-     and armhf."""]]
+ doc/news/version_5.20141203.mdwn view
@@ -0,0 +1,28 @@+git-annex 5.20141203 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * proxy: New command for direct mode repositories, allows bypassing+     the direct mode guard in a safe way to do all sorts of things+     including git revert, git mv, git checkout ...+   * undo: New command to undo the most recent change to a file+     or to the contents of a directory.+   * Add undo action to nautilus and konqueror integration.+   * diffdriver: New git-annex command, to make git external diff drivers+     work with annexed files.+   * pre-commit: Block partial commit of unlocked annexed file, since+     that left a typechange staged in index due to some infelicity of git's+     handling of partial commits.+   * Work around behavior change in lsof 4.88's -F output format.+   * S3: Switched to using the haskell aws library.+   * S3: No longer buffers entire files in memory when uploading without+     chunking.+   * S3: When built with a new enough version of the haskell aws library,+     supports doing multipart uploads, in order to store extremely large+     files in S3 when not using chunking.+   * Don't show "(gpg)" when decrypting the remote encryption cipher,+     since this could be taken to read that's the only time git-annex+     runs gpg, which is not the case.+   * Debian package is now maintained by Gergely Nagy.+   * Windows: Remove Alt+A keyboard shortcut, which turns out to have scope+     outside the menus.+   * Windows: Install ssh and other bundled programs to Git/cmd,+     instead of Git/bin, since the latter is not in the default msysgit PATH."""]]
+ doc/news/version_5/comment_1_ae8c464ccb7b7b218efd9151c5bbd1a5._comment view
@@ -0,0 +1,7 @@+[[!comment format=mdwn+ username="joey"+ subject="""comment 1"""+ date="2014-11-06T19:39:10Z"+ content="""+Yeah, typoed. Should not cause any big problem though.+"""]]
− doc/preferred_content/comment_11_4599023aa31ac3767a9c57b91b500f8d._comment 
@@ -1,35 +0,0 @@-[[!comment format=mdwn- username="https://www.google.com/accounts/o8/id?id=AItOawnDx6KWBl4PpP7qikNB7rp0hK_UvwQq_L0"- nickname="Александр"- subject="Dropping files from remotes"- date="2014-11-05T16:09:54Z"- content="""-I think I found a bug where git-annex assistant starts to drop files in remote repos when it shouldn't. It can be easily reproduced with latest version in Debian Jessy (5.20141024) like this:--Create 3 repos:--* A -- standard, archive-* m1 -- standard, manual-* m2 -- standard, manual--System has 3 files annexed: file1, file2, file3. Repo \"A\" has all three files, m1 has none, m2 has file3.--So, while in m1:--    m1 $ git annex find --want-drop --in m2-    file3--file3 shouln't be dropped from m2. There is no reason to do this. m2 is set as manual, and it shouldn't be touched in any case.--Now, let's get this file in m1:--    m1 $ git annex get file3-    get file3 (from m2...) ok-    (Recording state in git...)-    m1 $ git annex find --want-drop --in m2--So when 'file3' is present in local repo, it's not going to be dropped from m2.--I guess that rule 'present' works in local repo context while 'drop' acts on remote files.--"""]]
doc/special_remotes/S3.mdwn view
@@ -18,11 +18,11 @@ * `encryption` - One of "none", "hybrid", "shared", or "pubkey".   See [[encryption]]. +* `keyid` - Specifies the gpg key to use for [[encryption]].+ * `chunk` - Enables [[chunking]] when storing large files.   `chunk=1MiB` is a good starting point for chunking. -* `keyid` - Specifies the gpg key to use for [[encryption]].- * `embedcreds` - Optional. Set to "yes" embed the login credentials inside   the git repository, which allows other clones to also access them. This is   the default when gpg encryption is enabled; the credentials are stored@@ -33,7 +33,8 @@   embedcreds without gpg encryption.  * `datacenter` - Defaults to "US". Other values include "EU",-  "us-west-1", and "ap-southeast-1".+  "us-west-1", "us-west-2", "ap-southeast-1", "ap-southeast-2", and+  "sa-east-1".  * `storageclass` - Default is "STANDARD". If you have configured git-annex   to preserve multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"@@ -46,11 +47,24 @@   so by default, a bucket name is chosen based on the remote name   and UUID. This can be specified to pick a bucket name. +* `partsize` - Amazon S3 only accepts uploads up to a certian file size,+  and storing larger files requires a multipart upload process.++  Setting `partsize=1GiB` is recommended for Amazon S3 when not using+  chunking; this will cause multipart uploads to be done using parts+  up to 1GiB in size. Note that setting partsize to less than 100MiB+  will cause Amazon S3 to reject uploads.++  This is not enabled by default, since other S3 implementations may+  not support multipart uploads or have different limits,+  but can be enabled or changed at any time.+  time.+ * `fileprefix` - By default, git-annex places files in a tree rooted at the   top of the S3 bucket. When this is set, it's prefixed to the filenames   used. For example, you could set it to "foo/" in one special remote,   and to "bar/" in another special remote, and both special remotes could   then use the same bucket. -* `x-amz-*` are passed through as http headers when storing keys+* `x-amz-meta-*` are passed through as http headers when storing keys   in S3.
doc/thanks.mdwn view
@@ -132,7 +132,7 @@ Mica Semrick, Paul Staab, Rémi Vanicat, Martin Holtschneider, Jan Ivar Beddari, Peter Simons, Thomas Koch, Justin Geibel, Guillaume DELVIT, Shanti Bouchez, Oliver Brandt, François Deppierraz, Chad Walstrom, Tim Mattison,-Jakub Antoni Tyszko, Casa do Boneco, Florian Tham, +Jakub Antoni Tyszko, Casa do Boneco, Florian Tham, martin f. krafft, and 30 anonymous bitcoin users  With an especial thanks to the WikiMedia foundation,
doc/tips/centralized_git_repository_tutorial.mdwn view
@@ -47,7 +47,7 @@ 	(Recording state in git...) 	# git commit -a -m 'added a screencast I made' -Feel free the rename the files, etc, using normal git commands:+Feel free to rename the files, etc, using normal git commands:  	# git mv Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 Haskell_Amuse_Bouche.mp4 	# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
doc/tips/file_manager_integration.mdwn view
@@ -1,5 +1,6 @@ Integrating git-annex and your file manager provides an easy way to select-annexed files to get or drop.+annexed files to get or drop. The file manager can also be used to undo+changes to file managed by git-annex.  [[!toc]] @@ -25,14 +26,21 @@  ## XFCE (Thunar) -XFCE uses the Thunar file manager, which can also be easily configured to allow for custom actions. Just go to the "Configure custom actions..." item in the "Edit" menu, and create a custom action for get and drop with the following commands:+XFCE uses the Thunar file manager, which can also be easily configured to+allow for custom actions. Just go to the "Configure custom actions..." item+in the "Edit" menu, and create a custom action for get, drop, and undo with the+following commands:      git-annex drop --notify-start --notify-finish -- %F  for drop, and for get:      git-annex get --notify-start --notify-finish -- %F+    +and for undo: +    git-annex undo --notify-start --notify-finish -- %F+ This gives me the resulting config on disk, in `.config/Thunar/uca.xml`:      <action>@@ -68,7 +76,9 @@  ## OS X (Finder) -For OS X, it is possible to get context menus in Finder. Due to how OS X deals with sym links, one needs to operate on folders if using indirect mode. Direct mode operation has not been tested.+For OS X, it is possible to get context menus in Finder. Due to how OS X+deals with sym links, one needs to operate on folders if using indirect+mode. Direct mode operation has not been tested.  1. Open Automator and create a new Service. 2. Using the Drop down menus in the top create the sentence "Service receives selected folders in Finder.app" to have it work on folders. For direct mode operation it is probably reasonable to select "files or folders".@@ -81,7 +91,9 @@             cd "$(dirname "$f")" && git-annex get "$f"         done -The purpose of the first line is there to get git-annex on to the path. The reason for the for loop is in case multiple files or folders are marked when running the context menu command.+The purpose of the first line is there to get git-annex on to the path. The+reason for the for loop is in case multiple files or folders are marked+when running the context menu command.  Finally save the the workflow under the name for which it should be listed in the context menu.
+ doc/tips/publishing_your_files_to_the_public.mdwn view
@@ -0,0 +1,55 @@+# Creating a special S3 remote to hold files shareable by URL++(In this example, I'll assume you'll be creating a bucket in S3 named **public-annex** and a special remote in git-annex, which will store its files in the previous bucket, named **public-s3**, but change these names if you are going to do the thing for real)++First, in the AWS dashboard, go to (or create) the bucket you will use at S3 and add a public get policy to it:++    {+      "Version": "2008-10-17",+      "Statement": [+        {+          "Sid": "AllowPublicRead",+          "Effect": "Allow",+          "Principal": {+            "AWS": "*"+          },+          "Action": "s3:GetObject",+          "Resource": "arn:aws:s3:::public-annex/*"+        }+      ]+    }++Then set up your special [S3](http://git-annex.branchable.com/special_remotes/S3/) remote with (at least) these options:++    git annex initremote public-s3 type=s3 encryption=none bucket=public-annex chunk=0++This way git-annex will upload the files to this repo, (when you call `git annex copy [FILES...] --to public-s3`) without encrypting them and without chunking them, and, because of the policy of the bucket, they will be accessible by anyone with the link.++Following the example, the files will be accessible at `http://public-annex.s3.amazonaws.com/KEY` where `KEY` is the file key created by git-annex and which you can discover running++    git annex lookupkey FILEPATH++This way you can share a link to each file you have at your S3 remote.++___________________++## Sharing all links in a folder++To share all the links in a given folder, for example, you can go to that folder and run (this is an example with the _fish_ shell, but I'm sure you can do the same in _bash_, I just don't know exactly):++    for filename in (ls)+        echo $filename": https://public-annex.s3.amazonaws.com/"(git annex lookupkey $filename)+    end++## Sharing all links matching certain metadata++The same applies to all the filters you can do with git-annex.++For example, let's share links to all the files whose _author_'s name starts with "Mario" and are, in fact, stored at your public-s3 remote.+However, instead of just a list of links we will output a markdown-formatted list of the filenames linked to their S3 files:++    for filename in (git annex find --metadata "author=Mario*" --and --in public-s3)+       echo "* ["$filename"](https://public-annex.s3.amazonaws.com/"(git annex lookupkey $filename)")"+    end++Very useful.
+ doc/todo/S3_multipart_interruption_cleanup.mdwn view
@@ -0,0 +1,14 @@+When a multipart S3 upload is being made, and gets interrupted,+the parts remain in the bucket, and S3 may charge for them.++I am not sure what happens if the same object gets uploaded again. Is S3+nice enough to remove the old parts? I need to find out..++If not, this needs to be dealt with somehow. One way would be to configure an+expiry of the uploaded parts, but this is tricky as a huge upload could+take arbitrarily long. Another way would be to record the uploadid and the+etags of the parts, and then resume where it left off the next time the+object is sent to S3. (Or at least cancel the old upload; resume isn't+practical when uploading an encrypted object.) ++It could store that info in either the local FS or the git-annex branch.
doc/todo/direct_mode_undo.mdwn view
@@ -6,27 +6,41 @@  ## general approach -`git annex foo $gitcmd` could:+`git annex proxy $gitcmd` could:  1. check out a local clone of the repo-2. run "git $gitcmd" inside the cline-3. merge any changes from the clone back into the direct mode repo+2. run "git $gitcmd" inside the clone+3. Merge any changes from the clone back into the direct mode repo    and update the work tree the same as is done by `git annex merge`.+4. If a different branch was checked out in the clone, update the repo+   to have that same branch checked out.  This is a general bypass for the direct mode guard. It should work anywhere (even on FAT). It avoids problems like `git commit -a` being unsafe in direct mode, since running such a command in a local clone, which does not use direct mode is always safe. +Beyond handling undo, #4 means that this can be used to check +out past versions of the repo (eg, `git annex proxy checkout HEAD^^`)+ One problem with it is that it can only operate on changes that have been committed. If you've just accidentially deleted a file and want to undo that, and haven't run `git annex sync` to commit it, you can't revert it. -Also, while this is general, I don't know if the generality is useful.-What other changes, than revert, would it make sense to do with such a-command? It could be used to check out some other branch, but step 3 above-would merge that branch back into the direct mode repo.+The need to make a local clone will make it a bit slow, since the whole+work tree will need to be set up. It might be possible to reuse the clone+next time (after resetting it to reflect the current HEAD). +Some things like the reflog and local branches don't get cloned, so+git commands that try to act on those wouldn't work. Maybe it would be+better to make it use a separate work tree, but the same .git directory?+Then step #3 would instead update the direct mode work tree to refect+the new HEAD, and step #4 would not be needed.++> This is done.. But, I think an undo command would also be good+> to do, as a nicer user interface that can integrate well with a file+> manager. --[[Joey]]+ ## git annex undo  I don't want to recapitulate all of the git commands in git-annex for@@ -61,16 +75,12 @@ commit. Which a later run of undo can in turn revert. If it didn't commit, the history about the staged change that was reverted would be lost. -What about undoing changes to a whole directory? It first would-need to look through the git history to find every file under that-directory. And then it would behave as if it were passed all those-files. This would be useful for reverting `rm -rf`. But it could be very-expensive. And it could lead to surprising results, like undoing a lot-of unrelated changes when running on a bunch of files in a directory-that were changed at different times.+What about undoing changes to a whole directory? Recursively undoing+the last change to each file would be expensive, and likely confusing.+Instead, when a directory is passed, it could find the most recent commit+that touched files in that directory, and undo the changes to those files. -Maybe instead of letting a directory be passed, make undo with no-parameters revert all changes made in the most recent commit.+> [[done]] --[[Joey]]  Also, --depth could make undo look for an older commit than the most recent one to affect the specified file.
+ doc/todo/git_annex_open.mdwn view
@@ -0,0 +1,9 @@+I had an idea the other night that there could be a `git annex open` command. What this would do would be the following:++* if the file is already present locally: just `xdg-open` it+* if it is not present and cannot be streamed: `git annex get "$@" && xdg-open "$@"`+* if it can be streamed: `git annex get "$@"` and `xdg-open` when enough content has been streamed that we are confident it will play completely (unless network conditions change)++This would need some [[metadata]] support partly to guess if the file can be streamed, but also to find the content. It would also assume some more intelligence in `git annex get` where git annex would have progress information (maybe through [[chunking]]?).++How does that idea sound? --[[anarcat]]
+ doc/todo/git_annex_open/comment_1_67d90a1cb104d98e816354d96e1b0306._comment view
@@ -0,0 +1,21 @@+[[!comment format=mdwn+ username="joey"+ subject="""comment 1"""+ date="2014-11-10T17:11:52Z"+ content="""+Hmm, well, there have been requests for access to files as they're being+get-ed before. This does have the advantage of providing that,+but I don't know if I like trying in xdg-open, which might not always+be the way a user wants to open a file.++git-annex does have progress info available while a file is being+transferred. (If nothing else, it tends to know the key size, and can+see the file size.) However I don't know how it could tell if "enough"+of the file was available to stream it.++I guess what I'd be most comfortable with is putting in plumbing.+Like making `examinekey` be able to report the temporary file+that is used when a key is being downloaded. It might also make sense to+have a bit of plumbing that waits for a file being downloaded to get+to X% complete, or something.+"""]]
+ doc/users/james.mdwn view
@@ -0,0 +1,1 @@+hello, I am [[James Richardson|http://jamestechnotes.com/]].
git-annex.1 view
@@ -262,6 +262,21 @@ are on a video hosting site, and the video is downloaded. This allows importing e.g., youtube playlists. .IP+.IP "\fBundo [filename|directory] ...\fP"+When passed a filename, undoes the last change that was made to that+file.+.IP+When passed a directory, undoes the last change that was made to the+contents of that directory.+.IP+Running undo a second time will undo the undo, returning the working+tree to the same state it had before. In order for undoing an undo of+staged changes, any staged changes are first committed by the+undo command.+.IP+Note that this does not undo get/drop of a file's content; it only+operates on the file tree committed to git.+.IP .IP "\fBwatch\fP" Watches for changes to files in the current directory and its subdirectories, and takes care of automatically adding new files, as well as dealing with@@ -461,9 +476,9 @@ .IP As part of the switch to direct mode, any changed files will be committed. .IP-Note that git commands that operate on the work tree are often unsafe to-use in direct mode repositories, and can result in data loss or other-bad behavior.+Note that git commands that operate on the work tree will refuse to+run in direct mode repositories. Use \fBgit annex proxy\fP to safely run such+commands. .IP .IP "\fBindirect\fP" Switches a repository back from direct mode to the default, indirect mode.@@ -870,6 +885,24 @@ Most MATCHING OPTIONS can be used with findref, to limit the files it finds. However, the \-\-include and \-\-exclude options will not work. .IP+.IP "\fBproxy \-\- git cmd [options]\fP"+Only useful in a direct mode repository, this runs the specified git+command with a temporary work tree, and updates the working tree to+reflect any changes staged or committed by the git command.+.IP+For example, to revert the most recent change that was committed+to the repository:+.IP+git annex proxy \-\- git revert HEAD+.IP+To check out a past version of the repository:+.IP+git annex proxy \-\- git checkout HEAD^^+.IP+To rename a directory:+.IP+git annex proxy \-\- git mv mydir newname+.IP .IP "\fBresolvemerge\fP" Resolves a conflicted merge, by adding both conflicting versions of the file to the tree, using variants of their filename. This is done@@ -878,6 +911,20 @@ Note that only merge conflicts that involve an annexed file are resolved. Merge conflicts between two files that are not annexed will not be automatically resolved.+.IP+.IP "\fBdiffdriver\fP"+This is an external git diff driver shim. Normally, when using \fBgit diff\fP+with an external git driver, the symlinks to annexed files are not set up+right, so the external git driver cannot read them in order to perform+smart diffing of their contents. This command works around the problem,+by passing the fixed up files to the real external diff driver.+.IP+To use, just configure git to use "git\-annex diffdriver \-\- cmd params \-\-"+as the external diff command, where cmd is the real external diff+command you want to use, and params are any extra parameters to pass+to it. Note the trailing "\-\-", which is required.+.IP+For example, set \fBGIT_EXTERNAL_DIFF=git\-annex diffdriver \-\- j\-c\-diff \-\-\fP .IP .IP "\fBremotedaemon\fP" Detects when network remotes have received git pushes and fetches from them.
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20141125+Version: 5.20141203 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <joey@kitenet.net>@@ -107,7 +107,7 @@    containers, utf8-string, mtl (>= 2),    bytestring, old-locale, time, dataenc, SHA, process, json,    base (>= 4.5 && < 4.9), monad-control, exceptions (>= 0.6), transformers,-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,    SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3),    data-default, case-insensitive, http-conduit, http-types   CC-Options: -Wall@@ -151,7 +151,7 @@     CPP-Options: -DWITH_CRYPTOHASH    if flag(S3)-    Build-Depends: hS3+    Build-Depends: conduit, resourcet, conduit-extra, aws (>= 0.9.2)     CPP-Options: -DWITH_S3    if flag(WebDAV)@@ -203,7 +203,7 @@      yesod, yesod-default, yesod-static, yesod-form, yesod-core,      wai, wai-extra, warp, warp-tls,      blaze-builder, crypto-api, hamlet, clientsession,-     template-haskell, data-default, aeson, path-pieces,+     template-haskell, aeson, path-pieces,      shakespeare     CPP-Options: -DWITH_WEBAPP   if flag(Webapp) && flag (Webapp-secure)
standalone/android/install-haskell-packages view
@@ -8,6 +8,10 @@ # versions that have been gotten working. To update, delete the # cabal.config, run this script with an empty cabal and fix up the broken # patches, and then use cabal freeze to generate a new cabal.config.+#+# This only installs haskell packages for ghc-android. The host ghc+# also needs to have all the git-annex build deps installed, in similar+# versions. Those are installed from Debian packages using apt.  set -e
standalone/licences.gz view

binary file changed (60598 → 63144 bytes)

standalone/windows/build.sh view
@@ -61,7 +61,7 @@  # Build the installer cabal install nsis-ghc --make Build/NullSoftInstaller.hs+ghc -fforce-recomp --make Build/NullSoftInstaller.hs # Want to include cygwin programs in bundle, not others, since # it includes the cygwin libs that go with them. withcygpreferred Build/NullSoftInstaller.exe