packages feed

git-annex 3.20120629 → 3.20120721

raw patch · 104 files changed

+1929/−320 lines, 104 filesbinary-added

Files

Backend/SHA.hs view
@@ -1,6 +1,6 @@ {- git-annex SHA backend  -- - Copyright 2011 Joey Hess <joey@kitenet.net>+ - Copyright 2011,2012 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -12,7 +12,10 @@ import Types.Backend import Types.Key import Types.KeySource+ import qualified Build.SysConfig as SysConfig+import Data.Digest.Pure.SHA+import qualified Data.ByteString.Lazy as L  type SHASize = Int @@ -25,32 +28,19 @@ backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes  genBackend :: SHASize -> Maybe Backend-genBackend size-	| isNothing (shaCommand size) = Nothing-	| otherwise = Just b-	where-		b = Backend-			{ name = shaName size-			, getKey = keyValue size-			, fsckKey = Just $ checkKeyChecksum size-			}+genBackend size = Just $ Backend+	{ name = shaName size+	, getKey = keyValue size+	, fsckKey = Just $ checkKeyChecksum size+	}  genBackendE :: SHASize -> Maybe Backend-genBackendE size =-	case genBackend size of-		Nothing -> Nothing-		Just b -> Just $ b-			{ name = shaNameE size-			, getKey = keyValueE size-			}--shaCommand :: SHASize -> Maybe String-shaCommand 1 = SysConfig.sha1-shaCommand 256 = Just SysConfig.sha256-shaCommand 224 = SysConfig.sha224-shaCommand 384 = SysConfig.sha384-shaCommand 512 = SysConfig.sha512-shaCommand _ = Nothing+genBackendE size = do+	b <- genBackend size+	return $ b +		{ name = shaNameE size+		, getKey = keyValueE size+		}  shaName :: SHASize -> String shaName size = "SHA" ++ show size@@ -58,27 +48,48 @@ shaNameE :: SHASize -> String shaNameE size = shaName size ++ "E" -shaN :: SHASize -> FilePath -> Annex String-shaN size file = do+shaN :: SHASize -> FilePath -> Integer -> Annex String+shaN shasize file filesize = do 	showAction "checksum"-	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do-		sha <- fst . separate (== ' ') <$> hGetLine h-		if null sha-			then error $ command ++ " parse error"-			else return sha+	case shaCommand shasize filesize of+		Left sha -> liftIO $ sha <$> L.readFile file+		Right command -> liftIO $ runcommand command 	where-		command = fromJust $ shaCommand size+		runcommand command =+			pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do+				sha <- fst . separate (== ' ') <$> hGetLine h+				if null sha+					then error $ command ++ " parse error"+					else return sha +shaCommand :: SHASize -> Integer -> Either (L.ByteString -> String) String+shaCommand shasize filesize+	| shasize == 1 = use SysConfig.sha1 sha1+	| shasize == 256 = use SysConfig.sha256 sha256+	| shasize == 224 = use SysConfig.sha224 sha224+	| shasize == 384 = use SysConfig.sha384 sha384+	| shasize == 512 = use SysConfig.sha512 sha512+	| otherwise = error $ "bad sha size " ++ show shasize+	where+		use Nothing sha = Left $ showDigest . sha+		use (Just c) sha+			-- use builtin, but slower sha for small files+			-- benchmarking indicates it's faster up to+			-- and slightly beyond 50 kb files+			| filesize < 51200 = use Nothing sha+			| otherwise = Right c+ {- A key is a checksum of its contents. -} keyValue :: SHASize -> KeySource -> Annex (Maybe Key)-keyValue size source = do+keyValue shasize source = do 	let file = contentLocation source-	s <- shaN size file 	stat <- liftIO $ getFileStatus file+	let filesize = fromIntegral $ fileSize stat+	s <- shaN shasize file filesize 	return $ Just $ stubKey 		{ keyName = s-		, keyBackendName = shaName size-		, keySize = Just $ fromIntegral $ fileSize stat+		, keyBackendName = shaName shasize+		, keySize = Just filesize 		}  {- Extension preserving keys. -}@@ -86,25 +97,32 @@ keyValueE size source = keyValue size source >>= maybe (return Nothing) addE 	where 		addE k = return $ Just $ k-			{ keyName = keyName k ++ extension+			{ keyName = keyName k ++ selectExtension (keyFilename source) 			, keyBackendName = shaNameE size 			}-		naiveextension = takeExtension $ keyFilename source-		extension-			-- long or newline containing extensions are -			-- probably not really an extension-			| length naiveextension > 6 ||-			  '\n' `elem` naiveextension = ""-			| otherwise = naiveextension +selectExtension :: FilePath -> String+selectExtension f+	| null es = ""+	| otherwise = join "." ("":es)+	where+		es = filter (not . null) $ reverse $+			take 2 $ takeWhile shortenough $+			reverse $ split "." $ takeExtensions f+		shortenough e+			| '\n' `elem` e = False -- newline in extension?!+			| otherwise = length e <= 4 -- long enough for "jpeg"+ {- A key's checksum is checked during fsck. -} checkKeyChecksum :: SHASize -> Key -> FilePath -> Annex Bool checkKeyChecksum size key file = do 	fast <- Annex.getState Annex.fast-	present <- liftIO $ doesFileExist file-	if not present || fast-		then return True-		else check <$> shaN size file+	mstat <- liftIO $ catchMaybeIO $ getFileStatus file+	case (mstat, fast) of+		(Just stat, False) -> do+			let filesize = fromIntegral $ fileSize stat+			check <$> shaN size file filesize+		_ -> return True 	where 		check s 			| s == dropExtension (keyName key) = True
Build/Configure.hs view
@@ -28,15 +28,14 @@ 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" 	, TestCase "lsof" $ testCmd "lsof" "lsof -v >/dev/null 2>&1" 	, TestCase "ssh connection caching" getSshConnectionCaching-	] ++ shaTestCases False [1, 512, 224, 384] ++ shaTestCases True [256]+	] ++ shaTestCases [1, 256, 512, 224, 384] -shaTestCases :: Bool -> [Int] -> [TestCase]-shaTestCases required l = map make l+shaTestCases :: [Int] -> [TestCase]+shaTestCases l = map make l 	where-		make n = TestCase key $ selector key (shacmds n) "</dev/null"+		make n = TestCase key $ maybeSelectCmd key (shacmds n) "</dev/null" 			where 				key = "sha" ++ show n-		selector = if required then selectCmd else maybeSelectCmd 		shacmds n = concatMap (\x -> [x, osxpath </> x]) $ 			map (\x -> "sha" ++ show n ++ x) ["", "sum"] 		-- Max OSX puts GNU tools outside PATH, so look in@@ -73,7 +72,7 @@ getGitVersion :: Test getGitVersion = do 	(_, s) <- pipeFrom "git" ["--version"]-	let version = last $ words $ head $ lines s+	let version = unwords $ drop 2 $ words $ head $ lines s 	return $ Config "gitversion" (StringConfig version)  getSshConnectionCaching :: Test
CHANGELOG view
@@ -1,3 +1,20 @@+git-annex (3.20120721) unstable; urgency=low++  * get, move, copy: Now refuse to do anything when the requested file+    transfer is already in progress by another process.+  * status: Lists transfers that are currently in progress.+  * Fix passing --uuid to git-annex-shell.+  * When shaNsum commands cannot be found, use the Haskell SHA library+    (already a dependency) to do the checksumming. This may be slower,+    but avoids portability problems.+  * Use SHA library for files less than 50 kb in size, at which point it's+    faster than forking the more optimised external program.+  * SHAnE backends are now smarter about composite extensions, such as+    .tar.gz Closes: #680450+  * map: Write map.dot to .git/annex, which avoids watch trying to annex it.++ -- Joey Hess <joeyh@debian.org>  Sat, 21 Jul 2012 16:52:48 -0400+ git-annex (3.20120629) unstable; urgency=low    * cabal: Only try to use inotify on Linux.
CmdLine.hs view
@@ -30,8 +30,8 @@ type Flags = [Annex ()]  {- Runs the passed command line. -}-dispatch :: Bool -> Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()-dispatch fuzzyok allargs allcmds commonoptions header getgitrepo = do+dispatch :: Bool -> Params -> [Command] -> [Option] -> [(String, String)] -> String -> IO Git.Repo -> IO ()+dispatch fuzzyok allargs allcmds commonoptions fields header getgitrepo = do 	setupConsole 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo) 	case r of@@ -40,6 +40,7 @@ 			state <- Annex.new g 			(actions, state') <- Annex.run state $ do 				checkfuzzy+				forM_ fields $ \(f, v) -> Annex.setField f v 				sequence_ flags 				prepCommand cmd params 		 	tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdoneshot cmd]
Command.hs view
@@ -21,6 +21,7 @@ 	isBareRepo, 	numCopies, 	autoCopies,+	autoCopiesWith, 	module ReExported ) where @@ -112,8 +113,18 @@  - In auto mode, first checks that the number of known  - copies of the key is > or < than the numcopies setting, before running  - the action. -}-autoCopies :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart-autoCopies file key vs a = do+autoCopies :: FilePath -> Key -> (Int -> Int -> Bool) -> CommandStart -> CommandStart+autoCopies file key vs a = Annex.getState Annex.auto >>= go+	where+		go False = a+		go True = do+			numcopiesattr <- numCopies file+			needed <- getNumCopies numcopiesattr+			(_, have) <- trustPartition UnTrusted =<< Remote.keyLocations key+			if length have `vs` needed then a else stop++autoCopiesWith :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart+autoCopiesWith file key vs a = do 	numcopiesattr <- numCopies file 	Annex.getState Annex.auto >>= auto numcopiesattr 	where
Command/Copy.hs view
@@ -24,5 +24,5 @@ -- A copy is just a move that does not delete the source file. -- However, --auto mode avoids unnecessary copies. start :: Maybe Remote -> Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart-start to from file (key, backend) = autoCopies file key (<) $ \_numcopies ->+start to from file (key, backend) = autoCopies file key (<) $ 	Command.Move.start to from False file (key, backend)
Command/Drop.hs view
@@ -30,7 +30,7 @@ 	withFilesInGit $ whenAnnexed $ start from]  start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart-start from file (key, _) = autoCopies file key (>) $ \numcopies ->+start from file (key, _) = autoCopiesWith file key (>) $ \numcopies -> 	case from of 		Nothing -> startLocal file numcopies key 		Just remote -> do
Command/Fsck.hs view
@@ -94,7 +94,7 @@ 				( return True 				, ifM (Annex.getState Annex.fast) 					( return False-					, Remote.retrieveKeyFile remote key tmp+					, Remote.retrieveKeyFile remote key Nothing tmp 					) 				)
Command/Get.hs view
@@ -12,6 +12,7 @@ import qualified Remote import Annex.Content import qualified Command.Move+import Logs.Transfer  def :: [Command] def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek@@ -23,26 +24,26 @@  start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart start from file (key, _) = stopUnless (not <$> inAnnex key) $-	autoCopies file key (<) $ \_numcopies ->+	autoCopies file key (<) $ 		case from of-			Nothing -> go $ perform key+			Nothing -> go $ perform key file 			Just src -> 				-- get --from = copy --from 				stopUnless (Command.Move.fromOk src key) $-					go $ Command.Move.fromPerform src False key+					go $ Command.Move.fromPerform src False key file 	where 		go a = do 			showStart "get" file-			next a	+			next a -perform :: Key -> CommandPerform-perform key = stopUnless (getViaTmp key $ getKeyFile key) $+perform :: Key -> FilePath -> CommandPerform+perform key file = stopUnless (getViaTmp key $ getKeyFile key file) $ 	next $ return True -- no cleanup needed  {- Try to find a copy of the file in one of the remotes,  - and copy it to here. -}-getKeyFile :: Key -> FilePath -> Annex Bool-getKeyFile key file = dispatch =<< Remote.keyPossibilities key+getKeyFile :: Key -> FilePath -> FilePath -> Annex Bool+getKeyFile key file dest = dispatch =<< Remote.keyPossibilities key 	where 		dispatch [] = do 			showNote "not available"@@ -64,7 +65,7 @@ 			| Remote.hasKeyCheap r = 				either (const False) id <$> Remote.hasKey r key 			| otherwise = return True-		docopy r continue = do+		docopy r continue = download (Remote.uuid r) key (Just file) $ do 			showAction $ "from " ++ Remote.name r-			ifM (Remote.retrieveKeyFile r key file)+			ifM (Remote.retrieveKeyFile r key (Just file) dest) 				( return True , continue)
Command/Map.hs view
@@ -39,6 +39,8 @@  	umap <- uuidMap 	trusted <- trustGet Trusted+		+	file <- (</>) <$> fromRepo gitAnnexDir <*> pure "map.dot"  	liftIO $ writeFile file (drawMap rs umap trusted) 	next $ next $@@ -49,8 +51,6 @@ 				showOutput 				liftIO $ boolSystem "dot" [Param "-Tx11", File file] 			)-	where-		file = "map.dot"  {- Generates a graph for dot(1). Each repository, and any other uuids, are  - displayed as a node, and each of its remotes is represented as an edge@@ -203,7 +203,7 @@ 				Git.Config.hRead r  		configlist =-			onRemote r (pipedconfig, Nothing) "configlist" []+			onRemote r (pipedconfig, Nothing) "configlist" [] [] 		manualconfiglist = do 			sshparams <- sshToRepo r [Param sshcmd] 			liftIO $ pipedconfig "ssh" sshparams
Command/Move.hs view
@@ -16,6 +16,7 @@ import Annex.UUID import qualified Option import Logs.Presence+import Logs.Transfer  def :: [Command] def = [withOptions options $ command "move" paramPaths seek@@ -68,9 +69,9 @@ 		then stop -- not here, so nothing to do 		else do 			showMoveAction move file-			next $ toPerform dest move key-toPerform :: Remote -> Bool -> Key -> CommandPerform-toPerform dest move key = moveLock move key $ do+			next $ toPerform dest move key file+toPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform+toPerform dest move key file = moveLock move key $ do 	-- Checking the remote is expensive, so not done in the start step. 	-- In fast mode, location tracking is assumed to be correct, 	-- and an explicit check is not done, when copying. When moving,@@ -88,7 +89,8 @@ 			stop 		Right False -> do 			showAction $ "to " ++ Remote.name dest-			ok <- Remote.storeKey dest key+			ok <- upload (Remote.uuid dest) key (Just file) $+				Remote.storeKey dest key (Just file) 			if ok 				then finish 				else do@@ -118,7 +120,7 @@ 	where 		go = stopUnless (fromOk src key) $ do 			showMoveAction move file-			next $ fromPerform src move key+			next $ fromPerform src move key file fromOk :: Remote -> Key -> Annex Bool fromOk src key 	| Remote.hasKeyCheap src =@@ -129,13 +131,14 @@ 			u <- getUUID 			remotes <- Remote.keyPossibilities key 			return $ u /= Remote.uuid src && elem src remotes-fromPerform :: Remote -> Bool -> Key -> CommandPerform-fromPerform src move key = moveLock move key $+fromPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform+fromPerform src move key file = moveLock move key $ 	ifM (inAnnex key) 		( handle move True-		, do+		, download (Remote.uuid src) key (Just file) $ do 			showAction $ "from " ++ Remote.name src-			ok <- getViaTmp key $ Remote.retrieveKeyFile src key+			ok <- getViaTmp key $+				Remote.retrieveKeyFile src key (Just file) 			handle move ok 		) 	where
Command/RecvKey.hs view
@@ -12,6 +12,7 @@ import CmdLine import Annex.Content import Utility.RsyncFile+import Logs.Transfer  def :: [Command] def = [oneShot $ command "recvkey" paramKey seek@@ -21,14 +22,15 @@ seek = [withKeys start]  start :: Key -> CommandStart-start key = do-	whenM (inAnnex key) $ error "key is already present in annex"-	-	ok <- getViaTmp key (liftIO . rsyncServerReceive)-	if ok-		then do-			-- forcibly quit after receiving one key,-			-- and shutdown cleanly-			_ <- shutdown True-			liftIO exitSuccess-		else liftIO exitFailure+start key = ifM (inAnnex key)+	( error "key is already present in annex"+	, fieldTransfer Download key $ do+		ifM (getViaTmp key $ liftIO . rsyncServerReceive)+			( do+				-- forcibly quit after receiving one key,+				-- and shutdown cleanly+				_ <- shutdown True+				liftIO exitSuccess+			, liftIO exitFailure+			)+	)
Command/SendKey.hs view
@@ -1,6 +1,6 @@ {- git-annex command  -- - Copyright 2010 Joey Hess <joey@kitenet.net>+ - Copyright 2010,2012 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -11,6 +11,7 @@ import Command import Annex.Content import Utility.RsyncFile+import Logs.Transfer  def :: [Command] def = [oneShot $ command "sendkey" paramKey seek@@ -20,9 +21,12 @@ seek = [withKeys start]  start :: Key -> CommandStart-start key = do-	file <- inRepo $ gitAnnexLocation key-	whenM (inAnnex key) $-		liftIO $ rsyncServerSend file -- does not return-	warning "requested key is not present"-	liftIO exitFailure+start key = ifM (inAnnex key)+	( fieldTransfer Upload key $ do+		file <- inRepo $ gitAnnexLocation key+		liftIO $ ifM (rsyncServerSend file)+			( exitSuccess , exitFailure )+	, do+		warning "requested key is not present"+		liftIO exitFailure+	)
Command/Status.hs view
@@ -31,6 +31,7 @@ import Remote import Config import Utility.Percentage+import Logs.Transfer  -- a named computation that produces a statistic type Stat = StatState (Maybe (String, StatState String))@@ -70,6 +71,7 @@ 	, remote_list SemiTrusted "semitrusted" 	, remote_list UnTrusted "untrusted" 	, remote_list DeadTrusted "dead"+	, transfer_list 	, disk_size 	] slow_stats :: [Stat]@@ -169,6 +171,24 @@ 		lift Command.Unused.bloomBitsHashes  	return $ size ++ note++transfer_list :: Stat+transfer_list = stat "transfers in progress" $ nojson $ lift $ do+	uuidmap <- Remote.remoteMap id+	ts <- getTransfers+	if null ts+		then return "none"+		else return $ pp uuidmap "" $ sort ts+	where+		pp _ c [] = c+		pp uuidmap c ((t, i):xs) = "\n\t" ++ line uuidmap t i ++ pp uuidmap c xs+		line uuidmap t i = unwords+			[ show (transferDirection t) ++ "ing"+			, fromMaybe (show $ transferKey t) (associatedFile i)+			, if transferDirection t == Upload then "to" else "from"+			, maybe (fromUUID $ transferUUID t) Remote.name $+				M.lookup (transferUUID t) uuidmap+			]  disk_size :: Stat disk_size = stat "available local disk space" $ json id $ lift $
+ Fields.hs view
@@ -0,0 +1,32 @@+{- git-annex fields+ -+ - Copyright 2012 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Fields where++import Common.Annex+import qualified Annex++import Data.Char++{- A field, stored in Annex state, with a value sanity checker. -}+data Field = Field+	{ fieldName :: String+	, fieldCheck :: String -> Bool+	}++remoteUUID :: Field+remoteUUID = Field "remoteuuid" $+	-- does it look like a UUID?+	all (\c -> isAlphaNum c || c == '-')++associatedFile :: Field+associatedFile = Field "associatedfile" $ \f ->+	-- is the file a safe relative filename?+	not (isAbsolute f) && not ("../" `isPrefixOf` f)++getField :: Field -> Annex (Maybe String)+getField = Annex.getField . fieldName
Git/Merge.hs view
@@ -10,8 +10,10 @@ import Common import Git import Git.Command+import Git.Version  {- Avoids recent git's interactive merge. -} mergeNonInteractive :: Ref -> Repo -> IO Bool-mergeNonInteractive branch = runBool "merge"-	[Param "--no-edit", Param $ show branch]+mergeNonInteractive branch+	| older "1.7.7.6" = runBool "merge" [Param $ show branch]+	| otherwise = runBool "merge" [Param "--no-edit", Param $ show branch]
GitAnnex.hs view
@@ -143,4 +143,4 @@ header = "Usage: git-annex command [option ..]"  run :: [String] -> IO ()-run args = dispatch True args cmds options header Git.CurrentRepo.get+run args = dispatch True args cmds options [] header Git.CurrentRepo.get
GitAnnexShell.hs view
@@ -16,6 +16,7 @@ import Command import Annex.UUID import qualified Option+import Fields  import qualified Command.ConfigList import qualified Command.InAnnex@@ -47,7 +48,7 @@  options :: [OptDescr (Annex ())] options = Option.common ++-	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"+	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "local repository uuid" 	] 	where 		checkuuid expected = getUUID >>= check@@ -83,21 +84,40 @@ builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do 	checkNotReadOnly cmd-	dispatch False (cmd : filterparams params) cmds options header $+	let (params', fieldparams) = partitionParams params+	let fields = filter checkField $ parseFields fieldparams+	dispatch False (cmd : params') cmds options fields header $ 		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath  external :: [String] -> IO () external params = do 	checkNotLimited-	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $+	unlessM (boolSystem "git-shell" $ map Param $ "-c":fst (partitionParams params)) $ 		error "git-shell failed" --- Drop all args after "--".--- These tend to be passed by rsync and not useful.-filterparams :: [String] -> [String]-filterparams [] = []-filterparams ("--":_) = []-filterparams (a:as) = a:filterparams as+{- Parameters between two -- markers are field settings, in the form:+ - field=value field=value+ -+ - Parameters after the last -- are ignored, these tend to be passed by+ - rsync and not be useful.+ -}+partitionParams :: [String] -> ([String], [String])+partitionParams params+	| length segments < 2 = (segments !! 0, [])+	| otherwise = (segments !! 0, segments !! 1)+	where+		segments = segment (== "--") params++parseFields :: [String] -> [(String, String)]+parseFields = map (separate (== '='))++{- Only allow known fields to be set, ignore others.+ - Make sure that field values make sense. -}+checkField :: (String, String) -> Bool+checkField (field, value)+	| field == fieldName remoteUUID = fieldCheck remoteUUID value+	| field == fieldName associatedFile = fieldCheck associatedFile value+	| otherwise = False  failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds options
Locations.hs view
@@ -18,6 +18,7 @@ 	gitAnnexBadDir, 	gitAnnexBadLocation, 	gitAnnexUnusedLog,+	gitAnnexTransferDir, 	gitAnnexJournalDir, 	gitAnnexJournalLock, 	gitAnnexIndex,@@ -126,6 +127,11 @@ {- .git/annex/foounused is used to number possibly unused keys -} gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath gitAnnexUnusedLog prefix r = gitAnnexDir r </> (prefix ++ "unused")++{- .git/annex/transfer/ is used is used to record keys currently+ - being transferred. -}+gitAnnexTransferDir :: Git.Repo -> FilePath+gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r </> "transfer"  {- .git/annex/journal/ is used to journal changes made to the git-annex  - branch -}
+ Logs/Transfer.hs view
@@ -0,0 +1,188 @@+{- git-annex transfer information files and lock files+ -+ - Copyright 2012 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.Transfer where++import Common.Annex+import Annex.Perms+import Annex.Exception+import qualified Git+import Types.Remote+import qualified Fields++import System.Posix.Types+import Data.Time.Clock+import Data.Time.Clock.POSIX+import Data.Time+import System.Locale+import Control.Concurrent++{- Enough information to uniquely identify a transfer, used as the filename+ - of the transfer information file. -}+data Transfer = Transfer+	{ transferDirection :: Direction+	, transferUUID :: UUID+	, transferKey :: Key+	}+	deriving (Show, Eq, Ord)++{- Information about a Transfer, stored in the transfer information file.+ -+ - Note that the associatedFile may not correspond to a file in the local+ - git repository. It's some file, possibly relative to some directory,+ - of some repository, that was acted on to initiate the transfer.+ -}+data TransferInfo = TransferInfo+	{ startedTime :: Maybe POSIXTime+	, transferPid :: Maybe ProcessID+	, transferTid :: Maybe ThreadId+	, transferRemote :: Maybe Remote+	, bytesComplete :: Maybe Integer+	, associatedFile :: Maybe FilePath+	}+	deriving (Show, Eq, Ord)++data Direction = Upload | Download+	deriving (Eq, Ord)++instance Show Direction where+	show Upload = "upload"+	show Download = "download"++readDirection :: String -> Maybe Direction+readDirection "upload" = Just Upload+readDirection "download" = Just Download+readDirection _ = Nothing++upload :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a+upload u key file a = transfer (Transfer Upload u key) file a++download :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a+download u key file a = transfer (Transfer Download u key) file a++fieldTransfer :: Direction -> Key -> Annex a -> Annex a+fieldTransfer direction key a = do+	afile <- Fields.getField Fields.associatedFile+	maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a)+		=<< Fields.getField Fields.remoteUUID++{- Runs a transfer action. Creates and locks the lock file while the+ - action is running, and stores info in the transfer information+ - file. Will throw an error if the transfer is already in progress.+ -}+transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a+transfer t file a = do+	tfile <- fromRepo $ transferFile t+	createAnnexDirectory $ takeDirectory tfile+	mode <- annexFileMode+	info <- liftIO $ TransferInfo+		<$> (Just . utcTimeToPOSIXSeconds <$> getCurrentTime)+		<*> pure Nothing -- pid not stored in file, so omitted for speed+		<*> pure Nothing -- tid ditto+		<*> pure Nothing -- not 0; transfer may be resuming+		<*> pure Nothing+		<*> pure file+	bracketIO (prep tfile mode info) (cleanup tfile) a+	where+		prep tfile mode info = do+			fd <- openFd (transferLockFile tfile) ReadWrite (Just mode)+				defaultFileFlags { trunc = True }+			locked <- catchMaybeIO $+				setLock fd (WriteLock, AbsoluteSeek, 0, 0)+			when (locked == Nothing) $+				error $ "transfer already in progress"+			writeFile tfile $ writeTransferInfo info+			return fd+		cleanup tfile fd = do+			removeFile tfile+			removeFile $ transferLockFile tfile+			closeFd fd++{- If a transfer is still running, returns its TransferInfo. -}+checkTransfer :: Transfer -> Annex (Maybe TransferInfo)+checkTransfer t = do+	mode <- annexFileMode+	tfile <- fromRepo $ transferFile t+	mfd <- liftIO $ catchMaybeIO $+		openFd (transferLockFile tfile) ReadOnly (Just mode) defaultFileFlags+	case mfd of+		Nothing -> return Nothing -- failed to open file; not running+		Just fd -> do+			locked <- liftIO $+				getLock fd (WriteLock, AbsoluteSeek, 0, 0)+			liftIO $ closeFd fd+			case locked of+				Nothing -> return Nothing+				Just (pid, _) -> liftIO $+					flip catchDefaultIO Nothing $ do+						readTransferInfo pid +							<$> readFile tfile++{- Gets all currently running transfers. -}+getTransfers :: Annex [(Transfer, TransferInfo)]+getTransfers = do+	transfers <- catMaybes . map parseTransferFile <$> findfiles+	infos <- mapM checkTransfer transfers+	return $ map (\(t, Just i) -> (t, i)) $+		filter running $ zip transfers infos+	where+		findfiles = liftIO . dirContentsRecursive+			=<< fromRepo gitAnnexTransferDir+		running (_, i) = isJust i++{- The transfer information file to use for a given Transfer. -}+transferFile :: Transfer -> Git.Repo -> FilePath+transferFile (Transfer direction u key) r = gitAnnexTransferDir r+	</> show direction +	</> fromUUID u+	</> keyFile key++{- The transfer lock file corresponding to a given transfer info file. -}+transferLockFile :: FilePath -> FilePath+transferLockFile infofile = let (d,f) = splitFileName infofile in+	combine d ("lck." ++ f)++{- Parses a transfer information filename to a Transfer. -}+parseTransferFile :: FilePath -> Maybe Transfer+parseTransferFile file+	| "lck." `isPrefixOf` (takeFileName file) = Nothing+	| otherwise = case drop (length bits - 3) bits of+		[direction, u, key] -> Transfer+			<$> readDirection direction+			<*> pure (toUUID u)+			<*> fileKey key+		_ -> Nothing+	where+		bits = splitDirectories file++writeTransferInfo :: TransferInfo -> String+writeTransferInfo info = unlines+	-- transferPid is not included; instead obtained by looking at+	-- the process that locks the file.+	[ maybe "" show $ startedTime info+	-- bytesComplete is not included; changes too fast +	, fromMaybe "" $ associatedFile info -- comes last; arbitrary content+	]++readTransferInfo :: ProcessID -> String -> Maybe TransferInfo+readTransferInfo pid s =+	case bits of+		[time] -> TransferInfo+			<$> (Just <$> parsePOSIXTime time)+			<*> pure (Just pid)+			<*> pure Nothing+			<*> pure Nothing+			<*> pure Nothing+			<*> pure (if null filename then Nothing else Just filename)+		_ -> Nothing+	where+		(bits, filebits) = splitAt 1 $ lines s +		filename = join "\n" filebits++parsePOSIXTime :: String -> Maybe POSIXTime+parsePOSIXTime s = utcTimeToPOSIXSeconds+	<$> parseTime defaultTimeLocale "%s%Qs" s
Option.hs view
@@ -76,4 +76,3 @@ {- The flag or field name used for an option. -} name :: Option -> String name (Option _ o _ _) = Prelude.head o-
Remote/Bup.hs view
@@ -108,8 +108,8 @@ 	return $ bupParams "split" buprepo  		(os ++ [Param "-n", Param (bupRef k), src]) -store :: Git.Repo -> BupRepo -> Key -> Annex Bool-store r buprepo k = do+store :: Git.Repo -> BupRepo -> Key -> AssociatedFile -> Annex Bool+store r buprepo k _f = do 	src <- inRepo $ gitAnnexLocation k 	params <- bupSplitParams r buprepo k (File src) 	liftIO $ boolSystem "bup" params@@ -122,11 +122,11 @@ 		withEncryptedHandle cipher (L.readFile src) $ \h -> 			pipeBup params (Just h) Nothing -retrieve :: BupRepo -> Key -> FilePath -> Annex Bool-retrieve buprepo k f = do+retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool+retrieve buprepo k _f d = do 	let params = bupParams "join" buprepo [Param $ bupRef k] 	liftIO $ catchBoolIO $ do-		tofile <- openFile f WriteMode+		tofile <- openFile d WriteMode 		pipeBup params Nothing (Just tofile)  retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool
Remote/Directory.hs view
@@ -122,8 +122,8 @@ withStoredFiles :: ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool withStoredFiles = withCheckedFiles doesFileExist -store :: FilePath -> ChunkSize -> Key -> Annex Bool-store d chunksize k = do+store :: FilePath -> ChunkSize -> Key -> AssociatedFile -> Annex Bool+store d chunksize k _f = do 	src <- inRepo $ gitAnnexLocation k 	metered k $ \meterupdate ->  		storeHelper d chunksize k $ \dests ->@@ -242,8 +242,8 @@ 			preventWrite dir 			return (not $ null stored) -retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool-retrieve d chunksize k f = metered k $ \meterupdate ->+retrieve :: FilePath -> ChunkSize -> Key -> AssociatedFile -> FilePath -> Annex Bool+retrieve d chunksize k _ f = metered k $ \meterupdate -> 	liftIO $ withStoredFiles chunksize d k $ \files -> 		catchBoolIO $ do 			meteredWriteFile' meterupdate f files feeder
Remote/Git.hs view
@@ -21,6 +21,7 @@ import qualified Git.Construct import qualified Annex import Logs.Presence+import Logs.Transfer import Annex.UUID import qualified Annex.Content import qualified Annex.BranchState@@ -30,6 +31,7 @@ import Config import Init import Types.Key+import qualified Fields  remote :: RemoteType remote = RemoteType {@@ -110,7 +112,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo tryGitConfigRead r  	| not $ M.null $ Git.config r = return r -- already read-	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []+	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] [] 	| Git.repoIsHttp r = do 		headers <- getHttpHeaders 		store $ safely $ geturlconfig headers@@ -170,7 +172,7 @@ 						v -> return v 		checkremote = do 			showAction $ "checking " ++ Git.repoDescribe r-			onRemote r (check, unknown) "inannex" [Param (show key)]+			onRemote r (check, unknown) "inannex" [Param (show key)] [] 			where 				check c p = dispatch <$> safeSystem c p 				dispatch ExitSuccess = Right True@@ -217,16 +219,22 @@ 		[ Params "--quiet --force" 		, Param $ show key 		]+		[]  {- Tries to copy a key's content from a remote's annex to a file. -}-copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool-copyFromRemote r key file+copyFromRemote :: Git.Repo -> Key -> AssociatedFile -> FilePath -> Annex Bool+copyFromRemote r key file dest 	| not $ Git.repoIsUrl r = guardUsable r False $ do 		params <- rsyncParams r-		loc <- liftIO $ gitAnnexLocation key r-		rsyncOrCopyFile params loc file-	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file-	| Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) file+		u <- getUUID+		-- run copy from perspective of remote+		liftIO $ onLocal r $ do+			ensureInitialized+			loc <- inRepo $ gitAnnexLocation key+			upload u key file $+				rsyncOrCopyFile params loc dest+	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key dest file+	| Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) dest 	| otherwise = error "copying from non-ssh, non-http repo not supported"  copyFromRemoteCheap :: Git.Repo -> Key -> FilePath -> Annex Bool@@ -236,26 +244,28 @@ 		liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True 	| Git.repoIsSsh r = 		ifM (Annex.Content.preseedTmp key file)-			( copyFromRemote r key file+			( copyFromRemote r key Nothing file 			, return False 			) 	| otherwise = return False  {- Tries to copy a key's content to a remote's annex. -}-copyToRemote :: Git.Repo -> Key -> Annex Bool-copyToRemote r key+copyToRemote :: Git.Repo -> Key -> AssociatedFile -> Annex Bool+copyToRemote r key file 	| not $ Git.repoIsUrl r = guardUsable r False $ commitOnCleanup r $ do 		keysrc <- inRepo $ gitAnnexLocation key 		params <- rsyncParams r+		u <- getUUID 		-- run copy from perspective of remote 		liftIO $ onLocal r $ do 			ensureInitialized-			Annex.Content.saveState True `after`-				Annex.Content.getViaTmp key-					(rsyncOrCopyFile params keysrc)+			download u key file $+				Annex.Content.saveState True `after`+					Annex.Content.getViaTmp key+						(rsyncOrCopyFile params keysrc) 	| Git.repoIsSsh r = commitOnCleanup r $ do 		keysrc <- inRepo $ gitAnnexLocation key-		rsyncHelper =<< rsyncParamsRemote r False key keysrc+		rsyncHelper =<< rsyncParamsRemote r False key keysrc file 	| otherwise = error "copying to non-ssh repo not supported"  rsyncHelper :: [CommandParam] -> Annex Bool@@ -282,23 +292,26 @@  {- Generates rsync parameters that ssh to the remote and asks it  - to either receive or send the key's content. -}-rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]-rsyncParamsRemote r sending key file = do+rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> AssociatedFile -> Annex [CommandParam]+rsyncParamsRemote r sending key file afile = do+	u <- getUUID+	let fields = (Fields.remoteUUID, fromUUID u)+		: maybe [] (\f -> [(Fields.associatedFile, f)]) afile 	Just (shellcmd, shellparams) <- git_annex_shell r 		(if sending then "sendkey" else "recvkey")-		[ Param $ show key-		-- Command is terminated with "--", because-		-- rsync will tack on its own options afterwards,-		-- and they need to be ignored.-		, Param "--"-		]+		[ Param $ show key ]+		fields 	-- Convert the ssh command into rsync command line. 	let eparam = rsyncShell (Param shellcmd:shellparams) 	o <- rsyncParams r 	if sending-		then return $ o ++ eparam ++ [dummy, File file]-		else return $ o ++ eparam ++ [File file, dummy]+		then return $ o ++ rsyncopts eparam dummy (File file)+		else return $ o ++ rsyncopts eparam (File file) dummy 	where+		rsyncopts ps source dest+			| end ps == [dashdash] = ps ++ [source, dest]+			| otherwise = ps ++ [dashdash, source, dest]+		dashdash = Param "--" 		-- The rsync shell parameter controls where rsync 		-- goes, so the source/dest parameter can be a dummy value, 		-- that just enables remote rsync mode.@@ -325,7 +338,7 @@ 					Annex.Branch.commit "update" 			| otherwise = void $ do 				Just (shellcmd, shellparams) <--					git_annex_shell r "commit" []+					git_annex_shell r "commit" [] [] 				-- Throw away stderr, since the remote may not 				-- have a new enough git-annex shell to 				-- support committing.
Remote/Helper/Encryptable.hs view
@@ -59,14 +59,14 @@ 		cost = cost r + encryptedRemoteCostAdj 	} 	where-		store k = cip k >>= maybe-			(storeKey r k)+		store k f = cip k >>= maybe+			(storeKey r k f) 			(`storeKeyEncrypted` k)-		retrieve k f = cip k >>= maybe-			(retrieveKeyFile r k f)-			(\enck -> retrieveKeyFileEncrypted enck k f)-		retrieveCheap k f = cip k >>= maybe-			(retrieveKeyFileCheap r k f)+		retrieve k f d = cip k >>= maybe+			(retrieveKeyFile r k f d)+			(\enck -> retrieveKeyFileEncrypted enck k d)+		retrieveCheap k d = cip k >>= maybe+			(retrieveKeyFileCheap r k d) 			(\_ -> return False) 		withkey a k = cip k >>= maybe (a k) (a . snd) 		cip = cipherKey c
Remote/Helper/Hooks.hs view
@@ -27,8 +27,8 @@ addHooks' r starthook stophook = r' 	where 		r' = r-			{ storeKey = \k -> wrapper $ storeKey r k-			, retrieveKeyFile = \k f -> wrapper $ retrieveKeyFile r k f+			{ storeKey = \k f -> wrapper $ storeKey r k f+			, retrieveKeyFile = \k f d -> wrapper $ retrieveKeyFile r k f d 			, retrieveKeyFileCheap = \k f -> wrapper $ retrieveKeyFileCheap r k f 			, removeKey = \k -> wrapper $ removeKey r k 			, hasKey = \k -> wrapper $ hasKey r k
Remote/Helper/Ssh.hs view
@@ -1,6 +1,6 @@ {- git-annex remote access with ssh  -- - Copyright 2011 Joey Hess <joey@kitenet.net>+ - Copyright 2011.2012 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -13,6 +13,7 @@ import Config import Annex.UUID import Annex.Ssh+import Fields  {- Generates parameters to ssh to a repository's host and run a command.  - Caller is responsible for doing any neccessary shellEscaping of the@@ -25,9 +26,9 @@  {- Generates parameters to run a git-annex-shell command on a remote  - repository. -}-git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))-git_annex_shell r command params-	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)+git_annex_shell :: Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam]))+git_annex_shell r command params fields+	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts ++ fieldopts) 	| Git.repoIsSsh r = do 		uuid <- getRepoUUID r 		sshparams <- sshToRepo r [Param $ sshcmd uuid ]@@ -39,9 +40,16 @@ 		shellopts = Param command : File dir : params 		sshcmd uuid = unwords $ 			shellcmd : map shellEscape (toCommand shellopts) ++-			uuidcheck uuid+			uuidcheck uuid +++			map shellEscape (toCommand fieldopts) 		uuidcheck NoUUID = [] 		uuidcheck (UUID u) = ["--uuid", u]+		fieldopts+			| null fields = []+			| otherwise = fieldsep : map fieldopt fields ++ [fieldsep]+		fieldsep = Param "--"+		fieldopt (field, value) = Param $+			fieldName field ++ "=" ++ value  {- Uses a supplied function (such as boolSystem) to run a git-annex-shell  - command on a remote.@@ -53,9 +61,10 @@ 	-> (FilePath -> [CommandParam] -> IO a, a) 	-> String 	-> [CommandParam]+	-> [(Field, String)] 	-> Annex a-onRemote r (with, errorval) command params = do-	s <- git_annex_shell r command params+onRemote r (with, errorval) command params fields = do+	s <- git_annex_shell r command params fields 	case s of 		Just (c, ps) -> liftIO $ with c ps 		Nothing -> return errorval
Remote/Hook.hs view
@@ -101,8 +101,8 @@ 					return False 				) -store :: String -> Key -> Annex Bool-store h k = do+store :: String -> Key -> AssociatedFile -> Annex Bool+store h k _f = do 	src <- inRepo $ gitAnnexLocation k 	runHook h "store" k (Just src) $ return True @@ -112,8 +112,8 @@ 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp 	runHook h "store" enck (Just tmp) $ return True -retrieve :: String -> Key -> FilePath -> Annex Bool-retrieve h k f = runHook h "retrieve" k (Just f) $ return True+retrieve :: String -> Key -> AssociatedFile -> FilePath -> Annex Bool+retrieve h k _f d = runHook h "retrieve" k (Just d) $ return True  retrieveCheap :: String -> Key -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False
Remote/Rsync.hs view
@@ -99,8 +99,8 @@ 		use h = rsyncUrl o </> h k </> rsyncEscape o (f </> f)                 f = keyFile k -store :: RsyncOpts -> Key -> Annex Bool-store o k = rsyncSend o k <=< inRepo $ gitAnnexLocation k+store :: RsyncOpts -> Key -> AssociatedFile -> Annex Bool+store o k _f = rsyncSend o k <=< inRepo $ gitAnnexLocation k  storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do@@ -108,8 +108,8 @@ 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp 	rsyncSend o enck tmp -retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool-retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o+retrieve :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool+retrieve o k _ f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o 	-- use inplace when retrieving to support resuming 	[ Param "--inplace" 	, Param u@@ -117,11 +117,11 @@ 	]  retrieveCheap :: RsyncOpts -> Key -> FilePath -> Annex Bool-retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k f , return False )+retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k undefined f , return False )  retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do-	ifM (retrieve o enck tmp)+	ifM (retrieve o enck undefined tmp) 		( liftIO $ catchBoolIO $ do 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f 			return True@@ -169,7 +169,7 @@  - ensure that files are only moved into place once complete  -} partialParams :: CommandParam-partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"+partialParams = Params "--partial --partial-dir=.rsync-partial"  {- Runs an action in an empty scratch directory that can be used to build  - up trees for rsync. -}
Remote/S3.hs view
@@ -113,8 +113,8 @@ 					-- be human-readable 					M.delete "bucket" defaults -store :: Remote -> Key -> Annex Bool-store r k = s3Action r False $ \(conn, bucket) -> do+store :: Remote -> Key -> AssociatedFile -> Annex Bool+store r k _f = s3Action r False $ \(conn, bucket) -> do 	dest <- inRepo $ gitAnnexLocation k 	res <- liftIO $ storeHelper (conn, bucket) r k dest 	s3Bool res@@ -149,12 +149,12 @@ 		xheaders = filter isxheader $ M.assocs $ fromJust $ config r 		isxheader (h, _) = "x-amz-" `isPrefixOf` h -retrieve :: Remote -> Key -> FilePath -> Annex Bool-retrieve r k f = s3Action r False $ \(conn, bucket) -> do+retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool+retrieve r k _f d = s3Action r False $ \(conn, bucket) -> do 	res <- liftIO $ getObject conn $ bucketKey r bucket k 	case res of 		Right o -> do-			liftIO $ L.writeFile f $ obj_data o+			liftIO $ L.writeFile d $ obj_data o 			return True 		Left e -> s3Warning e
Remote/Web.hs view
@@ -51,21 +51,21 @@ 		remotetype = remote 	} -downloadKey :: Key -> FilePath -> Annex Bool-downloadKey key file = get =<< getUrls key+downloadKey :: Key -> AssociatedFile -> FilePath -> Annex Bool+downloadKey key _file dest = get =<< getUrls key 	where 		get [] = do 			warning "no known url" 			return False 		get urls = do 			showOutput -- make way for download progress bar-			downloadUrl urls file+			downloadUrl urls dest  downloadKeyCheap :: Key -> FilePath -> Annex Bool downloadKeyCheap _ _ = return False -uploadKey :: Key -> Annex Bool-uploadKey _ = do+uploadKey :: Key -> AssociatedFile -> Annex Bool+uploadKey _ _ = do 	warning "upload to web not supported" 	return False
Types/Remote.hs view
@@ -33,6 +33,9 @@ instance Eq (RemoteTypeA a) where 	x == y = typename x == typename y +{- A filename associated with a Key, for display to user. -}+type AssociatedFile = Maybe FilePath+ {- An individual remote. -} data RemoteA a = Remote { 	-- each Remote has a unique uuid@@ -42,9 +45,9 @@ 	-- Remotes have a use cost; higher is more expensive 	cost :: Int, 	-- Transfers a key to the remote.-	storeKey :: Key -> a Bool,+	storeKey :: Key -> AssociatedFile -> a Bool, 	-- retrieves a key's contents to a file-	retrieveKeyFile :: Key -> FilePath -> a Bool,+	retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> a Bool, 	-- retrieves a key's contents to a tmp file, if it can be done cheaply 	retrieveKeyFileCheap :: Key -> FilePath -> a Bool, 	-- removes a key's contents
+ Utility/Applicative.o view

binary file changed (absent → 1308 bytes)

Utility/Daemon.hs view
@@ -27,7 +27,7 @@ 			_ <- forkProcess child2 			out 		child2 = do-			maybe noop (lockPidFile True alreadyrunning) pidfile +			maybe noop (lockPidFile alreadyrunning) pidfile  			when changedirectory $ 				setCurrentDirectory "/" 			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags@@ -42,31 +42,44 @@ 		alreadyrunning = error "Daemon is already running." 		out = exitImmediately ExitSuccess -lockPidFile :: Bool -> IO () -> FilePath -> IO ()-lockPidFile write onfailure file = do-	fd <- openFd file ReadWrite (Just stdFileMode)-		defaultFileFlags { trunc = write }-	locked <- catchMaybeIO $ setLock fd (locktype, AbsoluteSeek, 0, 0)-	case locked of-		Nothing -> onfailure-		_ -> when write $ void $-			fdWrite fd =<< show <$> getProcessID+{- Locks the pid file, with an exclusive, non-blocking lock.+ - Runs an action on failure. On success, writes the pid to the file,+ - fully atomically. -}+lockPidFile :: IO () -> FilePath -> IO ()+lockPidFile onfailure file = do+	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags+	locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0)+	fd' <- openFd newfile ReadWrite (Just stdFileMode) defaultFileFlags+		{ trunc = True }+	locked' <- catchMaybeIO $ setLock fd' (WriteLock, AbsoluteSeek, 0, 0)+	case (locked, locked') of+		(Nothing, _) -> onfailure+		(_, Nothing) -> onfailure+		_ -> do+			_ <- fdWrite fd' =<< show <$> getProcessID+			renameFile newfile file+			closeFd fd 	where-		locktype-			| write = WriteLock-			| otherwise = ReadLock+		newfile = file ++ ".new"  {- Stops the daemon.  -  - The pid file is used to get the daemon's pid.  -- - To guard against a stale pid, try to take a nonblocking shared lock- - of the pid file. If this *fails*, the daemon must be running,- - and have the exclusive lock, so the pid file is trustworthy.+ - To guard against a stale pid, check the lock of the pid file,+ - and compare the process that has it locked with the file content.  -} stopDaemon :: FilePath -> IO ()-stopDaemon pidfile = lockPidFile False go pidfile-	where-		go = do-			pid <- readish <$> readFile pidfile-			maybe noop (signalProcess sigTERM) pid+stopDaemon pidfile = do+	fd <- openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags+	locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0)+	p <- readish <$> readFile pidfile+	case (locked, p) of+		(Nothing, _) -> noop+		(_, Nothing) -> noop+		(Just (pid, _), Just pid')+			| pid == pid' -> signalProcess sigTERM pid+			| otherwise -> error $+				"stale pid in " ++ pidfile ++ +				" (got " ++ show pid' ++ +				"; expected" ++ show pid ++ " )"
Utility/Directory.hs view
@@ -35,14 +35,15 @@ dirContents d = map (d </>) . filter (not . dirCruft) <$> getDirectoryContents d  {- Gets files in a directory, and then its subdirectories, recursively,- - and lazily. -}+ - and lazily. If the directory does not exist, no exception is thrown,+ - instead, [] is returned. -} dirContentsRecursive :: FilePath -> IO [FilePath] dirContentsRecursive topdir = dirContentsRecursive' topdir [""]  dirContentsRecursive' :: FilePath -> [FilePath] -> IO [FilePath] dirContentsRecursive' _ [] = return [] dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do-	(files, dirs') <- collect [] [] =<< dirContents (topdir </> dir)+	(files, dirs') <- collect [] [] =<< catchDefaultIO (dirContents (topdir </> dir)) [] 	files' <- dirContentsRecursive' topdir (dirs' ++ dirs) 	return (files ++ files') 	where
+ Utility/Directory.o view

binary file changed (absent → 17128 bytes)

+ Utility/Exception.o view

binary file changed (absent → 5624 bytes)

+ Utility/FileSystemEncoding.o view

binary file changed (absent → 4868 bytes)

Utility/Misc.hs view
@@ -35,3 +35,13 @@ {- Breaks out the first line. -} firstLine :: String-> String firstLine = takeWhile (/= '\n')++{- Splits a list into segments that are delimited by items matching+ - a predicate. (The delimiters are not included in the segments.) -}+segment :: (a -> Bool) -> [a] -> [[a]]+segment p l = map reverse $ go [] [] l+	where+		go c r [] = reverse $ c:r+		go c r (i:is)+			| p i = go [] (c:r) is+			| otherwise = go (i:c) r is
+ Utility/Misc.o view

binary file changed (absent → 5988 bytes)

+ Utility/Monad.o view

binary file changed (absent → 6164 bytes)

+ Utility/Mounts.hs view
@@ -0,0 +1,75 @@+{-# LINE 1 "Utility/Mounts.hsc" #-}+{- Interface to mtab (and fstab)+{-# LINE 2 "Utility/Mounts.hsc" #-}+ - + - Derived from hsshellscript, originally written by+ - Volker Wysk <hsss@volker-wysk.de>+ - + - Modified to support BSD and Mac OS X by+ - Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU LGPL version 2.1 or higher.+ -}++{-# LANGUAGE ForeignFunctionInterface #-}++module Utility.Mounts (+	Mntent(..),+	getMounts+) where++import Control.Monad+import Foreign+import Foreign.C+import GHC.IO hiding (finally, bracket)+import Prelude hiding (catch)+++{-# LINE 26 "Utility/Mounts.hsc" #-}++{- This is a stripped down mntent, containing only+ - fields available everywhere. -}+data Mntent = Mntent+	{ mnt_fsname :: String+	, mnt_dir :: String+	, mnt_type :: String+	} deriving (Read, Show, Eq, Ord)++getMounts :: IO [Mntent]+getMounts = do+	h <- c_mounts_start+	when (h == nullPtr) $+		throwErrno "getMounts"+	mntent <- getmntent h []+	_ <- c_mounts_end h+	return mntent++	where+		getmntent h c = do+			ptr <- c_mounts_next h+			if (ptr == nullPtr)+				then return $ reverse c+				else do+					mnt_fsname_str <- (\hsc_ptr -> peekByteOff hsc_ptr 0) ptr >>= peekCString+{-# LINE 51 "Utility/Mounts.hsc" #-}+					mnt_dir_str <- (\hsc_ptr -> peekByteOff hsc_ptr 4) ptr >>= peekCString+{-# LINE 52 "Utility/Mounts.hsc" #-}+					mnt_type_str <- (\hsc_ptr -> peekByteOff hsc_ptr 8) ptr >>= peekCString+{-# LINE 53 "Utility/Mounts.hsc" #-}+					let ent = Mntent+						{ mnt_fsname = mnt_fsname_str+						, mnt_dir = mnt_dir_str+						, mnt_type = mnt_type_str+						}+					getmntent h (ent:c)++{- Using unsafe imports because the C functions are belived to never block.+ - Note that getmntinfo is called with MNT_NOWAIT to avoid possibly blocking;+ - while getmntent only accesses a file in /etc (or /proc) that should not+ - block. -}+foreign import ccall unsafe "libmounts.h mounts_start" c_mounts_start+        :: IO (Ptr ())+foreign import ccall unsafe "libmounts.h mounts_next" c_mounts_next+        :: Ptr () -> IO (Ptr ())+foreign import ccall unsafe "libmounts.h mounts_end" c_mounts_end+        :: Ptr () -> IO CInt
+ Utility/PartialPrelude.o view

binary file changed (absent → 5448 bytes)

+ Utility/Path.o view

binary file changed (absent → 21888 bytes)

Utility/RsyncFile.hs view
@@ -22,9 +22,9 @@                  - string is a single quote. -} 		escape s = "'" ++  join "''" (split "'" s) ++ "'" -{- Runs rsync in server mode to send a file, and exits. -}-rsyncServerSend :: FilePath -> IO ()-rsyncServerSend file = rsyncExec $+{- Runs rsync in server mode to send a file. -}+rsyncServerSend :: FilePath -> IO Bool+rsyncServerSend file = rsync $ 	rsyncServerParams ++ [Param "--sender", File file]  {- Runs rsync in server mode to receive a file. -}@@ -47,11 +47,8 @@ rsync :: [CommandParam] -> IO Bool rsync = boolSystem "rsync" -rsyncExec :: [CommandParam] -> IO ()-rsyncExec params = executeFile "rsync" True (toCommand params) Nothing- {- Checks if an rsync url involves the remote shell (ssh or rsh).- - Use of such urls with rsync or rsyncExec requires additional shell+ - Use of such urls with rsync requires additional shell  - escaping. -} rsyncUrlIsShell :: String -> Bool rsyncUrlIsShell s
+ Utility/SafeCommand.o view

binary file changed (absent → 32116 bytes)

+ Utility/TempFile.o view

binary file changed (absent → 6216 bytes)

+ Utility/libmounts.o view

binary file changed (absent → 1284 bytes)

debian/changelog view
@@ -1,3 +1,20 @@+git-annex (3.20120721) unstable; urgency=low++  * get, move, copy: Now refuse to do anything when the requested file+    transfer is already in progress by another process.+  * status: Lists transfers that are currently in progress.+  * Fix passing --uuid to git-annex-shell.+  * When shaNsum commands cannot be found, use the Haskell SHA library+    (already a dependency) to do the checksumming. This may be slower,+    but avoids portability problems.+  * Use SHA library for files less than 50 kb in size, at which point it's+    faster than forking the more optimised external program.+  * SHAnE backends are now smarter about composite extensions, such as+    .tar.gz Closes: #680450+  * map: Write map.dot to .git/annex, which avoids watch trying to annex it.++ -- Joey Hess <joeyh@debian.org>  Sat, 21 Jul 2012 16:52:48 -0400+ git-annex (3.20120629) unstable; urgency=low    * cabal: Only try to use inotify on Linux.
+ doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn view
@@ -0,0 +1,23 @@+"git annex watch" will add the file generated by "git annex map", which is+probably not intended. Shouldn’t this file be created in /tmp or+.git/annex/ or somewhere else?++> Indeed, so [[done]] --[[Joey]]++    /tmp $ cd test/+    /tmp/test $ git init+    Initialized empty Git repository in /tmp/test/.git/+    /tmp/test $ git annex init+    init  ok+    (Recording state in git...)+    /tmp/test $ git annex watch+    /tmp/test $ git annex map+    map /tmp/test ok+    +      running: dot -Tx11 map.dot+    +    ok+    /tmp/test $ ls -l+    insgesamt 4+    lrwxrwxrwx 1 jojo jojo 180 Jul 15 23:36 map.dot -> .git/annex/objects/P3/76/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d+
+ doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn view
@@ -0,0 +1,8 @@+commit f20a40f breaks on OSX as mntent.h doesn't exist, the closet thing available to what mntent.h provides is getmntinfo(), it looks yet another bunch of ifdef's might be needed to work around OSX. This problem maybe similarly true with FreeBSD, libfam  seems to have worked around the issue - <http://oss.sgi.com/projects/fam/download/contrib/freebsd-mntent.patch>++hope the above report helps.++> Thanks, that was a very useful pointer. I couldn't figure out how to+> use Haskell's FFI to loop over the list of statfs structs returned by+> getmntinfo, so I incorporated that C code into a little library, +> and it seems to work ok. [[done]] --[[Joey]]
+ doc/bugs/git_annex_does_nothing_useful.mdwn view
@@ -0,0 +1,60 @@+As you can see, I'm running a pretty recent build of git-annex (ac799c3f363e0008b23e9c174e6fedc35e6fa92a),++    $ git annex version+    git-annex version: 3.20120630+    local repository version: 3+    default repository version: 3+    supported repository versions: 3+    upgrade supported from repository versions: 0 1 2++We have a file here which isn't currently available yet isn't+currently available (the link is shown in red),++    $ ls -l plot.py+    lrwxrwxrwx 1 ben ben 77 Jul  6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py+    $ ++Yet git-annex should be able to tell us where it is,++    $ git-annex whereis plot.py+    $ ++Hmm, well that's strange. What's happening here,++    $ git-annex whereis plot.py -d+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"]+    $ ++Alright, well maybe `git-annex get` will work,++    $ git annex get plot.py -d+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"]+    $ ls -l plot.py+    lrwxrwxrwx 1 ben ben 77 Jul  6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py++Nope, the link is still shown in red.++Alright, what about `git-annex copy`?++    $ git annex copy plot.py --from=goldnerlab --to=here -d +    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"]+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"]+    $ ls -l plot.py+    lrwxrwxrwx 1 ben ben 77 Jul  6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py++Still red.++Alright, what if I just try to get a non-existent file?++    $ git annex get adsflkah -d+    git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","adsflkah"]+    $++Alright, it didn't fail with an error, that's very strange. What is going on here?
+ doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn view
@@ -0,0 +1,41 @@+I had started a fresh repo to test out the watch command again on OSX and noticed that it's borked, I'm not sure when it was broken.++The snippet of the log message and command is++<pre>+$ git annex watch --foreground -d -v+watch . read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","git-annex"] +read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","--hash","refs/heads/git-annex"] +read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","log","refs/heads/git-annex..6702e5361146450800ae5af0b63e97bd9c55d70b","--oneline","-n1"] +chat: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","cat-file","--batch"] +(scanning...) call: git ["--git-dir=/Users/jtan+</pre>++I had run git-annex with a new repo with just doing a git init and git annex init, I just threw in one or two small text files to see if it was working. It just hangs and does nothing. I had also tried it out on one of my bigger repos and it does the same thing it just hangs at _(scanning...)_ There isn't much to go on, I wonder if it's hitting the [[Issue on OSX with some system limits]] or if its just a thread/fork issue on OSX.++It still hangs on the small repo even if I do++    $ sudo sysctl -w kern.maxfilesperproc=400000+    $ ulimit -n 2000++Also, just in case if you need it still (on a clean OSX 10.7 system)++<pre>+$ ulimit -a+core file size          (blocks, -c) 0+data seg size           (kbytes, -d) unlimited+file size               (blocks, -f) unlimited+max locked memory       (kbytes, -l) unlimited+max memory size         (kbytes, -m) unlimited+open files                      (-n) 256+pipe size            (512 bytes, -p) 1+stack size              (kbytes, -s) 8192+cpu time               (seconds, -t) unlimited+max user processes              (-u) 709+virtual memory          (kbytes, -v) unlimited+</pre>++Please close or merge this report if it's a duplicate.++> I've fixed this, it works in my (so far limited) tests. [[done]]+> --[[Joey]]
doc/design/assistant.mdwn view
@@ -1,5 +1,6 @@ The git-annex assistant is being [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).+[[Thanks]] to all my backers.  This is my design and plan for developing it. Still being fleshed out, still many ideas and use cases to add.@@ -12,12 +13,13 @@ * Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]] * Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]] * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]+* Months 7-11: user-driven features and polishing+* Month 12: "Windows purgatory" [[Windows]]  ## not yet on the map:  * [[desymlink]] * [[deltas]]-* In my overfunded nighmares: [[Windows]]  ## blog
− doc/design/assistant/.syncing.mdwn.swp

binary file changed (20480 → absent bytes)

doc/design/assistant/blog.mdwn view
@@ -2,4 +2,4 @@ [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). I'll be blogging about my progress here on a semi-daily basis. -[[!inline pages="page(design/assistant/blog/*)" show=30]]+[[!inline pages="page(design/assistant/blog/*)" show=0]]
doc/design/assistant/blog/day_10__lsof.mdwn view
@@ -23,7 +23,7 @@  At one point, something glorious happened, and it was always making exactly one commit for batch mode modifications of a lot of files (like untarring-them). Unfortunatly, I had to lose that gloriousness due to another+them). Unfortunately, I had to lose that gloriousness due to another potential race, which, while unlikely, would have made the program deadlock if it happened.  @@ -40,7 +40,7 @@  This works great! Starting up `git annex watch` when processes have files open is no longer a problem, and even if you're evil enough to try having-muliple processes open the same file, it will complain and not annex it+multiple processes open the same file, it will complain and not annex it until all the writers close it.  (Well, someone really evil could turn the write bit back on after git annex
doc/design/assistant/blog/day_12__freebsd_redux.mdwn view
@@ -3,13 +3,13 @@ understand kqueue fairly well -- there are some very tricky parts to the interface. -But... it still did't work. After building all this, my code was+But... it still didn't work. After building all this, my code was failing the same way that the [haskell kqueue library failed](https://github.com/hesselink/kqueue/issues/1) yesterday. I filed a [bug report with a testcase]().  Then I thought to ask on #haskell. Got sorted out in quick order! The-problem turns out to be that haskell's runtime has a peridic SIGALARM,+problem turns out to be that haskell's runtime has a periodic SIGALARM, that is interrupting my kevent call. It can be worked around with `+RTS -V0`, but I put in a fix to retry to kevent when it's interrupted.
doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn view
@@ -10,13 +10,13 @@ changes show up without delay. Eventually it'll need to support syncing between nodes that cannot directly contact one-another. Syncing needs to deal with nodes coming and going; one example of that is a USB drive being-plugged in, which should immediatly be synced, but network can also come+plugged in, which should immediately be synced, but network can also come and go, so it should periodically retry nodes it failed to sync with. To start with, I'll be focusing on fast syncing between directly connected nodes, but I have to keep this wider problem space in mind.  One problem with `git annex sync` is that it has to be run in both clones-in order for changes to fully propigate. This is because git doesn't allow+in order for changes to fully propagate. This is because git doesn't allow pushing changes into a non-bare repository; so instead it drops off a new branch in `.git/refs/remotes/$foo/synced/master`. Then when it's run locally it merges that new branch into `master`.
doc/design/assistant/blog/day_18__merging.mdwn view
@@ -12,7 +12,7 @@ So, instead, git-annex will use a regular `git merge`, and if it fails, it will fix up the conflicts. -That presented its own difficully, of finding which files in the tree+That presented its own difficulty, of finding which files in the tree conflict. `git ls-files --unmerged` is the way to do that, but its output is a quite raw form: @@ -21,9 +21,9 @@ 	100644 1eabec834c255a127e2e835dadc2d7733742ed9a 2	bar 	100644 36902d4d842a114e8b8912c02d239b2d7059c02b 3	bar -I had to stare at the rather inpenetrable documentation for hours and+I had to stare at the rather impenetrable documentation for hours and write a lot of parsing and processing code to get from that to these mostly-self expanatory data types:+self explanatory data types:  	data Conflicting v = Conflicting 	        { valUs :: Maybe v
doc/design/assistant/blog/day_19__random_improvements.mdwn view
@@ -35,7 +35,7 @@ 1. Uploads new data to every configured remote. Triggered by the watcher    thread when it adds content. Easy; just use a `TSet` of Keys to send. -2. Downloads new data from the cheapest remote that has it. COuld be +2. Downloads new data from the cheapest remote that has it. Could be     triggered by the    merger thread, after it merges in a git sync. Rather hard; how does it    work out what new keys are in the tree without scanning it all? Scan
+ doc/design/assistant/blog/day_20__data_transfer_design.mdwn view
@@ -0,0 +1,22 @@+Today is a planning day. I have only a few days left before I'm off to+Nicaragua for [DebConf](http://debconf12.debconf.org/), where I'll only+have smaller chunks of time without interruptions. So it's important to get+some well-defined smallish chunks designed that I can work on later. See+bulleted action items below (now moved to [[syncing]]. Each+should be around 1-2 hours unless it turns out to be 8 hours... :)++First, worked on writing down a design, and some data types, for data transfer+tracking (see [[syncing]] page). Found that writing down these simple data+types before I started slinging code has clarified things a lot for me.++Most importantly, I realized that I will need to modify `git-annex-shell`+to record on disk what transfers it's doing, so the assistant can get that+information and use it to both avoid redundant transfers (potentially a big+problem!), and later to allow the user to control them using the web app.++While eventually the user will be able to use the web app to prioritize+transfers, stop and start, throttle, etc, it's important to get the default+behavior right. So I'm thinking about things like how to prioritize uploads+vs downloads, when it's appropriate to have multiple downloads running at+once, etc.+
+ doc/design/assistant/blog/day_21__transfer_tracking.mdwn view
@@ -0,0 +1,28 @@+Worked today on two action items from my last blog post:++* on-disk transfers in progress information files (read/write/enumerate)+* locking for the files, so redundant transfer races can be detected,+  and failed transfers noticed++That's all done, and used by the `get`, `copy`, and `move` subcommands.++Also, I made `git-annex status` use that information to display any+file transfers that are currently in progress:++	joey@gnu:~/lib/sound/misc>git annex status+	[...]+	transfers in progress: +		downloading Vic-303.mp3 from leech++(Webapp, here we come!)++However... Files being sent or received by `git-annex-shell` don't yet+have this transfer info recorded. The problem is that to do so,+`git-annex-shell` will need to be run with a `--remote=` parameter. But+old versions will of course fail when run with such an unknown parameter.++This is a problem I last faced in December 2011 when adding the `--uuid=`+parameter. That time I punted and required the remote `git-annex-shell` be+updated to a new enough version to accept it. But as git-annex gets more widely+used and packaged, that's becoming less an option. I need to find a real+solution to this problem.
+ doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn view
@@ -0,0 +1,34 @@+Well, sometimes you just have to go for the hack. Trying to find a way+to add additional options to git-annex-shell without breaking backwards+compatibility, I noticed that it ignores all options after `--`, because+those tend to be random rsync options due to the way rsync runs it.++So, I've added a new class of options, that come in between, like+`-- opt=val opt=val ... --`++The parser for these will not choke on unknown options, unlike normal+getopt. So this let me add the additional info I needed to+pass to git-annex-shell to make it record transfer information. And+if I need to pass more info in the future, that's covered too.++It's ugly, but since only git-annex runs git-annex-shell, this is an+ugliness only I (and now you, dear reader) have to put up with.++Note to self: Command-line programs are sometimes an API, particularly+if designed to be called remotely, and so it makes sense consider+whether they are, and design expandability into them from day 1.++---++Anyway, we now have full transfer tracking in git-annex! Both sides of+a transfer know what's being transferred, and from where, and have+the info necessary to interrupt the transfer.++---++Also did some basic groundwork, adding a queue of transfers to perform,+and adding to the daemon's status information a map of currently running+transfers. ++Next up: The daemon will use inotify to notice new and deleted transfer+info files, and update its status info.
+ doc/design/assistant/blog/day_23__transfer_watching.mdwn view
@@ -0,0 +1,25 @@+Starting to travel, so limited time today.++Yet Another Thread added to the assistant, all it does is watch for changes+to transfer information files, and update the assistant's map of transfers+currently in progress. Now the assistant will know if some other repository+has connected to the local repo and is sending or receiving a file's+content.++This seemed really simple to write, it's just 78 lines of code. It worked+100% correctly the first time. :) But it's only so easy because I've got+this shiny new inotify hammer that I keep finding places to use in the+assistant.++Also, the new thread does some things that caused a similar thread (the+merger thread) to go into a MVar deadlock. Luckily, I spent much of +[day 19](day_19__random_improvements) investigating and fixing that+deadlock, even though it was not a problem at the time.++So, good.. I'm doing things right and getting to a place where rather+nontrivial features can be added easily.++--++Next up: Enough nonsense with tracking transfers... Time to start actually+transferring content around!
+ doc/design/assistant/blog/day_24__airport_digressions.mdwn view
@@ -0,0 +1,99 @@+In a series of airport layovers all day. Since I woke up at 3:45 am, +didn't feel up to doing serious new work, so instead I worked through some+OSX support backlog.++git-annex will now use Haskell's SHA library if the `sha256sum`+command is not available. That library is slow, but it's guaranteed to be+available; git-annex already depended on it to calculate HMACs.++Then I decided to see if it makes sense to use the SHA library+when adding smaller files. At some point, its slower implementation should+win over needing to fork and parse the output of `sha256sum`. This was +the first time I tried out Haskell's +[Criterion](http://hackage.haskell.org/package/criterion) benchmarker,+and I built this simple benchmark in short order.++[[!format haskell """+import Data.Digest.Pure.SHA+import Data.ByteString.Lazy as L+import Criterion.Main+import Common++testfile :: FilePath+testfile = "/tmp/bar" -- on ram disk++main = defaultMain+        [ bgroup "sha256"+                [ bench "internal" $ whnfIO internal+                , bench "external" $ whnfIO external+                ]+        ]++internal :: IO String+internal = showDigest . sha256 <$> L.readFile testfile++external :: IO String+external = pOpen ReadFromPipe "sha256sum" [testfile] $ \h ->+        fst . separate (== ' ') <$> hGetLine h+"""]]++The nice thing about benchmarking in Airports is when you're running a+benchmark locally, you don't want to do anything else with the computer,+so can alternate people watching, spacing out, and analizing results.++100 kb file:++	benchmarking sha256/internal+	mean: 15.64729 ms, lb 15.29590 ms, ub 16.10119 ms, ci 0.950+	std dev: 2.032476 ms, lb 1.638016 ms, ub 2.527089 ms, ci 0.950++	benchmarking sha256/external+	mean: 8.217700 ms, lb 7.931324 ms, ub 8.568805 ms, ci 0.950+	std dev: 1.614786 ms, lb 1.357791 ms, ub 2.009682 ms, ci 0.950++75 kb file:++	benchmarking sha256/internal+	mean: 12.16099 ms, lb 11.89566 ms, ub 12.50317 ms, ci 0.950+	std dev: 1.531108 ms, lb 1.232353 ms, ub 1.929141 ms, ci 0.950++	benchmarking sha256/external+	mean: 8.818731 ms, lb 8.425744 ms, ub 9.269550 ms, ci 0.950+	std dev: 2.158530 ms, lb 1.916067 ms, ub 2.487242 ms, ci 0.950++50 kb file:++	benchmarking sha256/internal+	mean: 7.699274 ms, lb 7.560254 ms, ub 7.876605 ms, ci 0.950+	std dev: 801.5292 us, lb 655.3344 us, ub 990.4117 us, ci 0.950++	benchmarking sha256/external+	mean: 8.715779 ms, lb 8.330540 ms, ub 9.102232 ms, ci 0.950+	std dev: 1.988089 ms, lb 1.821582 ms, ub 2.181676 ms, ci 0.950++10 kb file:++	benchmarking sha256/internal+	mean: 1.586105 ms, lb 1.574512 ms, ub 1.604922 ms, ci 0.950+	std dev: 74.07235 us, lb 51.71688 us, ub 108.1348 us, ci 0.950++	benchmarking sha256/external+	mean: 6.873742 ms, lb 6.582765 ms, ub 7.252911 ms, ci 0.950+	std dev: 1.689662 ms, lb 1.346310 ms, ub 2.640399 ms, ci 0.950++It's possible to get nice graphical reports out of Criterion, but +this is clear enough, so I stopped here. 50 kb seems a reasonable+cutoff point.++I also used this to benchmark the SHA256 in Haskell's Crypto package.+Surprisingly, it's a *lot* slower than even the Pure.SHA code.+On a 50 kb file:++	benchmarking sha256/Crypto+	collecting 100 samples, 1 iterations each, in estimated 6.073809 s+	mean: 69.89037 ms, lb 69.15831 ms, ub 70.71845 ms, ci 0.950+	std dev: 3.995397 ms, lb 3.435775 ms, ub 4.721952 ms, ci 0.950+++There's another Haskell library, [SHA2](http://hackage.haskell.org/package/SHA2),+which I should try some time.
+ doc/design/assistant/blog/day_25__transfer_queueing.mdwn view
@@ -0,0 +1,41 @@+So as not to bury the lead, I've been hard at work on my first day in+Nicaragua, and ** the git-annex assistant fully syncs files (including+their contents) between remotes now !! **++Details follow..++Made the committer thread queue Upload Transfers when new files+are added to the annex. Currently it tries to transfer the new content+to *every* remote; this inefficiency needs to be addressed later.++Made the watcher thread queue Download Transfers when new symlinks+appear that point to content we don't have. Typically, that will happen+after an automatic merge from a remote. This needs to be improved as it+currently adds Transfers from every remote, not just those that have the+content.++This was the second place that needed an ordered list of remotes+to talk to. So I cached such a list in the DaemonStatus state info.+This will also be handy later on, when the webapp is used to add new+remotes, so the assistant can know about them immediately.++Added YAT (Yet Another Thread), number 15 or so, the transferrer thread+that waits for transfers to be queued and runs them. Currently a naive+implementation, it runs one transfer at a time, and does not do anything+to recover when a transfer fails.++Actually transferring content requires YAT, so that the transfer+action can run in a copy of the Annex monad, without blocking+all the assistant's other threads from entering that monad while a transfer+is running. This is also necessary to allow multiple concurrent transfers+to run in the future.++This is a very tricky piece of code, because that thread will modify the+git-annex branch, and its parent thread has to invalidate its cache in+order to see any changes the child thread made. Hopefully that's the extent+of the complication of doing this. The only reason this was possible at all+is that git-annex already support multiple concurrent processes running+and all making independent changes to the git-annex branch, etc.++After all my groundwork this week, file content transferring is now+fully working!
+ doc/design/assistant/blog/day_26__dying_drives.mdwn view
@@ -0,0 +1,28 @@+My laptop's SSD died this morning. I had some work from yesterday+committed to the git repo on it, but not pushed as it didn't build.+Luckily I was able to get that off the SSD, which is now a read-only+drive -- even mounting it fails with fsck write errors.++Wish I'd realized the SSD was dying before the day before my trip to+Nicaragua..+Getting back to a useful laptop used most of my time and energy today.++I did manage to fix transfers to not block the rest of the assistant's+threads. Problem was that, without Haskell's threaded runtime, waiting+on something like a rsync command blocks all threads. To fix this,+transfers now are run in separate processes.++Also added code to allow multiple transfers to run at once. Each transfer+takes up a slot, with the number of free slots tracked by a `QSemN`.+This allows the transfer starting thread to block until a slot frees up,+and then run the transfer.++This needs to be extended to be aware of transfers initiated by remotes.+The transfer watcher thread should detect those starting and stopping +and update the `QSemN` accordingly. It would also be nice if transfers+initiated by remotes would be delayed when there are no free slots for them+... but I have not thought of a good way to do that.++There's a bug somewhere in the new transfer code, when two transfers are+queued close together, the second one is lost and doesn't happen.+Would debug this, but I'm spent for the day.
+ doc/design/assistant/blog/day_27__robust_transfers.mdwn view
@@ -0,0 +1,31 @@+Spent most of the day making file content transfers robust. There were lots+of bugs, hopefully I've fixed most of them. It seems to work well now,+even when I throw a lot of files at it.++One of the changes also sped up transfers; it no longer roundtrips to the+remote to verify it has a file. The idea here is that when the assistant is+running, repos should typically be fairly tightly synced to their remotes+by it, so some of the extra checks that the `move` command does are+unnecessary.++Also spent some time trying to use ghc's threaded runtime, but continue to+be baffled by the random hangs when using it. This needs fixing eventually;+all the assistant's threads can potentially be blocked when it's waiting on+an external command it has run.++Also changed how transfer info files are locked. The lock file is now+separate from the info file, which allows the TransferWatcher thread to+notice when an info file is created, and thus actually track transfers+initiated by remotes.++---++I'm fairly close now to merging the `assistant` branch into `master`.+The data syncing code is very brute-force, but it will work well enough+for a first cut.++Next I can either add some repository network mapping, and use graph+analysis to reduce the number of data transfers, or I can move on to the+[[webapp]]. Not sure yet which I'll do. It's likely that since DebConf+begins tomorrow I'll put off either of those big things until after the+conference.
+ doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn view
@@ -0,0 +1,17 @@+I didn't plan to work on git-annex much while at DebConf, because the conference+always prevents the kind of concentration I need. But I unexpectedly also had to deal+with [three dead drives](http://joeyh.name/blog/entry/I_am_become_Joey_destroyer_of_drives/)+and illness this week. ++That said, I have been trying to debug a problem with git-annex and Haskell's threaded+runtime all week. It just hangs, randomly. No luck so far isolating why, although I now+have a branch that hangs fairly reliably, and in which I am trying to whittle the entire+git-annex code base (all 18 thousand lines!) into a nice test case.++This threaded runtime problem doesn't affect the assistant yet, but if I want to use+Yesod in developing the webapp, I'll need the threaded runtime, and using the threaded+runtime in the assistant generally would make it more responsive and less hacky.++Since this is a task I can work on without much concentration, I'll probably keep beating+on it until I return home. Then I need to spend some quality thinking time on where+to go next in the assistant.
doc/design/assistant/blog/day_2__races.mdwn view
@@ -1,6 +1,6 @@ Last night I got `git annex watch` to also handle deletion of files. This was not as tricky as feared; the key is using `git rm --ignore-unmatch`,-which avoids most problimatic situations (such as a just deleted file+which avoids most problematic situations (such as a just deleted file being added back before git is run).  Also fixed some races when `git annex watch` is doing its startup scan of
+ doc/design/assistant/blog/day_36__minimal_test_case.mdwn view
@@ -0,0 +1,9 @@+Managed to find a minimal, 20 line test case for at least one of the ways+git-annex was hanging with GHC's threaded runtime. Sent it off to+haskell-cafe for analysis.+[thread](http://news.gmane.org/gmane.comp.lang.haskell.cafe)++Further managed to narrow the bug down to MissingH's use of logging code,+that git-annex doesn't use. [bug report](http://bugs.debian.org/681621).+So, I can at least get around this problem with a modified version of+MissingH. Hopefully that was the only thing causing the hangs I was seeing!
+ doc/design/assistant/blog/day_37__back.mdwn view
@@ -0,0 +1,64 @@+Back home and laptop is fixed.. back to work.++Warmup exercises:++* Went in to make it queue transfers when a broken symlink is received,+  only to find I'd already written code to do that, and forgotten about it.+  Heh. Did check that the git-annex branch is always sent first,+  which will ensure that code always knows where to transfer a key from.+  I had probably not considered this wrinkle when first writing the code;+  it worked by accident.++* Made the assistant check that a remote is known to have a key before+  queueing a download from it.++* Fixed a bad interaction between the `git annex map` command and the+  assistant.++----++Tried using a modified version of `MissingH` that doesn't use `HSLogger`+to make git-annex work with the threaded GHC runtime. Unfortunatly,+I am still seeing hangs in at least 3 separate code paths when+running the test suite. I may have managed to fix one of the hangs,+but have not grokked what's causing the others.++----++I now have access to a Mac OSX system, thanks to Kevin M. I've fixed+some portability problems in git-annex with it before, but today I tested+the assistant on it:++* Found a problem with the kqueue code that prevents incoming pushes from+  being noticed.++  The problem was that the newly added git ref file does not trigger an add+  event. The kqueue code saw a generic change event for the refs directory,+  but since the old file was being deleted and replaced by the new file,+  the kqueue code, which already had the old file in its cache, did not notice+  the file had been replaced.++  I fixed that by making the kqueue code also track the inode of each file.+  Currently that adds the overhead of a stat of each file, which could be+  avoided if haskell exposed the inode returned by `readdir`. Room to+  optimise this later...++* Also noticed that the kqueue code was not separating out file deletions+  from directory deletions. IIRC Jimmy had once mentioned a problem with file+  deletions not being noticed by the assistant, and this could be responsible+  for that, although the directory deletion code seems to handle them ok+  normally. It was making the transfer watching thread not notice when+  any transfers finished, for sure. I fixed this oversight, looking in the+  cache to see if there used to be a file or a directory, and running the+  appropriate hook.++Even with these fixes, the assistant does not yet reliably transfer file+contents on OSX. I think the problem is that with kqueue we're not+guaranteed to get an add event, and a deletion event for a transfer+info file -- if it's created and quickly deleted, the code that+synthensizes those events doesn't run in time to know it existed.+Since the transfer code relies on deletion events to tell when transfers+are complete, it stops sending files after the first transfer, if the+transfer ran so quickly it doesn't get the expected events. ++So, will need to work on OSX support some more...
+ doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn view
@@ -0,0 +1,66 @@+Beating my head against the threaded runtime some more. I can reproduce+one of the hangs consistently by running 1000 git annex add commands+in a loop. It hangs around 1% of the time, reading from `git cat-file`.++Interestingly, `git cat-file` is not yet running at this point -- +git-annex has forked a child process, but the child has not yet exec'd it.+Stracing the child git-annex, I see it stuck in a futex. Adding tracing,+I see the child never manages to run any code at all.++This really looks like the problem is once again in MissingH, which uses+`forkProcess`. Which happens to come with a big warning about being very+unsafe, in very subtle ways. Looking at the C code that the newer `process`+library uses when sparning a pipe to a process, it messes around with lots of+things; blocking signals, stopping a timer, etc. Hundreds of lines of C+code to safely start a child process, all doing things that MissingH omits.++That's the second time I've seemingly isolated a hang in the GHC threaded+runtime to MissingH.++And so I've started converting git-annex to use the new `process` library,+for running all its external commands. John Goerzen had mentioned `process`+to me once before when I found a nasty bug in MissingH, as the cool new+thing that would probably eliminate the `System.Cmd.Utils` part of MissingH,+but I'd not otherwise heard much about it. (It also seems to have the+benefit of supporting Windows.)++This is a big change and it's early days, but each time I see a hang, I'm+converting the code to use `process`, and so far the hangs have just gone+away when I do that.++---++Hours later... I've converted *all* of git-annex to use `process`.++In the er, process, the `--debug` switch stopped printing all the commands+it runs. I may try to restore that later.++I've not tested everything, but the test suite passes, even when+using the threaded runtime. **MILESTONE**++Looking forward to getting out of these weeds and back to useful work..++---++Hours later yet.... The `assistant` branch in git now uses the threaded+runtime. It works beautifully, using proper threads to run file transfers+in.++That should fix the problem I was seeing on OSX yesterday. Too tired to+test it now.++--++Amazingly, all the assistant's own dozen or so threads and thread+synch variables etc all work great under the threaded runtime. I had+assumed I'd see yet more concurrency problems there when switching to it,+but it all looks good. (Or whatever problems there are are subtle ones?)++I'm very relieved. The threaded logjam is broken! I had been getting+increasingly worried that not having the threaded runtime available would+make it very difficult to make the assistant perform really well, and cause+problems with the webapp, perhaps preventing me from using Yesod.++Now it looks like smooth sailing ahead. Still some hard problems, but+it feels like with inotify and kqueue and the threaded runtime all+dealt with, the really hard infrastructure-level problems are behind me.
+ doc/design/assistant/blog/day_40__dbus.mdwn view
@@ -0,0 +1,100 @@+Really productive day today, now that I'm out of the threaded runtime+tarpit!++First, brought back `--debug` logging, better than before! As part of that, I+wrote some 250 lines of code to provide a IMHO more pleasant interface to+`System.Process` (itself only 650 lines of code) that avoids all the+low-level setup, cleanup, and tuple unpacking. Now I can do things like+write to a pipe to a process, and ensure it exits nonzero, this easily:++	withHandle StdinHandle createProcessSuccess (proc "git" ["hash-object", "--stdin"]) $ \h ->+		hHutStr h objectdata++My interface also makes it easy to run nasty background processes,+reading their output lazily.++	lazystring <- withHandle StdoutHandle createBackgroundProcess (proc "find" ["/"]) hGetContents++Any true Haskellers are shuddering here, I really should be using+conduits or pipes, or something. One day..++----++The assistant needs to detect when removable drives are attached, and+sync with them. This is a reasonable thing to be working on at this point,+because it'll make the currently incomplete data transfer code fully usable+for the sneakernet use case, and firming that up will probably be a good+step toward handing other use cases involving data transfer over the+network, including cases where network remotes are transientely available.++So I've been playing with using dbus to detect mount events.+There's a very nice Haskell library to use dbus.++This simple program will detect removable drives being mounted, and+works on Xfce (as long as you have automounting enabled in its+configuration), and should also work on Gnome, and, probably, KDE:++[[!format haskell """+{-# LANGUAGE OverloadedStrings #-}++import Data.List (sort)+import DBus+import DBus.Client+import Control.Monad++main = do+	client <- connectSession+    +	listen client mountadded $ \s ->+		putStrLn (show s)++	forever $ getLine -- let listener thread run forever++	where+		mountadded = matchAny+			{ matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor"+			, matchMember = Just "MountAdded"+			}+"""]]++(Yeah... "org.gtk.Private.RemoteVolumeMonitor". There are so+many things wrong with that string. What does gtk have to do with+mounting a drive? Why is it Private? Bleagh. Should I only match+the "MountAdded" member and not the interface? Seems everyone who does+this relies on google to find other people who have cargo-culted it,+or just runs `dbus-monitor` and picks out things.+There seems to be no canonical list of events. Bleagh.)++----++Spent a while shaving a yak of needing a `getmntent` interface in Haskell.+Found one in a hsshellscript library; since that library is not packaged+in Debian, and I don't really want to depend on it, I extracted just+the mtab and fstab parts of it into a little library in git-annex.++----++I've started putting together a MountWatcher thread. On systems without+dbus (do OSX or the BSDs have dbus?), or if dbus is not running, it polls+`/etc/mtab` every 10 seconds for new mounts. When dbus is available,+it doesn't need the polling, and should notice mounts more quickly.++Open question: Should it still poll even when dbus is available? Some of us+like to mount our own drives, by hand and may have automounting disabled. It'd+be good if the assistant supported that. This might need a+`annex.no-dbus` setting, but I'd rather avoid needing such manual+configuration.++One idea is to do polling in addition to dbus, if `/etc/fstab` contains+mount points that seem to be removable drives, on which git remotes lives.+Or it could always do polling in addition to dbus, which is just some extra+work. Or, it could try to introspect dbus to see if mount events will+be generated.++The MountWatcher so far only detects new mounts and prints out what+happened. Next up: Do something in response to them. ++This will involve manipulating the Annex state to belatedly add the Remote+on the mount point.. tricky. And then, for Git Remotes, it should pull/push+the Remote to sync git data. Finally, for all remotes, it will need to+queue Transfers of file contents from/to the newly available Remote.
+ doc/design/assistant/blog/day_41__foo.mdwn view
@@ -0,0 +1,46 @@+I made the MountWatcher only use dbus if it sees a client connected to dbus+that it knows will send mount events, or if it can start up such a client+via dbus. (Fancy!) Otherwise it falls back to polling. This should be enough+to support users who manually mount things -- if they have gvfs+installed, it'll be used to detect their manual mounts, even when a desktop+is not running, and if they don't have gvfs, they get polling.++Also, I got the MountWatcher to work with KDE. Found a dbus event that's+emitted when KDE mounts a drive, and this is also used. If anyone with+some other desktop environment wants me to add support for it, and it uses+dbus, it should be easy: Run `dbus-monitor`, plug in a drive, get+it mounted, and send me a transcript.++Of course, it'd also be nice to support anything similar on OSX that can+provide mount event notifications. Not a priority though, since the polling+code will work.++---++Some OS X fixes today..++* Jimmy pointed out that my `getmntent` code broke the build on OSX again.+  Sorry about that.. I keep thinking Unix portability nightmares are a 80's+  thing, not a 2010's thing. Anyway, adapted a lot of hackish C code+  to emulate `getmntent` on BSD systems, and it seems to work. (I actually+  think the BSD interface to this is saner than Linux's, but I'd rather have+  either one than both, sigh..)+* Kqueue was blocking all the threads on OSX. This is fixed, and the+  assistant seems to be working on OSX again.++----++I put together a preliminary page thanking everyone who contributed to the+git-annex Kickstarter. [[thanks]] The wall-o-names is scary crazy humbling.++----++Improved `--debug` mode for the assistant, now every thread says whenever+it's doing anything interesting, and also there are timestamps.++----++Had been meaning to get on with syncing to drives when they're mounted, but+got sidetracked with the above. Maybe tomorrow. I did think through it+in some detail as I was waking up this morning, and think I have a pretty+good handle on it.
doc/design/assistant/blog/day_4__speed.mdwn view
@@ -16,7 +16,7 @@ (This will, in fact, be the start of the [[syncing]] phase of my roadmap!) There's lots of room here for smart behavior. Like, if a lot of changes are being made close together, wait for them to die down before committing. Or,-if it's been idle and a single file appears, commit it immediatly, since+if it's been idle and a single file appears, commit it immediately, since this is probably something the user wants synced out right away. I'll start with something stupid and then add the smarts.
doc/design/assistant/blog/day_5__committing.mdwn view
@@ -11,7 +11,7 @@  So next, I added some smarts to it. First, I wanted to stop it waking up every second when there was nothing to do, and instead blocking wait on a-change occuring. Secondly, I wanted it to know when past changes happened,+change occurring. Secondly, I wanted it to know when past changes happened, so it could detect batch mode scenarios, and avoid committing too frequently.  @@ -52,6 +52,6 @@                thisSecond t = now `diffUTCTime` t <= 1 """]] -Still some polishing to do to eliminate minor innefficiencies and deal+Still some polishing to do to eliminate minor inefficiencies and deal with more races, but this part of the git-annex assistant is now very usable, and will be going out to my beta testers soon!
doc/design/assistant/blog/day_6__polish.mdwn view
@@ -24,7 +24,7 @@ the index still have the old value.  Instead, I got creative. :) We can't trust what the index says about the-symlink, but if the index happens to contian a symlink that looks right,+symlink, but if the index happens to contain a symlink that looks right, we can trust that the SHA1 of its blob is the right SHA1, and reuse it when re-staging the symlink. Wham! Massive speedup!
doc/design/assistant/blog/day_7__bugfixes.mdwn view
@@ -10,7 +10,7 @@ tree of files is being moved or unpacked into the watched directory.  Instead, I made it only do the check during `git annex watch`'s initial-scan of the tree. This should be ok, because once it's running, you+scan of the tree. This should be OK, because once it's running, you won't be adding new  files to git anyway, since it'll automatically annex new files. This is good enough for now, but there are at least two problems with it:
doc/design/assistant/blog/day_8__speed.mdwn view
@@ -16,7 +16,7 @@ First impressions are important. :)  But what does "made recently" mean exactly? Well, my answer is possibly-overengineered, but most of it is really groundwork for things I'll need+over engineered, but most of it is really groundwork for things I'll need later anyway. I added a new data structure for tracking the status of the daemon, which is periodically written to disk by another thread (thread #6!) to `.git/annex/daemon.status` Currently it looks like this; I anticipate
doc/design/assistant/inotify.mdwn view
@@ -18,6 +18,17 @@   I may need to fork off multiple watcher processes to handle this.   See [[bugs/Issue_on_OSX_with_some_system_limits]]. +## todo++* Run niced and ioniced? Seems to make sense, this is a background job.+* configurable option to only annex files meeting certian size or+  filename criteria+* option to check files not meeting annex criteria into git directly,+  automatically+* honor .gitignore, not adding files it excludes (difficult, probably+  needs my own .gitignore parser to avoid excessive running of git commands+  to check for ignored files)+ ## beyond Linux  I'd also like to support OSX and if possible the BSDs.@@ -64,17 +75,6 @@   * <https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/fsevents.py> (good example program)  * Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.--## todo--- Run niced and ioniced? Seems to make sense, this is a background job.-- configurable option to only annex files meeting certian size or-  filename criteria-- option to check files not meeting annex criteria into git directly,-  automatically-- honor .gitignore, not adding files it excludes (difficult, probably-  needs my own .gitignore parser to avoid excessive running of git commands-  to check for ignored files)  ## the races
doc/design/assistant/progressbars.mdwn view
@@ -9,6 +9,6 @@ and use its size.  TODO: What about uploads? Will i have to parse rsync's progresss output?-Feed it via a named pipe? Ugh.+Feed it via a named pipe? Ugh. Check into librsync.  This is one of those potentially hidden but time consuming problems.
doc/design/assistant/syncing.mdwn view
@@ -1,27 +1,41 @@ Once files are added (or removed or moved), need to send those changes to all the other git clones, at both the git level and the key/value level. -## git syncing+## immediate action items -1. Can use `git annex sync`, which already handles bidirectional syncing.-   When a change is committed, launch the part of `git annex sync` that pushes-   out changes. **done**; changes are pushed out to all remotes in parallel-1. Watch `.git/refs/remotes/` for changes (which would be pushed in from-   another node via `git annex sync`), and run the part of `git annex sync`-   that merges in received changes, and follow it by the part that pushes out-   changes (sending them to any other remotes).-   [The watching can be done with the existing inotify code! This avoids needing-   any special mechanism to notify a remote that it's been synced to.]  -   **done**-1. Periodically retry pushes that failed.  **done** (every half an hour)-1. Also, detect if a push failed due to not being up-to-date, pull,-   and repush. **done**-2. Use a git merge driver that adds both conflicting files,-   so conflicts never break a sync. **done**-3. Investigate the XMPP approach like dvcs-autosync does, or other ways of+* At startup, and possibly periodically, look for files we have that+  location tracking indicates remotes do not, and enqueue Uploads for+  them. Also, enqueue Downloads for any files we're missing.+* After git sync, identify content that we don't have that is now available+  on remotes, and transfer. But first, need to ensure that when a remote+  receives content, and updates its location log, it syncs that update+  out.+* When MountWatcher detects a newly mounted drive, rescan git remotes+  in order to get ones on the drive, and do a git sync and file transfers+  to sync any repositories on it.++## longer-term TODO++* Test MountWatcher on Gnome (should work ok) and LXDE (dunno).+* git-annex needs a simple speed control knob, which can be plumbed+  through to, at least, rsync. A good job for an hour in an+  airport somewhere.+* Find a way to probe available outgoing bandwidth, to throttle so+  we don't bufferbloat the network to death.+* Investigate the XMPP approach like dvcs-autosync does, or other ways of    signaling a change out of band.-4. Add a hook, so when there's a change to sync, a program can be run+* Add a hook, so when there's a change to sync, a program can be run    and do its own signaling.+* --debug will show often unnecessary work being done. Optimise.+* This assumes the network is connected. It's often not, so the+  [[cloud]] needs to be used to bridge between LANs.+* Configurablity, including only enabling git syncing but not data transfer;+  only uploading new files but not downloading, and only downloading+  files in some directories and not others. See for use cases:+  [[forum/Wishlist:_options_for_syncing_meta-data_and_data]]+* speed up git syncing by using the cached ssh connection for it too+  (will need to use `GIT_SSH`, which needs to point to a command to run,+  not a shell command line)  ## data syncing @@ -46,45 +60,53 @@ get the basic functionality of the assistant to work. And we'll need this anyway. -### transfer tracking--	data ToTransfer = ToUpload Key | ToDownload Key-	type ToTransferChan = TChan [ToTransfer]--* ToUpload added by the watcher thread when it adds content.-* ToDownload added by the watcher thread when it seens new symlinks-  that lack content.--Transfer threads started/stopped as necessary to move data.-May sometimes want multiple threads downloading, or uploading, or even both.--	data TransferID = TransferThread ThreadID | TransferProcess Pid-	data Direction = Uploading | Downloading-	data Transfer = Transfer Direction Key TransferID EpochTime Integer-	-- add [Transfer] to DaemonStatus--The assistant needs to find out when `git-annex-shell` is receiving or-sending (triggered by another remote), so it can add data for those too.-This is important to avoid uploading content to a remote that is already-downloading it from us, or vice versa, as well as to in future let the web-app manage transfers as user desires. --For files being received, it can see the temp file, but other than lsof-there's no good way to find the pid (and I'd rather not kill blindly).--For files being sent, there's no filesystem indication. So git-annex-shell-(and other git-annex transfer processes) should write a status file to disk.--Can use file locking on these status files to claim upload/download rights,-which will avoid races.--This status file can also be updated periodically to show amount of transfer-complete (necessary for tracking uploads).--## other considerations+## done -It would be nice if, when a USB drive is connected,-syncing starts automatically. Use dbus on Linux?+1. Can use `git annex sync`, which already handles bidirectional syncing.+   When a change is committed, launch the part of `git annex sync` that pushes+   out changes. **done**; changes are pushed out to all remotes in parallel+1. Watch `.git/refs/remotes/` for changes (which would be pushed in from+   another node via `git annex sync`), and run the part of `git annex sync`+   that merges in received changes, and follow it by the part that pushes out+   changes (sending them to any other remotes).+   [The watching can be done with the existing inotify code! This avoids needing+   any special mechanism to notify a remote that it's been synced to.]  +   **done**+1. Periodically retry pushes that failed.  **done** (every half an hour)+1. Also, detect if a push failed due to not being up-to-date, pull,+   and repush. **done**+2. Use a git merge driver that adds both conflicting files,+   so conflicts never break a sync. **done** -This assumes the network is connected. It's often not, so the-[[cloud]] needs to be used to bridge between LANs.+* on-disk transfers in progress information files (read/write/enumerate)+  **done**+* locking for the files, so redundant transfer races can be detected,+  and failed transfers noticed **done**+* transfer info for git-annex-shell **done**+* update files as transfers proceed. See [[progressbars]]+  (updating for downloads is easy; for uploads is hard)+* add Transfer queue TChan **done**+* add TransferInfo Map to DaemonStatus for tracking transfers in progress.+  **done**+* Poll transfer in progress info files for changes (use inotify again!+  wow! hammer, meet nail..), and update the TransferInfo Map **done**+* enqueue Transfers (Uploads) as new files are added to the annex by+  Watcher. **done**+* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by+  Watcher. **done**+  (Note: Needs git-annex branch to be merged before the tree is merged,+  so it knows where to download from. Checked and this is the case.)+* Write basic Transfer handling thread. Multiple such threads need to be+  able to be run at once. Each will need its own independant copy of the +  Annex state monad. **done**+* Write transfer control thread, which decides when to launch transfers.+  **done**+* Transfer watching has a race on kqueue systems, which makes finished+  fast transfers not be noticed by the TransferWatcher. Which in turn+  prevents the transfer slot being freed and any further transfers+  from happening. So, this approach is too fragile to rely on for+  maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]],+  which would allow running something for sure when a transfer thread+  finishes. **done**+* Test MountWatcher on KDE, and add whatever dbus events KDE emits when+  drives are mounted. **done**
+ doc/design/assistant/thanks.mdwn view
@@ -0,0 +1,233 @@+The development of the git-annex assistant was made possible by the+generous donations of many people. I want to say "Thank You!" to each of+you individually, but until I meet all 951 of you, this page will have to+do. You have my most sincere thanks. --[[Joey]]++(If I got your name wrong, or you don't want it publically posted here,+email <joey@kitenet.net>.)++## Major Backers++These people are just inspiring in their enthusiasm and generosity to this+project.++* Jason Scott+* strager++## Beta Testers++Whole weeks of my time were made possible thanks to each of these+people, and their testing is invaluable to the development of+the git-annex assistant.++* Jimmy Tang+* David Pollak+* Pater+* Francois Marier+* Paul Sherwood+* Fred Epma+* Robert Ristroph+* Josh Triplett+* David Haslem+* AJ Ashton+* Svenne Krap+* Drew Hess++## Prioritizers++These forward-thinking people contributed generously just to help+set my priorities in which parts of the git-annex assistant were most+important to develop.++Paul C. Bryan, Paul Tötterman, Don Marti, Dean Thompson, Djoume, David Johnston+Asokan Pichai, Anders Østhus, Dominik Wagenknecht, Charlie Fox, Yazz D. Atlas,+fenchel, Erik Penninga, Richard Hartmann, Graham, Stan Yamane, Ben Skelton,+Ian McEwen, asc, Paul Tagliamonte, Sherif Abouseda, Igor Támara, Anne Wind,+Mesar Hameed, Brandur K. Holm Petersen, Takahiro Inoue, Kai Hendry,+Stephen Youndt, Lee Roberson, Ben Strawbridge, Andrew Greenberg, Alfred Adams+Andrew, Aaron De Vries, Monti Knazze, Jorge Canseco, Hamish, Mark Eichin++And special thanks to Kevin McKenzie, who also gave me a login to a Mac OSX+machine, which has proven invaluable.++## Other Backers++Most of the success of the Kickstarter is thanks to these folks. Some of+them spent significant amounts of money in the guise of getting some+swag. For others, being listed here, and being crucial to making the +git-annex assistant happen was reward enough. Large or small, these+contributions are, literally, my bread and butter this year.++Amitai Schlair, mvime, Romain Lenglet, James Petts, Jouni Uuksulainen,+Wichert Akkerman, Robert Bellus, Kasper Souren, rob, Michiel Buddingh',+Kevin, Rob Avina, Alon Levy, Vikash, Michael Alan Dorman, Harley Pig,+Andreas Olsson, Pietpiet, Christine Spang, Liz Young, Oleg Kosorukov,+Allard Hoeve, Valentin Haenel, Joost Baaij, Nathan Yergler, Nathan Howell,+Frédéric Schütz, Matti Eskelinen, Neil McGovern, Lane Lillquist, db48x,+Stuart Prescott, Mark Matienzo, KarlTheGood, leonm, Drew Slininger, +Andreas Fuchs, Conrad Parker, Johannes Engelke, Battlegarden, Justin Kelly,+Robin Wagner, Thad Ward, crenquis, Trudy Goold, Mike Cochrane, Adam Venturella,+Russell Foo, furankupan, Giorgio Occhioni, andy, mind, Mike Linksvayer,+Stefan Strahl, Jelmer Vernooij, Markus Fix, David Hicks, Justin Azoff,+Iain Nicol, Bob Ippolito, Thomas Lundstrøm, Jason Mandel, federico2,+Edd Cochran, Jose Ortega, Emmett Agnew, Rudy Garcia, Kodi, Nick Rusnov,+Michael Rubin, Tom de Grunt, Richard Murray, Peter, Suzanne Pierce, Jared+Marcotte, folk, Eamon, Jeff Richards, Leo Sutedja, dann frazier, Mikkel+kristiansen, Matt Thomas, Kilian Evang, Gergely Risko, Kristian Rumberg,+Peter Kropf, Mark Hepburn, greymont, D. Joe Anderson, Jeremy Zunker, ctebo,+Manuel Roumain, Jason Walsh, np, Shawn, Johan Tibell, Branden Tyree, Dinyar+Rabady, Andrew Mason, damond armstead, Ethan Aubin, TomTom Tommie, Jimmy+Kaplowitz, Steven Zakulec, mike smith, Jacob Kirkwood, Mark Hymers, Nathan+Collins, Asbjørn Sloth Tønnesen, Misty De Meo,+Jim Paris, Adam Sjøgren, miniBill, Taneli, Kumar Appaiah, Greg Grossmeier,+Sten Turpin, Otavio Salvador, Teemu Hukkanen, Brian Stengaard, bob walker,+bibeneus, andrelo, Yaroslav Halchenko, hesfalling, Tommy L, jlargentaye,+Serafeim Zanikolas, Don Armstrong, Chris Cormack, shayne.oneill, Radu+Raduta, Josh S, Robin Sheat, Henrik Mygind, kodx, Christian, Geoff+Crompton, Brian May, Olivier Berger, Filippo Gadotti, Daniel Curto-Millet,+Eskild Hustvedt, Douglas Soares de Andrade, Tom L, Michael Nacos, Michaël+P., William Roe, Joshua Honeycutt, Brian Kelly, Nathan Rasch, jorge, Martin+Galese, alex cox, Avery Brooks, David Whittington, Dan Martinez, Forrest+Sutton, Jouni K. Seppänen, Arnold Cano, Robert Beaty, Daniel, Kevin Savetz,+Randy, Ernie Braganza, Aaron Haviland, Brian Brunswick, asmw, sean, Michael+Williams, Alexander, Dougal Campbell, Robert Bacchas, Michael Lewis, Collin+Price, Wes Frazier, Matt Wall, Brandon Barclay, Derek van Vliet, Martin+DeMello, kitt hodsden, Stephen Kitt, Leif Bergman, Simon Lilburn, Michael+Prokop, Christiaan Conover, Nick Coombe, Tim Dysinger, Brandon Robinson,+Philip Newborough, keith, Mike Fullerton, Kyle, Phil Windley, Tyler Head,+George V. Reilly, Matthew, Ali Gündüz, Vasyl Diakonov, Paolo Capriotti,+allanfranta, Martin Haeberli, msingle, Vincent Sanders, Steven King, Dmitry+Gribanov, Brandon High, Ben Hughes, Mike Dank, JohnE, Diggory Hardy,+Michael Hanke, valhalla, Samuli Tuomola, Jeff Rau, Benjamin Lebsanft, John+Drago, James, Aidan Cooper, rondie, Paul Kohler, Matthew Knights, Aaron+Junod, Patrick R McDonald, Christopher Browne, Daniel Engel, John SJ+Anderson, Peter Sarossy, Mike Prasad, Christoph Ender, Jan Dittberner,+Zohar, Alexander Jelinek, stefan, Danny O'Brien, Matthew Thode, Nicole+Aptekar, maurice gaston, Chris Adams, Mike Klemencic, Reedy, Subito, Tobias+Gruetzmacher, Ole-Morten Duesund, André Koot, mp, gdop, Cole Scott, Blaker,+Matt Sottile, W. Craig Trader, Louis-Philippe Dubrule, Brian Boucheron,+Duncan Smith, Brenton Buchbach, Kyle Stevenson, Eliot Lash, Egon Elbre,+Praveen, williamji, Thomas Schreiber, Neil Ford, Ryan Pratt, Joshua Brand,+Peter Cox, Markus Engstrom, John Sutherland, Dean Bailey, Ed Summers,+Hillel Arnold, David Fiander, Kurt Yoder, Trevor Muñoz, keri, Ivan+Sergeyenko, Shad Bolling, Tal Kelrich, Steve Presley, gerald ocker, Essex+Taylor, Josh Thomson, Trevor Bramble, Lance Higgins, Frank Motta, Dirk+Kraft, soundray, Joe Haskins, nmjk, Apurva Desai, Colin Dean, docwhat,+Joseph Costello, Jst, flamsmark, Alex Lang, Bill Traynor, Anthony David,+Marc-André Lureau, AlNapp, Giovanni Moretti, John Lawrence, João Paulo+Pizani Flor, Jim Ray, Gregory Thrush, Alistair McGann, Andrew Wied,+Koutarou Furukawa, Xiscu Ignacio, Aaron Sachs, Matt, Quirijn, Chet+Williams, Chris Gray, Bruce Segal, Tom Conder, Louis Tovar, Alex Duryee,+booltox, d8uv, Decklin Foster, Rafael Cervantes, Micah R Ledbetter, Kevin+Sjöberg, Johan Strombom, Zachary Cohen, Jason Lewis, Yves Bilgeri, Ville+Aine, Mark Hurley, Marco Bonetti, Maximilian Haack, Hynek Schlawack,+Michael Leinartas, Andreas Liebschner, Duotrig, Nat Fairbanks, David+Deutsch, Colin Hayhurst, calca, Kyle Goodrick, Marc Bobillier, Robert+Snook, James Kim, Olivier Serres, Jon Redfern, Itai Tavor, Michael+Fladischer, Rob, Jan Schmid, Thomas H., Anders Söderbäck, Abhishek+Dasgupta, Jeff Goeke-Smith, Tommy Thorn, bonuswavepilot, Philipp Edelmann,+Nick, Alejandro Navarro Fulleda, Yann Lallemant, andrew brennan, +Dave Allen Barker Jr, Fabian, Lukas Anzinger, Carl Witty, Andy Taylor,+Andre Klärner, Andrew Chilton, Adam Gibbins, Alexander Wauck, Shane O'Dea,+Paul Waite, Iain McLaren, Maggie Ellen Robin Hess, Willard Korfhage,+Nicolas, Eric Nikolaisen, Magnus Enger, Philipp Kern, Andrew Alderwick,+Raphael Wimmer, Benjamin Schötz, Ana Guerrero, Pete, Pieter van der Eems,+Aaron Suggs, Fred Benenson, Cedric Howe, Lance Ivy, Tieg Zaharia, Kevin+Cooney, Jon Williams, Anton Kudris, Roman Komarov, Brad Hilton, Rick Dakan,+Adam Whitcomb, Paul Casagrande, Evgueni Baldin, Robert Sanders, Kagan+Kayal, Dean Gardiner, micah altman, Cameron Banga, Ross Mcnairn, Oscar+Vilaplana, Robin Graham, Dan Gervais, Jon Åslund, Ragan Webber, Noble Hays,+stephen brown, Sean True, Maciek Swiech, faser, eikenberry, Kai Laborenz,+Sergey Fedoseev, Chris Fournier, Svend Sorensen, Greg K, wojtek, Johan+Ribenfors, Anton, Benjamin, Oleg Tsarev, PsychoHazard, John Cochrane,+Kasper Lauritzen, Patrick Naish, Rob, Keith Nasman, zenmaster, David Royer,+Max Woolf, Dan Gabber, martin rhoads, Martin Schmidinger, Paul+Scott-Wilson, Tom Gromak, Andy Webster, Dale Webb, Jim Watson, Stephen+Hansen, Mircea, Dan Goodenberger, Matthias Fink, Andy Gott, Daniel, Jai+Nelson, Shrayas Rajagopal, Vladimir Rutsky, Alexander, Thorben Westerhuys,+hiruki, Tao Neuendorffer Flaherty, Elline, Marco Hegenberg, robert, Balda,+Brennen Bearnes, Richard Parkins, David Gwilliam, Mark Johnson, Jeff Eaton,+Reddawn90, Heather Pusey, Chris Heinz, Colin, Phatsaphong Thadabusapa,+valunthar, Michael Martinez, redlukas, Yury V. Zaytsev, Blake, Tobias+"betabrain" A., Leon, sopyer, Steve Burnett, bessarabov, sarble, krsch.com,+Jack Self, Jeff Welch, Sam Pettiford, Jimmy Stridh, Diego Barberá, David+Steele, Oscar Ciudad, John Braman, Jacob, Nick Jenkins, Ben Sullivan, Brian+Cleary, James Brosnahan, Darryn Ten, Alex Brem, Jonathan Hitchcock, Jan+Schmidle, Wolfrzx99, Steve Pomeroy, Matthew Sitton, Finkregh, Derek Reeve,+GDR!, Cory Chapman, Marc Olivier Chouinard, Andreas Ryland, Justin, Andreas+Persenius, Games That Teach, Walter Somerville, Bill Haecker, Brandon+Phenix, Justin Shultz, Colin Scroggins, Tim Goddard, Ben Margolin, Michael+Martinez, David Hobbs, Andre Le, Jason Roberts, Bob Lopez, Gert Van Gool,+Robert Carnel, Anders Lundqvist, Aniruddha Sathaye, Marco Gillies, Basti+von Bejga, Esko Arajärvi, Dominik Deobald, Pavel Yudaev, Fionn Behrens,+Davide Favargiotti, Perttu Luukko, Silvan Jegen, Marcelo Bittencourt,+Leonard Peris, smercer, Alexandre Dupas, Solomon Matthews, Peter Hogg,+Richard E. Varian, Ian Oswald, James W. Sarvey, Ed Grether, Frederic+Savard, Sebastian Nerz, Hans-Chr. Jehg, Matija Nalis, Josh DiMauro, Jason+Harris, Adam Byrtek, Tellef, Magnus, Bart Schuurmans, Giel van Schijndel,+Ryan, kiodos, Richard 'maddog' Collins, PawZubr, Jason Gassel, Alex+Boisvert, Richard Thompson, maddi bolton, csights, Aaron Bryson, Jason Chu,+Maxime Côté, Kineteka Systems, Joe Cabezas, Mike Czepiel, Rami Nasra,+Christian Simonsen, Wouter Beugelsdijk, Adam Gibson, Gal Buki, James+Marble, Alan Chambers, Bernd Wiehle, Simon Korzun, Daniel Glassey, Eero af+Heurlin, Mikael, Timo Engelhardt, Wesley Faulkner, Jay Wo, Mike Belle,+David Fowlkes Jr., Jimmy Tang, Karl-Heinz Strass, Ed Mullins, Sam Flint,+Hendrica, Mark Emer Anderson, Joshua Cole, Jan Gondol, Henrik Lindhe,+Albert Delgado, Patrick, Alexa Avellar, Chris, sebsn1349, Maxim Kachur,+Andrew Marshall, Navjot Narula, Alwin Mathew, Christian Mangelsdorf, Avi+Shevin, Kevin S., Guillermo Sanchez Estrada, Alex Krieger, Luca Meier, Will+Jessop, Nick Ruest, Lani Aung, Ulf Benjaminsson, Rudi Engelbrecht, Miles+Matton, Cpt_Threepwood, Adam Kuyper, reacocard, David Kilsheimer, Peter+Olson, Bill Fischofer, Prashant Shah, Simon Bonnick, Alexander Grudtsov,+Antoine Boegli, Richard Warren, Sebastian Rust, AlmostHuman, Timmy+Crawford, PC, Marek Belski, pontus, Douglas S Butts, Eric Wallmander, Joe+Pelar, VIjay, Trahloc, Vernon Vantucci, Matthew baya, Viktor Štujber,+Stephen Akiki, Daniil Zamoldinov, Atley, Chris Thomson, Jacob Briggs, Falko+Richter, Andy Schmitz, Sergi Sagas, Peder Refsnes, Jonatan, Ben, Bill+Niblock, Agustin Acuna, Jeff Curl, Tim Humphrey, bib, James Zarbock,+Lachlan Devantier, Michal Berg, Jeff Lucas, Sid Irish, Franklyn, Jared+Dickson, Olli Jarva, Adam Gibson, Lukas Loesche, Jukka Määttä, Alexander+Lin, Dao Tran, Kirk, briankb, Ryan Villasenor, Daniel Wong, barista, Tomas+Jungwirth, Jesper Hansen, Nivin Singh, Alessandro Tieghi, Billy Roessler,+Peter Fetterer, Pallav Laskar, jcherney, Tyler Wang, Steve, Gigahost, Beat+Wolf, Hannibal Skorepa, aktiveradio, Mark Nunnikhoven, Bret Comnes, Alan+Ruttenberg, Anthony DiSanti, Adam Warkiewicz, Brian Bowman, Jonathan, Mark+Filley, Tobias Mohr, Christian St. Cyr, j. faceless user, Karl Miller,+Thomas Taimre, Vikram, Jason Mountcastle, Jason, Paul Elliott, Alexander,+Stephen Farmer, rayslava, Peter Leurs, Sky Kruse, JP Reeves, John J Schatz,+Martin Sandmair, Will Thompson, John Hergenroeder, Thomas, Christophe+Ponsart, Wolfdog, Eagertolearn, LukasM, Federico Hernandez, Vincent Bernat,+Christian Schmidt, Cameron Colby Thomson, Josh Duff, James Brown, Theron+Trowbridge, Falke, Don Meares, tauu, Greg Cornford, Max Fenton, Kenneth+Reitz, Bruce Bensetler, Mark Booth, Herb Mann, Sindre Sorhus, Chris+Knadler, Daniel Digerås, Derek, Sin Valentine, Ben Gamari, david+lampenscherf, fardles, Richard Burdeniuk, Tobias Kienzler, Dawid Humbla,+Bruno Barbaroxa, D Malt, krivar, James Valleroy, Peter, Tim Geddings,+Matthias Holzinger, Hanen, Petr Vacek, Raymond, Griff Maloney, Andreas+Helveg Rudolph, Nelson Blaha, Colonel Fubar, Skyjacker Captain Gavin+Phoenix, shaun, Michael, Kari Salminen, Rodrigo Miranda, Alan Chan, Justin+Eugene Evans, Isaac, Ben Staffin, Matthew Loar, Magos, Roderik, Eugenio+Piasini, Nico B, Scott Walter, Lior Amsalem, Thongrop Rodsavas, Alberto de+Paola, Shawn Poulen, John Swiderski, lluks, Waelen, Mark Slosarek, Jim+Cristol, mikesol, Bilal Quadri, LuP, Allan Nicolson, Kevin Washington,+Isaac Wedin, Paul Anguiano, ldacruz, Jason Manheim, Sawyer, Jason+Woofenden, Joe Danziger, Declan Morahan, KaptainUfolog, Vladron, bart, Jeff+McNeill, Christian Schlotter, Ben McQuillan, Anthony, Julian, Martin O,+altruism, Eric Solheim, MarkS, ndrwc, Matthew, David Lehn, Matthew+Cisneros, Mike Skoglund, Kristy Carey++## Also thanks to++* The Kickstarter team, who have unleashed much good on the world.+* The Haskell developers, who toiled for 20 years in obscurity+  before most of us noticed them, and on whose giant shoulders I now stand,+  in awe of the view.+* The Git developers, for obvious reasons.+* All of git-annex's early adopters, who turned it from a personal+  toy project into something much more, and showed me the interest was there.+* Anna and Mark, for the loan of the video camera; as well as the rest of+  my family, for your support. Even when I couldn't explain what I was+  working on.+* The Hodges, for providing such a congenial place for me to live and work+  on these first world problems, while you're off helping people in the+  third world.
doc/download.mdwn view
@@ -18,6 +18,7 @@  The git repository has some branches: +* `assistant` contains the new change-tracking daemon * `ghc7.0` supports versions of ghc older than 7.4, which   had a major change to filename encoding. * `old-monad-control` is for systems that don't have a newer monad-control@@ -25,6 +26,7 @@ * `no-ifelse` avoids using the IFelse library   (merge it into master if you need it) * `no-bloom` avoids using bloom filters. (merge it into master if you need it)+* `no-s3` avoids using the S3 library (merge it into master if you need it) * `debian-stable` contains the latest backport of git-annex to Debian   stable. * `tweak-fetch` adds support for the git tweak-fetch hook, which has
+ doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn view
@@ -0,0 +1,1 @@+Are there any problems using submodules with git-annex? I have not tried it yet, I'm just asking.
+ doc/forum/Wishlist:_mark_remotes_offline.mdwn view
@@ -0,0 +1,12 @@+I have several remotes which are not always accessible. For example they can+be on hosts only accessible by LAN or on a portable hard drive which is not+plugged in. When running sync these remotes are checked as well, leading to+unnecessary error messages and possibly git-annex waiting for a few minutes+on each remote for a timeout.++In this situation it would be useful to mark some remotes as offline+(`git annex offline <remotename>`), so that git-annex would not even attempt+to contact them. Then, I could configure my system to automatically, for example,+mark a portable hard disk remote online when plugging it in, and offline when+unplugging it, and similarly marking remotes offline and online depending on+whether I have an internet connection or a connection to a specific network.
+ doc/forum/Wishlist:_options_for_syncing_meta-data_and_data.mdwn view
@@ -0,0 +1,13 @@+Since _transfer queueing_  and syncing of data works now in the assistant branch (been playing with it), there are times when I really don't want to sync the data, I would like to just sync meta-data and manually do a _get_ on files that I would want or selectively sync data in a subtree.++It would be nice to have the syncing/watch feature to have the option of syncing only *meta-data* or *meta-data and data*, I think this sort of option was already planned? It would also be nice to be able to automatically sync data for only a subtree.++My use case is, I have a big stash of files somewhere at home or work, and I want to keep what I am actually using on my laptop and be able to selectively just take a subtree or a set of subtree's of files. I would not always want to suck down all the data but still have the functionally to add files and push them upstream and sync meta-data.++that is...++> * Site A: big master annex in a server room with lots of disk (or machines), watches a directory and syncs both data and meta-data, it should always try and pull data from all it's child repos. That way I will always have a master copy of my data somewhere, it would be even nicer if I could have clones of the annex, where each annex is on a different machine which is configured to only sync a subtree of files so I can distribute my annex across different systems and disks.+>   * Site A: machine A: syncs Folder A+>   * Site A: machine B: syncs Folder B+>   * and so on with selectively syncing sites and directories+> * Laptop: has a clone of the annex, and watches a directory, syncs meta-data as usual and only uploads files to a remote (all or a designated one) but it never downloads files automatically or it should only occur inside a selected subtree.
+ doc/forum/pulling_from_encrypted_remote.mdwn view
@@ -0,0 +1,12 @@+Is there a way to pull from an encrypted remote?++Use case:++1. Have annex in an encrypted public rsync remote+2. Have USB stick with PGP keys (but not the annex repository)+3. Get to a new computer+4. Set up a new annex using the PGP keys I have.++1-3 work fine :) However, 4'th is the issue:++How would I do `git pull <remote>` for an encrypted remote? Is it possible?
doc/git-annex-shell.mdwn view
@@ -61,6 +61,14 @@   git-annex uses this to specify the UUID of the repository it was expecting   git-annex-shell to access, as a sanity check. +* -- fields=val fields=val.. --++  Additional fields may be specified this way, to retain compatability with+  past versions of git-annex-shell (that ignore these, but would choke+  on new dashed options).++  Currently used fields include remoteuuid= and associatedfile=+ # HOOK  After content is received or dropped from the repository by git-annex-shell,
doc/install/Fedora.mdwn view
@@ -1,7 +1,7 @@ Installation recipe for Fedora 14 thruough 17.  <pre>-sudo yum install ghc cabal-install+sudo yum install ghc cabal-install pcre-devel git clone git://git-annex.branchable.com/ git-annex cd git-annex git checkout ghc7.0@@ -15,4 +15,6 @@ Note: You can't just use `cabal install git-annex`, because Fedora does not yet ship ghc 7.4. -[Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)+* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)a+* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654)+* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656)
+ doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"+ nickname="Damien"+ subject="sha256 alternative"+ date="2012-06-30T14:34:11Z"+ content="""+in reply to comment 6: On my Mac (10.7.4) there's `/usr/bin/shasum -a 256 <file>` command that will produce the same output as `sha256sum <file>`.+"""]]
+ doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"+ nickname="Damien"+ subject="gnu commands"+ date="2012-07-01T17:03:57Z"+ content="""+…and another approach to the same problem: apparently git-annex also relies on the GNU coreutils (for instance, when doing `git annex get .`, `cp` complains about `illegal option -- -`). I do have the GNU coreutils installed with Homebrew, but they are all prefixed with `g`. So maybe you should try `gsha256sum` and `gcp` before `sha256sum` and `cp`, that seems like a more general solution.+"""]]
+ doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment view
@@ -0,0 +1,9 @@+[[!comment format=mdwn+ username="http://joeyh.name/"+ subject="comment 14"+ date="2012-07-04T12:43:54Z"+ content="""+@Damien, hmm, it should not be using any cp options, unless when it was built there was a cp in the path that supported some option like -p. Can you check with --debug what cp parameters it's trying to use?+++"""]]
+ doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment view
@@ -0,0 +1,9 @@+[[!comment format=mdwn+ username="http://joeyh.name/"+ subject="comment 15"+ date="2012-07-04T13:14:00Z"+ content="""+git-annex will now fall back to slower pure Haskell hashing code if `sha256sum`, etc programs are not in PATH. I'd still recommend installing the coreutils, as they're probably faster.++(The `shasum` command seems to come from a perl library, so I have not tried to make git-annex use that one.)+"""]]
− doc/news/version_3.20120611.mdwn 
@@ -1,6 +0,0 @@-git-annex 3.20120611 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * add: Prevent (most) modifications from being made to a file while it-     is being added to the annex.-   * initremote: Automatically describe a remote when creating it.-   * uninit: Refuse to run in a subdirectory. Closes: #[677076](http://bugs.debian.org/677076)"""]]
+ doc/news/version_3.20120721.mdwn view
@@ -0,0 +1,14 @@+git-annex 3.20120721 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * get, move, copy: Now refuse to do anything when the requested file+     transfer is already in progress by another process.+   * status: Lists transfers that are currently in progress.+   * Fix passing --uuid to git-annex-shell.+   * When shaNsum commands cannot be found, use the Haskell SHA library+     (already a dependency) to do the checksumming. This may be slower,+     but avoids portability problems.+   * Use SHA library for files less than 50 kb in size, at which point it's+     faster than forking the more optimised external program.+   * SHAnE backends are now smarter about composite extensions, such as+     .tar.gz Closes: #[680450](http://bugs.debian.org/680450)+   * map: Write map.dot to .git/annex, which avoids watch trying to annex it."""]]
+ doc/not/comment_1_ab41bec1ccc884e71780cb9458439170._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://bergey.dreamwidth.org/"+ ip="66.80.90.109"+ subject="git-media"+ date="2012-07-14T15:42:05Z"+ content="""+I haven't used git-media, but from the README it looks as though they now support several backends.  Might want to update the (very helpful!) comparison.+"""]]
+ doc/todo/assistant_git_sync_laddering.mdwn view
@@ -0,0 +1,8 @@+When the [[design/assistant]] is running on a pair of remotes, I've seen+them get out of sync, such that every pull and merge results in a conflict,+that then has to be auto-resolved. ++This seems similar to the laddering problem described in this old bug:+[[bugs/making_annex-merge_try_a_fast-forward]]++--[[Joey]]
+ doc/todo/assistant_threaded_runtime.mdwn view
@@ -0,0 +1,37 @@+The [[design/assistant]] would be better if git-annex used ghc's threaded+runtime (`ghc -threaded`).++Currently, whenever the assistant code runs some external command, all+threads are blocked waiting for it to finish.++For transfers, the assistant works around this problem by forking separate+upload processes, and not waiting on them until it sees an indication that+they have finished the transfer. While this works, it's messy.. threaded+would be better.++When pulling, pushing, and merging, the assistant runs external git+commands, and this does block all other threads. The threaded runtime would+really help here.++[[done]]; the assistant now builds with the threaded runtime.+Some work still remains to run certian long-running external git commands+in their own threads to prevent them blocking things, but that is easy to+do, now. --[[Joey]] ++---++Currently, git-annex seems unstable when built with the threaded runtime.+The test suite tends to hang when testing add. `git-annex` occasionally+hangs, apparently in a futex lock. This is not the assistant hanging, and+git-annex does not otherwise use threads, so this is surprising. --[[Joey]]++> I've spent a lot of time debugging this, and trying to fix it, in the+> "threaded" branch. There are still deadlocks. --[[Joey]]++---++It would be possible to not use the threaded runtime. Instead, we could+have a child process pool, with associated continuations to run after a+child process finishes. Then periodically do a nonblocking waitpid on each+process in the pool in turn (waiting for any child could break anything not+using the pool!). This is probably a last resort...
git-annex-shell.1 view
@@ -50,6 +50,13 @@ git\-annex uses this to specify the UUID of the repository it was expecting git\-annex\-shell to access, as a sanity check. .IP+.IP "\-\- fields=val fields=val.. \-\-"+Additional fields may be specified this way, to retain compatability with+past versions of git\-annex\-shell (that ignore these, but would choke+on new dashed options).+.IP+Currently used fields include remoteuuid= and associatedfile=+.IP .SH HOOK After content is received or dropped from the repository by git\-annex\-shell, it runs a hook, .git/hooks/annex\-content (or hooks/annex\-content on a bare
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 3.20120629+Version: 3.20120721 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess <joey@kitenet.net>
test.hs view
@@ -550,9 +550,6 @@ 	git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" 	-- --fast avoids it running graphviz, not a build dependency 	git_annex "map" ["--fast"] @? "map failed"-	doesFileExist "map.dot" @? "map.dot not generated"-	c <- readFile "map.dot"-	("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? ("map.dot bad content: " ++ c)  test_uninit :: Test test_uninit = "git-annex uninit" ~: intmpclonerepo $ do