packages feed

git-annex 5.20131230 → 5.20140107

raw patch · 111 files changed

+2735/−288 lines, 111 filesbinary-added

Files

Annex/Branch/Transitions.hs view
@@ -34,6 +34,8 @@ dropDead f content trustmap = case getLogVariety f of 	Just UUIDBasedLog -> ChangeFile $ 		UUIDBased.showLog id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLog Just content+	Just NewUUIDBasedLog -> ChangeFile $+		UUIDBased.showLogNew id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLogNew Just content 	Just (PresenceLog _) -> 		let newlog = Presence.compactLog $ dropDeadFromPresenceLog trustmap $ Presence.parseLog content 		in if null newlog
Annex/Content.hs view
@@ -15,6 +15,7 @@ 	getViaTmp, 	getViaTmpChecked, 	getViaTmpUnchecked,+	prepGetViaTmpChecked, 	withTmp, 	checkDiskSpace, 	moveAnnex,@@ -158,20 +159,31 @@ getViaTmpUnchecked = finishGetViaTmp (return True)  getViaTmpChecked :: Annex Bool -> Key -> (FilePath -> Annex Bool) -> Annex Bool-getViaTmpChecked check key action = do+getViaTmpChecked check key action = +	prepGetViaTmpChecked key $+		finishGetViaTmp check key action++{- Prepares to download a key via a tmp file, and checks that there is+ - enough free disk space.+ -+ - When the temp file already exists, count the space it is using as+ - free, since the download will overwrite it or resume.+ -+ - Wen there's enough free space, runs the download action.+ -}+prepGetViaTmpChecked :: Key -> Annex Bool -> Annex Bool+prepGetViaTmpChecked key getkey = do 	tmp <- fromRepo $ gitAnnexTmpLocation key -	-- Check that there is enough free disk space.-	-- When the temp file already exists, count the space-	-- it is using as free. 	e <- liftIO $ doesFileExist tmp 	alreadythere <- if e 		then fromIntegral . fileSize <$> liftIO (getFileStatus tmp) 		else return 0 	ifM (checkDiskSpace Nothing key alreadythere) 		( do+			-- The tmp file may not have been left writable 			when e $ thawContent tmp-			finishGetViaTmp check key action+			getkey 		, return False 		)
Assistant/Alert.hs view
@@ -5,7 +5,7 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE OverloadedStrings, CPP #-}+{-# LANGUAGE OverloadedStrings, CPP, BangPatterns #-}  module Assistant.Alert where @@ -367,8 +367,8 @@   where 	maxfilesshown = 10 -	(somefiles, counter) = splitcounter (dedupadjacent files)-	shortfiles = map (fromString . shortFile . takeFileName) somefiles+	(!somefiles, !counter) = splitcounter (dedupadjacent files)+	!shortfiles = map (fromString . shortFile . takeFileName) somefiles  	renderer alert = tenseWords $ msg : alertData alert ++ showcounter 	  where@@ -391,9 +391,9 @@ 			in (keep, length rest) 	 	combiner new old =-		let (fs, n) = splitcounter $+		let (!fs, n) = splitcounter $ 			dedupadjacent $ alertData new ++ alertData old-		    cnt = n + alertCounter new + alertCounter old+		    !cnt = n + alertCounter new + alertCounter old 		in old 			{ alertData = fs 			, alertCounter = cnt
Assistant/Alert/Utility.hs view
@@ -13,7 +13,7 @@  import qualified Data.Text as T import Data.Text (Text)-import qualified Data.Map as M+import qualified Data.Map.Strict as M  {- This is as many alerts as it makes sense to display at a time.  - A display might be smaller, or larger, the point is to not overwhelm the@@ -122,7 +122,7 @@ 	 		let (f, rest) = partition (\(_, a) -> isFiller a) l 			in drop bloat f ++ rest 	updatePrune = pruneBloat $ M.filterWithKey pruneSame $-		M.insertWith' const i al m+		M.insert i al m 	updateCombine combiner =  		let combined = M.mapMaybe (combiner al) m 		in if M.null combined
Assistant/DaemonStatus.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE BangPatterns #-}+ module Assistant.DaemonStatus where  import Assistant.Common@@ -23,7 +25,7 @@ import Data.Time.Clock.POSIX import Data.Time import System.Locale-import qualified Data.Map as M+import qualified Data.Map.Strict as M import qualified Data.Text as T  getDaemonStatus :: Assistant DaemonStatus@@ -37,7 +39,7 @@ 	dstatus <- getAssistant daemonStatusHandle 	liftIO $ do 		(s, b) <- atomically $ do-			r@(s, _) <- a <$> takeTMVar dstatus+			r@(!s, _) <- a <$> takeTMVar dstatus 			putTMVar dstatus s 			return r 		sendNotification $ changeNotifier s@@ -153,7 +155,8 @@ adjustTransfersSTM :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> STM () adjustTransfersSTM dstatus a = do 	s <- takeTMVar dstatus-	putTMVar dstatus $ s { currentTransfers = a (currentTransfers s) }+	let !v = a (currentTransfers s)+	putTMVar dstatus $ s { currentTransfers = v }  {- Checks if a transfer is currently running. -} checkRunningTransferSTM :: DaemonStatusHandle -> Transfer -> STM Bool@@ -168,7 +171,7 @@  - or if already present, updates it while preserving the old transferTid,  - transferPaused, and bytesComplete values, which are not written to disk. -} updateTransferInfo :: Transfer -> TransferInfo -> Assistant ()-updateTransferInfo t info = updateTransferInfo' $ M.insertWith' merge t info+updateTransferInfo t info = updateTransferInfo' $ M.insertWith merge t info   where 	merge new old = new 		{ transferTid = maybe (transferTid new) Just (transferTid old)@@ -213,8 +216,8 @@   where 	add s = (s { lastAlertId = i, alertMap = m }, i) 	  where-		i = nextAlertId $ lastAlertId s-		m = mergeAlert i alert (alertMap s)+		!i = nextAlertId $ lastAlertId s+		!m = mergeAlert i alert (alertMap s)  removeAlert :: AlertId -> Assistant () removeAlert i = updateAlert i (const Nothing)@@ -225,7 +228,9 @@ updateAlertMap :: (AlertMap -> AlertMap) -> Assistant () updateAlertMap a = notifyAlert `after` modifyDaemonStatus_ update   where-	update s = s { alertMap = a (alertMap s) }+	update s = +		let !m = a (alertMap s)+		in s { alertMap = m }  {- Displays an alert while performing an activity that returns True on  - success.
Assistant/Drop.hs view
@@ -102,10 +102,10 @@ 			)  	dropl fs n = checkdrop fs n Nothing $ \numcopies ->-		Command.Drop.startLocal afile numcopies key knownpresentremote+		Command.Drop.startLocal (Just afile) numcopies key knownpresentremote  	dropr fs r n  = checkdrop fs n (Just $ Remote.uuid r) $ \numcopies ->-		Command.Drop.startRemote afile numcopies key r+		Command.Drop.startRemote (Just afile) numcopies key r  	safely a = either (const False) id <$> tryAnnex a
Assistant/Install.hs view
@@ -96,6 +96,7 @@ 			lookup "GIT_ANNEX_STANDLONE_ENV" env 		restoreorig oldenv p@(k, _v) 			| k `elem` vars = case lookup ("ORIG_" ++ k) oldenv of-				Nothing -> Nothing-				(Just v') -> Just (k, v')+				(Just v')+					| not (null v') -> Just (k, v')+				_ -> Nothing 			| otherwise = Just p
Assistant/Monad.hs view
@@ -78,7 +78,7 @@ 	<*> newScanRemoteMap 	<*> newTransferQueue 	<*> newTransferSlots-	<*> newTransferrerPool+	<*> newTransferrerPool (checkNetworkConnections dstatus) 	<*> newFailedPushMap 	<*> newCommitChan 	<*> newChangePool
Assistant/Repair.hs view
@@ -113,23 +113,30 @@  -  - However, this could be on a network filesystem. Which is not very safe  - anyway (the assistant relies on being able to check when files have- - no writers to know when to commit them). Just in case, when the lock- - file appears stale, we delay for one minute, and check its size. If- - the size changed, delay for another minute, and so on. This will at- - least work to detect is another machine is writing out a new index- - file, since git does so by writing the new content to index.lock.+ - no writers to know when to commit them). Also, a few lock-file-ish+ - things used by git are not kept open, particularly MERGE_HEAD.  -+ - So, just in case, when the lock file appears stale, we delay for one+ - minute, and check its size. If the size changed, delay for another+ - minute, and so on. This will at work to detect when another machine+ - is writing out a new index file, since git does so by writing the+ - new content to index.lock.+ -  - Returns true if locks were cleaned up.  -} repairStaleGitLocks :: Git.Repo -> Assistant Bool repairStaleGitLocks r = do-	lockfiles <- filter (not . isInfixOf "gc.pid") -		. filter (".lock" `isSuffixOf`)-		<$> liftIO (findgitfiles r)+	lockfiles <- liftIO $ filter islock <$> findgitfiles r 	repairStaleLocks lockfiles 	return $ not $ null lockfiles   where 	findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) True . Git.localGitDir+	islock f+		| "gc.pid" `isInfixOf` f = False+		| ".lock" `isSuffixOf` f = True+		| takeFileName f == "MERGE_HEAD" = True+		| otherwise = False+ repairStaleLocks :: [FilePath] -> Assistant () repairStaleLocks lockfiles = go =<< getsizes   where
Assistant/ScanRemotes.hs view
@@ -13,7 +13,7 @@  import Data.Function import Control.Concurrent.STM-import qualified Data.Map as M+import qualified Data.Map.Strict as M  {- Blocks until there is a remote or remotes that need to be scanned.  -
Assistant/Ssh.hs view
@@ -149,7 +149,7 @@ 	sshdir <- sshDir 	let keyfile = sshdir </> "authorized_keys" 	ls <- lines <$> readFileStrict keyfile-	writeFile keyfile $ unlines $ filter (/= keyline) ls+	viaTmp writeSshConfig keyfile $ unlines $ filter (/= keyline) ls  {- Implemented as a shell command, so it can be run on remote servers over  - ssh.@@ -290,13 +290,15 @@ 	sshdir <- sshDir 	createDirectoryIfMissing True sshdir 	let configfile = sshdir </> "config"-	unlessM (catchBoolIO $ isInfixOf mangledhost <$> readFile configfile) $+	unlessM (catchBoolIO $ isInfixOf mangledhost <$> readFile configfile) $ do 		appendFile configfile $ unlines $ 			[ "" 			, "# Added automatically by git-annex" 			, "Host " ++ mangledhost 			] ++ map (\(k, v) -> "\t" ++ k ++ " " ++ v) 				(settings ++ config)+		setSshConfigMode configfile+ 	return $ sshdata { sshHostName = T.pack mangledhost }   where 	mangledhost = mangleSshHostName sshdata
Assistant/Sync.hs view
@@ -123,7 +123,8 @@ pushToRemotes :: Bool -> [Remote] -> Assistant [Remote] pushToRemotes notifypushes remotes = do 	now <- liftIO getCurrentTime-	syncAction remotes (pushToRemotes' now notifypushes)+	let remotes' = filter (not . remoteAnnexReadOnly . Remote.gitconfig) remotes+	syncAction remotes' (pushToRemotes' now notifypushes) pushToRemotes' :: UTCTime -> Bool -> [Remote] -> Assistant [Remote] pushToRemotes' now notifypushes remotes = do 	(g, branch, u) <- liftAnnex $ do
Assistant/TransferQueue.hs view
@@ -5,6 +5,8 @@  - Licensed under the GNU GPL version 3 or higher.  -} +{-# LANGUAGE BangPatterns #-}+ module Assistant.TransferQueue ( 	TransferQueue, 	Schedule(..),@@ -32,7 +34,7 @@ import Utility.TList  import Control.Concurrent.STM-import qualified Data.Map as M+import qualified Data.Map.Strict as M import qualified Data.Set as S  type Reason = String@@ -189,7 +191,7 @@ 				if acceptable info 					then do 						adjustTransfersSTM dstatus $-							M.insertWith' const t info+							M.insert t info 						return $ Just r 					else return Nothing @@ -217,7 +219,8 @@  dequeueTransfersSTM :: TransferQueue -> (Transfer -> Bool) -> STM [(Transfer, TransferInfo)] dequeueTransfersSTM q c = do-	(removed, ts) <- partition (c . fst) <$> readTList (queuelist q)-	void $ writeTVar (queuesize q) (length ts)+	!(removed, ts) <- partition (c . fst) <$> readTList (queuelist q)+	let !len = length ts+	void $ writeTVar (queuesize q) len 	setTList (queuelist q) ts 	return removed
Assistant/TransferrerPool.hs view
@@ -14,27 +14,45 @@  import qualified Command.TransferKeys as T -import Control.Concurrent.STM+import Control.Concurrent.STM hiding (check) import System.Process (create_group, std_in, std_out) import Control.Exception (throw) import Control.Concurrent -{- Runs an action with a Transferrer from the pool. -}+{- Runs an action with a Transferrer from the pool.+ -+ - Only one Transferrer is left running in the pool at a time.+ - So if this needed to start a new Transferrer, it's stopped when done.+ -} withTransferrer :: FilePath -> BatchCommandMaker -> TransferrerPool -> (Transferrer -> IO a) -> IO a withTransferrer program batchmaker pool a = do-	t <- maybe (mkTransferrer program batchmaker) (checkTransferrer program batchmaker)-		=<< atomically (tryReadTChan pool)+	(mi, leftinpool) <- atomically (popTransferrerPool pool)+	i@(TransferrerPoolItem (Just t) check) <- case mi of+		Nothing -> mkTransferrerPoolItem pool =<< mkTransferrer program batchmaker+		Just i -> checkTransferrerPoolItem program batchmaker i 	v <- tryNonAsync $ a t-	unlessM (putback t) $-		void $ forkIO $ stopTransferrer t+	if leftinpool == 0+		then atomically $ pushTransferrerPool pool i+		else do+			void $ forkIO $ stopTransferrer t+			atomically $ pushTransferrerPool pool $ TransferrerPoolItem Nothing check 	either throw return v-  where-  	putback t = atomically $ ifM (isEmptyTChan pool)-		( do-			writeTChan pool t-			return True-		, return False++{- Check if a Transferrer from the pool is still ok to be used.+ - If not, stop it and start a new one. -}+checkTransferrerPoolItem :: FilePath -> BatchCommandMaker -> TransferrerPoolItem -> IO TransferrerPoolItem+checkTransferrerPoolItem program batchmaker i = case i of+	TransferrerPoolItem (Just t) check -> ifM check+		( return i+		, do+			stopTransferrer t+			new check 		)+	TransferrerPoolItem Nothing check -> new check+  where+	new check = do+		t <- mkTransferrer program batchmaker+		return $ TransferrerPoolItem (Just t) check  {- Requests that a Transferrer perform a Transfer, and waits for it to  - finish. -}
Assistant/Types/TransferrerPool.hs view
@@ -1,4 +1,4 @@-{- A pool of "git-annex transferkeys" processes+{- A pool of "git-annex transferkeys" processes available for use  -  - Copyright 2013 Joey Hess <joey@kitenet.net>  -@@ -8,16 +8,60 @@ module Assistant.Types.TransferrerPool where  import Common.Annex+import Utility.NotificationBroadcaster+import Assistant.Types.DaemonStatus -import Control.Concurrent.STM+import Control.Concurrent.STM hiding (check) -type TransferrerPool = TChan Transferrer+{- This TMVar is never left empty. -}+type TransferrerPool = TMVar (MkCheckTransferrer, [TransferrerPoolItem]) +type CheckTransferrer = IO Bool+type MkCheckTransferrer = IO (IO Bool)++{- Each item in the pool may have a transferrer running, and has an+ - IO action that can be used to check if it's still ok to use the+ - transferrer. -}+data TransferrerPoolItem = TransferrerPoolItem (Maybe Transferrer) CheckTransferrer+ data Transferrer = Transferrer 	{ transferrerRead :: Handle 	, transferrerWrite :: Handle 	, transferrerHandle :: ProcessHandle 	} -newTransferrerPool :: IO TransferrerPool-newTransferrerPool = newTChanIO+newTransferrerPool :: MkCheckTransferrer -> IO TransferrerPool+newTransferrerPool c = newTMVarIO (c, [])++popTransferrerPool :: TransferrerPool -> STM (Maybe TransferrerPoolItem, Int)+popTransferrerPool p = do+	(c, l) <- takeTMVar p+	case l of+		[] -> do+			putTMVar p (c, [])+			return (Nothing, 0)+		(i:is) -> do+			putTMVar p (c, is)+			return $ (Just i, length is)++pushTransferrerPool :: TransferrerPool -> TransferrerPoolItem -> STM ()+pushTransferrerPool p i = do+	(c, l) <- takeTMVar p+	let l' = i:l+	putTMVar p (c, l')++{- Note that making a CheckTransferrer may allocate resources,+ - such as a NotificationHandle, so it's important that the returned+ - TransferrerPoolItem is pushed into the pool, and not left to be+ - garbage collected. -}+mkTransferrerPoolItem :: TransferrerPool -> Transferrer -> IO TransferrerPoolItem+mkTransferrerPoolItem p t = do+	mkcheck <- atomically $ fst <$> readTMVar p+	check <- mkcheck+	return $ TransferrerPoolItem (Just t) check++checkNetworkConnections :: DaemonStatusHandle -> MkCheckTransferrer+checkNetworkConnections dstatushandle = do+	dstatus <- atomically $ readTMVar dstatushandle+	h <- newNotificationHandle False (networkConnectedNotifier dstatus)+	return $ not <$> checkNotification h
Assistant/WebApp/Configurators/Ssh.hs view
@@ -278,9 +278,9 @@  {- Runs a ssh command; if it fails shows the user the transcript,  - and if it succeeds, runs an action. -}-sshSetup :: [String] -> String -> Handler Html -> Handler Html+sshSetup :: [String] -> Maybe String -> Handler Html -> Handler Html sshSetup opts input a = do-	(transcript, ok) <- liftIO $ sshTranscript opts (Just input)+	(transcript, ok) <- liftIO $ sshTranscript opts input 	if ok 		then a 		else showSshErr transcript@@ -383,7 +383,7 @@ 	 [ "-p", show (sshPort origsshdata) 	 , genSshHost (sshHostName origsshdata) (sshUserName origsshdata) 	 , remoteCommand-	 ] "" (a sshdata)+	 ] Nothing (a sshdata)   where 	remotedir = T.unpack $ sshDirectory sshdata 	remoteCommand = shellWrap $ intercalate "&&" $ catMaybes@@ -450,7 +450,7 @@ getMakeRsyncNetGCryptR sshdata NoRepoKey = whenGcryptInstalled $ 	withNewSecretKey $ getMakeRsyncNetGCryptR sshdata . RepoKey getMakeRsyncNetGCryptR sshdata (RepoKey keyid) = whenGcryptInstalled $-	sshSetup [sshhost, gitinit] [] $ makeGCryptRepo keyid sshdata+	sshSetup [sshhost, gitinit] Nothing $ makeGCryptRepo keyid sshdata   where 	sshhost = genSshHost (sshHostName sshdata) (sshUserName sshdata) 	gitinit = "git init --bare " ++ T.unpack (sshDirectory sshdata)@@ -498,7 +498,7 @@ 		, genSshHost (sshHostName sshdata) (sshUserName sshdata) 		, remotecommand 		]-	sshSetup sshopts (sshPubKey keypair) $ a sshdata+	sshSetup sshopts (Just $ sshPubKey keypair) $ a sshdata  isRsyncNet :: Maybe Text -> Bool isRsyncNet Nothing = False
Build/Configure.hs view
@@ -53,8 +53,8 @@ 	, (384, "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b") 	] -{- shaNsum are the program names used by coreutils. Some systems like OSX- - sometimes install these with 'g' prefixes.+{- shaNsum are the program names used by coreutils. Some systems+ - install these with 'g' prefixes.  -  - On some systems, shaN is used instead, but on other  - systems, it might be "hashalot", which does not produce@@ -74,13 +74,8 @@ 				then return $ Just c 				else search cmds 	-	shacmds n = concatMap (\x -> [x, 'g':x, osxpath </> x]) $+	shacmds n = concatMap (\x -> [x, 'g':x]) $ 		map (\x -> "sha" ++ show n ++ x) ["sum", ""]--	{- Max OSX sometimes puts GNU tools outside PATH, so look in-	 - the location it uses, and remember where to run them-	 - from. -}-	osxpath = "/opt/local/libexec/gnubin"  tmpDir :: String tmpDir = "tmp"
Build/LinuxMkLibs.hs view
@@ -141,4 +141,4 @@  - XXX Debian specific. -} glibcLibs :: IO [FilePath] glibcLibs = lines <$> readProcess "sh"-	["-c", "dpkg -L libc6 | egrep '\\.so|gconv'"]+	["-c", "dpkg -L libc6 libgcc1 | egrep '\\.so|gconv'"]
Build/NullSoftInstaller.hs view
@@ -163,4 +163,8 @@ 	, "cygp11-kit-0.dll"
 	, "cygtasn1-3.dll"
 	, "cygffi-6.dll"
+	, "cygbz2-1.dll"
+	, "cygreadline7.dll"
+	, "cygncursesw-10.dll"
+	, "cygusb0.dll"
 	]
CHANGELOG view
@@ -1,3 +1,32 @@+git-annex (5.20140107) unstable; urgency=medium++  * mirror: Support --all (and --unused).+  * external special remote protocol: Added GETUUID, GETWANTED, SETWANTED,+    SETSTATE, GETSTATE, DEBUG.+  * Windows: Fix bug in direct mode merge code that could cause files+    in subdirectories to go missing.+  * Windows: Avoid eating stdin when running ssh to add a authorized key,+    since this is used for password prompting.+  * Avoid looping if long-running git cat-file or git hash-object crashes+    and keeps crashing when restarted.+  * Assistant: Remove stale MERGE_HEAD files in lockfile cleanup.+  * Remotes can now be made read-only, by setting remote.<name>.annex-readonly+  * wanted, schedule: Avoid printing "ok" after requested value.+  * assistant: Ensure that .ssh/config and .ssh/authorized_keys are not+    group or world writable when writing to those files, as that can make+    ssh refuse to use them, if it allows another user to write to them.+  * addurl, importfeed: Honor annex.diskreserve as long as the size of the+    url can be checked.+  * add: Fix rollback when disk is completely full.+  * assistant: Fixed several minor memory leaks that manifested when+    adding a large number of files.+  * assistant: Start a new git-annex transferkeys process+    after a network connection change, so that remotes that use a persistent+    network connection are restarted.+  * Adjust Debian build deps to match current state of sparc, mipsel.++ -- Joey Hess <joeyh@debian.org>  Tue, 07 Jan 2014 12:22:18 -0400+ git-annex (5.20131230) unstable; urgency=medium    * Added new external special remote interface.
Command/Add.hs view
@@ -250,12 +250,12 @@  cleanup :: FilePath -> Key -> Maybe InodeCache -> Bool -> CommandCleanup cleanup file key mcache hascontent = do-	when hascontent $-		logStatus key InfoPresent 	ifM (isDirect <&&> pure hascontent) 		( do 			l <- inRepo $ gitAnnexLink file key 			stageSymlink file =<< hashSymlink l 		, addLink file key mcache 		)+	when hascontent $+		logStatus key InfoPresent 	return True
Command/AddUrl.hs view
@@ -28,7 +28,6 @@ import Annex.Content.Direct import Logs.Location import qualified Logs.Transfer as Transfer-import Utility.Daemon (checkDaemon) #ifdef WITH_QUVI import Annex.Quvi import qualified Utility.Quvi as Quvi@@ -108,14 +107,21 @@ 	ifM (pure relaxed <||> Annex.getState Annex.fast) 		( cleanup quviurl file key Nothing 		, do-			tmp <- fromRepo $ gitAnnexTmpLocation key-			showOutput-			ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do-				liftIO $ createDirectoryIfMissing True (parentDir tmp)-				downloadUrl [videourl] tmp-			if ok-				then cleanup quviurl file key (Just tmp)-				else return False+			{- Get the size, and use that to check+			 - disk space. However, the size info is not+			 - retained, because the size of a video stream+			 - might change and we want to be able to download+			 - it later. -}+			sizedkey <- addSizeUrlKey videourl key+			prepGetViaTmpChecked sizedkey $ do+				tmp <- fromRepo $ gitAnnexTmpLocation key+				showOutput+				ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do+					liftIO $ createDirectoryIfMissing True (parentDir tmp)+					downloadUrl [videourl] tmp+				if ok+					then cleanup quviurl file key (Just tmp)+					else return False 		) #endif @@ -152,45 +158,43 @@  download :: URLString -> FilePath -> Annex Bool download url file = do-	dummykey <- genkey-	tmp <- fromRepo $ gitAnnexTmpLocation dummykey-	showOutput-	ifM (runtransfer dummykey tmp)-		( do-			backend <- chooseBackend file-			let source = KeySource-				{ keyFilename = file-				, contentLocation = tmp-				, inodeCache = Nothing-				}-			k <- genKey source backend-			case k of-				Nothing -> return False-				Just (key, _) -> cleanup url file key (Just tmp)-		, return False-		)-  where 	{- Generate a dummy key to use for this download, before we can 	 - examine the file and find its real key. This allows resuming-	 - downloads, as the dummy key for a given url is stable.-	 --	 - If the assistant is running, actually hits the url here,-	 - to get the size, so it can display a pretty progress bar.-	 -}-	genkey = do-		pidfile <- fromRepo gitAnnexPidFile-		size <- ifM (liftIO $ isJust <$> checkDaemon pidfile)+	 - downloads, as the dummy key for a given url is stable. -}+	dummykey <- addSizeUrlKey url =<< Backend.URL.fromUrl url Nothing+	prepGetViaTmpChecked dummykey $ do+		tmp <- fromRepo $ gitAnnexTmpLocation dummykey+		showOutput+		ifM (runtransfer dummykey tmp) 			( do-				headers <- getHttpHeaders-				snd <$> Url.withUserAgent (Url.exists url headers)-			, return Nothing+				backend <- chooseBackend file+				let source = KeySource+					{ keyFilename = file+					, contentLocation = tmp+					, inodeCache = Nothing+					}+				k <- genKey source backend+				case k of+					Nothing -> return False+					Just (key, _) -> cleanup url file key (Just tmp)+			, return False 			)-		Backend.URL.fromUrl url size+  where   	runtransfer dummykey tmp =  		Transfer.download webUUID dummykey (Just file) Transfer.forwardRetry $ const $ do 			liftIO $ createDirectoryIfMissing True (parentDir tmp) 			downloadUrl [url] tmp-		++{- Hits the url to get the size, if available.+ -+ - This is needed to avoid exceeding the diskreserve when downloading,+ - and so the assistant can display a pretty progress bar.+ -}+addSizeUrlKey :: URLString -> Key -> Annex Key+addSizeUrlKey url key = do+	headers <- getHttpHeaders+	size <- snd <$> Url.withUserAgent (Url.exists url headers)+	return $ key { keySize = size }  cleanup :: URLString -> FilePath -> Key -> Maybe FilePath -> Annex Bool cleanup url file key mtmp = do
Command/Drop.hs view
@@ -18,6 +18,7 @@ import Config import qualified Option import Annex.Wanted+import Types.Key  def :: [Command] def = [withOptions [fromOption] $ command "drop" paramPaths seek@@ -34,21 +35,21 @@ start from file (key, _) = checkDropAuto from file key $ \numcopies -> 	stopUnless (checkAuto $ wantDrop False (Remote.uuid <$> from) (Just file)) $ 		case from of-			Nothing -> startLocal file numcopies key Nothing+			Nothing -> startLocal (Just file) numcopies key Nothing 			Just remote -> do 				u <- getUUID 				if Remote.uuid remote == u-					then startLocal file numcopies key Nothing-					else startRemote file numcopies key remote+					then startLocal (Just file) numcopies key Nothing+					else startRemote (Just file) numcopies key remote -startLocal :: FilePath -> Maybe Int -> Key -> Maybe Remote -> CommandStart-startLocal file numcopies key knownpresentremote = stopUnless (inAnnex key) $ do-	showStart "drop" file+startLocal :: AssociatedFile -> Maybe Int -> Key -> Maybe Remote -> CommandStart+startLocal afile numcopies key knownpresentremote = stopUnless (inAnnex key) $ do+	showStart "drop" (fromMaybe (key2file key) afile) 	next $ performLocal key numcopies knownpresentremote -startRemote :: FilePath -> Maybe Int -> Key -> Remote -> CommandStart-startRemote file numcopies key remote = do-	showStart ("drop " ++ Remote.name remote) file+startRemote :: AssociatedFile -> Maybe Int -> Key -> Remote -> CommandStart+startRemote afile numcopies key remote = do+	showStart ("drop " ++ Remote.name remote) (fromMaybe (key2file key) afile) 	next $ performRemote key numcopies remote  performLocal :: Key -> Maybe Int -> Maybe Remote -> CommandPerform
Command/ImportFeed.hs view
@@ -135,17 +135,18 @@ 		rundownload url (takeExtension url) $  			addUrlFile relaxed url 	QuviLink pageurl -> do-		mp <- withQuviOptions Quvi.query [Quvi.quiet, Quvi.httponly] pageurl-		case mp of-			Nothing -> return False-			Just page -> case headMaybe $ Quvi.pageLinks page of+		let quviurl = setDownloader pageurl QuviDownloader+		checkknown quviurl $ do+			mp <- withQuviOptions Quvi.query [Quvi.quiet, Quvi.httponly] pageurl+			case mp of 				Nothing -> return False-				Just link -> do-					let quviurl = setDownloader pageurl QuviDownloader-					let videourl = Quvi.linkUrl link-					checkknown videourl $-						rundownload videourl ("." ++ Quvi.linkSuffix link) $-							addUrlFileQuvi relaxed quviurl videourl+				Just page -> case headMaybe $ Quvi.pageLinks page of+					Nothing -> return False+					Just link -> do+						let videourl = Quvi.linkUrl link+						checkknown videourl $+							rundownload videourl ("." ++ Quvi.linkSuffix link) $+								addUrlFileQuvi relaxed quviurl videourl   where   	forced = Annex.getState Annex.force
Command/Mirror.hs view
@@ -18,18 +18,25 @@ import qualified Annex  def :: [Command]-def = [withOptions fromToOptions $ command "mirror" paramPaths seek-	SectionCommon "mirror content of files to/from another repository"]+def = [withOptions (fromToOptions ++ keyOptions) $+	command "mirror" paramPaths seek+		SectionCommon "mirror content of files to/from another repository"]  seek :: [CommandSeek] seek = 	[ withField toOption Remote.byNameWithUUID $ \to -> 	  withField fromOption Remote.byNameWithUUID $ \from ->+	  withKeyOptions (startKey Nothing to from Nothing) $ 	  withFilesInGit $ whenAnnexed $ start to from 	]  start :: Maybe Remote -> Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart start to from file (key, _backend) = do+	numcopies <- numCopies file+	startKey numcopies to from (Just file) key++startKey :: Maybe Int -> Maybe Remote -> Maybe Remote -> Maybe FilePath -> Key -> CommandStart+startKey numcopies to from afile key = do 	noAuto 	case (from, to) of 		(Nothing, Nothing) -> error "specify either --from or --to"@@ -40,19 +47,15 @@ 	noAuto = whenM (Annex.getState Annex.auto) $ 		error "--auto is not supported for mirror" 	mirrorto r = ifM (inAnnex key)-		( Command.Move.toStart r False (Just file) key-		, do-			numcopies <- numCopies file-			Command.Drop.startRemote file numcopies key r+		( Command.Move.toStart r False afile key+		, Command.Drop.startRemote afile numcopies key r 		) 	mirrorfrom r = do 		haskey <- Remote.hasKey r key 		case haskey of 			Left _ -> stop-			Right True -> Command.Get.start' (return True) Nothing key (Just file)+			Right True -> Command.Get.start' (return True) Nothing key afile 			Right False -> ifM (inAnnex key)-				( do-					numcopies <- numCopies file-					Command.Drop.startLocal file numcopies key Nothing+				( Command.Drop.startLocal afile numcopies key Nothing 				, stop 				)
Command/Schedule.hs view
@@ -8,10 +8,12 @@ module Command.Schedule where  import Common.Annex+import qualified Annex import Command import qualified Remote import Logs.Schedule import Types.ScheduledActivity+import Types.Messages  import qualified Data.Set as S @@ -37,6 +39,7 @@  performGet :: UUID -> CommandPerform performGet uuid = do+	Annex.setOutput QuietOutput 	s <- scheduleGet uuid 	liftIO $ putStrLn $ intercalate "; " $  		map fromScheduledActivity $ S.toList s
Command/Sync.hs view
@@ -203,7 +203,9 @@ pushRemote _remote Nothing = stop pushRemote remote (Just branch) = go =<< needpush   where-	needpush = anyM (newer remote) [syncBranch branch, Annex.Branch.name]+	needpush+		| remoteAnnexReadOnly (Types.Remote.gitconfig remote) = return False+		| otherwise = anyM (newer remote) [syncBranch branch, Annex.Branch.name] 	go False = stop 	go True = do 		showStart "push" (Remote.name remote)
Command/Wanted.hs view
@@ -8,9 +8,11 @@ module Command.Wanted where  import Common.Annex+import qualified Annex import Command import qualified Remote import Logs.PreferredContent+import Types.Messages  import qualified Data.Map as M @@ -36,6 +38,7 @@  performGet :: UUID -> CommandPerform performGet uuid = do+	Annex.setOutput QuietOutput 	m <- preferredContentMapRaw 	liftIO $ putStrLn $ fromMaybe "" $ M.lookup uuid m 	next $ return True
Git/Command.hs view
@@ -128,9 +128,14 @@  {- Runs a git command as a coprocess. -} gitCoProcessStart :: Bool -> [CommandParam] -> Repo -> IO CoProcess.CoProcessHandle-gitCoProcessStart restartable params repo = CoProcess.start restartable "git"+gitCoProcessStart restartable params repo = CoProcess.start numrestarts "git" 	(toCommand $ gitCommandLine params repo) 	(gitEnv repo)+  where+  	{- If a long-running git command like cat-file --batch+	 - crashes, it will likely start up again ok. If it keeps crashing+	 - 10 times, something is badly wrong. -}+	numrestarts = if restartable then 10 else 0  gitCreateProcess :: [CommandParam] -> Repo -> CreateProcess gitCreateProcess params repo =
Git/DiffTree.hs view
@@ -87,7 +87,7 @@ 		, srcsha = fromMaybe (error "bad srcsha") $ extractSha ssha 		, dstsha = fromMaybe (error "bad dstsha") $ extractSha dsha 		, status = s-		, file = asTopFilePath $ Git.Filename.decode f+		, file = asTopFilePath $ fromInternalGitPath $ Git.Filename.decode f 		} 	  where 		readmode = fst . Prelude.head . readOct
Logs.hs view
@@ -10,19 +10,21 @@ import Common.Annex import Types.Key -data LogVariety = UUIDBasedLog | PresenceLog Key+{- There are several varieties of log file formats. -}+data LogVariety = UUIDBasedLog | NewUUIDBasedLog | PresenceLog Key 	deriving (Show)  {- Converts a path from the git-annex branch into one of the varieties  - of logs used by git-annex, if it's a known path. -} getLogVariety :: FilePath -> Maybe LogVariety getLogVariety f-	| f `elem` uuidBasedLogs = Just UUIDBasedLog+	| f `elem` topLevelUUIDBasedLogs = Just UUIDBasedLog+	| isRemoteStateLog f = Just NewUUIDBasedLog 	| otherwise = PresenceLog <$> firstJust (presenceLogs f) -{- All the uuid-based logs stored in the git-annex branch. -}-uuidBasedLogs :: [FilePath]-uuidBasedLogs =+{- All the uuid-based logs stored in the top of the git-annex branch. -}+topLevelUUIDBasedLogs :: [FilePath]+topLevelUUIDBasedLogs = 	[ uuidLog 	, remoteLog 	, trustLog@@ -99,16 +101,29 @@ isUrlLog :: FilePath -> Bool isUrlLog file = urlLogExt `isSuffixOf` file +{- The filename of the remote state log for a given key. -}+remoteStateLogFile :: Key -> FilePath+remoteStateLogFile key = hashDirLower key </> keyFile key ++ remoteStateLogExt++remoteStateLogExt :: String+remoteStateLogExt = ".log.rmt"++isRemoteStateLog :: FilePath -> Bool+isRemoteStateLog path = remoteStateLogExt `isSuffixOf` path+ prop_logs_sane :: Key -> Bool prop_logs_sane dummykey = all id 	[ isNothing (getLogVariety "unknown") 	, expect isUUIDBasedLog (getLogVariety uuidLog) 	, expect isPresenceLog (getLogVariety $ locationLogFile dummykey) 	, expect isPresenceLog (getLogVariety $ urlLogFile dummykey)+	, expect isNewUUIDBasedLog (getLogVariety $ remoteStateLogFile dummykey) 	]   where   	expect = maybe False   	isUUIDBasedLog UUIDBasedLog = True 	isUUIDBasedLog _ = False+  	isNewUUIDBasedLog NewUUIDBasedLog = True+	isNewUUIDBasedLog _ = False 	isPresenceLog (PresenceLog k) = k == dummykey 	isPresenceLog _ = False
Logs/PreferredContent.hs view
@@ -19,9 +19,9 @@ import qualified Data.Map as M import qualified Data.Set as S import Data.Either-import Data.Time.Clock.POSIX  import Common.Annex+import Logs.PreferredContent.Raw import qualified Annex.Branch import qualified Annex import Logs@@ -36,15 +36,6 @@ import Logs.Remote import Types.StandardGroups -{- Changes the preferred content configuration of a remote. -}-preferredContentSet :: UUID -> String -> Annex ()-preferredContentSet uuid@(UUID _) val = do-	ts <- liftIO getPOSIXTime-	Annex.Branch.change preferredContentLog $-		showLog id . changeLog ts uuid val . parseLog Just-	Annex.changeState $ \s -> s { Annex.preferredcontentmap = Nothing }-preferredContentSet NoUUID _ = error "unknown UUID; cannot modify"- {- Checks if a file is preferred content for the specified repository  - (or the current repository if none is specified). -} isPreferredContent :: Maybe UUID -> AssumeNotPresent -> FilePath -> Bool -> Annex Bool@@ -71,15 +62,11 @@ 	Annex.changeState $ \s -> s { Annex.preferredcontentmap = Just m } 	return m -preferredContentMapRaw :: Annex (M.Map UUID String)-preferredContentMapRaw = simpleMap . parseLog Just-	<$> Annex.Branch.get preferredContentLog- {- This intentionally never fails, even on unparsable expressions,  - because the configuration is shared among repositories and newer  - versions of git-annex may add new features. Instead, parse errors  - result in a Matcher that will always succeed. -}-makeMatcher :: GroupMap -> M.Map UUID RemoteConfig -> UUID -> String -> FileMatcher+makeMatcher :: GroupMap -> M.Map UUID RemoteConfig -> UUID -> PreferredContentExpression -> FileMatcher makeMatcher groupmap configmap u expr 	| expr == "standard" = standardMatcher groupmap configmap u 	| null (lefts tokens) = Utility.Matcher.generate $ rights tokens@@ -95,7 +82,7 @@ 		getStandardGroup =<< u `M.lookup` groupsByUUID groupmap  {- Checks if an expression can be parsed, if not returns Just error -}-checkPreferredContentExpression :: String -> Maybe String+checkPreferredContentExpression :: PreferredContentExpression -> Maybe String checkPreferredContentExpression expr 	| expr == "standard" = Nothing 	| otherwise = case parsedToMatcher tokens of
+ Logs/PreferredContent/Raw.hs view
@@ -0,0 +1,31 @@+{- unparsed preferred content expressions+ -+ - Copyright 2012 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.PreferredContent.Raw where++import qualified Data.Map as M+import Data.Time.Clock.POSIX++import Common.Annex+import qualified Annex.Branch+import qualified Annex+import Logs+import Logs.UUIDBased+import Types.StandardGroups++{- Changes the preferred content configuration of a remote. -}+preferredContentSet :: UUID -> PreferredContentExpression -> Annex ()+preferredContentSet uuid@(UUID _) val = do+	ts <- liftIO getPOSIXTime+	Annex.Branch.change preferredContentLog $+		showLog id . changeLog ts uuid val . parseLog Just+	Annex.changeState $ \s -> s { Annex.preferredcontentmap = Nothing }+preferredContentSet NoUUID _ = error "unknown UUID; cannot modify"++preferredContentMapRaw :: Annex (M.Map UUID PreferredContentExpression)+preferredContentMapRaw = simpleMap . parseLog Just+	<$> Annex.Branch.get preferredContentLog
+ Logs/RemoteState.hs view
@@ -0,0 +1,33 @@+{- Remote state logs.+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Logs.RemoteState (+	getRemoteState,+	setRemoteState,+) where++import Common.Annex+import Logs+import Logs.UUIDBased+import qualified Annex.Branch++import qualified Data.Map as M+import Data.Time.Clock.POSIX++type RemoteState = String++setRemoteState :: UUID -> Key -> RemoteState -> Annex ()+setRemoteState u k s = do+	ts <- liftIO getPOSIXTime+	Annex.Branch.change (remoteStateLogFile k) $+		showLogNew id . changeLog ts u s . parseLogNew Just++getRemoteState :: UUID -> Key -> Annex (Maybe RemoteState)+getRemoteState u k = extract . parseLogNew Just+	<$> Annex.Branch.get (remoteStateLogFile k)+  where+	extract m = value <$> M.lookup u m
Logs/UUIDBased.hs view
@@ -6,8 +6,10 @@  - A line of the log will look like: "UUID[ INFO[ timestamp=foo]]"  - The timestamp is last for backwards compatability reasons,  - and may not be present on old log lines.+ -+ - New uuid based logs instead use the form: "timestamp UUID INFO"  - - - Copyright 2011 Joey Hess <joey@kitenet.net>+ - Copyright 2011-2013 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}@@ -17,8 +19,10 @@ 	LogEntry(..), 	TimeStamp(..), 	parseLog,+	parseLogNew, 	parseLogWithUUID, 	showLog,+	showLogNew, 	changeLog, 	addLog, 	simpleMap,@@ -56,6 +60,14 @@ 	showpair (k, LogEntry Unknown v) = 		unwords [fromUUID k, shower v] +showLogNew :: (a -> String) -> Log a -> String+showLogNew shower = unlines . map showpair . M.toList+  where+	showpair (k, LogEntry (Date p) v) =+		unwords [show p, fromUUID k, shower v]+	showpair (k, LogEntry Unknown v) =+		unwords ["0", fromUUID k, shower v]+ parseLog :: (String -> Maybe a) -> String -> Log a parseLog = parseLogWithUUID . const @@ -85,6 +97,17 @@ 		pdate s = case parseTime defaultTimeLocale "%s%Qs" s of 			Nothing -> Unknown 			Just d -> Date $ utcTimeToPOSIXSeconds d++parseLogNew :: (String -> Maybe a) -> String -> Log a+parseLogNew parser = M.fromListWith best . mapMaybe parse . lines+  where+	parse line = do+		let (ts, rest) = splitword line+		    (u, v) = splitword rest+		date <- Date . utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" ts+		val <- parser v+		Just (toUUID u, LogEntry date val)+	splitword = separate (== ' ')  changeLog :: POSIXTime -> UUID -> a -> Log a -> Log a changeLog t u v = M.insert u $ LogEntry (Date t) v
Makefile view
@@ -247,6 +247,7 @@ 	hdevtools check git-annex.hs -g -cpp -g -i -g -idist/build/git-annex/git-annex-tmp -g -i. -g -idist/build/autogen -g -Idist/build/autogen -g -Idist/build/git-annex/git-annex-tmp -g -IUtility -g -DWITH_TESTSUITE -g -DWITH_S3 -g -DWITH_ASSISTANT -g -DWITH_INOTIFY -g -DWITH_DBUS -g -DWITH_PAIRING -g -DWITH_XMPP -g -optP-include -g -optPdist/build/autogen/cabal_macros.h -g -odir -g dist/build/git-annex/git-annex-tmp -g -hidir -g dist/build/git-annex/git-annex-tmp -g -stubdir -g dist/build/git-annex/git-annex-tmp -g -threaded -g -Wall -g -XHaskell98 -g -XPackageImports  distributionupdate:+	git pull 	ghc --make Build/DistributionUpdate 	./Build/DistributionUpdate
Remote/External.hs view
@@ -18,6 +18,8 @@ import Crypto import Utility.Metered import Logs.Transfer+import Logs.PreferredContent.Raw+import Logs.RemoteState import Config.Cost import Annex.Content import Annex.UUID@@ -206,7 +208,7 @@ 	handleRemoteRequest (PROGRESS bytesprocessed) = 		maybe noop (\a -> liftIO $ a bytesprocessed) mp 	handleRemoteRequest (DIRHASH k) = -		sendMessage lck external $ VALUE $ hashDirMixed k+		send $ VALUE $ hashDirMixed k 	handleRemoteRequest (SETCONFIG setting value) = 		liftIO $ atomically $ do 			let v = externalConfig external@@ -215,7 +217,7 @@ 	handleRemoteRequest (GETCONFIG setting) = do 		value <- fromMaybe "" . M.lookup setting 			<$> liftIO (atomically $ readTMVar $ externalConfig external)-		sendMessage lck external $ VALUE value+		send $ VALUE value 	handleRemoteRequest (SETCREDS setting login password) = do 		c <- liftIO $ atomically $ readTMVar $ externalConfig external 		c' <- setRemoteCredPair' c (credstorage setting)@@ -225,12 +227,29 @@ 		c <- liftIO $ atomically $ readTMVar $ externalConfig external 		creds <- fromMaybe ("", "") <$>  			getRemoteCredPair c (credstorage setting)-		sendMessage lck external $ CREDS (fst creds) (snd creds)+		send $ CREDS (fst creds) (snd creds)+	handleRemoteRequest GETUUID = send $+		VALUE $ fromUUID $ externalUUID external+	handleRemoteRequest (SETWANTED expr) =+		preferredContentSet (externalUUID external) expr+	handleRemoteRequest GETWANTED = do+		expr <- fromMaybe "" . M.lookup (externalUUID external)+			<$> preferredContentMapRaw+		send $ VALUE expr+	handleRemoteRequest (SETSTATE key state) =+		setRemoteState (externalUUID external) key state+	handleRemoteRequest (GETSTATE key) = do+		state <- fromMaybe ""+			<$> getRemoteState (externalUUID external) key+		send $ VALUE state+	handleRemoteRequest (DEBUG msg) = liftIO $ debugM "external" msg 	handleRemoteRequest (VERSION _) = 		sendMessage lck external $ ERROR "too late to send VERSION"  	handleAsyncMessage (ERROR err) = error $ "external special remote error: " ++ err +	send = sendMessage lck external+ 	credstorage setting = CredPairStorage 		{ credPairFile = base 		, credPairEnvironment = (base ++ "login", base ++ "password")@@ -334,7 +353,9 @@ 	checkearlytermination Nothing = noop 	checkearlytermination (Just exitcode) = ifM (inPath cmd) 		( error $ unwords [ "failed to run", cmd, "(" ++ show exitcode ++ ")" ]-		, error $ cmd ++ " is not installed in PATH"+		, do+			path <- intercalate ":" <$> getSearchPath+			error $ cmd ++ " is not installed in PATH (" ++ path ++ ")" 		)  stopExternal :: External -> Annex ()
Remote/External/Types.hs view
@@ -33,6 +33,7 @@ import Common.Annex import Annex.Exception import Types.Key (file2key, key2file)+import Types.StandardGroups (PreferredContentExpression) import Utility.Metered (BytesProcessed(..)) import Logs.Transfer (Direction(..)) import Config.Cost (Cost)@@ -166,6 +167,12 @@ 	| GETCONFIG Setting 	| SETCREDS Setting String String 	| GETCREDS Setting+	| GETUUID+	| SETWANTED PreferredContentExpression+	| GETWANTED+	| SETSTATE Key String+	| GETSTATE Key+	| DEBUG String 	deriving (Show)  instance Receivable RemoteRequest where@@ -176,6 +183,12 @@ 	parseCommand "GETCONFIG" = parse1 GETCONFIG 	parseCommand "SETCREDS" = parse3 SETCREDS 	parseCommand "GETCREDS" = parse1 GETCREDS+	parseCommand "GETUUID" = parse0 GETUUID+	parseCommand "SETWANTED" = parse1 SETWANTED+	parseCommand "GETWANTED" = parse0 GETWANTED+	parseCommand "SETSTATE" = parse2 SETSTATE+	parseCommand "GETSTATE" = parse1 GETSTATE+	parseCommand "DEBUG" = parse1 DEBUG 	parseCommand _ = parseFail  -- Responses to RemoteRequest.
+ Remote/Helper/ReadOnly.hs view
@@ -0,0 +1,29 @@+{- Adds readonly support to remotes.+ -+ - Copyright 2013 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Helper.ReadOnly (adjustReadOnly) where++import Common.Annex+import Types.Remote++{- Adds support for read-only remotes, by replacing the+ - methods that write to a remote with dummies that fail.+ -+ - Note that disabling git pushes to remotes is not handled here.+ -}+adjustReadOnly :: Remote -> Remote+adjustReadOnly r+	| remoteAnnexReadOnly (gitconfig r) = r+		{ storeKey = \_ _ _ -> failbool+		, removeKey = \_ -> failbool+		, repairRepo = Nothing+		}+	| otherwise = r+  where+	failbool = do+		warning "this remote is readonly"+		return False
Remote/List.hs view
@@ -18,6 +18,7 @@ import Types.GitConfig import Annex.UUID import Remote.Helper.Hooks+import Remote.Helper.ReadOnly import qualified Git import qualified Git.Config @@ -89,7 +90,7 @@ 	let gc = extractRemoteGitConfig g (Git.repoDescribe r) 	let c = fromMaybe M.empty $ M.lookup u m 	mrmt <- generate t r u c gc-	return $ addHooks <$> mrmt+	return $ adjustReadOnly . addHooks <$> mrmt  {- Updates a local git Remote, re-reading its git config. -} updateRemote :: Remote -> Annex (Maybe Remote)
Types/GitConfig.hs view
@@ -97,6 +97,7 @@ 	, remoteAnnexCostCommand :: Maybe String 	, remoteAnnexIgnore :: Bool 	, remoteAnnexSync :: Bool+	, remoteAnnexReadOnly :: Bool 	, remoteAnnexTrustLevel :: Maybe String 	, remoteAnnexStartCommand :: Maybe String 	, remoteAnnexStopCommand :: Maybe String@@ -124,6 +125,7 @@ 	, remoteAnnexCostCommand = notempty $ getmaybe "cost-command" 	, remoteAnnexIgnore = getbool "ignore" False 	, remoteAnnexSync = getbool "sync" True+	, remoteAnnexReadOnly = getbool "readonly" False 	, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" 	, remoteAnnexStartCommand = notempty $ getmaybe "start-command" 	, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
Types/StandardGroups.hs view
@@ -12,6 +12,8 @@ import qualified Data.Map as M import Data.Maybe +type PreferredContentExpression = String+ data StandardGroup 	= ClientGroup 	| TransferGroup@@ -71,7 +73,7 @@ associatedDirectory _ _ = Nothing  {- See doc/preferred_content.mdwn for explanations of these expressions. -}-preferredContent :: StandardGroup -> String+preferredContent :: StandardGroup -> PreferredContentExpression preferredContent ClientGroup = lastResort $ 	"(exclude=*/archive/* and exclude=archive/*) or (" ++ notArchived ++ ")" preferredContent TransferGroup = lastResort $@@ -92,5 +94,5 @@   	 {- Most repositories want any content that is only on untrusted  - or dead repositories. -}-lastResort :: String -> String+lastResort :: String -> PreferredContentExpression lastResort s = "(" ++ s ++ ") or (not copies=semitrusted+:1)"
Utility/CoProcess.hs view
@@ -30,15 +30,15 @@ 	}  data CoProcessSpec = CoProcessSpec-	{ coProcessRestartable :: Bool+	{ coProcessNumRestarts :: Int 	, coProcessCmd :: FilePath 	, coProcessParams :: [String] 	, coProcessEnv :: Maybe [(String, String)] 	} -start :: Bool -> FilePath -> [String] -> Maybe [(String, String)] -> IO CoProcessHandle-start restartable cmd params env = do-	s <- start' $ CoProcessSpec restartable cmd params env+start :: Int -> FilePath -> [String] -> Maybe [(String, String)] -> IO CoProcessHandle+start numrestarts cmd params env = do+	s <- start' $ CoProcessSpec numrestarts cmd params env 	newMVar s  start' :: CoProcessSpec -> IO CoProcessState@@ -66,7 +66,7 @@ 				return   where   	restartable s a cont-		| coProcessRestartable (coProcessSpec s) =+		| coProcessNumRestarts (coProcessSpec s) > 0 = 			maybe restart cont =<< catchMaybeIO a 		| otherwise = cont =<< a 	restart = do@@ -75,7 +75,8 @@ 			hClose $ coProcessTo s 			hClose $ coProcessFrom s 		void $ waitForProcess $ coProcessPid s-		s' <- start' (coProcessSpec s)+		s' <- start' $ (coProcessSpec s)+			{ coProcessNumRestarts = coProcessNumRestarts (coProcessSpec s) - 1 } 		putMVar ch s' 		query ch send receive
Utility/Gpg.hs view
@@ -325,6 +325,8 @@ 		base <- getTemporaryDirectory 		dir <- mktmpdir $ base </> "gpgtmpXXXXXX" 		void $ setEnv var dir True+		-- For some reason, recent gpg needs a trustdb to be set up.+		_ <- pipeStrict [Params "--trust-model auto --update-trustdb"] [] 		_ <- pipeStrict [Params "--import -q"] $ unlines 			[testSecretKey, testKey] 		return dir
Utility/NotificationBroadcaster.hs view
@@ -21,15 +21,17 @@ 	notificationHandleFromId, 	sendNotification, 	waitNotification,+	checkNotification, ) where  import Common  import Control.Concurrent.STM-import Control.Concurrent.MSampleVar -{- One MSampleVar per client. The TMVar is never empty, so never blocks. -}-type NotificationBroadcaster = TMVar [MSampleVar ()]+{- One TMVar per client, which are empty when no notification is pending,+ - and full when a notification has been sent but not yet seen by the+ - client. The list TMVar is never empty, so never blocks. -}+type NotificationBroadcaster = TMVar [TMVar ()]  newtype NotificationId = NotificationId Int 	deriving (Read, Show, Eq, Ord)@@ -53,14 +55,13 @@ 	<$> pure b 	<*> addclient   where-	addclient = do+	addclient = atomically $ do 		s <- if force-			then newSV ()-			else newEmptySV-		atomically $ do-			l <- takeTMVar b-			putTMVar b $ l ++ [s]-			return $ NotificationId $ length l+			then newTMVar ()+			else newEmptyTMVar+		l <- takeTMVar b+		putTMVar b $ l ++ [s]+		return $ NotificationId $ length l  {- Extracts the identifier from a notification handle.  - This can be used to eg, pass the identifier through to a WebApp. -}@@ -76,11 +77,20 @@ 	l <- atomically $ readTMVar b 	mapM_ notify l   where-	notify s = writeSV s ()+	notify s = atomically $+		whenM (isEmptyTMVar s) $+			putTMVar s ()  {- Used by a client to block until a new notification is available since  - the last time it tried. -} waitNotification :: NotificationHandle -> IO () waitNotification (NotificationHandle b (NotificationId i)) = do 	l <- atomically $ readTMVar b-	readSV (l !! i)+	atomically $ takeTMVar (l !! i)++{- Used by a client to check if there has been a new notification since the+ - last time it checked, without blocking. -}+checkNotification :: NotificationHandle -> IO Bool+checkNotification (NotificationHandle b (NotificationId i)) = do+	l <- atomically $ readTMVar b+	maybe False (const True) <$> atomically (tryTakeTMVar (l !! i))
Utility/SshConfig.hs view
@@ -10,6 +10,7 @@ import Common import Utility.UserInfo import Utility.Tmp+import Utility.FileMode  import Data.Char import Data.Ord@@ -117,7 +118,19 @@ 		c <- readFileStrict configfile 		let c' = modifier c 		when (c /= c') $-			viaTmp writeFile configfile c'+			viaTmp writeSshConfig configfile c'++writeSshConfig :: FilePath -> String -> IO ()+writeSshConfig f s = do+	writeFile f s+	setSshConfigMode f++{- Ensure that the ssh config file lacks any group or other write bits, + - since ssh is paranoid about not working if other users can write+ - to one of its config files (.ssh/config and .ssh/authorized_keys) -}+setSshConfigMode :: FilePath -> IO ()+setSshConfigMode f = modifyFileMode f $+	removeModes [groupWriteMode, otherWriteMode]  sshDir :: IO FilePath sshDir = do
debian/changelog view
@@ -1,3 +1,32 @@+git-annex (5.20140107) unstable; urgency=medium++  * mirror: Support --all (and --unused).+  * external special remote protocol: Added GETUUID, GETWANTED, SETWANTED,+    SETSTATE, GETSTATE, DEBUG.+  * Windows: Fix bug in direct mode merge code that could cause files+    in subdirectories to go missing.+  * Windows: Avoid eating stdin when running ssh to add a authorized key,+    since this is used for password prompting.+  * Avoid looping if long-running git cat-file or git hash-object crashes+    and keeps crashing when restarted.+  * Assistant: Remove stale MERGE_HEAD files in lockfile cleanup.+  * Remotes can now be made read-only, by setting remote.<name>.annex-readonly+  * wanted, schedule: Avoid printing "ok" after requested value.+  * assistant: Ensure that .ssh/config and .ssh/authorized_keys are not+    group or world writable when writing to those files, as that can make+    ssh refuse to use them, if it allows another user to write to them.+  * addurl, importfeed: Honor annex.diskreserve as long as the size of the+    url can be checked.+  * add: Fix rollback when disk is completely full.+  * assistant: Fixed several minor memory leaks that manifested when+    adding a large number of files.+  * assistant: Start a new git-annex transferkeys process+    after a network connection change, so that remotes that use a persistent+    network connection are restarted.+  * Adjust Debian build deps to match current state of sparc, mipsel.++ -- Joey Hess <joeyh@debian.org>  Tue, 07 Jan 2014 12:22:18 -0400+ git-annex (5.20131230) unstable; urgency=medium    * Added new external special remote interface.
debian/control view
@@ -10,11 +10,10 @@ 	libghc-pcre-light-dev, 	libghc-sha-dev, 	libghc-cryptohash-dev,-	libghc-regex-tdfa-dev [!mips !mipsel !s390], 	libghc-dataenc-dev, 	libghc-utf8-string-dev, 	libghc-hs3-dev (>= 0.5.6),-	libghc-dav-dev (>= 0.3) [amd64 i386 kfreebsd-amd64 kfreebsd-i386 powerpc sparc],+	libghc-dav-dev (>= 0.3) [amd64 i386 kfreebsd-amd64 kfreebsd-i386 powerpc], 	libghc-quickcheck2-dev, 	libghc-monad-control-dev (>= 0.3), 	libghc-monadcatchio-transformers-dev,@@ -51,10 +50,11 @@ 	libghc-async-dev, 	libghc-http-dev, 	libghc-feed-dev,-	libghc-tasty-dev,-	libghc-tasty-hunit-dev,-	libghc-tasty-quickcheck-dev,-	lsof,+	libghc-regex-tdfa-dev [!mipsel !s390],+	libghc-tasty-dev [!mipsel !sparc],+	libghc-tasty-hunit-dev [!mipsel !sparc],+	libghc-tasty-quickcheck-dev [!mipsel !sparc],+	lsof [!kfreebsd-i386 !kfreebsd-amd64], 	ikiwiki, 	perlmagick, 	git (>= 1:1.8.4),
+ doc/bugs/Assistant_from_standalone_5.20131230_64bit_crashes_on_startup.mdwn view
@@ -0,0 +1,31 @@+The assistant from the standalone 64bit build crashes upon startup.++### Version information++     [0 zerodogg@firefly annexed]$ git annex version+     git-annex version: 5.20131230-g4aa88d8+     build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus XMPP Feeds Quvi TDFA+     key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL+     remote types: git gcrypt S3 bup directory rsync web webdav glacier hook external+     local repository version: 5+     supported repository version: 5+     upgrade supported from repository versions: 0 1 2 4+     [0 zerodogg@firefly annexed]$ uname -a+     Linux firefly 3.12.5-302.fc20.x86_64 #1 SMP Tue Dec 17 20:42:32 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux+     [0 zerodogg@firefly annexed]$ cat /etc/fedora-release +     Fedora release 20 (Heisenbug)+     [0 zerodogg@firefly annexed]$ ++### Error++     [0 zerodogg@firefly annexed]$ git annex webapp+     Launching web browser on file:///home/zerodogg/Documents/annexed/.git/annex/webapp.html+     +     (process:6045): GLib-CRITICAL **: g_slice_set_config: assertion 'sys_page_size == 0' failed+     libgcc_s.so.1 must be installed for pthread_cancel to work+     error: git-annex died of signal 6+     [134 zerodogg@firefly annexed]$ ls -l /usr/lib64/libgcc_s.so.1+     lrwxrwxrwx. 1 root root 28 2013-12-18 10:54 /usr/lib64/libgcc_s.so.1 -> libgcc_s-4.8.2-20131212.so.1+     [0 zerodogg@firefly annexed]$ ++> [[fixed|done]] --[[Joey]]
+ doc/bugs/Assistant_lost_dbus_connection_spamming_log.mdwn view
@@ -0,0 +1,78 @@+## what+After starting the assistant, daemon.log is being spammed with "accept: unsupported operation (Function not implemented)" due to dbus failing.++## version+[[!format sh """+$ git-annex version+git-annex version: 5.20131224-g6ca5271+"""]]+(The armel standalone)++## question+Is this detrimental to the assistant?++## partial daemon.log+[[!format sh """+[2014-01-04 23:03:34 PST] main: starting assistant version 5.20131224-g6ca5271+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+[                                                                                                       +2f0a 1i 4ld-eb0du1 s-t 0of4 a ai2dl3de: d0w;3a :tf3ca5hl  lPoiSnnT g]d  ibTrarecacknt sotfroey r mS/tcva+olnifeingt/Egrirto-raMnensesxa g(e/ v=a r\"/rsuenrCvliiceenst/:h oumneasb/lger etgo/ .dceotnefrimgi/ngei ++xi s tl o(sNto  dsbuucsh  cfoinlnee cotri odni;r efcatlolriyn)g)+back to polling (SocketError {socketErrorMessage = "connect: does not exist (No such file or directory)"+                                                                                                        +Already up-to-date.                                                                                     +accept: unsupported operation (Function not implemented)+(scanning...) [2014-01-04 23:03:36 PST] Watcher: Performing startup scan                                +accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+Everything up-to-date                                                                                   +accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+accept: unsupported operation (Function not implemented)+                                                                                                        +  lost dbus connection; falling back to polling (SocketError {socketErrorMessage = "connect: does not exist (No such file or directory)", socketErrorFatal = True, socketErrorAddress = Just (Address "unix:path=/var/run/dbus/system_bus_socket")})+"""]]
doc/bugs/Backup_repository_doesn__39__t_get_all_files.mdwn view
@@ -1,3 +1,5 @@+[[!meta title="assistant should try sending old/deleted versions of files to backup repositories"]]+ ### Please describe the problem. Backup group repositories don't download all "unused" files when running the webapp.
doc/bugs/OSX_app_issues.mdwn view
@@ -1,6 +1,5 @@-This is a collection of problem reports for the standalone OSX app.-If you have a problem using it, post it here. --[[Joey]] +This was a collection of problem reports for the standalone OSX app.+Please post separate bug reports going forward; this bug report is+[[closed|done]] --[[Joey]]  (Some things that should be fixed now have been moved to [[old]].)--[[!tag /design/assistant/OSX]]
doc/bugs/OSX_build_broken.mdwn view
@@ -9,3 +9,5 @@ ### What version of git-annex are you using? On what operating system?  The current version as downloaded from http://downloads.kitenet.net/git-annex/OSX/current/10.7.5_Lion/git-annex.dmg on OSX 10.7.5++> OS Lion builds are no longer being made. [[done]] --[[Joey]]
doc/bugs/TransferScanner_crash_on_Android.mdwn view
@@ -22,3 +22,7 @@ [2013-07-16 15:19:26 NZST] TransferScanner: warning TransferScanner crashed: unknown response from git cat-file ("refs/heads/git-annex:289/20f/SHA256E-s85883241--3bf01cfd6a422f9b661ed335e6142bbdaf899cd71587bb3cc812256064c7071e missing",refs/heads/git-annex:289/20f/SHA256E-s85883241--3bf01cfd6a422f9b661ed335e6142bbdaf899cd71587bb3cc812256064c7071e\00\00\00\00\00(7991 more elided)\00.log) # End of transcript or log. """]]++> Lacking other information about this problem, and with the assistant+> able to detect problems and run `git annex repair` to fix up corrupt git+> repos, I suppose this is [[done]] --[[Joey]]
+ doc/bugs/USB_drive_not_syncing.mdwn view
@@ -0,0 +1,516 @@+### Please describe the problem.+A USB drive could not be synced, whether as a client or transfer repository. The misbehaving part appears to be at this point in the logs:++error: Ref refs/heads/synced/git-annex is at ff94266684335f9b152282fe620ef086e067afba but expected 0000000000000000000000000000000000000000++"git annex get" also fails as shown in the (full) log message pasted below.++### What steps will reproduce the problem?+Not sure. Re-inserting the same drive gives the same problem. As the very first time using git annex (and not knowing git that well too), I need guidance in narrowing down this problem. If this bug has been fixed in the mean time, that is also fine.++### What version of git-annex are you using? On what operating system?+git-annex version: 4.20131106~bpo70+1+build flags: Assistant Webapp Pairing Testsuite S3 Inotify DBus XMPP Feeds Quvi TDFA+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL+remote types: git gcrypt S3 bup directory rsync web glacier hook+local repository version: 3+default repository version: 3+supported repository versions: 3 4+upgrade supported from repository versions: 0 1 2+++apg-get installed from Debian wheezy backports. uname -a gives:++Linux im1 3.2.0-4-686-pae #1 SMP Debian 3.2.51-1 i686 GNU/Linux+++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++[2014-01-05 21:01:40 CST] main: starting assistant version 4.20131106~bpo70+1+[2014-01-05 21:01:40 CST] Cronner: You should enable consistency checking to protect your data. +(Recording state in git...)+(scanning...) [2014-01-05 21:01:40 CST] Watcher: Performing startup scan+(started...) [2014-01-05 21:03:56 CST] Committer: Adding README.txt cassandra-env.sh cassandra..roperties cassandra..roperties cassandra.yaml++add conf/README.txt (checksum...) ok+add conf/README.txt (checksum...) ok+add conf/cassandra-env.sh (checksum...) ok+add conf/cassandra-env.sh (checksum...) ok+add conf/cassandra-rackdc.properties (checksum...) ok+add conf/cassandra-rackdc.properties (checksum...) ok+add conf/cassandra-topology.properties (checksum...) ok+add conf/cassandra-topology.properties (checksum...) ok+add conf/cassandra.yaml (checksum...) ok+add conf/cassandra.yaml (checksum...) [2014-01-05 21:03:57 CST] Committer: Adding commitlog..roperties cqlshrc.sample log4j-ser..roperties log4j-too..roperties+ok+add conf/commitlog_archiving.properties (checksum...) ok+add conf/commitlog_archiving.properties (checksum...) ok+add conf/cqlshrc.sample (checksum...) ok+add conf/cqlshrc.sample (checksum...) ok+add conf/log4j-server.properties (checksum...) ok+add conf/log4j-server.properties (checksum...) ok+add conf/log4j-tools.properties (checksum...) ok+add conf/log4j-tools.properties (checksum...) [2014-01-05 21:03:58 CST] Committer: Committing changes to git+[2014-01-05 21:09:46 CST] Committer: Committing changes to git+ok+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+Launching web browser on file:///home/tyc20/Desktop/annex/.git/annex/webapp.html+[2014-01-05 21:11:46 CST] main: starting assistant version 4.20131106~bpo70+1+[2014-01-05 21:11:46 CST] Cronner: You should enable consistency checking to protect your data. +(scanning...) [2014-01-05 21:11:46 CST] Watcher: Performing startup scan+(started...) +(scanning...) [2014-01-05 21:13:17 CST] Watcher: Performing startup scan+[2014-01-05 21:13:17 CST] Committer: Committing changes to git+(Recording state in git...)+(started...) [2014-01-05 21:13:18 CST] Committer: Committing changes to git+[2014-01-05 21:16:19 CST] Committer: Adding cassandra.thrift++(Recording state in git...)+add conf/cassandra.thrift (checksum...) [2014-01-05 21:16:19 CST] Committer: Committing changes to git+[2014-01-05 21:16:20 CST] Committer: Adding cassandra.thrift+ok+(Recording state in git...)+(Recording state in git...)+add conf/cassandra.thrift (checksum...) [2014-01-05 21:16:20 CST] Committer: Committing changes to git+[2014-01-05 21:16:29 CST] Committer: Committing changes to git+[2014-01-05 21:16:40 CST] Committer: Committing changes to git+[2014-01-05 21:17:56 CST] Committer: Adding new-name.properties+[2014-01-05 21:17:56 CST] Committer: Committing changes to git+ok+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+ok+(Recording state in git...)+(Recording state in git...)++  conf/Smalltalk-80 The language and its implementation (Bluebook).pdf still has writers, not adding++  conf/Smalltalk-80 The language and its implementation (Bluebook).pdf still has writers, not adding+[2014-01-05 21:27:34 CST] Committer: Committing changes to git+[2014-01-05 21:28:09 CST] Committer: Adding The Essen..tions.pdf+(Recording state in git...)+(Recording state in git...)+add conf/The Essence of Compiling with Continuations.pdf (checksum...) ok+add conf/The Essence of Compiling with Continuations.pdf (checksum...) [2014-01-05 21:28:09 CST] Committer: Committing changes to git+[2014-01-05 21:28:49 CST] Committer: Committing changes to git+gpg: new configuration file `/home/tyc20/.gnupg/gpg.conf' created+gpg: WARNING: options in `/home/tyc20/.gnupg/gpg.conf' are not yet active during this run+ok+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)+(Recording state in git...)++  Detected a crippled filesystem.++  Detected a filesystem without fifo support.++  Disabling ssh connection caching.+[2014-01-05 21:35:09 CST] main: Syncing with ADATAUFD +warning: no common commits+From /media/A-DATA UFD/annex+ * [new branch]      git-annex  -> ADATAUFD/git-annex+[2014-01-05 21:35:09 CST] Pusher: Syncing with ADATAUFD +Already up-to-date.+error: Ref refs/heads/synced/git-annex is at ff94266684335f9b152282fe620ef086e067afba but expected 0000000000000000000000000000000000000000+error: Ref refs/heads/synced/master is at 745c01118e8684b4826f2fb53368faf0568fcafc but expected 0000000000000000000000000000000000000000+remote: error: failed to lock refs/heads/synced/git-annex        +To /media/A-DATA UFD/annex+ * [new branch]      git-annex -> synced/git-annex+ * [new branch]      master -> synced/master+remote: error: failed to lock refs/heads/synced/master        +To /media/A-DATA UFD/annex+ ! [remote rejected] git-annex -> synced/git-annex (failed to lock)+ ! [remote rejected] master -> synced/master (failed to lock)+error: failed to push some refs to '/media/A-DATA UFD/annex'+Already up-to-date.+Everything up-to-date++cassandra-rackdc.properties++        1199 100%    0.00kB/s    0:00:00  +        1199 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1290 bytes  received 31 bytes  2642.00 bytes/sec+total size is 1199  speedup is 0.91+[2014-01-05 21:35:13 CST] Transferrer: Uploaded cassandra..roperties+[2014-01-05 21:35:13 CST] Pusher: Syncing with ADATAUFD ++cassandra-topology.properties++        1358 100%    0.00kB/s    0:00:00  +        1358 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1451 bytes  received 31 bytes  2964.00 bytes/sec+total size is 1358  speedup is 0.92+[2014-01-05 21:35:13 CST] Transferrer: Uploaded cassandra..roperties++cassandra.thrift++       32768  93%    0.00kB/s    0:00:00  +       35216 100%    2.33MB/s    0:00:00 (xfer#1, to-check=0/1)++sent 35300 bytes  received 31 bytes  70662.00 bytes/sec+total size is 35216  speedup is 1.00+[2014-01-05 21:35:14 CST] Transferrer: Uploaded cassandra.thrift++commitlog_archiving.properties++        2328 100%    0.00kB/s    0:00:00  +        2328 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 2422 bytes  received 31 bytes  4906.00 bytes/sec+total size is 2328  speedup is 0.95+[2014-01-05 21:35:15 CST] Transferrer: Uploaded commitlog..roperties++cqlshrc.sample++        1324 100%    0.00kB/s    0:00:00  +        1324 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1402 bytes  received 31 bytes  2866.00 bytes/sec+total size is 1324  speedup is 0.92+[2014-01-05 21:35:15 CST] Transferrer: Uploaded cqlshrc.sample++new-name.properties++        1850 100%    0.00kB/s    0:00:00  +        1850 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1933 bytes  received 31 bytes  3928.00 bytes/sec+total size is 1850  speedup is 0.94+[2014-01-05 21:35:16 CST] Transferrer: Uploaded new-name.properties+To /media/A-DATA UFD/annex+   ff94266..d2dcc21  git-annex -> synced/git-annex+[2014-01-05 21:35:19 CST] Pusher: Syncing with ADATAUFD +To /media/A-DATA UFD/annex+   d2dcc21..c9f93f8  git-annex -> synced/git-annex+[2014-01-05 21:42:39 CST] Committer: Adding Smalltalk..book).pdf+(Recording state in git...)+(merging ADATAUFD/git-annex into git-annex...)+(Recording state in git...)+++(Recording state in git...)+(Recording state in git...)+add Smalltalk-80 The language and its implementation (Bluebook).pdf (checksum...) [2014-01-05 21:42:40 CST] Committer: Committing changes to git+[2014-01-05 21:42:40 CST] Pusher: Syncing with ADATAUFD ++Smalltalk-80 The language and its implementation (Bluebook).pdf++       32768   0%    0.00kB/s    0:00:00  Already up-to-date.++     8847360  26%    8.39MB/s    0:00:02  +    21037056  62%   10.01MB/s    0:00:01  To /media/A-DATA UFD/annex+   c9f93f8..1f1bfe9  git-annex -> synced/git-annex+   745c011..6c6149b  master -> synced/master+Already up-to-date.++    31129600  92%    9.88MB/s    0:00:00  +    33558574 100%    9.85MB/s    0:00:03 (xfer#1, to-check=0/1)+[2014-01-05 21:42:45 CST] Pusher: Syncing with ADATAUFD +Everything up-to-date++sent 33562798 bytes  received 31 bytes  3196459.90 bytes/sec+total size is 33558574  speedup is 1.00+[2014-01-05 21:42:51 CST] Transferrer: Uploaded Smalltalk..book).pdf+[2014-01-05 21:42:51 CST] Pusher: Syncing with ADATAUFD +To /media/A-DATA UFD/annex+   1f1bfe9..07ac8cf  git-annex -> synced/git-annex+[2014-01-05 21:46:18 CST] main: Syncing with annex2 +warning: no common commits+From /home/tyc20/Desktop/annex2+ * [new branch]      git-annex  -> annex2/git-annex+ * [new branch]      master     -> annex2/master+Already up-to-date!+Merge made by the 'recursive' strategy.+[2014-01-05 21:46:19 CST] Pusher: Syncing with annex2, ADATAUFD +Already up-to-date.+To /home/tyc20/Desktop/annex2+ * [new branch]      git-annex -> synced/git-annex+ * [new branch]      master -> synced/master+error: Ref refs/heads/synced/git-annex is at cf8e5160f57d8279770613851ee75fb6b7c1d381 but expected 0000000000000000000000000000000000000000+remote: error: failed to lock refs/heads/synced/git-annex        +error: Ref refs/heads/synced/master is at 8de8929c1e6712613f9e81faba88c2e464c809a7 but expected 0000000000000000000000000000000000000000+remote: error: failed to lock refs/heads/synced/master        +To /home/tyc20/Desktop/annex2+ ! [remote rejected] git-annex -> synced/git-annex (failed to lock)+ ! [remote rejected] master -> synced/master (failed to lock)+error: failed to push some refs to '/home/tyc20/Desktop/annex2'+Already up-to-date.+[2014-01-05 21:46:20 CST] Transferrer: Uploaded Smalltalk..book).pdf+ok+(Recording state in git...)+(Recording state in git...)+++(Recording state in git...)+(Recording state in git...)+(merging annex2/git-annex into git-annex...)+(Recording state in git...)++++drop ADATAUFD Smalltalk-80 The language and its implementation (Bluebook).pdf [2014-01-05 21:46:20 CST] Transferrer: Uploaded cassandra..roperties+ok+drop ADATAUFD conf/cassandra-rackdc.properties [2014-01-05 21:46:21 CST] Transferrer: Uploaded cassandra..roperties+ok+drop ADATAUFD conf/cassandra-topology.properties To /media/A-DATA UFD/annex+   07ac8cf..cf8e516  git-annex -> synced/git-annex+   6c6149b..8de8929  master -> synced/master+From /home/tyc20/Desktop/annex2+   0f740b9..47ed905  git-annex  -> annex2/git-annex+[2014-01-05 21:46:23 CST] Transferrer: Uploaded cassandra.thrift+Already up-to-date.+ok++(merging annex2/git-annex into git-annex...)+(Recording state in git...)+drop ADATAUFD conf/cassandra.thrift To /home/tyc20/Desktop/annex2+   cf8e516..859a06b  git-annex -> synced/git-annex+[2014-01-05 21:46:25 CST] Transferrer: Uploaded commitlog..roperties+ok+drop ADATAUFD conf/commitlog_archiving.properties [2014-01-05 21:46:25 CST] Transferrer: Uploaded cqlshrc.sample+ok+drop ADATAUFD conf/cqlshrc.sample [2014-01-05 21:46:26 CST] Transferrer: Uploaded new-name.properties+ok+drop ADATAUFD conf/new-name.properties [2014-01-05 21:46:27 CST] Pusher: Syncing with annex2, ADATAUFD +To /home/tyc20/Desktop/annex2+   859a06b..da020f9  git-annex -> synced/git-annex+To /media/A-DATA UFD/annex+   cf8e516..da020f9  git-annex -> synced/git-annex+Updating 8de8929..2bfeaab+Fast-forward+ conf/{new-name.properties => new-name2.properties} | 0+ 1 file changed, 0 insertions(+), 0 deletions(-)+ rename conf/{new-name.properties => new-name2.properties} (100%)+[2014-01-05 21:47:23 CST] Committer: Committing changes to git+[2014-01-05 21:47:23 CST] Pusher: Syncing with annex2, ADATAUFD +To /home/tyc20/Desktop/annex2+   da020f9..1649bb6  git-annex -> synced/git-annex+Already up-to-date.+[2014-01-05 21:47:24 CST] Committer: Committing changes to git+To /media/A-DATA UFD/annex+   da020f9..1649bb6  git-annex -> synced/git-annex+   8de8929..2bfeaab  master -> synced/master+Already up-to-date.+[2014-01-05 21:47:37 CST] Pusher: Syncing with annex2, ADATAUFD +Everything up-to-date+Everything up-to-date++Smalltalk-80 The language and its implementation (Bluebook).pdf++       32768   0%    0.00kB/s    0:00:00  +    10354688  30%    9.84MB/s    0:00:02  +    20709376  61%    9.86MB/s    0:00:01  +    29851648  88%    9.48MB/s    0:00:00  +    33558574 100%    9.33MB/s    0:00:03 (xfer#1, to-check=0/1)++sent 33562798 bytes  received 31 bytes  3532929.37 bytes/sec+total size is 33558574  speedup is 1.00+[2014-01-05 21:52:09 CST] Transferrer: Uploaded Smalltalk..book).pdf+[2014-01-05 21:52:09 CST] Pusher: Syncing with annex2, ADATAUFD ++cassandra-rackdc.properties++        1199 100%    0.00kB/s    0:00:00  +        1199 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1290 bytes  received 31 bytes  2642.00 bytes/sec+total size is 1199  speedup is 0.91+To /home/tyc20/Desktop/annex2+   1649bb6..7329695  git-annex -> synced/git-annex+[2014-01-05 21:52:13 CST] Transferrer: Uploaded cassandra..roperties++cassandra-topology.properties++        1358 100%    0.00kB/s    0:00:00  +        1358 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1451 bytes  received 31 bytes  2964.00 bytes/sec+total size is 1358  speedup is 0.92+[2014-01-05 21:52:13 CST] Transferrer: Uploaded cassandra..roperties++cassandra.thrift++       32768  93%    0.00kB/s    0:00:00  +       35216 100%    2.33MB/s    0:00:00 (xfer#1, to-check=0/1)++sent 35300 bytes  received 31 bytes  23554.00 bytes/sec+total size is 35216  speedup is 1.00+[2014-01-05 21:52:14 CST] Transferrer: Uploaded cassandra.thrift++commitlog_archiving.properties++        2328 100%    0.00kB/s    0:00:00  +        2328 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 2422 bytes  received 31 bytes  4906.00 bytes/sec+total size is 2328  speedup is 0.95+[2014-01-05 21:52:15 CST] Transferrer: Uploaded commitlog..roperties++cqlshrc.sample++        1324 100%    0.00kB/s    0:00:00  +        1324 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1402 bytes  received 31 bytes  2866.00 bytes/sec+total size is 1324  speedup is 0.92+[2014-01-05 21:52:15 CST] Transferrer: Uploaded cqlshrc.sample++new-name2.properties++        1850 100%    0.00kB/s    0:00:00  +        1850 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 1934 bytes  received 31 bytes  3930.00 bytes/sec+total size is 1850  speedup is 0.94+[2014-01-05 21:52:17 CST] Transferrer: Uploaded new-name2..roperties+To /media/A-DATA UFD/annex+   1649bb6..7329695  git-annex -> synced/git-annex+[2014-01-05 21:52:21 CST] Pusher: Syncing with annex2, ADATAUFD +To /home/tyc20/Desktop/annex2+   7329695..a7286da  git-annex -> synced/git-annex+To /media/A-DATA UFD/annex+   7329695..a7286da  git-annex -> synced/git-annex+Updating 2bfeaab..40ceb79+Fast-forward+ conf/commitlog_archiving.properties | 1 -+ 1 file changed, 1 deletion(-)+ delete mode 120000 conf/commitlog_archiving.properties+[2014-01-05 21:55:05 CST] Pusher: Syncing with annex2 +[2014-01-05 21:55:05 CST] Committer: Committing changes to git+To /home/tyc20/Desktop/annex2+   a7286da..688d27e  git-annex -> synced/git-annex+Already up-to-date.+[2014-01-05 21:55:07 CST] Pusher: Syncing with annex2 +Everything up-to-date+[2014-01-05 21:55:58 CST] MountWatcher: You should enable consistency checking to protect your data in ADATAUFD. +[2014-01-05 21:55:58 CST] MountWatcher: Syncing with ADATAUFD +To /media/A-DATA UFD/annex+   a7286da..688d27e  git-annex -> synced/git-annex+   2bfeaab..40ceb79  master -> synced/master+Already up-to-date.+++++In addition, this is what the drive looks like from the console.++tyc20@im1:/media/A-DATA UFD/annex$ git annex get+(merging synced/git-annex into git-annex...)+(Recording state in git...)+get SHA256E-s9400--8dce230c548a40fe9722ef1cf9444a54ae726a7420391a8eae937c8beaff6d96.sh (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s1228--7e4190c2da12aa55c0f2598688d38723716bc11809ade1e594eae000ca40742c (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s298--d60cff88544f63e6da177d98ef18a593268df6d4fcfd1247443ce3656cb185ba.txt (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s26331--69bbae153239fa57b752bc931345275f36b608b26a3e9542ab4fabf21e05e57c.pdf (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s31688--fea8690cf82baea3365ff692108742fdec308638bf6d66a8765bdc582e6ebb2c.yaml (not available) +  No other repository is known to contain the file.+failed+git-annex: get: 5 failed+tyc20@im1:/media/A-DATA UFD/annex$ ls+annex  branches  config  description  HEAD  hooks  info  objects  refs+tyc20@im1:/media/A-DATA UFD/annex$ git annex sync+commit  +ok+pull im1 +From /home/tyc20/Desktop/annex+ * [new branch]      git-annex  -> im1/git-annex+ * [new branch]      master     -> im1/master+ * [new branch]      synced/git-annex -> im1/synced/git-annex+ * [new branch]      synced/master -> im1/synced/master+ok+push im1 +Counting objects: 73, done.+Delta compression using up to 2 threads.+Compressing objects: 100% (17/17), done.+Writing objects: 100% (24/24), 1.54 KiB | 0 bytes/s, done.+Total 24 (delta 16), reused 0 (delta 0)+To /home/tyc20/Desktop/annex+   688d27e..2819176  git-annex -> synced/git-annex+ok+tyc20@im1:/media/A-DATA UFD/annex$ git annex sync+commit  +ok+pull im1 +remote: Counting objects: 10, done.+remote: Compressing objects: 100% (4/4), done.+remote: Total 5 (delta 3), reused 0 (delta 0)+Unpacking objects: 100% (5/5), done.+From /home/tyc20/Desktop/annex+   688d27e..c28d1a4  git-annex  -> im1/git-annex+   2819176..c28d1a4  synced/git-annex -> im1/synced/git-annex+ok+(merging im1/git-annex into git-annex...)+tyc20@im1:/media/A-DATA UFD/annex$ git annex get+get SHA256E-s9400--8dce230c548a40fe9722ef1cf9444a54ae726a7420391a8eae937c8beaff6d96.sh (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s1228--7e4190c2da12aa55c0f2598688d38723716bc11809ade1e594eae000ca40742c (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s298--d60cff88544f63e6da177d98ef18a593268df6d4fcfd1247443ce3656cb185ba.txt (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s26331--69bbae153239fa57b752bc931345275f36b608b26a3e9542ab4fabf21e05e57c.pdf (not available) +  No other repository is known to contain the file.+failed+get SHA256E-s31688--fea8690cf82baea3365ff692108742fdec308638bf6d66a8765bdc582e6ebb2c.yaml (not available) +  No other repository is known to contain the file.+failed+git-annex: get: 5 failed+tyc20@im1:/media/A-DATA UFD/annex$ mount -l+sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)+proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)+udev on /dev type devtmpfs (rw,relatime,size=10240k,nr_inodes=128012,mode=755)+devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)+tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=103412k,mode=755)+/dev/disk/by-uuid/25f084fb-c946-4c0d-bf0b-65ae4b68a7c6 on / type ext3 (rw,relatime,errors=remount-ro,barrier=1,data=ordered)+tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)+tmpfs on /run/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=285260k)+none on /mnt/share type vboxsf (rw,nodev,relatime)+rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)+/dev/sr0 on /media/VBOXADDITIONS_4.3.6_91406 type iso9660 (ro,nosuid,nodev,relatime,uid=1000,gid=1000,iocharset=utf8,mode=0400,dmode=0500,uhelper=udisks) [VBOXADDITIONS_4.3.6_91406]+/dev/sdb1 on /media/A-DATA UFD type vfat (rw,nosuid,nodev,relatime,uid=1000,gid=1000,fmask=0022,dmask=0077,codepage=cp437,iocharset=utf8,shortname=mixed,showexec,utf8,flush,errors=remount-ro,uhelper=udisks) [A-DATA UFD]+++tyc20@im1:/media/A-DATA UFD/annex$ ls -la ~/Desktop/annex/conf/+total 60+drwxr-xr-x 2 tyc20 tyc20  4096 Jan  5 21:55 .+drwxr-xr-x 4 tyc20 tyc20  4096 Jan  5 21:42 ..+-rw-r--r-- 1 tyc20 tyc20  1199 Jul 28 09:14 cassandra-rackdc.properties+-rw-r--r-- 1 tyc20 tyc20 35216 Jul 28 09:14 cassandra.thrift+-rw-r--r-- 1 tyc20 tyc20  1358 Jul 28 09:14 cassandra-topology.properties+-rw-r--r-- 1 tyc20 tyc20  1324 Jul 28 09:14 cqlshrc.sample+-rw-r--r-- 1 tyc20 tyc20  1850 Jul 28 09:14 new-name2.properties+tyc20@im1:/media/A-DATA UFD/annex$ ls -la ~/Desktop/annex/+total 32828+drwxr-xr-x 4 tyc20 tyc20     4096 Jan  5 21:42 .+drwxr-xr-x 4 tyc20 tyc20     4096 Jan  5 21:45 ..+drwxr-xr-x 2 tyc20 tyc20     4096 Jan  5 21:55 conf+drwxr-xr-x 9 tyc20 tyc20     4096 Jan  5 22:01 .git+-rwxrwxrwx 1 tyc20 tyc20 33558574 Aug 27  2007 Smalltalk-80 The language and its implementation (Bluebook).pdf+tyc20@im1:/media/A-DATA UFD/annex$ ++# End of transcript or log.+"""]]
+ doc/bugs/assistant_eats_all_CPU.mdwn view
@@ -0,0 +1,520 @@+### Please describe the problem.++After running for a while, the assistant maxes my CPU. I have evidence+here that it is just sitting there, doing apparently nothing (transfers+are finished), eating precious CPU.++Nowadays, I do not let the assistant run on its own, otherwise it makes+my office noisy because the CPU fan starts.++This makes the assistant and the webapp basically unuseable for me:+if i want to use its feature, I must remember to shut down the daemon+every time I am done with it, which seems contrary to the spirit of+the project. :)++### What steps will reproduce the problem?++It's a little unclear. There was a peculiar crash here (power cable+yanked out) during a `git annex copy`, which *was* eating up all CPU,+but mostly through I/O.++That is the purple block in this graph:++![CPU usage during the last 24 hours][cpu-day.png]++Power was yanked out about right after that purple block was gone:++![Uptime in the last 24 hours][uptime-day.png]++Files were copied to an external drive at the time of the crash,+which explains the high iowait. The last lines in daemon.log are:++[[!format txt """+sent 184184937 bytes  received 31 bytes  13643330.96 bytes/sec+total size is 184162304  speedup is 1.00+[2013-12-31 23:56:47 EST] Transferrer: Uploaded American ..Queen.avi+[2013-12-31 23:56:47 EST] Pusher: Syncing with backup+To /backup/srv/video+   5155be4..7d904d5  git-annex -> synced/git-annex+"""]]++And daemon.status:++[[!format txt """+lastRunning:1388569856.366405s+scanComplete:True+sanityCheckRunning:False+lastSanityCheck:1387169250.801117s+"""]]++When the machine rebooted, I logged in, and resumed the transfer using+`git annex copy`. The assistant was also started automatically during+my session, but I wasn't aware of that until this morning (7AM) when i+came back to the terminal and noticed how noisy it was.++At that point i did `killall git-annex` and CPU usage returned to+normal, as the frequency of the CPU:++![CPU frequency scaling in the last 24 hours][cpuspeed-day.png]++I am trying to see if I can reproduce this problem reliably - I +just started the assistant again now, and while the CPU usage is+already high, I will forgive it for now because it's doing its +initial scans... Let's see what happens in the afternoon! --[[anarcat]]++Update: I seem to be able to reproduce. I just started the webapp and+the scans were performed. They are now finished, yet the CPU usage is +still high:++[[!format txt """+top - 14:10:20 up 1 day,  1:16,  7 users,  load average: 5,51, 3,61, 2,53+Tasks: 292 total,   2 running, 288 sleeping,   0 stopped,   2 zombie+%Cpu(s): 60,9 us,  7,8 sy,  0,6 ni, 29,5 id,  0,8 wa,  0,0 hi,  0,5 si,  0,0 st+KiB Mem:   8162960 total,  6943100 used,  1219860 free,  1065464 buffers+KiB Swap:  4022268 total,    15452 used,  4006816 free,  1983252 cached++  PID USER      PR  NI  VIRT  RES  SHR S  %CPU %MEM    TIME+  COMMAND+18364 anarcat   20   0  152m  27m  18m S  53,7  0,3   0:35.61 git-annex+ 8388 anarcat   20   0  192m  54m  21m S  47,8  0,7  12:55.19 git-annex+18671 anarcat   20   0  157m  33m  18m S  29,9  0,4   0:22.65 git-annex+"""]]++An strace on 18364 yields:++[[!format txt """+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 537245076}) = 0+[pid 18364] <... futex resumed> )       = ? ERESTARTSYS (To be restarted)+[pid 18364] --- SIGVTALRM (Virtual timer expired) @ 0 (0) ---+[pid 18364] rt_sigreturn(0x1a)          = 202+[pid 18364] futex(0x3de703c, FUTEX_WAIT_PRIVATE, 51, NULL <unfinished ...>+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 256339453}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 537670797}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 256748972}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 538079838}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 257202502}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 538533443}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 257611085}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 538941839}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 258018398}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 539349100}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 258433069}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 539763939}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 258863864}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 540195885}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 259295204}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 540626505}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 259729523}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 541060751}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 260131616}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 541463041}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 260539761}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 541871201}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 260970889}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 542302047}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 261378036}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 542709166}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 261780507}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 543111819}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 262194032}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 543525143}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 262610040}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 543941528}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 263042272}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 544374016}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18366] rt_sigprocmask(SIG_BLOCK, [TERM], [], 8) = 0+[pid 18366] clock_gettime(0x2 /* CLOCK_??? */, {82, 263532359}) = 0+[pid 18366] clock_gettime(0x3 /* CLOCK_??? */, {79, 544864814}) = 0+[pid 18366] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0+[pid 18364] <... futex resumed> )       = ? ERESTARTSYS (To be restarted)+[pid 18364] --- SIGVTALRM (Virtual timer expired) @ 0 (0) ---+[pid 18364] rt_sigreturn(0x1a)          = 202+[pid 18364] futex(0x3de703c, FUTEX_WAIT_PRIVATE, 51, NULL <unfinished ...>+"""]]++ [cpu-day.png]: http://paste.anarc.at/cpu-day.png+ [uptime-day.png]: http://paste.anarc.at/uptime-day.png+ [cpuspeed-day.png]: http://paste.anarc.at/cpuspeed-day.png++### What version of git-annex are you using? On what operating system?++Debian wheezy.++[[!format sh """+anarcat@marcos:~$ git annex version+git-annex version: 5.20131109-gf2cb5b9+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL+remote types: git gcrypt S3 bup directory rsync web webdav glacier hook+"""]]++### Please provide any additional information below.++This *may* be related to [[bugs/webapp_takes_100__37___of_the_cpu/]],+but I didn't observe both problems simultaneously so far - ie. the+webapp doesn't need to run for this bug to happen.++Here's the output of top:++[[!format sh """+top - 04:55:01 up 16:01, 10 users,  load average: 3,96, 3,72, 3,59+Tasks: 293 total,   6 running, 284 sleeping,   0 stopped,   3 zombie+%Cpu(s): 87,4 us, 10,9 sy,  0,6 ni,  0,1 id,  0,3 wa,  0,0 hi,  0,7 si,  0,0 st+KiB Mem:   8162960 total,  6847120 used,  1315840 free,  1084756 buffers+KiB Swap:  4022268 total,     1320 used,  4020948 free,  2041108 cached++  PID USER      PR  NI  VIRT  RES  SHR S  %CPU %MEM    TIME+  COMMAND+ 8099 anarcat   20   0  144m  28m  18m S  34,0  0,4 460:10.96 git-annex+ 6525 anarcat   20   0  436m 295m  18m S  28,4  3,7 448:50.71 git-annex+11146 munin     20   0 25376 6540 1896 R  28,4  0,1   0:00.05 munin-update+ 8045 anarcat   20   0  165m  41m  18m S  22,7  0,5 455:14.41 git-annex+11142 root      20   0 32316 7356 2016 R  22,7  0,1   0:00.05 ikisite+11144 aegir     20   0  161m  16m 6608 R  22,7  0,2   0:00.05 php+ 2649 liquidso  20   0  222m  54m 5516 S   5,7  0,7  40:40.61 liquidsoap+11149 anarcat   20   0 24448 1552 1072 R   5,7  0,0   0:00.01 top+    1 root      20   0 10644  784  652 S   0,0  0,0   0:01.10 init+    2 root      20   0     0    0    0 S   0,0  0,0   0:00.01 kthreadd+    3 root      20   0     0    0    0 S   0,0  0,0   2:10.09 ksoftirqd/0+    6 root      rt   0     0    0    0 S   0,0  0,0   0:00.22 migration/0+    7 root      rt   0     0    0    0 S   0,0  0,0   0:00.59 watchdog/0+    8 root      rt   0     0    0    0 S   0,0  0,0   0:00.25 migration/1+   10 root      20   0     0    0    0 S   0,0  0,0   2:11.10 ksoftirqd/1+   12 root      rt   0     0    0    0 S   0,0  0,0   0:00.17 watchdog/1+   13 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 cpuset+   14 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 khelper+   15 root      20   0     0    0    0 S   0,0  0,0   0:00.00 kdevtmpfs+   16 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 netns+   17 root      20   0     0    0    0 S   0,0  0,0   0:00.17 sync_supers+   18 root      20   0     0    0    0 S   0,0  0,0   0:00.00 bdi-default+   19 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kintegrityd+   20 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kblockd+   22 root      20   0     0    0    0 S   0,0  0,0   0:00.06 khungtaskd+   23 root      20   0     0    0    0 S   0,0  0,0   3:04.23 kswapd0+   24 root      25   5     0    0    0 S   0,0  0,0   0:00.00 ksmd+   25 root      39  19     0    0    0 S   0,0  0,0   0:00.00 khugepaged+   26 root      20   0     0    0    0 S   0,0  0,0   0:00.00 fsnotify_mark+   27 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 crypto+  113 root      20   0     0    0    0 S   0,0  0,0   0:00.01 khubd+  133 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ata_sff+  153 root      20   0     0    0    0 S   0,0  0,0   0:05.68 scsi_eh_0+  158 root      20   0     0    0    0 S   0,0  0,0   0:00.00 scsi_eh_1+  160 root      20   0     0    0    0 S   0,0  0,0   0:00.09 kworker/u:2+  161 root      20   0     0    0    0 S   0,0  0,0   0:00.00 scsi_eh_2+  162 root      20   0     0    0    0 S   0,0  0,0   0:00.00 scsi_eh_3+  163 root      20   0     0    0    0 S   0,0  0,0   0:01.43 kworker/u:3+  294 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  295 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kcryptd_io+  296 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kcryptd+  305 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  307 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  311 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  313 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  341 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  342 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kcryptd_io+  343 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kcryptd+  352 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  354 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  356 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  358 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  361 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kdmflush+  400 root      20   0     0    0    0 S   0,0  0,0   0:00.00 jbd2/dm-1-8+  401 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ext4-dio-unwrit+  468 root      20   0     0    0    0 S   0,0  0,0   0:02.19 kworker/1:2+  538 root      20   0 21804 1804  796 S   0,0  0,0   0:00.10 udevd+  705 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kpsmoused+  878 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 hd-audio0+  890 postfix   20   0 44216 2460 1912 S   0,0  0,0   0:00.00 pickup+  939 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 kvm-irqfd-clean+ 1150 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 firewire+ 1719 root      20   0     0    0    0 S   0,0  0,0   0:03.40 flush-254:3+ 1786 root      20   0     0    0    0 S   0,0  0,0   0:00.17 jbd2/dm-2-8+ 1787 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ext4-dio-unwrit+ 1788 root      20   0     0    0    0 S   0,0  0,0   0:06.60 jbd2/dm-3-8+ 1789 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ext4-dio-unwrit+ 1790 root      20   0     0    0    0 S   0,0  0,0   0:02.56 jbd2/dm-4-8+ 1791 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ext4-dio-unwrit+ 1792 root      20   0     0    0    0 S   0,0  0,0   0:01.19 jbd2/dm-8-8+ 1793 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 ext4-dio-unwrit+ 2004 anarcat   20   0     0    0    0 Z   0,0  0,0   0:17.57 chromium+ 2379 root      20   0  115m 1796  928 S   0,0  0,0   0:02.16 rsyslogd+ 2435 icecast2  20   0  148m 4440 2780 S   0,0  0,1   0:56.15 icecast2+ 2488 root      20   0  4112  644  500 S   0,0  0,0   0:00.66 acpid+ 2513 daemon    20   0 16668  156    0 S   0,0  0,0   0:00.00 atd+ 2536 proxy     20   0  6884  344  196 S   0,0  0,0   0:00.00 polipo+ 2573 messageb  20   0 31488 2548  884 S   0,0  0,0   0:00.79 dbus-daemon+ 2626 bind      20   0  111m  15m 2276 S   0,0  0,2   0:00.46 named+ 2694 root      20   0 82308 3112 2520 S   0,0  0,0   0:00.03 gdm3+ 2729 root      20   0 99,3m 4652 3608 S   0,0  0,1   0:00.02 gdm-simple-slav+ 2731 root      20   0  178m  27m 9908 S   0,0  0,3  40:30.80 Xorg+ 2738 root      20   0  282m  14m 6576 S   0,0  0,2   0:02.98 apache2+ 2763 root      20   0     0    0    0 S   0,0  0,0   0:00.92 flush-254:4+ 2779 root      20   0 21000 1164  904 S   0,0  0,0   0:00.00 bluetoothd+ 2784 avahi     20   0 34236 1876 1460 S   0,0  0,0   0:00.15 avahi-daemon+ 2786 avahi     20   0 34032  440  184 S   0,0  0,0   0:00.00 avahi-daemon+ 2792 root      10 -10     0    0    0 S   0,0  0,0   0:00.00 krfcommd+ 2829 bitlbee   20   0 45216 1136  712 S   0,0  0,0   0:00.00 bitlbee+ 2973 root      20   0  136m 3572 2892 S   0,0  0,0   0:00.37 accounts-daemon+ 2974 clamav    20   0 45432 2172 1260 S   0,0  0,0   0:26.77 freshclam+ 2997 root      20   0  141m 4552 3072 S   0,0  0,1   0:00.21 polkitd+ 3021 root      20   0 18876  984  740 S   0,0  0,0   0:00.48 cron+ 3062 root      20   0 75948 3436 2304 S   0,0  0,0   0:00.03 cupsd+ 3064 colord    20   0  152m 8672 3344 S   0,0  0,1   0:00.10 colord+ 3069 root      20   0  126m 4116 2964 S   0,0  0,1   0:00.16 console-kit-dae+ 3071 colord    20   0  300m  12m 7932 S   0,0  0,2   0:01.23 colord-sane+ 3171 nobody    10 -10 44328 1672 1048 S   0,0  0,0   0:00.88 gpsd+ 3178 root      20   0  9204 3508  380 S   0,0  0,0   0:39.47 haveged+ 3192 ikiwiki-  20   0 13772  864  728 S   0,0  0,0   0:00.00 git-daemon+ 3310 anarcat   20   0 91856  27m 2820 S   0,0  0,3   0:03.30 xterm+ 3314 anarcat   20   0 24004 5452 1876 S   0,0  0,1   0:00.54 bash+ 3336 root      20   0  152m 4172 3300 S   0,0  0,1   0:00.07 upowerd+ 3437 root      20   0     0    0    0 S   0,0  0,0   0:00.00 scsi_eh_6+ 3438 root      20   0     0    0    0 S   0,0  0,0   6:52.70 usb-storage+ 3470 anarcat   20   0 61648 2516 1912 S   0,0  0,0   0:00.01 gvfs-gphoto2-vo+ 3474 anarcat   20   0 78024 2564 2048 S   0,0  0,0   0:02.04 gvfs-afc-volume+ 3530 root      20   0 52400  10m 2064 S   0,0  0,1   0:03.34 munin-node+ 3599 rtkit     21   1 39632 1344 1112 S   0,0  0,0   0:00.30 rtkit-daemon+ 3721 root      20   0  4176  700  560 S   0,0  0,0   0:00.01 mysqld_safe+ 3899 anarcat   20   0 85216  21m 2872 S   0,0  0,3   0:00.76 xterm+ 3903 anarcat   20   0 23216 4688 1804 S   0,0  0,1   0:00.58 bash+ 4108 mysql     30  10  434m 137m 4396 S   0,0  1,7   1:53.14 mysqld+ 4109 root      20   0  4084  628  524 S   0,0  0,0   0:00.00 logger+ 4209 anarcat   20   0  129m  23m  18m S   0,0  0,3   1:06.16 git-annex+ 4226 root      20   0  183m 4888 3504 S   0,0  0,1   0:00.08 gdm-session-wor+ 4240 anarcat   20   0 21360 2572 2188 S   0,0  0,0   0:00.34 git+ 4431 root      20   0     0    0    0 S   0,0  0,0   0:00.00 kauditd+ 4447 anarcat   20   0  4176  664  548 S   0,0  0,0   0:00.03 sh+ 4530 ntp       20   0 41052 2284 1644 S   0,0  0,0   0:02.71 ntpd+ 4593 postgres  20   0  102m 9008 7680 S   0,0  0,1   0:01.04 postgres+ 4667 anarcat   20   0 12384 1072  704 S   0,0  0,0   0:00.11 ssh-agent+ 4668 anarcat   20   0 20164  372  156 S   0,0  0,0   0:01.03 gpg-agent+ 4671 anarcat   20   0 24184  552  304 S   0,0  0,0   0:00.00 dbus-launch+ 4672 anarcat   20   0 30668 1592  736 S   0,0  0,0   0:01.10 dbus-daemon+ 4704 anarcat   20   0 49456 2740 2216 S   0,0  0,0   0:00.13 xfconfd+ 4711 anarcat   20   0 63740 3124 2312 S   0,0  0,0   0:02.61 xscreensaver+ 4713 anarcat   20   0  156m 8232 5856 S   0,0  0,1   0:06.14 xfce4-session+ 4714 postgres  20   0  102m 1952  616 S   0,0  0,0   0:06.79 postgres+ 4715 postgres  20   0  102m 1704  376 S   0,0  0,0   0:04.95 postgres+ 4716 postgres  20   0  102m 3068 1128 S   0,0  0,0   0:01.32 postgres+ 4717 postgres  20   0 72884 1800  416 S   0,0  0,0   0:00.99 postgres+ 4735 postgrey  20   0 66452  14m 3136 S   0,0  0,2   0:00.40 /usr/sbin/postg+ 4753 anarcat   20   0  124m 4060 2916 S   0,0  0,0   0:00.02 xfsettingsd+ 4754 anarcat   20   0  166m 6496 5204 S   0,0  0,1   0:00.07 Thunar+ 4778 anarcat   20   0 63364 2564 2096 S   0,0  0,0   0:00.03 gvfsd+ 4783 anarcat   20   0  305m  14m 9568 S   0,0  0,2   0:52.46 xfce4-panel+ 4790 anarcat   20   0  156m 7968 6232 S   0,0  0,1   0:00.23 xfdesktop+ 4791 root      20   0 21516  804  496 S   0,0  0,0   0:00.00 dovecot+ 4837 anarcat   20   0  148m 4296 2768 S   0,0  0,1   0:11.87 xfce4-settings-+ 4856 anarcat   20   0 26620 1248  988 S   0,0  0,0   1:19.60 xcompmgr+ 4873 anarcat   20   0  115m 8004 3872 S   0,0  0,1   0:07.42 awesome+ 4887 anarcat   20   0  267m  12m 8664 S   0,0  0,2   0:02.51 update-notifier+ 4899 anarcat   20   0  183m 5868 4708 S   0,0  0,1   0:00.18 polkit-gnome-au+ 4905 anarcat   20   0 56320 4180 2040 S   0,0  0,1   0:00.10 gconfd-2+ 4908 anarcat    9 -11  300m 6912 4184 S   0,0  0,1   0:08.00 pulseaudio+ 4916 root      20   0  126m 4208 3056 S   0,0  0,1   0:02.46 udisks-daemon+ 4917 root      20   0 47408  776  428 S   0,0  0,0   0:06.61 udisks-daemon+ 4921 anarcat   20   0  113m 3200 2448 S   0,0  0,0   0:00.00 gconf-helper+ 4972 anarcat   20   0  145m 7432 5880 S   0,0  0,1   0:00.09 panel-16-systra+ 4974 anarcat   20   0  148m 9328 7348 S   0,0  0,1   2:40.37 xfce4-weather-p+ 4975 anarcat   20   0  174m  11m 8312 S   0,0  0,1   0:42.26 xfce4-xkb-plugi+ 4978 mpd       20   0  503m  29m 2920 S   0,0  0,4   1:24.97 mpd+ 5029 mpd       20   0  256m 6412 4008 S   0,0  0,1   0:11.74 pulseaudio+ 5030 dovecot   20   0 13044  768  592 S   0,0  0,0   0:00.00 anvil+ 5031 root      20   0 13172  944  628 S   0,0  0,0   0:00.00 log+ 5034 root      20   0 23156 2536  768 S   0,0  0,0   0:00.00 config+ 5040 mpd       20   0  113m 3192 2448 S   0,0  0,0   0:00.00 gconf-helper+ 5234 root      20   0     0    0    0 S   0,0  0,0   0:01.19 kworker/0:1+ 5745 root      20   0 37788 2472 1912 S   0,0  0,0   0:00.68 master+ 5756 postfix   20   0 44368 2700 2100 S   0,0  0,0   0:00.27 qmgr+ 5807 root      20   0 49848 1216  608 S   0,0  0,0   0:00.05 sshd+ 5908 debian-t  20   0 56008  21m 2988 S   0,0  0,3   0:21.87 tor+ 5928 debian-t  30  10  190m  31m 2560 S   0,0  0,4   7:35.56 transmission-da+ 5948 anarcat   20   0 71776 8204 2860 S   0,0  0,1   0:05.76 xterm+ 5954 anarcat   20   0 22956 4540 1772 S   0,0  0,1   0:00.10 bash+ 6019 spampd    20   0  147m  62m 1844 S   0,0  0,8   0:00.55 spampd+ 6049 root       0 -20     0    0    0 S   0,0  0,0   0:00.00 iprt+ 6085 root      20   0  3948   84    0 S   0,0  0,0   0:00.00 minissdpd+ 6095 root      20   0 17404  908  744 S   0,0  0,0   0:00.00 getty+ 6096 root      20   0 17404  908  744 S   0,0  0,0   0:00.00 getty+ 6097 root      20   0 17404  904  744 S   0,0  0,0   0:00.00 getty+ 6098 root      20   0 17404  896  744 S   0,0  0,0   0:00.00 getty+ 6099 root      20   0 17404  900  744 S   0,0  0,0   0:00.00 getty+ 6100 root      20   0 17404  904  744 S   0,0  0,0   0:00.00 getty+ 6108 anarcat   20   0 24248 1040  816 S   0,0  0,0   0:00.08 screen+ 6109 anarcat   20   0 25864 2640  796 S   0,0  0,0   0:05.37 screen+ 6110 anarcat   20   0 22868 4220 1624 S   0,0  0,1   0:00.06 bash+ 6153 root      20   0     0    0    0 S   0,0  0,0   0:00.00 flush-254:8+ 6174 anarcat   20   0 22872 4224 1624 S   0,0  0,1   0:00.07 bash+ 6238 anarcat   20   0  103m  12m 5232 S   0,0  0,2   0:20.80 irssi+ 6252 anarcat   20   0  212m  40m  13m S   0,0  0,5   1:09.74 emacs23+ 6534 anarcat   20   0 22876 4400 1716 S   0,0  0,1   0:00.07 bash+ 6598 anarcat   20   0  302m 163m 3864 S   0,0  2,0  17:19.63 offlineimap+ 6616 anarcat   20   0 22408 2648 2228 S   0,0  0,0   0:05.04 git+ 6662 root      20   0     0    0    0 S   0,0  0,0   0:01.25 kworker/1:1+ 6666 root      20   0     0    0    0 S   0,0  0,0   0:00.00 flush-254:2+ 6897 anarcat   20   0 1011m 159m  47m S   0,0  2,0   4:54.69 chromium+ 6902 postfix   20   0 46744 3396 2400 S   0,0  0,0   0:00.09 tlsmgr+ 6903 anarcat   20   0  283m 7952 2788 S   0,0  0,1   0:05.03 chromium+ 6904 anarcat   20   0  6240  356  272 S   0,0  0,0   0:00.00 chrome-sandbox+ 6905 anarcat   20   0  304m  17m  12m S   0,0  0,2   0:00.02 chromium+ 6908 anarcat   20   0  320m 5800  764 S   0,0  0,1   0:00.16 chromium+ 6950 anarcat   20   0  955m  31m  16m S   0,0  0,4   0:01.68 chromium+ 6955 anarcat   20   0  953m  33m  17m S   0,0  0,4   0:01.25 chromium+ 6960 anarcat   20   0  959m  36m  15m S   0,0  0,5   0:08.32 chromium+ 6965 anarcat   20   0  957m  34m  14m S   0,0  0,4   0:10.25 chromium+ 6970 anarcat   20   0 1029m 114m  17m S   0,0  1,4   0:46.99 chromium+ 6975 anarcat   20   0  953m  26m  14m S   0,0  0,3   0:00.53 chromium+ 6980 anarcat   20   0  959m  35m  16m S   0,0  0,4   0:14.68 chromium+ 6985 anarcat   20   0  963m  45m  17m S   0,0  0,6   0:22.59 chromium+ 6990 anarcat   20   0  952m  25m  13m S   0,0  0,3   0:00.70 chromium+ 6995 anarcat   20   0  960m  36m  18m S   0,0  0,5   0:27.21 chromium+ 7040 anarcat   20   0  979m  69m  22m S   0,0  0,9   0:05.41 chromium+ 7055 anarcat   20   0  966m  51m  20m S   0,0  0,6   0:04.88 chromium+ 7063 anarcat   20   0  967m  42m  18m S   0,0  0,5   0:04.46 chromium+ 7072 anarcat   20   0  967m  49m  19m S   0,0  0,6   0:05.17 chromium+ 7078 anarcat   20   0  971m  53m  19m S   0,0  0,7   0:05.68 chromium+ 7087 anarcat   20   0  963m  51m  19m S   0,0  0,7   0:04.75 chromium+ 7096 anarcat   20   0  961m  48m  18m S   0,0  0,6   0:04.75 chromium+ 7107 anarcat   20   0  161m 4440 3252 S   0,0  0,1   0:02.90 gvfs-gdu-volume+ 7127 anarcat   20   0  967m  55m  22m S   0,0  0,7   1:07.92 chromium+ 7183 anarcat   20   0  912m  55m  20m S   0,0  0,7   0:04.97 chromium+ 7213 anarcat   20   0  964m  47m  19m S   0,0  0,6   0:04.69 chromium+ 7235 anarcat   20   0  965m  65m  21m S   0,0  0,8   0:05.26 chromium+ 7258 anarcat   20   0  961m  57m  20m S   0,0  0,7   0:05.39 chromium+ 7285 anarcat   20   0 18652 2484 1000 S   0,0  0,0   0:01.25 git+ 7434 anarcat   39  19 18124 1960  964 S   0,0  0,0   0:00.29 git+ 7445 www-data  20   0  141m  24m 5620 S   0,0  0,3   0:00.71 munin-cgi-graph+ 7452 anarcat   20   0  980m  63m  22m S   0,0  0,8   0:02.50 chromium+ 7492 anarcat   20   0  978m  64m  21m S   0,0  0,8   0:03.74 chromium+ 7498 anarcat   20   0  978m  61m  21m S   0,0  0,8   0:01.42 chromium+ 7518 anarcat   20   0  984m  63m  22m S   0,0  0,8   0:04.38 chromium+ 7532 anarcat   20   0  973m  56m  21m S   0,0  0,7   0:02.28 chromium+ 7813 root      20   0     0    0    0 S   0,0  0,0   0:01.22 kworker/0:2+ 7899 root      20   0     0    0    0 S   0,0  0,0   0:00.96 kworker/1:0+ 8058 anarcat   20   0 24488 1460 1104 S   0,0  0,0   0:28.34 git+ 8107 anarcat   20   0 17412 1384 1100 S   0,0  0,0   0:00.00 git+ 8130 anarcat   20   0 23792 7572 1004 S   0,0  0,1   0:41.63 git+ 8236 anarcat   39  19 21152 4940  964 S   0,0  0,1   0:00.71 git+ 8405 anarcat   20   0 17368 1364 1108 S   0,0  0,0   0:00.54 git+ 8407 anarcat   20   0 17464 1336  964 S   0,0  0,0   0:00.03 git+ 8610 anarcat   20   0 78228  14m 2860 S   0,0  0,2   0:00.37 xterm+ 8614 anarcat   20   0 23084 4548 1796 S   0,0  0,1   0:00.28 bash+ 8689 anarcat   20   0 72992 9224 2860 S   0,0  0,1   0:00.38 xterm+ 8693 anarcat   20   0 22876 4380 1696 S   0,0  0,1   0:00.07 bash+ 8735 root      20   0 99,3m 4616 3576 S   0,0  0,1   0:00.02 gdm-simple-slav+ 8736 root      20   0  142m  15m 6664 S   0,0  0,2   0:00.56 Xorg+ 8755 Debian-g  20   0 30196 1184  736 S   0,0  0,0   0:00.01 dbus-daemon+ 8756 root      20   0  119m 4548 3432 S   0,0  0,1   0:00.08 gdm-session-wor+ 8757 root      20   0 55872 2116 1572 S   0,0  0,0   0:00.00 sudo+ 8759 aegir     20   0 20640 2248 1660 S   0,0  0,0   0:00.07 bash+ 8769 Debian-g  20   0  266m  10m 7720 S   0,0  0,1   0:00.07 gnome-session+ 8773 Debian-g  20   0  502m  24m  10m S   0,0  0,3   0:00.85 gnome-settings-+ 8780 Debian-g  20   0 51980 2236 1836 S   0,0  0,0   0:00.00 gvfsd+ 8784 Debian-g   9 -11  237m 6044 3644 S   0,0  0,1   0:00.07 pulseaudio+ 8785 Debian-g  20   0  113m 3196 2448 S   0,0  0,0   0:00.00 gconf-helper+ 8787 Debian-g  20   0 52956 3756 1932 S   0,0  0,0   0:00.06 gconfd-2+ 8790 Debian-g  20   0  279m 9124 6632 S   0,0  0,1   0:00.12 metacity+ 8795 Debian-g  20   0  171m 9328 7056 S   0,0  0,1   0:00.04 polkit-gnome-au+ 8796 Debian-g  20   0  362m  16m  11m S   0,0  0,2   0:00.68 gdm-simple-gree+ 8812 root      20   0 72360 3916 2948 S   0,0  0,0   0:00.08 gdm-session-wor+ 9129 root      20   0     0    0    0 S   0,0  0,0   0:00.74 kworker/0:0+ 9311 www-data  20   0  288m  44m  33m S   0,0  0,6   0:07.55 apache2+10405 www-data  20   0  287m  42m  32m S   0,0  0,5   0:06.75 apache2+10406 www-data  20   0  287m  44m  34m S   0,0  0,6   0:07.02 apache2+10409 www-data  20   0  287m  41m  31m S   0,0  0,5   0:06.51 apache2+10412 www-data  20   0  287m  42m  32m S   0,0  0,5   0:06.82 apache2+10413 www-data  20   0  291m  40m  27m S   0,0  0,5   0:06.52 apache2+10415 www-data  20   0  287m  39m  29m S   0,0  0,5   0:06.67 apache2+10416 www-data  20   0  288m  42m  31m S   0,0  0,5   0:06.28 apache2+10417 www-data  20   0  290m  50m  38m S   0,0  0,6   0:07.43 apache2+10878 root      20   0     0    0    0 S   0,0  0,0   0:00.02 kworker/0:3+11037 anarcat   20   0  4180  584  488 S   0,0  0,0   0:00.00 sh+11038 anarcat   20   0 62760 4964 3244 S   0,0  0,1   0:00.48 notmuch+11040 anarcat   20   0  4180  652  544 S   0,0  0,0   0:00.00 post-new+11133 root      20   0 52576 1468 1052 S   0,0  0,0   0:00.00 cron+11134 root      20   0 52576 1464 1052 S   0,0  0,0   0:00.00 cron+11135 root      20   0 52576 1468 1052 S   0,0  0,0   0:00.00 cron+11138 aegir     20   0  4176  576  484 S   0,0  0,0   0:00.00 sh+11139 root      20   0  4176  580  484 S   0,0  0,0   0:00.00 sh+11141 munin     20   0  4176  576  480 S   0,0  0,0   0:00.00 sh+11143 root      20   0  4240  324  248 S   0,0  0,0   0:00.00 xargs+11145 munin     20   0  4176  576  484 S   0,0  0,0   0:00.00 munin-cron+11166 anarcat   20   0 61364 2988 2468 R   0,0  0,0   0:00.00 notmuch+11167 anarcat   20   0 61364  752  212 R   0,0  0,0   0:00.00 notmuch+12004 www-data  20   0  292m  52m  37m S   0,0  0,7   0:09.05 apache2+12172 anarcat   20   0  4092  580  480 S   0,0  0,0   0:00.00 emacsclient+12465 spampd    20   0  149m  65m 2840 S   0,0  0,8   0:01.41 spampd+12468 spampd    20   0  150m  67m 2840 S   0,0  0,8   0:01.07 spampd+12895 anarcat   20   0  970m  51m  21m S   0,0  0,6   0:01.80 chromium+13761 anarcat   20   0     0    0    0 Z   0,0  0,0   0:00.00 git+16056 root      20   0 21800 1464  452 S   0,0  0,0   0:00.00 udevd+16057 root      20   0 21800 1340  332 S   0,0  0,0   0:00.00 udevd+16655 www-data  20   0  147m 8232  700 S   0,0  0,1   0:00.04 apache2+17006 anarcat   20   0  526m  29m  17m S   0,0  0,4   0:19.95 gmpc+17959 www-data  20   0 70828  24m 2172 S   0,0  0,3   1:22.37 munin-cgi-html+18082 anarcat   20   0 71776 8164 2820 S   0,0  0,1   0:00.12 xterm+18086 anarcat   20   0 23048 4664 1804 S   0,0  0,1   0:00.18 bash+18571 anarcat   20   0  258m  11m 8296 S   0,0  0,1   0:00.11 xfrun4+19692 spampd    20   0  148m  64m 2832 S   0,0  0,8   0:00.63 spampd+25574 anarcat   20   0  952m  31m  15m S   0,0  0,4   0:00.71 chromium+25590 anarcat   20   0     0    0    0 Z   0,0  0,0   0:41.75 chromium+"""]]++And here's the process tree:++[[!format sh """+$ ps -O start xf | grep git-annex+ 8610 13:03:21 S ?        00:00:00 xterm -class UXTerm -title uxterm -u8+ 8614 13:03:21 S pts/4    00:00:00  \_ bash+12172 13:13:15 S pts/4    00:00:00      \_ emacsclient --alternate-editor=vim -c install.hostmaster.inc+ 8099 13:00:20 S ?        07:40:31 /usr/local/bin/git-annex assistant --startdelay=5s+ 8107 13:00:20 S ?        00:00:00  \_ git --git-dir=/srv/books/.git --work-tree=/srv/books -c core.bare=false cat-file --batch+ 8405 13:00:28 S ?        00:00:00  \_ git --git-dir=/srv/books/.git --work-tree=/srv/books -c core.bare=false cat-file --batch+ 8407 13:00:28 S ?        00:00:00  \_ git --git-dir=/srv/books/.git --work-tree=/srv/books -c core.bare=false check-ignore -z --stdin --verbose --non-matching+ 8045 13:00:11 S ?        07:35:36 /usr/local/bin/git-annex assistant --startdelay=5s+ 8058 13:00:11 S ?        00:00:28  \_ git --git-dir=/srv/mp3/.git --work-tree=/srv/mp3 cat-file --batch+ 8130 13:00:21 S ?        00:00:41  \_ git --git-dir=/srv/mp3/.git --work-tree=/srv/mp3 check-ignore -z --stdin --verbose --non-matching+ 8236 13:00:24 S ?        00:00:00  \_ git --git-dir=/srv/mp3/.git --work-tree=/srv/mp3 check-attr -z --stdin annex.backend annex.numcopies --+ 7107 12:58:40 S ?        00:00:02 /usr/lib/gvfs/gvfs-gdu-volume-monitor+ 6525 12:57:25 S ?        07:29:13 /usr/local/bin/git-annex assistant --startdelay=5s+ 6616 12:57:34 S ?        00:00:05  \_ git --git-dir=/srv/video/.git --work-tree=/srv/video cat-file --batch+ 7285 12:58:45 S ?        00:00:01  \_ git --git-dir=/srv/video/.git --work-tree=/srv/video check-ignore -z --stdin --verbose --non-matching+ 7434 12:58:51 S ?        00:00:00  \_ git --git-dir=/srv/video/.git --work-tree=/srv/video check-attr -z --stdin annex.backend annex.numcopies --+ 4209 19:30:17 S ?        00:01:06  \_ /usr/local/bin/git-annex transferkeys --readfd 25 --writefd 24+ 4240 19:30:17 S ?        00:00:00      \_ git --git-dir=/srv/video/.git --work-tree=/srv/video cat-file --batch+13761 23:56:38 Z ?        00:00:00      \_ [git] <defunct>+ 6252 12:56:59 S ?        00:01:09 /usr/bin/emacs23+"""]]
+ doc/bugs/assistant_unable_to_auth___40__windows__41__.mdwn view
@@ -0,0 +1,83 @@+[[!meta title="windows assistant fails to set up rsync.net authorized keys"]]++### Please describe the problem.+Having set up a key pair, the assistant is unable to use it to log in. The console only shows a single attempt to log in, when it's adding the key to the remote machine's authorized_keys file. This is evidently failing, as the key is not actually added to authorized_keys, possibly because the shell script it tries to execute is malformed (crazy quoting, I can't untangle it), or possibly because it failed to ask for a password (I expected it to, but it didn't prompt for one). Also, once I do add the key to authorized_keys, I can ssh in using it but asking the assistant to retry still fails.++### What steps will reproduce the problem?+Set up a remote repository.++### What version of git-annex are you using? On what operating system?+Windows 7, Version: 5.20131230-g192d991 ++### Please provide any additional information below.++    Failed to make repository+    Something went wrong setting up the repository on the remote server.+    Transcript:+    Permission denied, please try again.+    Permission denied, please try again.+    Permission denied (publickey,gssapi-keyex,gssapi-with-mic,password).++[[!format sh """+[2014-01-01 03:28:05 Eastern Standard Time] call: ssh-keygen ["-P","","-f","C:\\+Users\\FAMILY~1\\AppData\\Local\\Temp\\git-annex-keygen.5\\key"]+Generating public/private rsa key pair.+cygwin warning:+  MS-DOS style path detected: C:\Users\FAMILY~1\AppData\Local\Temp\git-annex-key+gen.5\key+  Preferred POSIX equivalent is: /cygdrive/c/Users/FAMILY~1/AppData/Local/Temp/g+it-annex-keygen.5/key+  CYGWIN environment variable option "nodosfilewarning" turns off this warning.+  Consult the user's guide for more details about POSIX paths:+    http://cygwin.com/cygwin-ug-net/using.html#using-pathnames+Your identification has been saved in C:\Users\FAMILY~1\AppData\Local\Temp\git-a+nnex-keygen.5\key.+Your public key has been saved in C:\Users\FAMILY~1\AppData\Local\Temp\git-annex+-keygen.5\key.pub.+The key fingerprint is:+c7:0f:81:19:c8:39:cc:e1:d3:6a:62:9a:d5:d2:9f:c1 Familyroom@Familyroom-PC+The key's randomart image is:++--[ RSA 2048]----++|     +.+.        |+|     .B. +       |+|      o.+ .      |+|     o + . .     |+|    = = E +      |+|   = + . + o     |+|  o     o   .    |+|                 |+|                 |++-----------------++[2014-01-01 03:28:07 Eastern Standard Time] chat: ssh ["-p","22","rlbrooks@celeb+dil.db48x.net","sh -c 'mkdir -p '\"'\"'annex/test'\"'\"'&&cd '\"'\"'annex/test'\+"'\"'&&if [ ! -d .git ]; then git init --bare --shared && git config receive.den+yNonFastforwards false; fi&&git annex init&&mkdir -p ~/.ssh&&if [ ! -e ~/.ssh/gi+t-annex-shell ]; then (echo '\"'\"'#!/bin/sh'\"'\"';echo '\"'\"'set -e'\"'\"';ec+ho '\"'\"'if [ \"x$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then'\"'\"';echo '\"'\"'ex+ec git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"'\"'\"';echo '\"'\"'else'\"'\"';e+cho '\"'\"'exec git-annex-shell -c \"$@\"'\"'\"';echo '\"'\"'fi'\"'\"') > ~/.ssh+/git-annex-shell; fi&&chmod 700 ~/.ssh/git-annex-shell&&touch ~/.ssh/authorized_+keys&&chmod 600 ~/.ssh/authorized_keys&&echo '\"'\"'command=\"GIT_ANNEX_SHELL_DI+RECTORY='\"'\"'\"'\"'\"'\"'\"'\"'annex/test'\"'\"'\"'\"'\"'\"'\"'\"' ~/.ssh/git-+annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding ssh-rsa A+AAAB3NzaC1yc2EAAAADAQABAAABAQDeB64i+DhYBCq0mdet6sBr1JCtYR5zNc0E9NYaZbBVRBcpZDCRs+AS1z8mjyEnb49y7AnNgsx/f7xAWtyFHFr8AbrYnaHmJYicix0R6inDXZ42WJTusHVW47/1ysErHmEusZ+KWXNzN5bu8bOoz0FqCh4Ymo3QjgMzk6+PoWqSCmwewuX++RkB6q1IN1VTMqJguPtdQpTp3hCIV6GENf6+HvnLUQ59JKg1sgt5UA88zDAMlXicmUeuHRSW7nVlVIBHFDJW8f5CyF66wiuGTLdhhMlaOqpUqIEHubPF+ZE4iTOC/uuOdWjGgySX681ub4seKyz7h8uDePPo0uz0BXJpt9vT Familyroom@Familyroom-PC\n'\+"'\"' >>~/.ssh/authorized_keys'"]+"""]]++[[!format sh """+$ ssh rlbrooks@git-annex-celebdil.db48x.net-rlbrooks_annex+git-annex-shell: bad parameters++Usage: git-annex-shell [-c] command [parameters ...] [option ..]++Options:+           --force         allow actions that may lose annexed data+  -F       --fast          avoid slow operations+  -a       --auto          automatic mode+  -q       --quiet         avoid verbose output+etc+"""]]
+ doc/bugs/assistant_using_the_incorrect_path_on_windows__63__.mdwn view
@@ -0,0 +1,42 @@+### Please describe the problem.+The assistant (or rather the programs it's running) appears to be using unix-style paths instead of cygwin paths, causing several types of errors. A selection of items from the terminal:++[[!format sh """+Launching web browser on file://C:\Users\Familyroom\annex\.git\annex\webapp.html+[2013-12-31 01:14:05 Eastern Standard Time] read: ssh-keygen ["-F","celebdil.db4+8x.net"]+ssh-keygen: /home/Familyroom/.ssh/known_hosts: No such file or directory+[2013-12-31 01:14:05 Eastern Standard Time] read: ssh ["-oNumberOfPasswordPrompt+s=0","-oStrictHostKeyChecking=no","-n","-p","22","rlbrooks@celebdil.db48x.net","+sh -c 'echo git-annex-probe loggedin;if which git-annex-shell; then echo git-ann+ex-probe git-annex-shell; fi;if which git; then echo git-annex-probe git; fi;if+which rsync; then echo git-annex-probe rsync; fi;if which ~/.ssh/git-annex-shell+; then echo git-annex-probe ~/.ssh/git-annex-shell; fi;cd '\"'\"'annex'\"'\"' &&+ git config --list'"]+[2013-12-31 01:14:08 Eastern Standard Time] read: ssh-keygen ["-F","celebdil.db4+8x.net"]+ssh-keygen: /home/Familyroom/.ssh/known_hosts: No such file or directory+[2013-12-31 01:14:08 Eastern Standard Time] read: ssh ["-oStrictHostKeyChecking=+no","-n","-p","22","rlbrooks@celebdil.db48x.net","sh -c 'echo git-annex-probe lo+ggedin;if which git-annex-shell; then echo git-annex-probe git-annex-shell; fi;i+f which git; then echo git-annex-probe git; fi;if which rsync; then echo git-ann+ex-probe rsync; fi;if which ~/.ssh/git-annex-shell; then echo git-annex-probe ~/+.ssh/git-annex-shell; fi;cd '\"'\"'annex'\"'\"' && git config --list'"]+rlbrooks@celebdil.db48x.net's password:+[2013-12-31 01:15:03 Eastern Standard Time] read: gpg ["--quiet","--trust-model"+,"always","--with-colons","--list-secret-keys","--fixed-list-mode"]+gpg: WARNING: using insecure memory!+gpg: please see http://www.gnupg.org/documentation/faqs.html for more informatio+n+gpg: fatal: can't create directory `/home/Familyroom/.gnupg': No such file or di+rectory+secmem usage: 0/0 bytes in 0/0 blocks of pool 0/65536+"""]]++Naturally, I don't have a /home/Familyroom, although I do have a /c/Users/Familyroom.++### What steps will reproduce the problem?+Start creating a remote repository.++### What version of git-annex are you using? On what operating system?+Windows 7, git-annex version 5.20131230-g192d991
+ doc/bugs/endless_loop_when_PWD_gone.mdwn view
@@ -0,0 +1,16 @@+`git annex get` becomes very unhappy as soon as `$PWD` is gone.+The easiest way is to `git checkout git-annex` while you are getting in another tab.++This results in git-annex entering an endless loop:++    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory+    fatal: Unable to read current working directory: No such file or directory++> [[fixed|done]] --[[Joey]]
doc/bugs/fatal:_git-write-tree:_error_building_trees.mdwn view
@@ -99,3 +99,5 @@ After sync'ing on the server, it appears that this has been the case for quite some time (based off of what symlinks were created).  Lastly: Joey, this is probably what caused that weird behavior in the webapp where it showed the bad transfer each day after the fsck at noon. I never diagnosed that more but I bet I won't see it tomorrow.++[[!tag moreinfo]]
+ doc/bugs/git-annex_add_hangs_indefinitely_on_all_files.mdwn view
@@ -0,0 +1,32 @@+### Please describe the problem.++Running "git annex add" on a file of any size (whether 10KB or 1GB) hangs forever, must be killed with ^C, and does not add the file.++"git annex unannex" still works, as do other commands like "git annex status", "git annex whereis", etc.++### What steps will reproduce the problem?++Running "git annex add" from this computer on any git annex repository on any hard drive.++### What version of git-annex are you using? On what operating system?++I am using 3.20121127 on Arch Linux with git version 1.8.5.2.++My hard drive has several gigabytes of free space, and I have no trouble using git itself for development in other repositories.++I have been using this for over a year with no problem; I first noticed this about a week ago, and it must have started within the last ~3 weeks at most.+++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++$git annex add test.txt+add test.txt ++# End of transcript or log.+"""]]++> [[done]]
doc/bugs/git-annex_opens_too_many_files.mdwn view
@@ -38,3 +38,5 @@  # End of transcript or log. """]]++> [[fixed|done]] --[[Joey]]
− doc/bugs/git_annex_get___60__file__62___should_verify_file_hash.mdwn 
@@ -1,32 +0,0 @@-### Please describe the problem.-git annex get fileName-  should perform a hash check on the file content before adding to the local repository---### What steps will reproduce the problem?-Two scenarios:-1) Malicious user and owner of repository being pulled from can edit his/her local .git/annex/objects directory -to alter the file content. For src code, this could be to insert a bug, insert a backdoor, or for example -to replace an image file artifact for a website, with a pornographic image.-The user pulling the file content with "git annex get fileName"  might not be aware of the file contents -until they actually examine the file or perform an fsck or commit locally. -In the meantime a kiddy porn image could be sitting in their repository or a src code backdoor can get incorporated and deployed etc.-2) a file could also simply get corrupted during download. An inherent hash check during the 'annex get' would-point out the problem immediately.-To reproduce:  create repoNum1, and clone it to create repoNum2. manual edit/replace content in repoNum1/.git/annex/objects/...-then perform a 'git annex get <fileName>' from repoNum2 on the file that has been manipulated ---### What version of git-annex are you using? On what operating system?-3.2012112ubuntu2 on running linux mint ---### Please provide any additional information below.--Aside: Thanks Joey - this is fantastic work you are doing. You have really improved git. The ability to checkout -an entire tree - but selectively get only the content actually needed is a real killer feature.-Kudos and again many many thanks-M.---# End of transcript or log.-"""]]
+ doc/bugs/gsha256sum_crashes.mdwn view
@@ -0,0 +1,92 @@+[[!meta title="SIGILL on Mavericks"]]++### Please describe the problem.++gsha256sum crashes continiously, so that excessive logs must be removed and the notebook is not responsive.++Mac OS Mavericks++### What steps will reproduce the problem?++I cannot reproduce it. My guess is that it has problems with special characters in filenames of my music repository.++### What version of git-annex are you using? On what operating system?++Version: 5.20131221-g00d1673+Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash ++### Please provide any additional information below.++++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log++[2014-01-01 11:50:02 CET] main: starting assistant version 5.20131221-g00d1673+[2014-01-01 11:53:43 CET] TransferScanner: Syncing with sokrates.local_Musik +Already up-to-date.++(scanning...) [2014-01-01 11:53:45 CET] Watcher: Performing startup scan+Already up-to-date.+++  The installed version of git is too old for .gitignores to be honored by git-annex.+To ssh://juh@git-annex-sokrates.local-juh_Musik/~/Musik/+   0ec1344..c7bc58c  git-annex -> synced/git-annex++.DS_Store+           0   0%    0.00kB/s    0:00:00+       21508 100%  238.68kB/s    0:00:00 (xfer#1, to-check=0/1)++sent 42 bytes  received 21602 bytes  14429.33 bytes/sec+total size is 21508  speedup is 0.99+[2014-01-01 12:10:03 CET] Transferrer: Downloaded .DS_Store+[2014-01-01 12:10:03 CET] Pusher: Syncing with sokrates.local_Musik +To ssh://juh@git-annex-sokrates.local-juh_Musik/~/Musik/+   c7bc58c..c3ad19a  git-annex -> synced/git-annex+(Recording state in git...)+(started...) [2014-01-01 12:33:25 CET] Committer: Adding 15 - 23 s.. drøm.ogg 04 Cuatro..ielo.flac 05 Compadrón.flac 07 Buscan..dón.flac 10 Adiós..lita.flac 12 Bésam..Boca.flac 14 Nunca ..idar.flac 03 Adiós..ngo).flac 06 Selecc..ngo).flac and 2745 other files++add Edvard Grieg/Grieg Edition/Complete Piano Music/Complete Piano Music, Volume X (feat. piano_ Geir Henning Braaten)/15 - 23 småstykker for klaver, EG 104 No. 7- En drøm.ogg Committer crashed: gsha256sum parse error+[2014-01-01 12:33:27 CET] Committer: warning Committer crashed: gsha256sum parse error++.DS_Store+           0   0%    0.00kB/s    0:00:00+        6148 100%    5.86MB/s    0:00:00 (xfer#1, to-check=0/1)++sent 42 bytes  received 6242 bytes  4189.33 bytes/sec+total size is 6148  speedup is 0.98+[2014-01-01 12:49:45 CET] Transferrer: Downloaded .DS_Store+[2014-01-01 12:49:45 CET] Pusher: Syncing with sokrates.local_Musik +[2014-01-01 12:53:42 CET] NetWatcherFallback: Syncing with sokrates.local_Musik +From ssh://git-annex-sokrates.local-juh_Musik/~/Musik+   0ec1344..c3ad19a  git-annex  -> sokrates.local_Musik/git-annex+fatal: Unable to create '/home/juh/Musik/.git/refs/heads/synced/git-annex.lock': Die Datei existiert bereits.++If no other git process is currently running, this probably means a+git process crashed in this repository earlier. Make sure no other git+process is running and remove the file manually to continue.+git-annex-shell: git-shell failed+fatal: The remote end hung up unexpectedly+fatal: The remote end hung up unexpectedly+To ssh://juh@git-annex-sokrates.local-juh_Musik/~/Musik/+   c3ad19a..e6da014  git-annex -> synced/git-annex+From ssh://git-annex-sokrates.local-juh_Musik/~/Musik+   c3ad19a..e6da014  git-annex  -> sokrates.local_Musik/git-annex+Everything up-to-date+[2014-01-01 13:11:59 CET] UpgradeWatcher: A new version of git-annex has been installed. ++"""]]++I can send you the crash report of hsha256sum if you need it.++> Fine, we'll stop using this on OSX, since git-annex has its own builtin+> and not even much slower hashers. done--[[Joey]]++>> I'm told that other programs, such as wget, also crash with SIGILL.+>> Are there multiple OSX architectures with different instructions sets or+>> what? --[[Joey]]++>>> Fixed and confirmed fixed now. gsha256sum etc added back to+>>> OSX app. [[done]] --[[Joey]]
+ doc/bugs/illegal_hardware_instruction_on_OSX_for_5.20131230-g9a495e6.mdwn view
@@ -0,0 +1,25 @@+### Please describe the problem.++git-annex exists with "illegal hardware instruction" on OSX++### What steps will reproduce the problem?++git annex webapp++### What version of git-annex are you using? On what operating system?++git annex 5.20131230-g9a495e6+OSX 10.9.1++### Please provide any additional information below.++[[!format sh """++start git assistant with "git annex web app"+safari open tab, but shows connection refused+git annex crashed with "illegal hardware instruction"++# End of transcript or log.+"""]]++> [[fixed|done]] --[[Joey]]
doc/bugs/import_memleak_from_the_assistant.mdwn view
@@ -1596,3 +1596,13 @@  # End of transcript or log. """]]++> [[Fixed|done]]. This was several garden-variety haskell laziness leaks,+> all fixed by adding strictness annotations.+>+> Before: [[leakbefore.png]]+> After: [[leakafter.png]]+> +> Looks like I got them all, and it returns to running in constant space+> after adding and uploading the files (which can take memory porportional+> to the number of files that were added/changed at once). --[[Joey]]
+ doc/bugs/import_memleak_from_the_assistant/leakafter.png view

binary file changed (absent → 43348 bytes)

+ doc/bugs/import_memleak_from_the_assistant/leakbefore.png view

binary file changed (absent → 24006 bytes)

+ doc/bugs/info_file_contains_wrong_version.mdwn view
@@ -0,0 +1,20 @@+### Please describe the problem.++The script producing the .info file along with the binary tarball seems to generate the wrong version.+++### What steps will reproduce the problem?++    % curl https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz.info+    GitAnnexDistribution {distributionUrl = "https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz",+    distributionKey = Key {keyName = "2a01f853b08dc0f47392b50cda41d5f589f46413073fdf4ffd5b817d6638a502.tar.gz", keyBackendName = "SHA256E",+    keySize = Just 27578790, keyMtime = Nothing}, distributionVersion = "5.20131222",+    distributionReleasedate = 2013-12-30 18:56:44.908259 UTC, distributionUrgentUpgrade = Nothing}        +++`distributionVersion` is 5.20131222, but `git annex version` returns 5.20131230-g4aa88d8.++This may look anecdotical, but at least the `git-annex-bin` package of Arch Linux uses the .info data to retrieve the version information.+++> [[fixed|done]] --[[Joey]]
+ doc/bugs/interference_with_Dropbox_results_in_data_loss.mdwn view
@@ -0,0 +1,48 @@+### Please describe the problem.++When working with git-annex in a folder managed by Dropbox, the folders names in .git/annex/objects/ are mangled by Dropbox. This leads to lost files.++This is not a git-annex bug, but the problem is severe and it would be helpful when git-annex can deal with it somehow. Possibilities are:++* include a warning in documentation (this will be probably found after someone encounters the problem actually)+* issue a warning when running in a Dropbox managed folder+* avoid using filenames that differs in case only (this would make git-annex also more robust against corruption i.e. when a repository is copied over filesystems/protocols without filename case support)++Dropbox file name limitations: [[https://www.dropbox.com/help/145/en]]++### What steps will reproduce the problem?+    cd ~/Dropbox+    mkdir test_annex+    cd test_annex+    git init+    git annex init "test_annex"+    cp ~/somefiles/* .+    git annex add *+    # wait for Dropbox to upload and mangle folder names in .git/annex/objects/+    find -L . -type l | wc -l # print number of broken symlinks+    git annex fsck # prints all the missing files+++### What version of git-annex are you using? On what operating system?++* git-annex 3.20130207+* Fedora 19++### Please provide any additional information below.++[[!format sh """+# folder names mangled by Dropbox++$ ls .git/annex/objects/+Ff/                      M9 (Case Conflict (1))/  x8 (Case Conflict (1))/+Ff (Case Conflict)/      pF/                      x9/+Ff (Case Conflict (1))/  pF (Case Conflict)/      x9 (Case Conflict)/+Ff (Case Conflict (2))/  pF (Case Conflict (1))/  Zf/+Ff (Case Conflict (3))/  pF (Case Conflict (2))/  Zf (Case Conflict)/+Ff (Case Conflict (4))/  pF (Case Conflict (3))/  Zf (Case Conflict (1))/+Ff (Case Conflict (5))/  pF (Case Conflict (4))/  Zf (Case Conflict (2))/+Ff (Case Conflict (6))/  PG/                      Zf (Case Conflict (3))/+...++# End of transcript or log.+"""]]
doc/bugs/microsd__47__thumbdrives_seem_to_die_when_using_the_ARM_build.mdwn view
@@ -34,3 +34,5 @@  # End of transcript or log. """]]++[[done]]
+ doc/bugs/webapp_takes_100__37___of_the_cpu.mdwn view
@@ -0,0 +1,41 @@+### Please describe the problem.++I just started git-annex, and I can hear the fan of my CPU. That doesn't happen unless the CPU is maxed, and indeed:++[[!format txt """+top - 10:09:53 up 2 days, 13:13, 10 users,  load average: 2,49, 2,41, 1,59+Tasks: 306 total,   3 running, 302 sleeping,   0 stopped,   1 zombie+%Cpu(s): 57,2 us,  7,9 sy,  1,2 ni, 32,1 id,  1,3 wa,  0,0 hi,  0,3 si,  0,0 st+KiB Mem:   8162960 total,  7480272 used,   682688 free,  1107616 buffers+KiB Swap:  4022268 total,    23384 used,  3998884 free,  3469028 cached++  PID USER      PR  NI  VIRT  RES  SHR S  %CPU %MEM    TIME+  COMMAND+27841 anarcat   20   0  191m  45m  20m S  92,3  0,6   4:32.68 /usr/local/bin/git-annex webapp+"""]]++### What steps will reproduce the problem?++I am not sure. I just started the webapp, and it seems to be scanning two huge repositories (160GB and 800GB) with lots of files. The larger one is also in the process of being sync'd to the backup drive.++### What version of git-annex are you using? On what operating system?++My usual bastard build from hell:++[[!format txt """+git-annex version: 5.20131109-gf2cb5b9+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL+remote types: git gcrypt S3 bup directory rsync web webdav glacier hook+local repository version: 3+default repository version: 3+supported repository versions: 3 5+upgrade supported from repository versions: 0 1 2 4+"""]]++### Please provide any additional information below.++Note that I had to close the chromium tab that was displaying the actual webapp UI, because that was taking another 50% of a CPU (I have a dual core).++Note also that things calmed down since git-annex started transfering larger files - the webapp only takes 75% of the CPU now. ;) And chromium is negligible. But there clearly seems to be a degenerate case where a lot of small files get transfered that seem to freakout the web UI. -- [[anarcat]]++Oh, and just copying the files using `git annex copy --to backup` doesn't use 100% of the CPU.
doc/bugs/webdav_special_remote_fails_on_android___40__box.com__41__.mdwn view
@@ -28,3 +28,5 @@ I don't see any mention of WebDAV in daemon.log and friends.    It seems like NODELAY should be optional, as it's just a performance optimization (and disabling packet coalescing may be battery-hostile on a mobile device, too).++> Should be fixed in the last release, [[done]] --[[Joey]]
doc/builds.mdwn view
@@ -15,9 +15,6 @@ <h2>OSX Mavericks</h2> <iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/"> </iframe>-<h2>OSX Lion</h2>-<iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/">-</iframe> <h2>Windows</h2> <iframe width=1024 height=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/"> </iframe>
doc/design/external_special_remote_protocol.mdwn view
@@ -209,6 +209,32 @@   or a file.   (git-annex replies with "CREDS User Password". If no creds are found,   User and Password are both empty.)+* `GETUUID`  +  Queries for the UUID of the special remote being used.  +  (git-annex replies with VALUE followed by the UUID.)+* `SETWANTED PreferredContentExpression`  +  Can be used to set the preferred content of a repository. Normally+  this is not configured by a special remote, but it may make sense+  in some situations to hint at the kind of content that should be stored+  in the special remote. Note that if a unparsable expression is set,+  git-annex will ignore it.+* `GETWANTED`  +  Gets the current preferred content setting of the repository.+  (git-annex replies with VALUE followed by the preferred content+  expression.)+* `SETSTATE Key Value`  +  Can be used to store some form of state for a Key. The state stored+  can be anything this remote needs to store, in any format.+  It is stored in the git-annex branch. Note that this means that if+  multiple repositories are using the same special remote, and store+  different state, whichever one stored the state last will win. Also,+  it's best to avoid storing much state, since this will bloat the+  git-annex branch. Most remotes will not need to store any state.+* `GETSTATE Key`  +  Gets any state that has been stored for the key.  +  (git-annex replies with VALUE followed by the state.)+* `DEBUG message`+  Tells git-annex to display the message if --debug is enabled.  ## general messages @@ -222,6 +248,22 @@   git-annex will not talk to it any further. If the program receives   an ERROR from git-annex, it can exit with its own ERROR. +## long running network connections++Since an external special remote is started only when git-annex needs to+access the remote, and then left running, it's ok to open a network+connection in the PREPARE stage, and continue to use that network+connection as requests are made.++If you're unable to open a network connection, or the connection closes,+perhaps because the network is down, it's ok to fail to perform any+requests. Or you can try to reconnect when a new request is made.++Note that the external special remote program may be left running for+quite a long time, especially when the git-annex assistant is using it.+The assistant will detect when the system connects to a network, and will+start a new process the next time it needs to use a remote.+ ## TODO  * When storing encrypted files stream the file up/down the pipe, rather@@ -232,8 +274,6 @@   the remote. However, \n and probably \0 need to be escaped somehow in the   file data, which adds complication. * uuid discovery during INITREMOTE.-* uuid verification during PREPARE (so, for example, it can check if a-  removable drive repo has the expected uuid) * Support for splitting files into chunks. * Support for getting and setting the list of urls that can be associated   with a key.
doc/design/roadmap.mdwn view
@@ -7,12 +7,12 @@ * Month 1 [[!traillink assistant/encrypted_git_remotes]] * Month 2 [[!traillink assistant/disaster_recovery]] * Month 3 user-driven features and polishing [[todo/direct_mode_guard]] [[assistant/upgrading]]-* **Month 4 Windows porting, [[todo/support_for_writing_external_special_remotes]] & encourage others to contribute to git-annex**-* Month 5 [[!traillink assistant/xmpp_security]]-* Month 6 Windows assistant and webapp+* Month 4 [[Windows_webapp|assistant/Windows]], Linux arm, [[todo/support_for_writing_external_special_remotes]]+* **Month 5 user-driven features and polishing**+* Month 6 [[!traillink assistant/xmpp_security]] * Month 7 user-driven features and polishing * Month 8 [[!traillink assistant/gpgkeys]] [[!traillink assistant/sshpassword]]-* Month 9 get [[assistant/Android]] out of beta+* Month 9 get [[assistant/Android]] and Windows out of beta * Month 10 user-driven features and polishing * Month 11 [[!traillink assistant/chunks]] [[!traillink assistant/deltas]] * Month 12 user-driven features and polishing
doc/devblog/day_88__lazy_sunday.mdwn view
@@ -4,7 +4,7 @@ just simplifies things to have only one supported annex.version.  Added youtube playlist support to git-annex. Seems I had almost all the-peices needed, and didn't know it. Only about a dozen lines of code!+pieces needed, and didn't know it. Only about a dozen lines of code!  Added PREPARE-FAILURE support to the external special remote interface.
+ doc/devblog/day_89__reflections.mdwn view
@@ -0,0 +1,18 @@+Worked on bug report and forum backlog (24 messages left), and made a few+bug fixes. The main one was a fix for a Windows-specific direct mode merge+bug.++This month didn't go entirely to plan. I had not expected to work on the+Windows assistant and webapp and get it so close to fully working. Nor had+I expected to spend time and make significant progress on porting git-annex+to *Linux* -- particularly to embedded NAS devices! I had hoped to+encourage some others to develop git-annex, but only had one bite from a+student and it didn't work out. Meanwhile, automatically rewarding+committers with bitcoin is an interesting alternative approach to possibly+motivating contributors, and I would like to set that up, but the software+is new and I haven't had time yet. The only thing that went exactly as+planned was the external special remote implementation.++A special surprise this month is that I have started hearing privately from+several institutions that are starting using git-annex in interesting ways.+Hope I can share details of some of that 2014!
+ doc/devblog/day_90__slow_start.mdwn view
@@ -0,0 +1,23 @@+Various work on Debian, OSX, and Windows stuff. Mostly uninteresting, but+took most of the day.++Made `git annex mirror --all` work. I can see why I left it out; when the+mirroring wants to drop an object, in --all mode it doesn't have an+associated file in the tree, so it cannot look at the annex.numcopies in+gitattributes. Same reason why `git annex drop --all` is not implemented.+But decided to go ahead and only use other numcopies configuration for+mirroring.++Added GETWANTED and SETWANTED to the external special remote protocol, and+that is as far as I want to go on adding git-annex plumbing stuff to the+protocol. I expect Tobias will release a boatload of special remotes+updated to the new protocol soon, which seems to prove it has everything+that could reasonably be needed.++This is a nice public git-annex repository containing a growing collection+of tech conference videos. <https://github.com/RichiH/conference_proceedings>++Did some design work on [[todo/untracked_remotes]], which I think will turn+out to be read-only remotes. Being able to clone a repository and use+git-annex in the clone without anything leaking back upstream is often+desirable when using public repository, or a repository with many users.
+ doc/devblog/day_91__wintry_mix.mdwn view
@@ -0,0 +1,11 @@+Implemented read-only remotes. This may not cover every use case around+wanting to clone a repository and use git-annex without leaking the+existence of your clone back to it, but I think it hits most of them in a+quite easy way, and allows for some potentially interesting stuff like+partitioned networks of git-annex repositories.++Zooko and I have been talking things over (for rather too long),+and I think have now agreed on a how a more advanced git-annex+Tahoe-LAFS special remote should work. This includes storing the+tahoe file-caps in the git-annex branch. So, I really need to add that+per-special-remote data storage feature I've been thinking about.
+ doc/devblog/day_92-93__reconnection.mdwn view
@@ -0,0 +1,34 @@+Yesterday, added per-remote, per-key state storage. This is exported via+the external special remote protocol, and I expect to use it at least for+Tahoe Lafs.++Also, made the assistant write ssh config files with better permissions,+so ssh won't refuse to use them. (The only case I know of where that+happened was on Windows.)++Today, made `addurl` and `importfeed` honor annex.diskreserve. Found out about+this the hard way, when an importfeed cron job filled up my server with+youtube videos. I should probably also make `import` honor+annex.diskreserve.++---++I've been working, so far inconclusively, on making the assistant+deal with remotes that might open a long duration network connection.+Problem being that if the connection is lost, and the remote is not smart+enough to reconnect, all further use of it could fail.++In a `restarttransferrer` branch, I have made the assistant start separate+`transferkeys` processes for each remote. So if a remote starts to+fail, the assistant can stop its `transferkeys` process, and restart it,+solving the problem.++But, if a resource needed for a remote is not available, this degrades to+every transfer attempt to that remote restarting it. So I don't know if this is the+right approach.++Other approaches being considered include asking that implementors of+external special remotes deal with reconnection themselves (Tobias, do you+deal with this in your remotes?), or making the+assistant only restart failing remotes after it detects there's been +a network connection change.
+ doc/devblog/day_94__leaks.mdwn view
@@ -0,0 +1,12 @@+Spent ages tracking down a memory leak in the assistant that showed up when+a lot of files were added. Turned out to be a standard haskell laziness+induced problem, fixed by adding strictness annotations. Actually there+were several of them, that leaked at different rates. Eventually, I seem to+have gotten them all fixed:++Before: [[bugs/import_memleak_from_the_assistant/leakbefore.png]]+After: [[bugs/import_memleak_from_the_assistant/leakafter.png]]++Also fixed a bug in `git annex add` when the disk was completely full. It+could sometimes in that situation move the file from the work tree to+.git/annex/objects and fail to put the symlink in place.
+ doc/devblog/day_95__reconnection_revisited.mdwn view
@@ -0,0 +1,13 @@+Taught the assistant to stop reusing an existing `git annex transferkeys`+process after it detects a network connection change. I don't think this is+a complete solution to what to do about long-duration network connections+in remotes. For one thing a remote could take a long time to time out+when the network is disconnected, and block other transfers (eg to+local drives) in the meantime. But at least if a remote loses its network+connection and does not try to reconnect on its own, and so is continually+failing, this will get it back into a working state eventually.++Also, fixed a problem with the OSX Mavericks build, it seems that the+versions of wget and coreutils stuff that I was including in it were built+by homebrew with full optimisations turned on, so didn't work on some CPUs.+Replaced those with portable builds.
doc/ekg.mdwn view
@@ -12,3 +12,20 @@ more than one git-annex process with EKG support at the same time can result in some "resource busy (Address already in use)" messages -- but git-annex will continue to work.++----++## full profiling++For the really tricky memory leaks, here's how to make a profiling build of+git-annex.++1. `cabal configure` with only the flags you really need+2. `cabal build --ghc-options="-prof -auto-all -caf-all"`+  This will probably fail due to some missing profiling libraries.+  You have to get the profiling versions of all needed haskell libraries+  installed somehow.+3. Run git-annex with the special flags `+RTS -hc -p`+4. Reproduce the memory leak problem.+5. If the assistant was run, stop it.+6. `hp2ps -e8in -c git-annex.hp` to generate a .ps graph of memory usage.
+ doc/forum/Controlling_content_on_mobile_device.mdwn view
@@ -0,0 +1,46 @@+I have some questions about how to best manage content on mobile+devices.  This is probably more of a wishlist discussion, although I+haven't had enough experience with the assistant and preferred content+features enough to know how much of this might already be possible.++(While I did donate to both campaigns, I also didn't donate at the+"personal git-annex assistant" level, so I also know not to expect+personal undivided attention :) . Any tips would be appreciated.)++In short, I don't think git-annex really fits with how I'd like to use+it on my Android phone.  My current setup is as follows:++- I have a large repository with various media files.  This is stored+  on several computers, at least one of which generally has a copy of+  everything.  I use a centralized bare repo to automatically+  synchronize between computers with "`git annex sync origin`" using+  cron, and manually control what is on each one using "`copy --to+  host`", "`get`", "`drop`", etc.  It is *awesome*.++What I would like for Android is:++- The phone keeps in sync with the origin repository.++- On the phone, I'd like to browse through and choose content to get+  or drop.  I could use the command-line "`get`" and "`drop`" for+  this, of course, but being able to do it through the webapp would be+  better, as typing on a phone is no fun.++- From _off_ the phone, I'd like to be able to choose which content+  should be _on_ the phone.  For example, using the command line or+  webapp on my desktop, I'd like to be able to choose media files that+  should be downloaded to (or removed from) the phone.  However, I+  don't want to have to move files between directories in the repository+  to achieve this.++- The only useful content that gets created on the phone is photos and+  videos.  Those should get uploaded to the main server, but only the+  most recent files should remain on the phone by default (say,+  anything older than 2 weeks gets dropped).  Both on the phone, and+  off the phone, I'd be able to override that using the previously+  described features, so that certain photos remain on the phone+  forever, or get dropped earlier.++Thoughts?  I feel like git-annex is close to being able to do this.++-jim
@@ -0,0 +1,11 @@+So, I'm using git-annex to manage a group of external hard drives that I use to store a bunch of big files.++When syncing tonight, though, it decided to take a whole subdirectory and replace the symlinks with the content directly when merging, or something.++The remote side is not like that, and I didn't tell it to do anything like that.+It just seems to feel like that's the thing to do, then it syncs and fails because suddenly git has many gigs of objects, which is what git-annex is supposed to solve.++I've reset master a bunch of times, and it keeps choosing that path, but I don't know why.+I've also reset master and synced/master a few times. I'm not fully sure what that one actually encodes, so I didn't want to touch it too much, but setting it to the same thing as master didn't seem to fix things either.++Any idea why?
+ doc/forum/Gitolite_problems.mdwn view
@@ -0,0 +1,11 @@+I am having some issues trying to make git-annex work with the latest version of Gitolite, +When running git annex sync, or git annex copy --to I get.+"""+Failed to get annex.uuid configuration of repository origin++Instead, got: "annex.uuid=\ncore.gcrypt-id=\n"+"""++Has anyone found any solution to this? Also, I am using the latest version of Gitolite, because I wanted to see whether it would work.+Thanks, Zack.+EDIT: It might also be worthy to note that I did have a go at following the guide on this site.
+ doc/forum/How_do_I_do_with_.gitrefs__47_____63__.mdwn view
@@ -0,0 +1,1 @@+There is .gitrefs directory in my annexed repo while I am unaware. Should I do `git add .gitrefs/` or `git annex add .gitrefs/` or just ignore?
+ doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__.mdwn view
@@ -0,0 +1,65 @@+I have just done a sync in a direct mode repo:++    git-annex sync sshremote++It has quite a few conflicts (both sides adding the same file with same content). But somehow the merge won't conclude. Here's some log:++    $ git-annex sync sshremote+    commit  ok+    pull sshremote +    warning: no common commits+    remote: Counting objects: 32162, done.+    remote: Compressing objects: 100% (31192/31192), done.+    remote: Total 32162 (delta 3), reused 32162 (delta 3)+    Receiving objects: 100% (32162/32162), 3.39 MiB | 1.61 MiB/s, done.+    Resolving deltas: 100% (3/3), done.+    From sshremote:/repo+     * [new branch]      annex/direct/master -> sshremote/annex/direct/master+     * [new branch]      git-annex  -> sshremote/git-annex+     * [new branch]      master     -> sshremote/master+     * [new branch]      synced/master -> sshremote/synced/master++    Auto-merging file1+    CONFLICT (add/add): Merge conflict in file1+    Auto-merging file2+    CONFLICT (add/add): Merge conflict in file2+    Auto-merging file3+    CONFLICT (add/add): Merge conflict in file3+    Auto-merging file4+    CONFLICT (add/add): Merge conflict in file4+    Automatic merge failed; fix conflicts and then commit the result.+    file1: needs merge+    file2: needs merge+    file3: needs merge+    file4: needs merge+    ^C++I waited for a few minutes, and checked that both cpu and disk activity is near zero on both server. So I interrupted it, thinking it might be some locking issue. ++But now I cannot sync with the remote anymore because the merge is still in process. How should I resolve the conflicts? I could do it manually if I cannot use annex's auto merge.++    $ git-annex sync sshremote+    commit  ok+    pull sshremote ++    fatal: You have not concluded your merge (MERGE_HEAD exists).+    Please, commit your changes before you can merge.++    fatal: You have not concluded your merge (MERGE_HEAD exists).+    Please, commit your changes before you can merge.+    failed++    $ git merge --abort+    fatal: This operation must be run in a work tree++Update: this is running on OSX with:++    git-annex version: 5.20131221-g00d1673+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL+    remote types: git gcrypt S3 bup directory rsync web webdav glacier hook+    local repository version: 3+    default repository version: 3+    supported repository versions: 3 5+    upgrade supported from repository versions: 0 1 2 4+
+ doc/forum/Portable_version_of_git-annex_for_windows.mdwn view
@@ -0,0 +1,22 @@+Problem: I don't have admin access on all the computers I want to plug my USB with git-annex on.+Solution: Make git & git-annex portable.++The only catch is I'm not familiar enough with git to be able to actually tell if the more specialized git-annex working properly. The commands execute without crashing so I suspect it's fine but I was hoping someone else more competent could try this out and let me know if I it's working so I can add it to the tips section. Thanks in advance!++Ingredients:++* Portable version of msysgit: <http://code.google.com/p/msysgit/downloads/list>++* Windows git-annex installer: <http://git-annex.branchable.com/install/>++* 7-zip (portable version optional): <http://portableapps.com/apps/utilities/7-zip_portable>++Steps:++1. Use 7-zip to extract the contents of the git-annex installer into their own folder.++2. Extract the portable version of git and copy all files extracted from git-annex installer into the "cmd" folder.++3. Run it!++Edit: the markdown is eating my attempts to create a numbered list and making them a paragraph instead. Apologies!
+ doc/forum/how_do_i_manually_sync_my_external_drive__63__.mdwn view
@@ -0,0 +1,9 @@+I am working on rehauling my backup scripts here, and I was originally doing an rsync of everything to an external drive, but now I think I can be smarted and skip my annexes in that rsync, and use git-annex superpowers to do the sync instead.++I have added this to my backup script:++    ( cd /srv/video && git annex copy --to backup . )++And it works, so yaaay. :) However, I feel it could be faster. This seems to check each file one at a time, but doesn't git-annex keep a state of the remote internally, which would allow it to copy over only the missing files, without checking if each file is present individually?++I mean, it's still pretty fast considering the dataset, but I wonder if there isn't some fast/easier way. Keep in mind I am hesitant of using the assistant for this because I am [[confused_about_external_drives]], the [[bugs/webapp takes 100% of the cpu]] and the [[bugs/assistant eats all CPU]]. I would prefer to script this anyways. --[[anarcat]]
+ doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up.mdwn view
@@ -0,0 +1,27 @@+I am running git-annex on Mac OS 10.9 and syncing to a Linux device that was attached to a Synology NAS.  Both were running December versions of git-annex.++While working on an application in IntelliJ my workspace.xml file disappeared.  It went from being a real file to being a symlink to a location that didn't exist.++The symlink looks like this: workspace.xml -> ../../../.git/annex/objects/M2/8W/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml++I tried git-annex fsck and I get this message: ** No known copies exist of XXXXXXX/workspace.xml++"XXXXXXX" is a path I needed to obscure I can see it in the git-annex repo but it is a reference to a symlink that doesn't exist++It was mentioned in another thread ( http://git-annex.branchable.com/forum/git_annex_on_osx_only_creating_symlinks__63____63__/ ) that this could happen if I did a forced drop but I do not use git-annex on the command line.  I'm using the assistant and having it manage all of my files for me.++I tried looking at the log for it and it shows this:++<pre>+timmattison$ /Applications/git-annex.app/Contents/MacOS/git-annex log XXXXXXX/.idea/workspace.xml +- 2014-01-03 19:49:58 XXXXXXX/.idea/workspace.xml | d6747880-b355-4d41-b4e5-d1ad3afcb4a5 -- timmattison@MacBook-Pro.local:~/Desktop/annex++ 2014-01-03 17:08:32 XXXXXXX/.idea/workspace.xml | d6747880-b355-4d41-b4e5-d1ad3afcb4a5 -- timmattison@MacBook-Pro.local:~/Desktop/annex+(Recording state in git...)+</pre>++So I can see that the git-annex thinks the file was deleted but what really happened is that IntelliJ just modified it.  workspace.xml files are modified very often when working in IntelliJ.++Is this a known bug that has been fixed in the new version (2013-12-30 or later)?++Thanks,+Tim
doc/git-annex.mdwn view
@@ -181,11 +181,16 @@   repository. If a file's content is present in the source repository, it is   copied to the destination repository. If a file's content is not present in   the source repository, it will be dropped from the destination repository-  when possible.+  when the numcopies setting allows.    Note that mirror does not sync the git repository, but only the file   contents. +  Also, --all may be specified to mirror all objects stored in the git+  annex, not only objects used by currently existing files. However, this+  bypasses checking the .gitattributes annex.numcopies setting when+  dropping files.+ * `addurl [url ...]`    Downloads each url to its own file, which is added to the annex.@@ -1269,6 +1274,12 @@    If set to `false`, prevents git-annex sync (and the git-annex assistant)   from syncing with this remote.++* `remote.<name>.annex-readonly`++  If set to `true`, prevents git-annex from making changes to a remote.+  This both prevents git-annex sync from pushing changes, and prevents+  storing or removing files from read-only remote.  * `remote.<name>.annexUrl`
doc/install/Linux_standalone.mdwn view
@@ -5,9 +5,6 @@ This tarball should work on most Linux systems. It has basically no dependencies and is self-contained. -The armel version can be installed on NAS devices and other embedded ARM-linux systems.- * i386: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz) * amd64: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz) * armel: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-armel.tar.gz)@@ -21,8 +18,11 @@ PATH. This lets you use `git annex`, without overriding your system's own versions of git, etc. -Note: This is a last resort. Most Linux users should instead-[[install]] git-annex from their distribution of choice.+The armel version can be installed on NAS devices and other embedded ARM+linux systems.++* [[tips/Synology_NAS_and_git_annex]]+* [[forum_thread|forum/new_linux_arm_tarball_build]]  ## autobuilds
doc/install/OSX.mdwn view
@@ -5,7 +5,7 @@  * 10.9 Mavericks: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.9_Mavericks/git-annex.dmg) * 10.8.2 Mountain Lion: [git-annex.dmg.bz2](https://downloads.kitenet.net/git-annex/OSX/current/10.8.2_Mountain_Lion/git-annex.dmg.bz2) **warning: not being updated any longer**-* 10.7.5 Lion: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.7.5_Lion/git-annex.dmg)+* 10.7.5 Lion: [git-annex.dmg](https://downloads.kitenet.net/git-annex/OSX/current/10.7.5_Lion/git-annex.dmg) **warning: not being updated any longer**  To run the [[git-annex_assistant|/assistant]], just install the app, look for the icon, and start it up. @@ -19,12 +19,6 @@ several more. Handy if you don't otherwise have git installed.  ## autobuilds--[Jimmy Tang](http://www.sgenomics.org/~jtang/) autobuilds-the app for OSX Lion.--* [autobuild of git-annex.dmg](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/ref/master/git-annex.dmg) ([build logs](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/))-  * [past builds](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/sha1/) -- directories are named from the commitid's  [[Joey]] autobuilds the app for Mavericks.
doc/internals.mdwn view
@@ -39,6 +39,10 @@ This branch operates on objects exclusively. No file names will ever be stored in this branch. +The files stored in this branch are all designed to be auto-merged+using git's [[union merge driver|git-union-merge]]. So each line+has a timestamp, to allow the most recent information to be identified.+ ### `uuid.log`  Records the UUIDs of known repositories, and associates them with a@@ -110,7 +114,7 @@ ## `aaa/bbb/*.log`  These log files record [[location_tracking]] information-for file contents. Again these are placed in two levels of subdirectories+for file contents. These are placed in two levels of subdirectories for hashing. See [[hashing]] for details.  The name of the key is the filename, and the content@@ -122,14 +126,22 @@ 	1287290776.765152s 1 e605dca6-446a-11e0-8b2a-002170d25c55 	1287290767.478634s 0 26339d22-446b-11e0-9101-002170d25c55 -These files are designed to be auto-merged using git's [[union merge driver|git-union-merge]].-The timestamps allow the most recent information to be identified.- ## `aaa/bbb/*.log.web`  These log files record urls used by the [[web_special_remote|special_remotes/web]]. Their format is similar to the location tracking files, but with urls rather than UUIDs.++## `aaa/bbb/*.log.rmt`++These log files are used by remotes that need to record their own state+about keys. Each remote can store one line of data about a key, in+its own format.++Example:++	1287290776.765152s e605dca6-446a-11e0-8b2a-002170d25c55 blah blah+	1287290767.478634s 26339d22-446b-11e0-9101-002170d25c55 foo=bar  ## `schedule.log`
doc/links/other_stuff.mdwn view
@@ -4,4 +4,5 @@ * [[privacy]] * [[what git annex is not|not]] * [[related_software]]+* [[public git-annex repos|publicrepos]] * [[sitemap]]
− doc/news/version_5.20131127.mdwn 
@@ -1,24 +0,0 @@-git-annex 5.20131127 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * webapp: Detect when upgrades are available, and upgrade if the user-     desires.-     (Only when git-annex is installed using the prebuilt binaries-     from git-annex upstream, not from eg Debian.)-   * assistant: Detect when the git-annex binary is modified or replaced,-     and either prompt the user to restart the program, or automatically-     restart it.-   * annex.autoupgrade configures both the above upgrade behaviors.-   * Added support for quvi 0.9. Slightly suboptimal due to limitations in its-     interface compared with the old version.-   * Bug fix: annex.version did not get set on automatic upgrade to v5 direct-     mode repo, so the upgrade was performed repeatedly, slowing commands down.-   * webapp: Fix bug that broke switching between local repositories-     that use the new guarded direct mode.-   * Android: Fix stripping of the git-annex binary.-   * Android: Make terminal app show git-annex version number.-   * Android: Re-enable XMPP support.-   * reinject: Allow to be used in direct mode.-   * Futher improvements to git repo repair. Has now been tested in tens-     of thousands of intentionally damaged repos, and successfully-     repaired them all.-   * Allow use of --unused in bare repository."""]]
+ doc/news/version_5.20140107.mdwn view
@@ -0,0 +1,26 @@+git-annex 5.20140107 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * mirror: Support --all (and --unused).+   * external special remote protocol: Added GETUUID, GETWANTED, SETWANTED,+     SETSTATE, GETSTATE, DEBUG.+   * Windows: Fix bug in direct mode merge code that could cause files+     in subdirectories to go missing.+   * Windows: Avoid eating stdin when running ssh to add a authorized key,+     since this is used for password prompting.+   * Avoid looping if long-running git cat-file or git hash-object crashes+     and keeps crashing when restarted.+   * Assistant: Remove stale MERGE\_HEAD files in lockfile cleanup.+   * Remotes can now be made read-only, by setting remote.&lt;name&gt;.annex-readonly+   * wanted, schedule: Avoid printing "ok" after requested value.+   * assistant: Ensure that .ssh/config and .ssh/authorized\_keys are not+     group or world writable when writing to those files, as that can make+     ssh refuse to use them, if it allows another user to write to them.+   * addurl, importfeed: Honor annex.diskreserve as long as the size of the+     url can be checked.+   * add: Fix rollback when disk is completely full.+   * assistant: Fixed several minor memory leaks that manifested when+     adding a large number of files.+   * assistant: Start a new git-annex transferkeys process+     after a network connection change, so that remotes that use a persistent+     network connection are restarted.+   * Adjust Debian build deps to match current state of sparc, mipsel."""]]
+ doc/publicrepos.mdwn view
@@ -0,0 +1,17 @@+git-annex repositories are often kept private, but here's a list of some of+the public repositories that you can clone to try out git-annex.++* [downloads.kitenet.net](http://downloads.kitenet.net/)  +  `git clone https://downloads.kitenet.net/.git/`  +  Various downloads of things produced by Joey Hess, including git-annex+  builds.+* [debconf-share](https://downloads.kitenet.net/.git/)  +  `git clone http://annex.debconf.org/debconf-share/.git/`  +  [DebConf](http://debconf.org/) Media, photos, videos, etc.+* [conference-proceedings](https://github.com/RichiH/conference_proceedings)  +  `git clone https://github.com/RichiH/conference_proceedings.git`  +  A growing collection of videos of technology conferences.+  Submit a pull request to add your own!++This is a wiki -- add your own public repository to the list!+See [[tips/centralized_git_repository_tutorial]].
+ doc/tips/Synology_NAS_and_git_annex.mdwn view
@@ -0,0 +1,59 @@+# How to use git-annex on a Synology NAS++This is known to work with DSM 4.3-3810 Update 1 and git-annex standalone version 5.20131224-g6ca5271.++## Installation Steps++(1) In the DSM Package Center, install Git, which is available from Synology (no third-party repository needed).++(2) Download the latest [[standalone|install/Linux_standalone]] git-annex build for Linux on armel.++(3) Extract it somewhere sensible (eg: a bin/ directory your users home directory)++(4) Go into the git-annex.linux directory and ./runshell. You can now run git-annex as you normally would.++## How to sync with the Synology NAS++### On the Synology+(1) Setup port forwarding and associated dynamic dns, if applicable. Many good guides online for this.++(2) Setup ssh key based authentication with the Synology for each computer you want to sync with it. You want a specific key that is used only by git-annex, for each computer. Again, many good guides online.++(3) In the Synology .ssh/authorized_keys file for your account, add (substituing your username)+[[!format sh """+command="/home/$yourusername/.ssh/git-annex-shell"+"""]]+to the beginning of the line. Eg, it would look like this:+[[!format sh """+command="/home/greg/.ssh/git-annex-shell" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1yE96E/JQNPt0ziiNYJRvndCvLK4uG5h/SNYoAIBF1uH6L7VYAt3HWVqSyi3BcV70WDZ/yWgtNzbrcir46JpvEHMcvYaXLbANwoDGNjG/gsz7kP/8VUxZ6hG3P3ICuwnqVum5+rYXm6oj3xzWPfTRhhRoDZLOQdevSNpdGNaa/lSg8Vuq2suHwjQlQb8AIUuCZmS5cm6XwoUq/jJtN4LTuTPqMjzA6NkdhWM2Kigi9jPQBFborkYBPMphmZwBZiVnhsH1XpaOff+mP03D2gF/huC+b1vbWQstjuehUbY59rvJ4ijb3810Uq2ep7dwLagmILtX5GbL+GS64pAn9sIP annex-othercomputer+"""]]++(4) the git-annex-shell script in your .ssh should be created for you aftering your initial ./runshell++(5) Double check that the script points to the correct directory of where your extracted git-annex.linux lives.++### On the other computers - the manual way+(1) See step 2 above about creating the specific git-annex ssh keys.++(2) In your .ssh/config, create an alias for your Synology that includes specifying the right sshkey. For example, mine looks like:+[[!format sh """+Host synologyhost+    HostName mydynamicdomain.no-ip.org+    IdentityFile /home/greg/.ssh/annex_rsa+"""]]++(3) Now when you clone the git repo from the Synology, or add it as a remote, do the following:+[[!format sh """+git clone greg@synologyhost:/absolute/path/to/annexname annexname+"""]]+or++[[!format sh """+git remote add synology greg@synologyhost:/absolute/path/to/annexname+"""]]++(4) Run git-annex sync++### On the other computers - Using the assistant++(1) Use the webapp to add the remote. I'm not sure if there are any gotchas here as I have not done it this way yet.
+ doc/todo/commit_in_direct_mode.mdwn view
@@ -0,0 +1,9 @@+As of right now, the only way to commit changes to direct repositories is `git annex sync [remote]`.+There is no way to specify what directory to operate on.+When moving around files on a larger scale, the ability to commit specific subsets of changes would be rather nice.++`git annex commit [path]` or `git annex sync [remote] -- [path]` would probably make sense.+++Thanks,+Richard
+ doc/todo/git_annex_get___60__file__62___should_verify_file_hash.mdwn view
@@ -0,0 +1,32 @@+### Please describe the problem.+git annex get fileName-  should perform a hash check on the file content before adding to the local repository+++### What steps will reproduce the problem?+Two scenarios:+1) Malicious user and owner of repository being pulled from can edit his/her local .git/annex/objects directory +to alter the file content. For src code, this could be to insert a bug, insert a backdoor, or for example +to replace an image file artifact for a website, with a pornographic image.+The user pulling the file content with "git annex get fileName"  might not be aware of the file contents +until they actually examine the file or perform an fsck or commit locally. +In the meantime a kiddy porn image could be sitting in their repository or a src code backdoor can get incorporated and deployed etc.+2) a file could also simply get corrupted during download. An inherent hash check during the 'annex get' would+point out the problem immediately.+To reproduce:  create repoNum1, and clone it to create repoNum2. manual edit/replace content in repoNum1/.git/annex/objects/...+then perform a 'git annex get <fileName>' from repoNum2 on the file that has been manipulated +++### What version of git-annex are you using? On what operating system?+3.2012112ubuntu2 on running linux mint +++### Please provide any additional information below.++Aside: Thanks Joey - this is fantastic work you are doing. You have really improved git. The ability to checkout +an entire tree - but selectively get only the content actually needed is a real killer feature.+Kudos and again many many thanks+M.+++# End of transcript or log.+"""]]
doc/todo/untracked_remotes.mdwn view
@@ -7,3 +7,21 @@  It seems that such an untracked repository would need to automatically consider itself untrusted. Is that enough to avoid losing data?++> [[done]]; set remote.<name>.annex-readonly=true to prevent+> git-annex from pushing changes to the remote, or modifying the contents+> of the remote in any way.+> +> Note that I am intentionally not making this feature be about security.+> The remote can still tell if you're connecting to it, and indeed if it+> really wants to, and git-annex-shell is being used on the remote, it can+> determine your local repository's uuid.+> +> This allows for some complicated setups. For example, a public repository+> P can be a readonly remote of a clone on your laptop L, and L in turn has+> another, non-readonly remote D on a removable drive. This allows L and D+> to keep track of which files one-another have, without leaking this info+> to P. But note that if L adds P as a remote, it also has to mark it+> readonly, to avoid leaking data.+> +> --[[Joey]]
+ doc/todo/wishlist:_derived_content_support.mdwn view
@@ -0,0 +1,8 @@+I handle some video and music files with git annex and it would be awesome if git annex supported tracking of relations between files.++Example:+I have music in flac on my htpc, but i think flac is way overkill to have on my Android device, so i transcode them to ogg/mp3 on my htpc and then sync them to my Android device.++Transcoding is a good example for this type of feature, but there might be other uses too.++The basic thing is to know a file is a "copy" of another file, but its a generated result, so the actual file is not important, but the source is.
+ doc/todo/wishlist:_git_annex_info_UUID.mdwn view
@@ -0,0 +1,8 @@+All repos contain some level of information about all other tracked repositories.+It would be nice if I could see that info, preferably with a timestamp telling me when the last sync happened.++`git annex info UUID/name` would be suggestion.+++Thanks,+Richard
+ doc/todo/wishlist:_git_annex_info_UUID/comment_2._comment view
@@ -0,0 +1,8 @@+One piece of information that's sometimes useful, but not always, is to get a count of keys present in another remote plus the size of the remote.++Thus, I could verify that some repos are empty, archive repos have every single file, etc etc.++I still think that info is best suited for `git annex info name/UUID` as it's more volatile than what `git annex vicfg` displays.+++Richard
+ doc/todo/wishlist:_pack_metadata_in_direct_mode.mdwn view
@@ -0,0 +1,3 @@+The metadata storage for direct mode (V3) is this. In directory .git/annex/objects, there is one .map for all annexed file, and one .cache for all files in the working tree. Both are small files, containing only 1 line or a few lines. I have a repo with lots of photos, and this created lots of small files. I believe this will cause many performance issues. ++It would be great if these files are packed, maybe also in the git pack files format.
+ doc/users/sameerds.mdwn view
@@ -0,0 +1,3 @@+Extremely grateful user of git-annex.++[[http://sameer.sbuddhe.net/]]
git-annex.1 view
@@ -165,11 +165,16 @@ repository. If a file's content is present in the source repository, it is copied to the destination repository. If a file's content is not present in the source repository, it will be dropped from the destination repository-when possible.+when the numcopies setting allows. .IP Note that mirror does not sync the git repository, but only the file contents. .IP+Also, \-\-all may be specified to mirror all objects stored in the git+annex, not only objects used by currently existing files. However, this+bypasses checking the .gitattributes annex.numcopies setting when+dropping files.+.IP .IP "\fBaddurl [url ...]\fP" Downloads each url to its own file, which is added to the annex. .IP@@ -1139,6 +1144,11 @@ .IP "\fBremote.<name>.annex\-sync\fP" If set to \fBfalse\fP, prevents git\-annex sync (and the git\-annex assistant) from syncing with this remote.+.IP+.IP "\fBremote.<name>.annex\-readonly\fP"+If set to \fBtrue\fP, prevents git\-annex from making changes to a remote.+This both prevents git\-annex sync from pushing changes, and prevents+storing or removing files from read\-only remote. .IP .IP "\fBremote.<name>.annexUrl\fP" Can be used to specify a different url than the regular \fBremote.<name>.url\fP
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20131230+Version: 5.20140107 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <joey@kitenet.net>