packages feed

git-annex (empty) → 3.20110702

raw patch · 602 files changed

+22546/−0 lines, 602 filesdep +MissingHdep +SHAdep +basebuild-type:Customsetup-changedbinary-added

Dependencies added: MissingH, SHA, base, bytestring, containers, curl, dataenc, directory, extensible-exceptions, filepath, hS3, haskell98, hslogger, mtl, network, old-locale, pcre-light, process, time, unix, utf8-string

This diff is very large; some files are shown as “too large to diff”. Download the raw patch for the complete diff.

Files

+ .Branch.hs.swp view

binary file changed (absent → 16384 bytes)

+ .gitattributes view
@@ -0,0 +1,1 @@+debian/changelog merge=dpkg-mergechangelogs
+ .gitignore view
@@ -0,0 +1,19 @@+*.hi+*.o+test+configure+SysConfig.hs+git-annex+git-annex-shell+git-union-merge+git-annex.1+git-annex-shell.1+git-union-merge.1+doc/.ikiwiki+html+*.tix+.hpc+Touch.hs+StatFS.hs+Remote/S3.hs+dist
+ Annex.hs view
@@ -0,0 +1,102 @@+{- git-annex monad+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Annex (+	Annex,+	AnnexState(..),+	new,+	run,+	eval,+	getState,+	changeState,+	gitRepo+) where++import Control.Monad.State++import qualified Git+import Git.Queue+import Types.Backend+import Types.Remote+import Types.Crypto+import Types.BranchState+import Types.TrustLevel+import Types.UUID++-- git-annex's monad+type Annex = StateT AnnexState IO++-- internal state storage+data AnnexState = AnnexState+	{ repo :: Git.Repo+	, backends :: [Backend Annex]+	, supportedBackends :: [Backend Annex]+	, remotes :: [Remote Annex]+	, repoqueue :: Queue+	, quiet :: Bool+	, force :: Bool+	, fast :: Bool+	, branchstate :: BranchState+	, forcebackend :: Maybe String+	, forcenumcopies :: Maybe Int+	, defaultkey :: Maybe String+	, toremote :: Maybe String+	, fromremote :: Maybe String+	, exclude :: [String]+	, forcetrust :: [(UUID, TrustLevel)]+	, trustmap :: Maybe TrustMap+	, cipher :: Maybe Cipher+	}++newState :: [Backend Annex] -> Git.Repo -> AnnexState+newState allbackends gitrepo = AnnexState+	{ repo = gitrepo+	, backends = []+	, remotes = []+	, supportedBackends = allbackends+	, repoqueue = empty+	, quiet = False+	, force = False+	, fast = False+	, branchstate = startBranchState+	, forcebackend = Nothing+	, forcenumcopies = Nothing+	, defaultkey = Nothing+	, toremote = Nothing+	, fromremote = Nothing+	, exclude = []+	, forcetrust = []+	, trustmap = Nothing+	, cipher = Nothing+	}++{- Create and returns an Annex state object for the specified git repo. -}+new :: Git.Repo -> [Backend Annex] -> IO AnnexState+new gitrepo allbackends =+	newState allbackends `liftM` (liftIO . Git.configRead) gitrepo++{- performs an action in the Annex monad -}+run :: AnnexState -> Annex a -> IO (a, AnnexState)+run = flip runStateT+eval :: AnnexState -> Annex a -> IO a+eval = flip evalStateT++{- Gets a value from the internal state, selected by the passed value+ - constructor. -}+getState :: (AnnexState -> a) -> Annex a+getState = gets++{- Applies a state mutation function to change the internal state. + -+ - Example: changeState $ \s -> s { quiet = True }+ -}+changeState :: (AnnexState -> AnnexState) -> Annex ()+changeState = modify++{- Returns the git repository being acted on -}+gitRepo :: Annex Git.Repo+gitRepo = getState repo
+ AnnexQueue.hs view
@@ -0,0 +1,47 @@+{- git-annex command queue+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module AnnexQueue (+	add,+	flush,+	flushWhenFull+) where++import Control.Monad.State (liftIO)+import Control.Monad (when, unless)++import Annex+import Messages+import qualified Git.Queue+import Utility++{- Adds a git command to the queue, possibly running previously queued+ - actions if enough have accumulated. -}+add :: String -> [CommandParam] -> FilePath -> Annex ()+add command params file = do+	q <- getState repoqueue+	store $ Git.Queue.add q command params file++{- Runs the queue if it is full. Should be called periodically. -}+flushWhenFull :: Annex ()+flushWhenFull = do+	q <- getState repoqueue+	when (Git.Queue.full q) $ flush False++{- Runs (and empties) the queue. -}+flush :: Bool -> Annex ()+flush silent = do+	q <- getState repoqueue+	unless (0 == Git.Queue.size q) $ do+		unless silent $+			showSideAction "Recording state in git..."+		g <- gitRepo+		q' <- liftIO $ Git.Queue.flush g q+		store q'++store :: Git.Queue.Queue -> Annex ()+store q = changeState $ \s -> s { repoqueue = q }
+ Backend.hs view
@@ -0,0 +1,200 @@+{- git-annex key-value storage backends+ -+ - git-annex uses a key-value abstraction layer to allow files contents to be+ - stored in different ways. In theory, any key-value storage system could be+ - used to store the file contents, and git-annex would then retrieve them+ - as needed and put them in `.git/annex/`.+ - + - When a file is annexed, a key is generated from its content and/or metadata.+ - This key can later be used to retrieve the file's content (its value). This+ - key generation must be stable for a given file content, name, and size.+ - + - Multiple pluggable backends are supported, and more than one can be used+ - to store different files' contents in a given repository.+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Backend (+	list,+	storeFileKey,+	retrieveKeyFile,+	removeKey,+	hasKey,+	fsckKey,+	upgradableKey,+	lookupFile,+	chooseBackends,+	keyBackend,+	lookupBackendName,+	maybeLookupBackendName+) where++import Control.Monad.State (liftIO, when)+import System.IO.Error (try)+import System.FilePath+import System.Posix.Files+import System.Directory++import Locations+import qualified Git+import qualified Annex+import Types+import Types.Key+import qualified Types.Backend as B+import Messages+import Content+import DataUnits++{- List of backends in the order to try them when storing a new key. -}+list :: Annex [Backend Annex]+list = do+	l <- Annex.getState Annex.backends -- list is cached here+	if not $ null l+		then return l+		else do+			s <- getstandard+			d <- Annex.getState Annex.forcebackend+			handle d s+	where+		parseBackendList l [] = l+		parseBackendList bs s = map (lookupBackendName bs) $ words s+		handle Nothing s = return s+		handle (Just "") s = return s+		handle (Just name) s = do+			bs <- Annex.getState Annex.supportedBackends+			let l' = (lookupBackendName bs name):s+			Annex.changeState $ \state -> state { Annex.backends = l' }+			return l'+		getstandard = do+			bs <- Annex.getState Annex.supportedBackends+			g <- Annex.gitRepo+			return $ parseBackendList bs $+				Git.configGet g "annex.backends" ""++{- Looks up a backend in a list. May fail if unknown. -}+lookupBackendName :: [Backend Annex] -> String -> Backend Annex+lookupBackendName bs s = maybe unknown id $ maybeLookupBackendName bs s+	where+		unknown = error $ "unknown backend " ++ s+maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex)+maybeLookupBackendName bs s =+	if 1 /= length matches+		then Nothing+		else Just $ head matches+	where matches = filter (\b -> s == B.name b) bs++{- Attempts to store a file in one of the backends. -}+storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))+storeFileKey file trybackend = do+	bs <- list+	let bs' = maybe bs (:bs) trybackend+	storeFileKey' bs' file+storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))+storeFileKey' [] _ = return Nothing+storeFileKey' (b:bs) file = maybe nextbackend store =<< (B.getKey b) file+	where+		nextbackend = storeFileKey' bs file+		store key = do+			stored <- (B.storeFileKey b) file key+			if (not stored)+				then nextbackend+				else return $ Just (key, b)++{- Attempts to retrieve an key from one of the backends, saving it to+ - a specified location. -}+retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool+retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest++{- Removes a key from a backend. -}+removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool+removeKey backend key numcopies = (B.removeKey backend) key numcopies++{- Checks if a key is present in its backend. -}+hasKey :: Key -> Annex Bool+hasKey key = do+	backend <- keyBackend key+	(B.hasKey backend) key++{- Checks a key for problems. -}+fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool+fsckKey backend key file numcopies = do+	size_ok <- checkKeySize key+	backend_ok <-(B.fsckKey backend) key file numcopies+	return $ size_ok && backend_ok++{- Checks if a key is upgradable to a newer representation. -}+upgradableKey :: Backend Annex -> Key -> Annex Bool+upgradableKey backend key = (B.upgradableKey backend) key++{- Looks up the key and backend corresponding to an annexed file,+ - by examining what the file symlinks to. -}+lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))+lookupFile file = do+	bs <- Annex.getState Annex.supportedBackends+	tl <- liftIO $ try getsymlink+	case tl of+		Left _ -> return Nothing+		Right l -> makekey bs l+	where+		getsymlink = do+			l <- readSymbolicLink file+			return $ takeFileName l+		makekey bs l = maybe (return Nothing) (makeret bs l) (fileKey l)+		makeret bs l k =+			case maybeLookupBackendName bs bname of+					Just backend -> return $ Just (k, backend)+					Nothing -> do+						when (isLinkToAnnex l) $+							warning skip+						return Nothing+			where+				bname = keyBackendName k+				skip = "skipping " ++ file ++ +					" (unknown backend " ++ bname ++ ")"++{- Looks up the backends that should be used for each file in a list.+ - That can be configured on a per-file basis in the gitattributes file.+ -}+chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))]+chooseBackends fs = do+	g <- Annex.gitRepo+	forced <- Annex.getState Annex.forcebackend+	if forced /= Nothing+		then do+			l <- list+			return $ map (\f -> (f, Just $ head l)) fs+		else do+			bs <- Annex.getState Annex.supportedBackends+			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs+			return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs++{- Returns the backend to use for a key. -}+keyBackend :: Key -> Annex (Backend Annex)+keyBackend key = do+	bs <- Annex.getState Annex.supportedBackends+	return $ lookupBackendName bs $ keyBackendName key++{- The size of the data for a key is checked against the size encoded in+ - the key's metadata, if available. -}+checkKeySize :: Key -> Annex Bool+checkKeySize key = do+	g <- Annex.gitRepo+	let file = gitAnnexLocation g key+	present <- liftIO $ doesFileExist file+	case (present, keySize key) of+		(_, Nothing) -> return True+		(False, _) -> return True+		(True, Just size) -> do+			stat <- liftIO $ getFileStatus file+			let size' = fromIntegral (fileSize stat)+			if size == size'+				then return True+				else do+					dest <- moveBad key+					warning $ "Bad file size (" +++						compareSizes storageUnits True size size' ++ +						"); moved to " ++ dest+					return False
+ Backend/File.hs view
@@ -0,0 +1,220 @@+{- git-annex pseudo-backend+ -+ - This backend does not really do any independant data storage,+ - it relies on the file contents in .git/annex/ in this repo,+ - and other accessible repos.+ -+ - This is an abstract backend; name, getKey and fsckKey have to be implemented+ - to complete it.+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Backend.File (backend, checkKey) where++import Data.List+import Data.String.Utils++import Types.Backend+import LocationLog+import qualified Remote+import qualified Git+import Content+import qualified Annex+import Types+import UUID+import Messages+import Trust+import Types.Key++backend :: Backend Annex+backend = Backend {+	name = mustProvide,+	getKey = mustProvide,+	storeFileKey = dummyStore,+	retrieveKeyFile = copyKeyFile,+	removeKey = checkRemoveKey,+	hasKey = inAnnex,+	fsckKey = checkKeyOnly,+	upgradableKey = checkUpgradableKey+}++mustProvide :: a+mustProvide = error "must provide this field"++{- Storing a key is a no-op. -}+dummyStore :: FilePath -> Key -> Annex Bool+dummyStore _ _ = return True++{- Try to find a copy of the file in one of the remotes,+ - and copy it to here. -}+copyKeyFile :: Key -> FilePath -> Annex Bool+copyKeyFile key file = do+	remotes <- Remote.keyPossibilities key+	if null remotes+		then do+			showNote "not available"+			showLocations key []+			return False+		else trycopy remotes remotes+	where+		trycopy full [] = do+			showTriedRemotes full+			showLocations key []+			return False+		trycopy full (r:rs) = do+			probablythere <- probablyPresent r+			if probablythere+				then docopy r (trycopy full rs)+				else trycopy full rs+		-- This check is to avoid an ugly message if a remote is a+		-- drive that is not mounted.+		probablyPresent r =+			if Remote.hasKeyCheap r+				then do+					res <- Remote.hasKey r key+					case res of+						Right b -> return b+						Left _ -> return False+				else return True+		docopy r continue = do+			showNote $ "from " ++ Remote.name r ++ "..."+			copied <- Remote.retrieveKeyFile r key file+			if copied+				then return True+				else continue++{- Checks remotes to verify that enough copies of a key exist to allow+ - for a key to be safely removed (with no data loss), and fails with an+ - error if not. -}+checkRemoveKey :: Key -> Maybe Int -> Annex Bool+checkRemoveKey key numcopiesM = do+	force <- Annex.getState Annex.force+	if force || numcopiesM == Just 0+		then return True+		else do+			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key+			untrusteduuids <- trustGet UnTrusted+			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)+			numcopies <- getNumCopies numcopiesM+			findcopies numcopies trusteduuids tocheck []+	where+		findcopies need have [] bad+			| length have >= need = return True+			| otherwise = notEnoughCopies need have bad+		findcopies need have (r:rs) bad+			| length have >= need = return True+			| otherwise = do+				let u = Remote.uuid r+				let dup = u `elem` have+				haskey <- Remote.hasKey r key+				case (dup, haskey) of+					(False, Right True)	-> findcopies need (u:have) rs bad+					(False, Left _)		-> findcopies need have rs (r:bad)+					_			-> findcopies need have rs bad+		notEnoughCopies need have bad = do+			unsafe+			showLongNote $+				"Could only verify the existence of " +++				show (length have) ++ " out of " ++ show need ++ +				" necessary copies"+			showTriedRemotes bad+			showLocations key have+			hint+			return False+		unsafe = showNote "unsafe"+		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"++showLocations :: Key -> [UUID] -> Annex ()+showLocations key exclude = do+	g <- Annex.gitRepo+	u <- getUUID g+	uuids <- keyLocations key+	untrusteduuids <- trustGet UnTrusted+	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) +	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)+	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted+	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped+	showLongNote $ message ppuuidswanted ppuuidsskipped+	where+		filteruuids list x = filter (`notElem` x) list+		message [] [] = "No other repository is known to contain the file."+		message rs [] = "Try making some of these repositories available:\n" ++ rs+		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us+		message rs us = message rs [] ++ message [] us++showTriedRemotes :: [Remote.Remote Annex] -> Annex ()+showTriedRemotes [] = return ()	+showTriedRemotes remotes =+	showLongNote $ "Unable to access these remotes: " +++		(join ", " $ map Remote.name remotes)++{- If a value is specified, it is used; otherwise the default is looked up+ - in git config. forcenumcopies overrides everything. -}+getNumCopies :: Maybe Int -> Annex Int+getNumCopies v = +	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)+	where+		use (Just n) = return n+		use Nothing = do+			g <- Annex.gitRepo+			return $ read $ Git.configGet g config "1"+		config = "annex.numcopies"++{- Ideally, all keys have file size metadata. Old keys may not. -}+checkUpgradableKey :: Key -> Annex Bool+checkUpgradableKey key+	| keySize key == Nothing = return True+	| otherwise = return False++{- This is used to check that numcopies is satisfied for the key on fsck.+ - This trusts data in the the location log, and so can check all keys, even+ - those with data not present in the current annex.+ -+ - The passed action is first run to allow backends deriving this one+ - to do their own checks.+ -}+checkKey :: (Key -> Annex Bool) -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool+checkKey a key file numcopies = do+	a_ok <- a key+	copies_ok <- checkKeyNumCopies key file numcopies+	return $ a_ok && copies_ok++checkKeyOnly :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool+checkKeyOnly = checkKey (\_ -> return True)++checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool+checkKeyNumCopies key file numcopies = do+	needed <- getNumCopies numcopies+	locations <- keyLocations key+	untrusted <- trustGet UnTrusted+	let untrustedlocations = intersect untrusted locations+	let safelocations = filter (`notElem` untrusted) locations+	let present = length safelocations+	if present < needed+		then do+			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations+			warning $ missingNote (filename file key) present needed ppuuids+			return False+		else return True+	where+		filename Nothing k = show k+		filename (Just f) _ = f++missingNote :: String -> Int -> Int -> String -> String+missingNote file 0 _ [] = +		"** No known copies exist of " ++ file+missingNote file 0 _ untrusted =+		"Only these untrusted locations may have copies of " ++ file +++		"\n" ++ untrusted +++		"Back it up to trusted locations with git-annex copy."+missingNote file present needed [] =+		"Only " ++ show present ++ " of " ++ show needed ++ +		" trustworthy copies exist of " ++ file +++		"\nBack it up with git-annex copy."+missingNote file present needed untrusted = +		missingNote file present needed [] +++		"\nThe following untrusted locations may also have copies: " +++		"\n" ++ untrusted
+ Backend/SHA.hs view
@@ -0,0 +1,127 @@+{- git-annex SHA backend+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Backend.SHA (backends) where++import Control.Monad.State+import Data.String.Utils+import System.Cmd.Utils+import System.IO+import System.Directory+import Data.Maybe+import System.Posix.Files+import System.FilePath++import qualified Backend.File+import Messages+import qualified Annex+import Locations+import Content+import Types+import Types.Backend+import Types.Key+import Utility+import qualified SysConfig++type SHASize = Int++sizes :: [Int]+sizes = [1, 256, 512, 224, 384]++backends :: [Backend Annex]+-- order is slightly significant; want sha1 first ,and more general+-- sizes earlier+backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes++genBackend :: SHASize -> Maybe (Backend Annex)+genBackend size+	| shaCommand size == Nothing = Nothing+	| otherwise = Just b+	where+		b = Backend.File.backend +			{ name = shaName size+			, getKey = keyValue size+			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size+			}++genBackendE :: SHASize -> Maybe (Backend Annex)+genBackendE size =+	case genBackend size of+		Nothing -> Nothing+		Just b -> Just $ b+			{ name = shaNameE size+			, getKey = keyValueE size+			}++shaCommand :: SHASize -> Maybe String+shaCommand 1 = SysConfig.sha1+shaCommand 256 = SysConfig.sha256+shaCommand 224 = SysConfig.sha224+shaCommand 384 = SysConfig.sha384+shaCommand 512 = SysConfig.sha512+shaCommand _ = Nothing++shaName :: SHASize -> String+shaName size = "SHA" ++ show size++shaNameE :: SHASize -> String+shaNameE size = shaName size ++ "E"++shaN :: SHASize -> FilePath -> Annex String+shaN size file = do+	showNote "checksum..."+	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do+		line <- hGetLine h+		let bits = split " " line+		if null bits+			then error $ command ++ " parse error"+			else return $ head bits+	where+		command = fromJust $ shaCommand size++{- A key is a checksum of its contents. -}+keyValue :: SHASize -> FilePath -> Annex (Maybe Key)+keyValue size file = do+	s <- shaN size file	+	stat <- liftIO $ getFileStatus file+	return $ Just $ stubKey+		{ keyName = s+		, keyBackendName = shaName size+		, keySize = Just $ fromIntegral $ fileSize stat+		}++{- Extension preserving keys. -}+keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)+keyValueE size file = keyValue size file >>= maybe (return Nothing) addE+	where+		addE k = return $ Just $ k+			{ keyName = keyName k ++ extension+			, keyBackendName = shaNameE size+			}+		naiveextension = takeExtension file+		extension = +			if length naiveextension > 6+				then "" -- probably not really an extension+				else naiveextension++-- A key's checksum is checked during fsck.+checkKeyChecksum :: SHASize -> Key -> Annex Bool+checkKeyChecksum size key = do+	g <- Annex.gitRepo+	fast <- Annex.getState Annex.fast+	let file = gitAnnexLocation g key+	present <- liftIO $ doesFileExist file+	if (not present || fast)+		then return True+		else do+			s <- shaN size file+			if s == dropExtension (keyName key)+				then return True+				else do+					dest <- moveBad key+					warning $ "Bad file content; moved to " ++ dest+					return False
+ Backend/WORM.hs view
@@ -0,0 +1,43 @@+{- git-annex "WORM" backend -- Write Once, Read Many+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Backend.WORM (backends) where++import Control.Monad.State+import System.FilePath+import System.Posix.Files++import qualified Backend.File+import Types.Backend+import Types+import Types.Key++backends :: [Backend Annex]+backends = [backend]++backend :: Backend Annex+backend = Backend.File.backend {+	name = "WORM",+	getKey = keyValue+}++{- The key includes the file size, modification time, and the+ - basename of the filename.+ -+ - That allows multiple files with the same names to have different keys,+ - while also allowing a file to be moved around while retaining the+ - same key.+ -}+keyValue :: FilePath -> Annex (Maybe Key)+keyValue file = do+	stat <- liftIO $ getFileStatus file+	return $ Just $ Key {+		keyName = takeFileName file,+		keyBackendName = name backend,+		keySize = Just $ fromIntegral $ fileSize stat,+		keyMtime = Just $ modificationTime stat+	}
+ BackendList.hs view
@@ -0,0 +1,19 @@+{- git-annex backend list+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module BackendList (allBackends) where++-- When adding a new backend, import it here and add it to the list.+import qualified Backend.WORM+import qualified Backend.SHA+import Types++allBackends :: [Backend Annex]+allBackends = concat +	[ Backend.WORM.backends+	, Backend.SHA.backends+	]
+ Base64.hs view
@@ -0,0 +1,18 @@+{- Simple Base64 access+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Base64 (toB64, fromB64) where++import Codec.Binary.Base64+import Data.Bits.Utils++toB64 :: String -> String		+toB64 = encode . s2w8++fromB64 :: String -> String+fromB64 s = maybe bad w82s $ decode s+	where bad = error "bad base64 encoded data"
+ Branch.hs view
@@ -0,0 +1,359 @@+{- management of the git-annex branch+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Branch (+	create,+	update,+	get,+	change,+	commit,+	files,+	refExists,+	hasOrigin,+	name	+) where++import Control.Monad (when, unless, liftM)+import Control.Monad.State (liftIO)+import System.FilePath+import System.Directory+import Data.String.Utils+import System.Cmd.Utils+import Data.Maybe+import Data.List+import System.IO+import System.IO.Binary+import qualified Data.ByteString.Char8 as B++import Types.BranchState+import qualified Git+import qualified Git.UnionMerge+import qualified Annex+import Utility+import Types+import Messages+import Locations++type GitRef = String++{- Name of the branch that is used to store git-annex's information. -}+name :: GitRef+name = "git-annex"++{- Fully qualified name of the branch. -}+fullname :: GitRef+fullname = "refs/heads/" ++ name++{- Branch's name in origin. -}+originname :: GitRef+originname = "origin/" ++ name++{- Converts a fully qualified git ref into a short version for human+ - consumptiom. -}+shortref :: GitRef -> String+shortref = remove "refs/heads/" . remove "refs/remotes/"+	where+		remove prefix s+			| prefix `isPrefixOf` s = drop (length prefix) s+			| otherwise = s++{- A separate index file for the branch. -}+index :: Git.Repo -> FilePath+index g = gitAnnexDir g </> "index"++{- Populates the branch's index file with the current branch contents.+ - + - Usually, this is only done when the index doesn't yet exist, and+ - the index is used to build up changes to be commited to the branch,+ - and merge in changes from other branches.+ -}+genIndex :: Git.Repo -> IO ()+genIndex g = Git.UnionMerge.ls_tree g fullname >>= Git.UnionMerge.update_index g++{- Runs an action using the branch's index file. -}+withIndex :: Annex a -> Annex a+withIndex = withIndex' False+withIndex' :: Bool -> Annex a -> Annex a+withIndex' bootstrapping a = do+	g <- Annex.gitRepo+	let f = index g+	reset <- liftIO $ Git.useIndex f++	e <- liftIO $ doesFileExist f+	unless e $ do+		unless bootstrapping $ create+		liftIO $ createDirectoryIfMissing True $ takeDirectory f+		liftIO $ unless bootstrapping $ genIndex g++	r <- a+	liftIO reset+	return r++withIndexUpdate :: Annex a -> Annex a+withIndexUpdate a = update >> withIndex a++getState :: Annex BranchState+getState = Annex.getState Annex.branchstate++setState :: BranchState -> Annex ()+setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }++setCache :: FilePath -> String -> Annex ()+setCache file content = do+	state <- getState+	setState state { cachedFile = Just file, cachedContent = content }++invalidateCache :: Annex ()+invalidateCache = do+	state <- getState+	setState state { cachedFile = Nothing, cachedContent = "" }++getCache :: FilePath -> Annex (Maybe String)+getCache file = getState >>= handle+	where+		handle state+			| cachedFile state == Just file =+				return $ Just $ cachedContent state+			| otherwise = return Nothing++{- Creates the branch, if it does not already exist. -}+create :: Annex ()+create = unlessM (refExists fullname) $ do+	g <- Annex.gitRepo+	e <- hasOrigin+	if e+		then liftIO $ Git.run g "branch" [Param name, Param originname]+		else withIndex' True $+			liftIO $ Git.commit g "branch created" fullname []++{- Stages the journal, and commits staged changes to the branch. -}+commit :: String -> Annex ()+commit message = whenM stageJournalFiles $ do+	g <- Annex.gitRepo+	withIndex $ liftIO $ Git.commit g message fullname [fullname]++{- Ensures that the branch is up-to-date; should be called before+ - data is read from it. Runs only once per git-annex run. -}+update :: Annex ()+update = do+	state <- getState+	unless (branchUpdated state) $ withIndex $ do+		{- Since branches get merged into the index, it's important to+		 - first stage the journal into the index. Otherwise, any+		 - changes in the journal would later get staged, and might+		 - overwrite changes made during the merge.+		 -+		 - It would be cleaner to handle the merge by updating the+		 - journal, not the index, with changes from the branches.+		 -}+		staged <- stageJournalFiles++		g <- Annex.gitRepo+		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]+		let refs = map (last . words) (lines r)+		updated <- catMaybes `liftM` mapM updateRef refs+		unless (null updated && not staged) $ liftIO $+			Git.commit g "update" fullname (fullname:updated)+		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }+		invalidateCache++{- Does origin/git-annex exist? -}+hasOrigin :: Annex Bool+hasOrigin = refExists originname++{- Checks if a git ref exists. -}+refExists :: GitRef -> Annex Bool+refExists ref = do+	g <- Annex.gitRepo+	liftIO $ Git.runBool g "show-ref"+		[Param "--verify", Param "-q", Param ref]++{- Ensures that a given ref has been merged into the index. -}+updateRef :: GitRef -> Annex (Maybe String)+updateRef ref+	| ref == fullname = return Nothing+	| otherwise = do+		g <- Annex.gitRepo+		-- checking with log to see if there have been changes+		-- is less expensive than always merging+		diffs <- liftIO $ Git.pipeRead g [+			Param "log",+			Param (name++".."++ref),+			Params "--oneline -n1"+			]+		if (null diffs)+			then return Nothing+			else do+				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."+				-- By passing only one ref, it is actually+				-- merged into the index, preserving any+				-- changes that may already be staged.+				--+				-- However, any changes in the git-annex+				-- branch that are *not* reflected in the+				-- index will be removed. So, documentation+				-- advises users not to directly modify the+				-- branch.+				liftIO $ Git.UnionMerge.merge g [ref]+				return $ Just ref++{- Records changed content of a file into the journal. -}+change :: FilePath -> String -> Annex ()+change file content = do+	setJournalFile file content+	setCache file content++{- Gets the content of a file on the branch, or content from the journal, or+ - staged in the index.+ -+ - Returns an empty string if the file doesn't exist yet. -}+get :: FilePath -> Annex String+get file = do+	cached <- getCache file+	case cached of+		Just content -> return content+		Nothing -> do+			j <- getJournalFile file+			case j of+				Just content -> do+					setCache file content+					return content+				Nothing -> withIndexUpdate $ do+					content <- catFile file+					setCache file content+					return content++{- Uses git cat-file in batch mode to read the content of a file.+ -+ - Only one process is run, and it persists and is used for all accesses. -}+catFile :: FilePath -> Annex String+catFile file = do+	state <- getState+	maybe (startup state) ask (catFileHandles state)+	where+		startup state = do+			g <- Annex.gitRepo+			let cmd = Git.gitCommandLine g+				[Param "cat-file", Param "--batch"]+			let gitcmd = join " " ("git" : toCommand cmd)+			(_, from, to) <- liftIO $ hPipeBoth "sh"+				-- want stderr on stdin to see error messages+				["-c", "exec " ++ gitcmd ++ " 2>&1"]+			setState state { catFileHandles = Just (from, to) }+			ask (from, to)+		ask (from, to) = liftIO $ do+			let want = fullname ++ ":" ++ file+			hPutStrLn to want+			hFlush to+			header <- hGetLine from+			case words header of+				[sha, blob, size]+					| length sha == Git.shaSize &&+					  blob == "blob" -> handle from size+					| otherwise -> empty+				_ -> empty+		handle from size = case reads size of+			[(bytes, "")] -> readcontent from bytes+			_ -> empty+		readcontent from bytes = do+			content <- B.hGet from bytes+			c <- hGetChar from+			when (c /= '\n') $+				error "missing newline from git cat-file"+			return $ B.unpack content+		empty = return ""++{- Lists all files on the branch. There may be duplicates in the list. -}+files :: Annex [FilePath]+files = withIndexUpdate $ do+	g <- Annex.gitRepo+	bfiles <- liftIO $ Git.pipeNullSplit g+		[Params "ls-tree --name-only -r -z", Param fullname]+	jfiles <- getJournalFiles+	return $ jfiles ++ bfiles++{- Records content for a file in the branch to the journal.+ -+ - Using the journal, rather than immediatly staging content to the index+ - avoids git needing to rewrite the index after every change. -}+setJournalFile :: FilePath -> String -> Annex ()+setJournalFile file content = do+	g <- Annex.gitRepo+	liftIO $ catch (write g) $ const $ do+		createDirectoryIfMissing True $ gitAnnexJournalDir g+		createDirectoryIfMissing True $ gitAnnexTmpDir g+		write g+	where+		-- journal file is written atomically+		write g = do+			let jfile = journalFile g file+			let tmpfile = gitAnnexTmpDir g </> takeFileName jfile+			writeBinaryFile tmpfile content+			renameFile tmpfile jfile++{- Gets journalled content for a file in the branch. -}+getJournalFile :: FilePath -> Annex (Maybe String)+getJournalFile file = do+	g <- Annex.gitRepo+	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)+		(const $ return Nothing)++{- List of journal files. -}+getJournalFiles :: Annex [FilePath]+getJournalFiles = getJournalFilesRaw >>= return . map fileJournal++getJournalFilesRaw :: Annex [FilePath]+getJournalFilesRaw = do+	g <- Annex.gitRepo+	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)+		(const $ return [])+	return $ filter (\f -> f /= "." && f /= "..") fs++{- Stages all journal files into the index, and returns True if the index+ - was modified. -}+stageJournalFiles :: Annex Bool+stageJournalFiles = do+	l <- getJournalFilesRaw+	if null l+		then return False+		else do+			g <- Annex.gitRepo+			withIndex $ liftIO $ stage g l+			return True+	where+		stage g fs = do+			let dir = gitAnnexJournalDir g+			let paths = map (dir </>) fs+			-- inject all the journal files directly into git+			-- in one quick command+			(h, s) <- Git.pipeWriteRead g [Param "hash-object",+				Param "-w", Param "--stdin-paths"] $ unlines paths+			-- update the index, also in just one command+			Git.UnionMerge.update_index g $+				index_lines (lines s) $ map fileJournal fs+			forceSuccess h+			mapM_ removeFile paths+		index_lines shas fs = map genline $ zip shas fs+		genline (sha, file) = Git.UnionMerge.update_index_line sha file++{- Produces a filename to use in the journal for a file on the branch.+ -+ - The journal typically won't have a lot of files in it, so the hashing+ - used in the branch is not necessary, and all the files are put directly+ - in the journal directory.+ -}+journalFile :: Git.Repo -> FilePath -> FilePath+journalFile repo file = gitAnnexJournalDir repo </> concatMap mangle file+	where+		mangle '/' = "_"+		mangle '_' = "__"+		mangle c = [c]++{- Converts a journal file (relative to the journal dir) back to the+ - filename on the branch. -}+fileJournal :: FilePath -> FilePath+fileJournal = replace "//" "_" . replace "_" "/"
+ CmdLine.hs view
@@ -0,0 +1,108 @@+{- git-annex command line parsing and dispatch+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module CmdLine (+	dispatch,+	usage,+	shutdown+) where++import System.IO.Error (try)+import System.Console.GetOpt+import Control.Monad.State (liftIO)+import Control.Monad (when)++import qualified Annex+import qualified AnnexQueue+import qualified Git+import Content+import Types+import Command+import BackendList+import Version+import Options+import Messages+import UUID++{- Runs the passed command line. -}+dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()+dispatch args cmds options header gitrepo = do+	setupConsole+	state <- Annex.new gitrepo allBackends+	(actions, state') <- Annex.run state $ parseCmd args header cmds options+	tryRun state' $ [startup] ++ actions ++ [shutdown]++{- Parses command line, stores configure flags, and returns a + - list of actions to be run in the Annex monad. -}+parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool]+parseCmd argv header cmds options = do+	(flags, params) <- liftIO $ getopt+	when (null params) $ error $ "missing command" ++ usagemsg+	case lookupCmd (head params) of+		[] -> error $ "unknown command" ++ usagemsg+		[command] -> do+			_ <- sequence flags+			when (cmdusesrepo command) $+				checkVersion+			prepCommand command (drop 1 params)+		_ -> error "internal error: multiple matching commands"+	where+		getopt = case getOpt Permute options argv of+			(flags, params, []) ->+				return (flags, params)+			(_, _, errs) ->+				ioError (userError (concat errs ++ usagemsg))+		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds+		usagemsg = "\n\n" ++ usage header cmds options++{- Usage message with lists of commands and options. -}+usage :: String -> [Command] -> [Option] -> String+usage header cmds options =+	usageInfo (header ++ "\n\nOptions:") options +++		"\nCommands:\n" ++ cmddescs+	where+		cmddescs = unlines $ map (indent . showcmd) cmds+		showcmd c =+			cmdname c +++			pad (longest cmdname + 1) (cmdname c) +++			cmdparams c +++			pad (longest cmdparams + 2) (cmdparams c) +++			cmddesc c+		pad n s = replicate (n - length s) ' '+		longest f = foldl max 0 $ map (length . f) cmds++{- Runs a list of Annex actions. Catches IO errors and continues+ - (but explicitly thrown errors terminate the whole command).+ -}+tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()+tryRun state actions = tryRun' state 0 actions+tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO ()+tryRun' state errnum (a:as) = do+	result <- try $ Annex.run state $ do+		AnnexQueue.flushWhenFull+		a+	case result of+		Left err -> do+			Annex.eval state $ showErr err+			tryRun' state (errnum + 1) as+		Right (True,state') -> tryRun' state' errnum as+		Right (False,state') -> tryRun' state' (errnum + 1) as+tryRun' _ errnum [] = do+	when (errnum > 0) $ error $ show errnum ++ " failed"++{- Actions to perform each time ran. -}+startup :: Annex Bool+startup = do+	prepUUID+	return True++{- Cleanup actions. -}+shutdown :: Annex Bool+shutdown = do+	saveState+	liftIO $ Git.reap+	return True
+ Command.hs view
@@ -0,0 +1,271 @@+{- git-annex commands+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command where++import Control.Monad.State (liftIO)+import System.Directory+import System.Posix.Files+import Control.Monad (filterM, liftM, when)+import System.Path.WildMatch+import Text.Regex.PCRE.Light.Char8+import Data.List+import Data.Maybe+import Data.String.Utils++import Types+import qualified Backend+import Messages+import qualified Annex+import qualified Git+import qualified Git.LsFiles as LsFiles+import Utility+import Types.Key++{- A command runs in four stages.+ -+ - 0. The seek stage takes the parameters passed to the command,+ -    looks through the repo to find the ones that are relevant+ -    to that command (ie, new files to add), and generates+ -    a list of start stage actions. -}+type CommandSeek = [String] -> Annex [CommandStart]+{- 1. The start stage is run before anything is printed about the+  -   command, is passed some input, and can early abort it+  -   if the input does not make sense. It should run quickly and+  -   should not modify Annex state. -}+type CommandStart = Annex (Maybe CommandPerform)+{- 2. The perform stage is run after a message is printed about the command+ -    being run, and it should be where the bulk of the work happens. -}+type CommandPerform = Annex (Maybe CommandCleanup)+{- 3. The cleanup stage is run only if the perform stage succeeds, and it+ -    returns the overall success/fail of the command. -}+type CommandCleanup = Annex Bool+{- Some helper functions are used to build up CommandSeek and CommandStart+ - functions. -}+type CommandSeekStrings = CommandStartString -> CommandSeek+type CommandStartString = String -> CommandStart+type CommandSeekWords = CommandStartWords -> CommandSeek+type CommandStartWords = [String] -> CommandStart+type CommandSeekKeys = CommandStartKey -> CommandSeek+type CommandStartKey = Key -> CommandStart+type BackendFile = (FilePath, Maybe (Backend Annex))+type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek+type CommandStartBackendFile = BackendFile -> CommandStart+type AttrFile = (FilePath, String)+type CommandSeekAttrFiles = CommandStartAttrFile -> CommandSeek+type CommandStartAttrFile = AttrFile -> CommandStart+type CommandSeekNothing = CommandStart -> CommandSeek+type CommandStartNothing = CommandStart++data Command = Command {+	cmdusesrepo :: Bool,+	cmdname :: String,+	cmdparams :: String,+	cmdseek :: [CommandSeek],+	cmddesc :: String+}++{- Most commands operate on files in a git repo. -}+repoCommand :: String -> String -> [CommandSeek] -> String -> Command+repoCommand = Command True++{- Others can run anywhere. -}+standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command+standaloneCommand = Command False++{- For start and perform stages to indicate what step to run next. -}+next :: a -> Annex (Maybe a)+next a = return $ Just a++{- Or to indicate nothing needs to be done. -}+stop :: Annex (Maybe a)+stop = return Nothing++{- Prepares a list of actions to run to perform a command, based on+ - the parameters passed to it. -}+prepCommand :: Command -> [String] -> Annex [Annex Bool]+prepCommand Command { cmdseek = seek } params = do+	lists <- mapM (\s -> s params) seek+	return $ map doCommand $ concat lists++{- Runs a command through the start, perform and cleanup stages -}+doCommand :: CommandStart -> CommandCleanup+doCommand = start+	where+		start   = stage $ maybe (return True) perform+		perform = stage $ maybe (showEndFail >> return False) cleanup+		cleanup = stage $ \r -> showEndResult r >> return r+		stage a b = b >>= a++notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)+notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file++isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)+isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file++notBareRepo :: Annex a -> Annex a+notBareRepo a = do+	g <- Annex.gitRepo+	when (Git.repoIsLocalBare g) $ do+		error "You cannot run this subcommand in a bare repository."+	a++{- These functions find appropriate files or other things based on a+   user's parameters, and run a specified action on them. -}+withFilesInGit :: CommandSeekStrings+withFilesInGit a params = do+	repo <- Annex.gitRepo+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params+	liftM (map a) $ filterFiles files+withAttrFilesInGit :: String -> CommandSeekAttrFiles+withAttrFilesInGit attr a params = do+	repo <- Annex.gitRepo+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params+	liftM (map a) $ liftIO $ Git.checkAttr repo attr files+withBackendFilesInGit :: CommandSeekBackendFiles+withBackendFilesInGit a params = do+	repo <- Annex.gitRepo+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params+	files' <- filterFiles files+	backendPairs a files'+withFilesMissing :: CommandSeekStrings+withFilesMissing a params = do+	files <- liftIO $ filterM missing params+	liftM (map a) $ filterFiles files+	where+		missing f = do+			e <- doesFileExist f+			return $ not e+withFilesNotInGit :: CommandSeekBackendFiles+withFilesNotInGit a params = do+	repo <- Annex.gitRepo+	force <- Annex.getState Annex.force+	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params+	newfiles' <- filterFiles newfiles+	backendPairs a newfiles'+withWords :: CommandSeekWords+withWords a params = return [a params]+withStrings :: CommandSeekStrings+withStrings a params = return $ map a params+withFilesToBeCommitted :: CommandSeekStrings+withFilesToBeCommitted a params = do+	repo <- Annex.gitRepo+	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params+	liftM (map a) $ filterFiles tocommit+withFilesUnlocked :: CommandSeekBackendFiles+withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged+withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles+withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged+withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles+withFilesUnlocked' typechanged a params = do+	-- unlocked files have changed type from a symlink to a regular file+	repo <- Annex.gitRepo+	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params+	unlockedfiles <- liftIO $ filterM notSymlink $+		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles+	unlockedfiles' <- filterFiles unlockedfiles+	backendPairs a unlockedfiles'+withKeys :: CommandSeekKeys+withKeys a params = return $ map a $ map parse params+	where+		parse p = maybe (error "bad key") id $ readKey p+withTempFile :: CommandSeekStrings+withTempFile a params = return $ map a params+withNothing :: CommandSeekNothing+withNothing a [] = return [a]+withNothing _ _ = error "This command takes no parameters."++backendPairs :: CommandSeekBackendFiles+backendPairs a files = liftM (map a) $ Backend.chooseBackends files++{- Filter out files those matching the exclude glob pattern,+ - if it was specified. -}+filterFiles :: [FilePath] -> Annex [FilePath]+filterFiles l = do+	exclude <- Annex.getState Annex.exclude+	if null exclude+		then return l+		else return $ filter (notExcluded $ wildsRegex exclude) l+	where+		notExcluded r f = isNothing $ match r f []++wildsRegex :: [String] -> Regex+wildsRegex ws = compile regex []+	where+		regex = "^(" ++ alternatives ++ ")"+		alternatives = join "|" $ map wildToRegex ws++{- filter out symlinks -}	+notSymlink :: FilePath -> IO Bool+notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f++{- Descriptions of params used in usage messages. -}+paramRepeating :: String -> String+paramRepeating s = s ++ " ..."+paramOptional :: String -> String+paramOptional s = "[" ++ s ++ "]"+paramPair :: String -> String -> String+paramPair a b = a ++ " " ++ b+paramPath :: String+paramPath = "PATH"+paramKey :: String+paramKey = "KEY"+paramDesc :: String+paramDesc = "DESC"+paramNumber :: String+paramNumber = "NUMBER"+paramRemote :: String+paramRemote = "REMOTE"+paramGlob :: String+paramGlob = "GLOB"+paramName :: String+paramName = "NAME"+paramType :: String+paramType = "TYPE"+paramKeyValue :: String+paramKeyValue = "K=V"+paramNothing :: String+paramNothing = ""++{- The Key specified by the --key parameter. -}+cmdlineKey :: Annex Key+cmdlineKey  = do+	k <- Annex.getState Annex.defaultkey+	case k of+		Nothing -> nokey+		Just "" -> nokey+		Just kstring -> maybe badkey return $ readKey kstring+	where+		nokey = error "please specify the key with --key"+		badkey = error "bad key"++{- Given an original list of files, and an expanded list derived from it,+ - ensures that the original list's ordering is preserved. + -+ - The input list may contain a directory, like "dir" or "dir/". Any+ - items in the expanded list that are contained in that directory will+ - appear at the same position as it did in the input list.+ -}+preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]+-- optimisation, only one item in original list, so no reordering needed+preserveOrder [_] new = new+preserveOrder orig new = collect orig new+	where+		collect [] n = n+		collect [_] n = n -- optimisation+		collect (l:ls) n = found ++ collect ls rest+			where (found, rest)=partition (l `dirContains`) n++{- Runs an action that takes a list of FilePaths, and ensures that + - its return list preserves order.+ -+ - This assumes that it's cheaper to call preserveOrder on the result,+ - than it would be to run the action separately with each param. In the case+ - of git file list commands, that assumption tends to hold.+ -}+runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]+runPreserveOrder a files = liftM (preserveOrder files) (a files)
+ Command/Add.hs view
@@ -0,0 +1,68 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Add where++import Control.Monad.State (liftIO)+import System.Posix.Files++import Command+import qualified Annex+import qualified AnnexQueue+import qualified Backend+import LocationLog+import Types+import Content+import Messages+import Utility+import Touch++command :: [Command]+command = [repoCommand "add" paramPath seek "add files to annex"]++{- Add acts on both files not checked into git yet, and unlocked files. -}+seek :: [CommandSeek]+seek = [withFilesNotInGit start, withFilesUnlocked start]++{- The add subcommand annexes a file, storing it in a backend, and then+ - moving it into the annex directory and setting up the symlink pointing+ - to its content. -}+start :: CommandStartBackendFile+start pair@(file, _) = notAnnexed file $ do+	s <- liftIO $ getSymbolicLinkStatus file+	if (isSymbolicLink s) || (not $ isRegularFile s)+		then stop+		else do+			showStart "add" file+			next $ perform pair++perform :: BackendFile -> CommandPerform+perform (file, backend) = do+	stored <- Backend.storeFileKey file backend+	case stored of+		Nothing -> stop+		Just (key, _) -> do+			moveAnnex key file+			next $ cleanup file key++cleanup :: FilePath -> Key -> CommandCleanup+cleanup file key = do+	logStatus key InfoPresent++	link <- calcGitLink file key+	liftIO $ createSymbolicLink link file++	-- touch the symlink to have the same mtime as the file it points to+	s <- liftIO $ getFileStatus file+	let mtime = modificationTime s+	liftIO $ touch file (TimeSpec mtime) False++	force <- Annex.getState Annex.force+	if force+		then AnnexQueue.add "add" [Param "-f", Param "--"] file+		else AnnexQueue.add "add" [Param "--"] file+	return True
+ Command/AddUrl.hs view
@@ -0,0 +1,82 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.AddUrl where++import Control.Monad.State (liftIO, when)+import Network.URI+import Data.String.Utils+import System.Directory++import Command+import qualified Backend+import qualified Remote.Web+import qualified Command.Add+import qualified Annex+import Messages+import Content+import PresenceLog+import Types.Key+import Locations+import Utility++command :: [Command]+command = [repoCommand "addurl" paramPath seek "add urls to annex"]++seek :: [CommandSeek]+seek = [withStrings start]++start :: CommandStartString+start s = do+	let u = parseURI s+	case u of+		Nothing -> error $ "bad url " ++ s+		Just url -> do+			file <- liftIO $ url2file url+			showStart "addurl" file+			next $ perform s file+			+perform :: String -> FilePath -> CommandPerform+perform url file = do+	g <- Annex.gitRepo+	showNote $ "downloading " ++ url+	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }+	let tmp = gitAnnexTmpLocation g dummykey+	liftIO $ createDirectoryIfMissing True (parentDir tmp)+	ok <- Remote.Web.download [url] tmp+	if ok+		then do+			[(_, backend)] <- Backend.chooseBackends [file]+			stored <- Backend.storeFileKey tmp backend+			case stored of+				Nothing -> stop+				Just (key, _) -> do+					moveAnnex key tmp+					Remote.Web.setUrl key url InfoPresent+					next $ Command.Add.cleanup file key+		else stop++url2file :: URI -> IO FilePath+url2file url = do+	let parts = filter safe $ split "/" $ uriPath url+	if null parts+		then fallback+		else do+			let file = last parts+			e <- doesFileExist file+			if e then fallback else return file+	where+		fallback = do+			let file = replace "/" "_" $ show url+			e <- doesFileExist file+			when e $ error "already have this url"+			return file+		safe s+			| null s = False+			| s == "." = False+			| s == ".." = False+			| otherwise = True
+ Command/ConfigList.hs view
@@ -0,0 +1,28 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.ConfigList where++import Control.Monad.State (liftIO)++import Annex+import Command+import UUID++command :: [Command]+command = [standaloneCommand "configlist" paramNothing seek+		"outputs relevant git configuration"]++seek :: [CommandSeek]+seek = [withNothing start]++start :: CommandStartNothing+start = do+	g <- Annex.gitRepo+	u <- getUUID g+	liftIO $ putStrLn $ "annex.uuid=" ++ u+	stop
+ Command/Copy.hs view
@@ -0,0 +1,19 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Copy where++import Command+import qualified Command.Move++command :: [Command]+command = [repoCommand "copy" paramPath seek+	"copy content of files to/from another repository"]++-- A copy is just a move that does not delete the source file.+seek :: [CommandSeek]+seek = [withFilesInGit $ Command.Move.start False]
+ Command/Describe.hs view
@@ -0,0 +1,36 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Describe where++import Command+import qualified Remote+import UUID+import Messages++command :: [Command]+command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek+	"change description of a repository"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	let (name, description) =+		case ws of+			(n:d) -> (n,unwords d)+			_ -> error "Specify a repository and a description."+	+	showStart "describe" name+	u <- Remote.nameToUUID name+	next $ perform u description++perform :: UUID -> String -> CommandPerform+perform u description = do+	describeUUID u description+	next $ return True
+ Command/Drop.hs view
@@ -0,0 +1,49 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Drop where++import Command+import qualified Backend+import LocationLog+import Types+import Content+import Messages+import Utility++command :: [Command]+command = [repoCommand "drop" paramPath seek+	"indicate content of files not currently wanted"]++seek :: [CommandSeek]+seek = [withAttrFilesInGit "annex.numcopies" start]++{- Indicates a file's content is not wanted anymore, and should be removed+ - if it's safe to do so. -}+start :: CommandStartAttrFile+start (file, attr) = isAnnexed file $ \(key, backend) -> do+	inbackend <- Backend.hasKey key+	if inbackend+		then do+			showStart "drop" file+			next $ perform key backend numcopies+		else stop+	where+		numcopies = readMaybe attr :: Maybe Int++perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform+perform key backend numcopies = do+	success <- Backend.removeKey backend key numcopies+	if success+		then next $ cleanup key+		else stop++cleanup :: Key -> CommandCleanup+cleanup key = do+	whenM (inAnnex key) $ removeAnnex key+	logStatus key InfoMissing+	return True
+ Command/DropKey.hs view
@@ -0,0 +1,44 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.DropKey where++import Command+import qualified Annex+import LocationLog+import Types+import Content+import Messages++command :: [Command]+command = [repoCommand "dropkey" (paramRepeating paramKey) seek+	"drops annexed content for specified keys"] ++seek :: [CommandSeek]+seek = [withKeys start]++start :: CommandStartKey+start key = do+	present <- inAnnex key+	force <- Annex.getState Annex.force+	if not present+		then stop+		else if not force+			then error "dropkey is can cause data loss; use --force if you're sure you want to do this"+			else do+				showStart "dropkey" (show key)+				next $ perform key++perform :: Key -> CommandPerform+perform key = do+	removeAnnex key+	next $ cleanup key++cleanup :: Key -> CommandCleanup+cleanup key = do+	logStatus key InfoMissing+	return True
+ Command/DropUnused.hs view
@@ -0,0 +1,91 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.DropUnused where++import Control.Monad.State (liftIO)+import qualified Data.Map as M+import System.Directory+import Data.Maybe++import Command+import Types+import Messages+import Locations+import qualified Annex+import qualified Command.Drop+import qualified Command.Move+import qualified Remote+import qualified Git+import Backend+import Types.Key+import Utility++type UnusedMap = M.Map String Key++command :: [Command]+command = [repoCommand "dropunused" (paramRepeating paramNumber) seek+	"drop unused file content"]++seek :: [CommandSeek]+seek = [withUnusedMaps]++{- Read unused logs once, and pass the maps to each start action. -}+withUnusedMaps :: CommandSeek+withUnusedMaps params = do+	unused <- readUnusedLog ""+	unusedbad <- readUnusedLog "bad"+	unusedtmp <- readUnusedLog "tmp"+	return $ map (start (unused, unusedbad, unusedtmp)) params++start :: (UnusedMap, UnusedMap, UnusedMap) -> CommandStartString+start (unused, unusedbad, unusedtmp) s = notBareRepo $ search+	[ (unused, perform)+	, (unusedbad, performOther gitAnnexBadLocation)+	, (unusedtmp, performOther gitAnnexTmpLocation)+	]+	where+		search [] = stop+		search ((m, a):rest) = do+			case M.lookup s m of+				Nothing -> search rest+				Just key -> do+					showStart "dropunused" s+					next $ a key++perform :: Key -> CommandPerform+perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote+	where+		dropremote name = do+			r <- Remote.byName name+			showNote $ "from " ++ Remote.name r ++ "..."+			next $ Command.Move.fromCleanup r True key+		droplocal = do+			backend <- keyBackend key+			Command.Drop.perform key backend (Just 0) -- force drop++performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform+performOther filespec key = do+	g <- Annex.gitRepo+	let f = filespec g key+	liftIO $ whenM (doesFileExist f) $ removeFile f+	next $ return True++readUnusedLog :: FilePath -> Annex UnusedMap+readUnusedLog prefix = do+	g <- Annex.gitRepo+	let f = gitAnnexUnusedLog prefix g+	e <- liftIO $ doesFileExist f+	if e+		then do+			l <- liftIO $ readFile f+			return $ M.fromList $ map parse $ lines l+		else return $ M.empty+	where+		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)+			where+				ws = words line
+ Command/Find.hs view
@@ -0,0 +1,27 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Find where++import Control.Monad.State (liftIO)++import Command+import Content+import Utility++command :: [Command]+command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek+	"lists available files"]++seek :: [CommandSeek]+seek = [withFilesInGit start]++{- Output a list of files. -}+start :: CommandStartString+start file = isAnnexed file $ \(key, _) -> do+	whenM (inAnnex key) $ liftIO $ putStrLn file+	stop
+ Command/Fix.hs view
@@ -0,0 +1,48 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Fix where++import Control.Monad.State (liftIO)+import System.Posix.Files+import System.Directory++import Command+import qualified AnnexQueue+import Utility+import Content+import Messages++command :: [Command]+command = [repoCommand "fix" paramPath seek+	"fix up symlinks to point to annexed content"]++seek :: [CommandSeek]+seek = [withFilesInGit start]++{- Fixes the symlink to an annexed file. -}+start :: CommandStartString+start file = isAnnexed file $ \(key, _) -> do+	link <- calcGitLink file key+	l <- liftIO $ readSymbolicLink file+	if link == l+		then stop+		else do+			showStart "fix" file+			next $ perform file link++perform :: FilePath -> FilePath -> CommandPerform+perform file link = do+	liftIO $ createDirectoryIfMissing True (parentDir file)+	liftIO $ removeFile file+	liftIO $ createSymbolicLink link file+	next $ cleanup file++cleanup :: FilePath -> CommandCleanup+cleanup file = do+	AnnexQueue.add "add" [Param "--"] file+	return True
+ Command/FromKey.hs view
@@ -0,0 +1,50 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.FromKey where++import Control.Monad.State (liftIO)+import System.Posix.Files+import System.Directory+import Control.Monad (unless)++import Command+import qualified AnnexQueue+import Utility+import qualified Backend+import Content+import Messages+import Types.Key++command :: [Command]+command = [repoCommand "fromkey" paramPath seek+	"adds a file using a specific key"]++seek :: [CommandSeek]+seek = [withFilesMissing start]++start :: CommandStartString+start file = notBareRepo $ do+	key <- cmdlineKey+	inbackend <- Backend.hasKey key+	unless inbackend $ error $+		"key ("++keyName key++") is not present in backend"+	showStart "fromkey" file+	next $ perform file++perform :: FilePath -> CommandPerform+perform file = do+	key <- cmdlineKey+	link <- calcGitLink file key+	liftIO $ createDirectoryIfMissing True (parentDir file)+	liftIO $ createSymbolicLink link file+	next $ cleanup file++cleanup :: FilePath -> CommandCleanup+cleanup file = do+	AnnexQueue.add "add" [Param "--"] file+	return True
+ Command/Fsck.hs view
@@ -0,0 +1,82 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Fsck where++import Control.Monad (when)+import Control.Monad.State (liftIO)++import Command+import qualified Backend+import qualified Annex+import UUID+import Types+import Messages+import Utility+import Content+import LocationLog+import Locations++command :: [Command]+command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek+	"check for problems"]++seek :: [CommandSeek]+seek = [withAttrFilesInGit "annex.numcopies" start]++start :: CommandStartAttrFile+start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do+	showStart "fsck" file+	next $ perform key file backend numcopies+	where+		numcopies = readMaybe attr :: Maybe Int++perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform+perform key file backend numcopies = do+	-- the location log is checked first, so that if it has bad data+	-- that gets corrected+	locationlogok <- verifyLocationLog key file+	backendok <- Backend.fsckKey backend key (Just file) numcopies+	if locationlogok && backendok+		then next $ return True+		else stop++{- Checks that the location log reflects the current status of the key,+   in this repository only. -}+verifyLocationLog :: Key -> FilePath -> Annex Bool+verifyLocationLog key file = do+	g <- Annex.gitRepo+	present <- inAnnex key+	+	-- Since we're checking that a key's file is present, throw+	-- in a permission fixup here too.+	when present $ liftIO $ do+		let f = gitAnnexLocation g key+		preventWrite f+		preventWrite (parentDir f)++	u <- getUUID g+        uuids <- keyLocations key++	case (present, u `elem` uuids) of+		(True, False) -> do+				fix g u InfoPresent+				-- There is no data loss, so do not fail.+				return True+		(False, True) -> do+				fix g u InfoMissing+				warning $+					"** Based on the location log, " ++ file+					++ "\n** was expected to be present, " +++					"but its content is missing."+				return False+		_ -> return True+	+	where+		fix g u s = do+			showNote "fixing location log"+			logChange g key u s
+ Command/Get.hs view
@@ -0,0 +1,45 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Get where++import Command+import qualified Backend+import qualified Annex+import qualified Remote+import Types+import Content+import Messages+import qualified Command.Move++command :: [Command]+command = [repoCommand "get" paramPath seek+		"make content of annexed files available"]++seek :: [CommandSeek]+seek = [withFilesInGit start]++start :: CommandStartString+start file = isAnnexed file $ \(key, backend) -> do+	inannex <- inAnnex key+	if inannex+		then stop+		else do+			showStart "get" file+			from <- Annex.getState Annex.fromremote+			case from of+				Nothing -> next $ perform key backend+				Just name -> do+					src <- Remote.byName name+					next $ Command.Move.fromPerform src False key++perform :: Key -> Backend Annex -> CommandPerform+perform key backend = do+	ok <- getViaTmp key (Backend.retrieveKeyFile backend key)+	if ok+		then next $ return True -- no cleanup needed+		else stop
+ Command/InAnnex.hs view
@@ -0,0 +1,28 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.InAnnex where++import Control.Monad.State (liftIO)+import System.Exit++import Command+import Content++command :: [Command]+command = [repoCommand "inannex" (paramRepeating paramKey) seek+		"checks if keys are present in the annex"]++seek :: [CommandSeek]+seek = [withKeys start]++start :: CommandStartKey+start key = do+	present <- inAnnex key+	if present+		then stop+		else liftIO $ exitFailure
+ Command/Init.hs view
@@ -0,0 +1,72 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Init where++import Control.Monad.State (liftIO)+import Control.Monad (when, unless)+import System.Directory++import Command+import qualified Annex+import qualified Git+import qualified Branch+import UUID+import Version+import Messages+import Types+import Utility+	+command :: [Command]+command = [standaloneCommand "init" paramDesc seek+		"initialize git-annex with repository description"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	when (null description) $+		error "please specify a description of this repository\n"+	showStart "init" description+	next $ perform description+	where+		description = unwords ws++perform :: String -> CommandPerform+perform description = do+	Branch.create+	g <- Annex.gitRepo+	u <- getUUID g+	setVersion+	describeUUID u description+	unless (Git.repoIsLocalBare g) $+		gitPreCommitHookWrite g+	next $ return True++{- set up a git pre-commit hook, if one is not already present -}+gitPreCommitHookWrite :: Git.Repo -> Annex ()+gitPreCommitHookWrite repo = do+	exists <- liftIO $ doesFileExist hook+	if exists+		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"+		else liftIO $ do+			viaTmp writeFile hook preCommitScript+			p <- getPermissions hook+			setPermissions hook $ p {executable = True}+	where+		hook = preCommitHook repo++preCommitHook :: Git.Repo -> FilePath+preCommitHook repo = +	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"++preCommitScript :: String+preCommitScript = +		"#!/bin/sh\n" +++		"# automatically configured by git-annex\n" ++ +		"git annex pre-commit .\n"
+ Command/InitRemote.hs view
@@ -0,0 +1,105 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.InitRemote where++import qualified Data.Map as M+import Control.Monad (when)+import Control.Monad.State (liftIO)+import Data.Maybe+import Data.String.Utils++import Command+import qualified Remote+import qualified Types.Remote as R+import Types+import UUID+import Messages++command :: [Command]+command = [repoCommand "initremote"+	(paramPair paramName $+		paramOptional $ paramRepeating $ paramKeyValue) seek+	"sets up a special (non-git) remote"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	when (null ws) $ needname++	(u, c) <- findByName name+	let fullconfig = M.union config c	+	t <- findType fullconfig++	showStart "initremote" name+	next $ perform t u $ M.union config c++	where+		name = head ws+		config = Remote.keyValToConfig $ tail ws+		needname = do+			let err s = error $ "Specify a name for the remote. " ++ s+			names <- remoteNames+			if null names+				then err ""+				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names+			++perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform+perform t u c = do+	c' <- R.setup t u c+	next $ cleanup u c'++cleanup :: UUID -> R.RemoteConfig -> CommandCleanup+cleanup u c = do+	Remote.configSet u c+        return True++{- Look up existing remote's UUID and config by name, or generate a new one -}+findByName :: String -> Annex (UUID, R.RemoteConfig)+findByName name = do+	m <- Remote.readRemoteLog+	maybe generate return $ findByName' name m+	where+		generate = do+			uuid <- liftIO $ genUUID+			return (uuid, M.insert nameKey name M.empty)++findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)+findByName' n m = if null matches then Nothing else Just $ head matches+	where+		matches = filter (matching . snd) $ M.toList m+		matching c = case M.lookup nameKey c of+			Nothing -> False+			Just n'+				| n' == n -> True+				| otherwise -> False++remoteNames :: Annex [String]+remoteNames = do+	m <- Remote.readRemoteLog+	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m++{- find the specified remote type -}+findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)+findType config = maybe unspecified specified $ M.lookup typeKey config+	where+		unspecified = error "Specify the type of remote with type="+		specified s = case filter (findtype s) Remote.remoteTypes of+			[] -> error $ "Unknown remote type " ++ s+			(t:_) -> return t+		findtype s i = R.typename i == s++{- The name of a configured remote is stored in its config using this key. -}+nameKey :: String+nameKey = "name"++{- The type of a remote is stored in its config using this key. -}+typeKey :: String+typeKey = "type"
+ Command/Lock.hs view
@@ -0,0 +1,37 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Lock where++import Control.Monad.State (liftIO)+import System.Directory++import Command+import Messages+import qualified AnnexQueue+import Utility+	+command :: [Command]+command = [repoCommand "lock" paramPath seek "undo unlock command"]++seek :: [CommandSeek]+seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]++{- Undo unlock -}+start :: CommandStartBackendFile+start (file, _) = do+	showStart "lock" file+	next $ perform file++perform :: FilePath -> CommandPerform+perform file = do+	liftIO $ removeFile file+	-- Checkout from HEAD to get rid of any changes that might be +	-- staged in the index, and get back to the previous symlink to+	-- the content.+	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] file+	next $ return True -- no cleanup needed
+ Command/Map.hs view
@@ -0,0 +1,226 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Map where++import Control.Monad.State (liftIO)+import Control.Exception.Extensible+import System.Cmd.Utils+import qualified Data.Map as M+import Data.List.Utils++import Command+import qualified Annex+import qualified Git+import Messages+import Types+import Utility+import UUID+import Trust+import Ssh+import qualified Dot++-- a link from the first repository to the second (its remote)+data Link = Link Git.Repo Git.Repo++command :: [Command]+command = [repoCommand "map" paramNothing seek "generate map of repositories"]++seek :: [CommandSeek]+seek = [withNothing start]++start :: CommandStartNothing+start = do+	g <- Annex.gitRepo+	rs <- spider g++	umap <- uuidMap+	trusted <- trustGet Trusted++	liftIO $ writeFile file (drawMap rs umap trusted)+	showLongNote $ "running: dot -Tx11 " ++ file+	showProgress+	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]+	next $ next $ return r+	where+		file = "map.dot"++{- Generates a graph for dot(1). Each repository, and any other uuids, are+ - displayed as a node, and each of its remotes is represented as an edge+ - pointing at the node for the remote.+ -+ - The order nodes are added to the graph matters, since dot will draw+ - the first ones near to the top and left. So it looks better to put+ - the repositories first, followed by uuids that were not matched+ - to a repository.+ -}+drawMap :: [Git.Repo] -> (M.Map UUID String) -> [UUID] -> String+drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others+	where+		repos = map (node umap rs) rs+		ruuids = ts ++ map getUncachedUUID rs+		others = map (unreachable . uuidnode) $+			filter (`notElem` ruuids) (M.keys umap)+		trusted = map (trustworthy . uuidnode) ts+		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap++hostname :: Git.Repo -> String+hostname r+	| Git.repoIsUrl r = Git.urlHost r+	| otherwise = "localhost"++basehostname :: Git.Repo -> String+basehostname r = head $ split "." $ hostname r++{- A name to display for a repo. Uses the name from uuid.log if available,+ - or the remote name if not. -}+repoName :: (M.Map UUID String) -> Git.Repo -> String+repoName umap r+	| null repouuid = fallback+	| otherwise = M.findWithDefault fallback repouuid umap+	where+		repouuid = getUncachedUUID r+		fallback = maybe "unknown" id $ Git.repoRemoteName r++{- A unique id for the node for a repo. Uses the annex.uuid if available. -}+nodeId :: Git.Repo -> String+nodeId r =+	case (getUncachedUUID r) of+		"" -> Git.repoLocation r+		u -> u++{- A node representing a repo. -}+node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String+node umap fullinfo r = unlines $ n:edges+	where+		n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $+			decorate $ Dot.graphNode (nodeId r) (repoName umap r)+		edges = map (edge umap fullinfo r) (Git.remotes r)+		decorate+			| Git.configMap r == M.empty = unreachable+			| otherwise = reachable++{- An edge between two repos. The second repo is a remote of the first. -}+edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	+edge umap fullinfo from to =+	Dot.graphEdge (nodeId from) (nodeId fullto) edgename+	where+		-- get the full info for the remote, to get its UUID+		fullto = findfullinfo to+		findfullinfo n =+			case (filter (same n) fullinfo) of+				[] -> n+				(n':_) -> n'+		{- Only name an edge if the name is different than the name+		 - that will be used for the destination node, and is+		 - different from its hostname. (This reduces visual clutter.) -}+		edgename = maybe Nothing calcname $ Git.repoRemoteName to+		calcname n+			| n == repoName umap fullto || n == hostname fullto = Nothing+			| otherwise = Just n++unreachable :: String -> String+unreachable = Dot.fillColor "red"+reachable :: String -> String+reachable = Dot.fillColor "white"+trustworthy :: String -> String+trustworthy = Dot.fillColor "green"++{- Recursively searches out remotes starting with the specified repo. -}+spider :: Git.Repo -> Annex [Git.Repo]+spider r = spider' [r] []+spider' :: [Git.Repo] -> [Git.Repo] -> Annex [Git.Repo]+spider' [] known = return known+spider' (r:rs) known+	| any (same r) known = spider' rs known+	| otherwise = do+		r' <- scan r++		-- The remotes will be relative to r', and need to be+		-- made absolute for later use.+		let remotes = map (absRepo r') (Git.remotes r')+		let r'' = Git.remotesAdd r' remotes++		spider' (rs ++ remotes) (r'':known)++absRepo :: Git.Repo -> Git.Repo -> Git.Repo+absRepo reference r+	| Git.repoIsUrl reference = Git.localToUrl reference r+	| otherwise = r++{- Checks if two repos are the same. -}+same :: Git.Repo -> Git.Repo -> Bool+same a b+	| both Git.repoIsSsh = matching Git.urlAuthority && matching Git.workTree+	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show+	| neither Git.repoIsSsh = matching Git.workTree+	| otherwise = False+		+	where+		matching t = t a == t b+		both t = t a && t b+		neither t = not (t a) && not (t b)++{- reads the config of a remote, with progress display -}+scan :: Git.Repo -> Annex Git.Repo+scan r = do+	showStart "map" $ Git.repoDescribe r+	v <- tryScan r+	case v of+		Just r' -> do+			showEndOk+			return r'+		Nothing -> do+			showEndFail+			return r++{- tries to read the config of a remote, returning it only if it can+ - be accessed -}+tryScan :: Git.Repo -> Annex (Maybe Git.Repo)+tryScan r+	| Git.repoIsSsh r = sshscan+	| Git.repoIsUrl r = return Nothing+	| otherwise = safely $ Git.configRead r+	where+		safely a = do+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))+			case result of+				Left _ -> return Nothing+				Right r' -> return $ Just r'+		pipedconfig cmd params = safely $+			pOpen ReadFromPipe cmd (toCommand params) $+				Git.hConfigRead r++		configlist =+			onRemote r (pipedconfig, Nothing) "configlist" []+		manualconfiglist = do+			let sshcmd =+				"cd " ++ shellEscape(Git.workTree r) ++ " && " +++				"git config --list"+			sshparams <- sshToRepo r [Param sshcmd]+			liftIO $ pipedconfig "ssh" sshparams++		-- First, try sshing and running git config manually,+		-- only fall back to git-annex-shell configlist if that+		-- fails.+		-- +		-- This is done for two reasons, first I'd like this+		-- subcommand to be usable on non-git-annex repos.+		-- Secondly, configlist doesn't include information about+		-- the remote's remotes.+		sshscan = do+			sshnote+			v <- manualconfiglist+			case v of+				Nothing -> do+					sshnote+					configlist+				ok -> return ok++		sshnote = do+			showNote "sshing..."+			showProgress
+ Command/Merge.hs view
@@ -0,0 +1,29 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Merge where++import Command+import qualified Branch+import Messages++command :: [Command]+command = [repoCommand "merge" paramNothing seek+		"auto-merges remote changes into the git-annex branch"]++seek :: [CommandSeek]+seek = [withNothing start]++start :: CommandStartNothing+start = do+	showStart "merge" "."+	next perform++perform :: CommandPerform+perform = do+	Branch.update+	next $ return True
+ Command/Migrate.hs view
@@ -0,0 +1,75 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Migrate where++import Control.Monad.State (liftIO)+import System.Posix.Files+import System.Directory+import System.FilePath++import Command+import qualified Annex+import qualified Backend+import Locations+import Types+import Content+import Messages+import Utility+import qualified Command.Add++command :: [Command]+command = [repoCommand "migrate" paramPath seek "switch data to different backend"]++seek :: [CommandSeek]+seek = [withBackendFilesInGit start]++start :: CommandStartBackendFile+start (file, b) = isAnnexed file $ \(key, oldbackend) -> do+	exists <- inAnnex key+	newbackend <- choosebackend b+	upgradable <- Backend.upgradableKey oldbackend key+	if (newbackend /= oldbackend || upgradable) && exists+		then do+			showStart "migrate" file+			next $ perform file key newbackend+		else stop+	where+		choosebackend Nothing = do+			backends <- Backend.list+			return $ head backends+		choosebackend (Just backend) = return backend++perform :: FilePath -> Key -> Backend Annex -> CommandPerform+perform file oldkey newbackend = do+	g <- Annex.gitRepo++	-- Store the old backend's cached key in the new backend+	-- (the file can't be stored as usual, because it's already a symlink).+	-- The old backend's key is not dropped from it, because there may+	-- be other files still pointing at that key.+	let src = gitAnnexLocation g oldkey+	let tmpfile = gitAnnexTmpDir g </> takeFileName file+	liftIO $ createLink src tmpfile+	stored <- Backend.storeFileKey tmpfile $ Just newbackend+	liftIO $ cleantmp tmpfile+	case stored of+		Nothing -> stop+		Just (newkey, _) -> do+			ok <- getViaTmpUnchecked newkey $ \t -> do+				-- Make a hard link to the old backend's+				-- cached key, to avoid wasting disk space.+				liftIO $ unlessM (doesFileExist t) $ createLink src t+				return True+			if ok+				then do+					-- Update symlink to use the new key.+					liftIO $ removeFile file+					next $ Command.Add.cleanup file newkey+				else stop+	where+		cleantmp t = whenM (doesFileExist t) $ removeFile t
+ Command/Move.hs view
@@ -0,0 +1,151 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Move where++import Control.Monad (when)++import Command+import qualified Command.Drop+import qualified Annex+import LocationLog+import Types+import Content+import qualified Remote+import UUID+import Messages++command :: [Command]+command = [repoCommand "move" paramPath seek+	"move content of files to/from another repository"]++seek :: [CommandSeek]+seek = [withFilesInGit $ start True]++{- Move (or copy) a file either --to or --from a repository.+ -+ - This only operates on the cached file content; it does not involve+ - moving data in the key-value backend. -}+start :: Bool -> CommandStartString+start move file = do+	to <- Annex.getState Annex.toremote+	from <- Annex.getState Annex.fromremote+	case (from, to) of+		(Nothing, Nothing) -> error "specify either --from or --to"+		(Nothing, Just name) -> do+			dest <- Remote.byName name+			toStart dest move file+		(Just name, Nothing) -> do+			src <- Remote.byName name+			fromStart src move file+		(_ ,  _) -> error "only one of --from or --to can be specified"++showAction :: Bool -> FilePath -> Annex ()+showAction True file = showStart "move" file+showAction False file = showStart "copy" file++{- Used to log a change in a remote's having a key. The change is logged+ - in the local repo, not on the remote. The process of transferring the+ - key to the remote, or removing the key from it *may* log the change+ - on the remote, but this cannot be relied on. -}+remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()+remoteHasKey remote key present	= do+	let remoteuuid = Remote.uuid remote+	g <- Annex.gitRepo+	logChange g key remoteuuid status+	where+		status = if present then InfoPresent else InfoMissing++{- Moves (or copies) the content of an annexed file to a remote.+ -+ - If the remote already has the content, it is still removed from+ - the current repository.+ -+ - Note that unlike drop, this does not honor annex.numcopies.+ - A file's content can be moved even if there are insufficient copies to+ - allow it to be dropped.+ -}+toStart :: Remote.Remote Annex -> Bool -> CommandStartString+toStart dest move file = isAnnexed file $ \(key, _) -> do+	g <- Annex.gitRepo+	u <- getUUID g+	ishere <- inAnnex key+	if not ishere || u == Remote.uuid dest+		then stop -- not here, so nothing to do+		else do+			showAction move file+			next $ toPerform dest move key+toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform+toPerform dest move key = do+	-- Checking the remote is expensive, so not done in the start step.+	-- In fast mode, location tracking is assumed to be correct,+	-- and an explicit check is not done, when copying. When moving,+	-- it has to be done, to avoid inaverdent data loss.+	fast <- Annex.getState Annex.fast+	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)+	isthere <- if fastcheck+		then do+			remotes <- Remote.keyPossibilities key+			return $ Right $ dest `elem` remotes+		else Remote.hasKey dest key+	case isthere of+		Left err -> do+			showNote $ show err+			stop+		Right False -> do+			showNote $ "to " ++ Remote.name dest ++ "..."+			ok <- Remote.storeKey dest key+			if ok+				then next $ toCleanup dest move key+				else do+					when fastcheck $+						warning "This could have failed because --fast is enabled."+					stop+		Right True -> next $ toCleanup dest move key+toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup+toCleanup dest move key = do+	remoteHasKey dest key True+	if move+		then Command.Drop.cleanup key+		else return True++{- Moves (or copies) the content of an annexed file from a remote+ - to the current repository.+ -+ - If the current repository already has the content, it is still removed+ - from the remote.+ -}+fromStart :: Remote.Remote Annex -> Bool -> CommandStartString+fromStart src move file = isAnnexed file $ \(key, _) -> do+	g <- Annex.gitRepo+	u <- getUUID g+	remotes <- Remote.keyPossibilities key+	if (u == Remote.uuid src) || (null $ filter (== src) remotes)+		then stop+		else do+			showAction move file+			next $ fromPerform src move key+fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform+fromPerform src move key = do+	ishere <- inAnnex key+	if ishere+		then next $ fromCleanup src move key+		else do+			showNote $ "from " ++ Remote.name src ++ "..."+			ok <- getViaTmp key $ Remote.retrieveKeyFile src key+			if ok+				then next $ fromCleanup src move key+				else stop -- fail+fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup+fromCleanup src True key = do+	ok <- Remote.removeKey src key+	-- better safe than sorry: assume the src dropped the key+	-- even if it seemed to fail; the failure could have occurred+	-- after it really dropped it+	remoteHasKey src key False+	return ok+fromCleanup _ False _ = return True
+ Command/PreCommit.hs view
@@ -0,0 +1,31 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.PreCommit where++import Command+import qualified Command.Add+import qualified Command.Fix++command :: [Command]+command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]++{- The pre-commit hook needs to fix symlinks to all files being committed.+ - And, it needs to inject unlocked files into the annex. -}+seek :: [CommandSeek]+seek = [withFilesToBeCommitted Command.Fix.start,+	withFilesUnlockedToBeCommitted start]++start :: CommandStartBackendFile+start pair = next $ perform pair++perform :: BackendFile -> CommandPerform+perform pair@(file, _) = do+	ok <- doCommand $ Command.Add.start pair+	if ok+		then next $ return True+		else error $ "failed to add " ++ file ++ "; canceling commit"
+ Command/RecvKey.hs view
@@ -0,0 +1,37 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.RecvKey where++import Control.Monad.State (liftIO)+import System.Exit++import Command+import CmdLine+import Content+import Utility+import RsyncFile++command :: [Command]+command = [repoCommand "recvkey" paramKey seek+	"runs rsync in server mode to receive content"]++seek :: [CommandSeek]+seek = [withKeys start]++start :: CommandStartKey+start key = do+	whenM (inAnnex key) $ error "key is already present in annex"+	+	ok <- getViaTmp key (liftIO . rsyncServerReceive)+	if ok+		then do+			-- forcibly quit after receiving one key,+			-- and shutdown cleanly so queued git commands run+			_ <- shutdown+			liftIO exitSuccess+		else liftIO exitFailure
+ Command/Semitrust.hs view
@@ -0,0 +1,33 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Semitrust where++import Command+import qualified Remote+import UUID+import Trust+import Messages++command :: [Command]+command = [repoCommand "semitrust" (paramRepeating paramRemote) seek+	"return repository to default trust level"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	let name = unwords ws+	showStart "semitrust" name+	u <- Remote.nameToUUID name+	next $ perform u++perform :: UUID -> CommandPerform+perform uuid = do+	trustSet uuid SemiTrusted+	next $ return True
+ Command/SendKey.hs view
@@ -0,0 +1,35 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.SendKey where++import Control.Monad.State (liftIO)+import System.Exit++import Locations+import qualified Annex+import Command+import Content+import Utility+import RsyncFile+import Messages++command :: [Command]+command = [repoCommand "sendkey" paramKey seek+	"runs rsync in server mode to send content"]++seek :: [CommandSeek]+seek = [withKeys start]++start :: CommandStartKey+start key = do+	g <- Annex.gitRepo+	let file = gitAnnexLocation g key+	whenM (inAnnex key) $+		liftIO $ rsyncServerSend file -- does not return+	warning "requested key is not present"+	liftIO exitFailure
+ Command/SetKey.hs view
@@ -0,0 +1,50 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.SetKey where++import Control.Monad.State (liftIO)++import Command+import Utility+import LocationLog+import Content+import Messages++command :: [Command]+command = [repoCommand "setkey" (paramRepeating paramKey) seek+	"sets annexed content for a key using a temp file"]++seek :: [CommandSeek]+seek = [withTempFile start]++{- Sets cached content for a key. -}+start :: CommandStartString+start file = do+	showStart "setkey" file+	next $ perform file++perform :: FilePath -> CommandPerform+perform file = do+	key <- cmdlineKey+	-- the file might be on a different filesystem, so mv is used+	-- rather than simply calling moveToObjectDir; disk space is also+	-- checked this way.+	ok <- getViaTmp key $ \dest -> do+		if dest /= file+			then liftIO $+				boolSystem "mv" [File file, File dest]+			else return True+	if ok+		then next cleanup+		else error "mv failed!"++cleanup :: CommandCleanup+cleanup = do+	key <- cmdlineKey+	logStatus key InfoPresent+	return True
+ Command/Status.hs view
@@ -0,0 +1,185 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Status where++import Control.Monad.State+import Data.Maybe+import System.IO+import Data.List+import qualified Data.Map as M++import qualified Annex+import qualified Types.Backend as B+import qualified Types.Remote as R+import qualified Remote+import qualified Command.Unused+import qualified Git+import Command+import Types+import DataUnits+import Content+import Types.Key+import Locations++-- a named computation that produces a statistic+type Stat = StatState (Maybe (String, StatState String))++-- cached info that multiple Stats may need+data StatInfo = StatInfo+	{ keysPresentCache :: (Maybe (SizeList Key))+	, keysReferencedCache :: (Maybe (SizeList Key))+	}++-- a state monad for running Stats in+type StatState = StateT StatInfo Annex++-- a list with a known length+-- (Integer is used for the length to avoid+-- blowing up if someone annexed billions of files..)+type SizeList a = ([a], Integer)++sizeList :: [a] -> SizeList a+sizeList l = (l, genericLength l)++command :: [Command]+command = [repoCommand "status" paramNothing seek+	"shows status information about the annex"]++seek :: [CommandSeek]+seek = [withNothing start]++{- Order is significant. Less expensive operations, and operations+ - that share data go together.+ -}+faststats :: [Stat]+faststats = +	[ supported_backends+	, supported_remote_types+	, tmp_size+	, bad_data_size+	]+slowstats :: [Stat]+slowstats =+	[ local_annex_keys+	, local_annex_size+	, total_annex_keys+	, total_annex_size+	, backend_usage+	]++start :: CommandStartNothing+start = do+	fast <- Annex.getState Annex.fast+	let todo = if fast then faststats else faststats ++ slowstats+	evalStateT (mapM_ showStat todo) (StatInfo Nothing Nothing)+	stop++stat :: String -> StatState String -> Stat+stat desc a = return $ Just (desc, a)++nostat :: Stat+nostat = return $ Nothing++showStat :: Stat -> StatState ()+showStat s = calc =<< s+	where+		calc (Just (desc, a)) = do+			liftIO $ putStr $ desc ++ ": "+			liftIO $ hFlush stdout+			liftIO . putStrLn =<< a+		calc Nothing = return ()++supported_backends :: Stat+supported_backends = stat "supported backends" $+	lift (Annex.getState Annex.supportedBackends) >>=+		return . unwords . (map B.name)++supported_remote_types :: Stat+supported_remote_types = stat "supported remote types" $+	return $ unwords $ map R.typename Remote.remoteTypes++local_annex_size :: Stat+local_annex_size = stat "local annex size" $+	cachedKeysPresent >>= keySizeSum++total_annex_size :: Stat+total_annex_size = stat "total annex size" $+	cachedKeysReferenced >>= keySizeSum++local_annex_keys :: Stat+local_annex_keys = stat "local annex keys" $ +	return . show . snd =<< cachedKeysPresent++total_annex_keys :: Stat+total_annex_keys = stat "total annex keys" $+	return . show . snd =<< cachedKeysReferenced++tmp_size :: Stat+tmp_size = staleSize "temporary directory size" gitAnnexTmpDir++bad_data_size :: Stat+bad_data_size = staleSize "bad keys size" gitAnnexBadDir++backend_usage :: Stat+backend_usage = stat "backend usage" $+	return . usage =<< cachedKeysReferenced+	where+		usage (ks, _) = pp "" $ sort $ map swap $ splits ks+		splits :: [Key] -> [(String, Integer)]+		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks+		tcount k = (keyBackendName k, 1)+		swap (a, b) = (b, a)+		pp c [] = c+		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs+++cachedKeysPresent :: StatState (SizeList Key)+cachedKeysPresent = do+	s <- get+	case keysPresentCache s of+		Just v -> return v+		Nothing -> do+			keys <- lift $ getKeysPresent+			let v = sizeList keys+			put s { keysPresentCache = Just v }+			return v++cachedKeysReferenced :: StatState (SizeList Key)+cachedKeysReferenced = do+	s <- get+	case keysReferencedCache s of+		Just v -> return v+		Nothing -> do+			keys <- lift $ Command.Unused.getKeysReferenced+			-- A given key may be referenced repeatedly.+			-- nub does not seem too slow (yet)..+			let v = sizeList $ nub keys+			put s { keysReferencedCache = Just v }+			return v++keySizeSum :: SizeList Key -> StatState String+keySizeSum (keys, len) = do+	let knownsize = catMaybes $ map keySize keys+	let total = roughSize storageUnits False $ foldl (+) 0 knownsize+	let missing = len - genericLength knownsize+	return $ total +++		if missing > 0+			then aside $ "but " ++ show missing ++ " keys have unknown size"+			else ""++staleSize :: String -> (Git.Repo -> FilePath) -> Stat+staleSize label dirspec = do+	keys <- lift (Command.Unused.staleKeys dirspec)+	if null keys+		then nostat+		else stat label $ do+			s <- keySizeSum $ sizeList keys+			return $ s ++ aside "clean up with git-annex unused"++aside :: String -> String+aside s = "\t(" ++ s ++ ")"
+ Command/Trust.hs view
@@ -0,0 +1,33 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Trust where++import Command+import qualified Remote+import Trust+import UUID+import Messages++command :: [Command]+command = [repoCommand "trust" (paramRepeating paramRemote) seek+	"trust a repository"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	let name = unwords ws+	showStart "trust" name+	u <- Remote.nameToUUID name+	next $ perform u++perform :: UUID -> CommandPerform+perform uuid = do+	trustSet uuid Trusted+	next $ return True
+ Command/Unannex.hs view
@@ -0,0 +1,75 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Unannex where++import Control.Monad.State (liftIO)+import Control.Monad (unless)+import System.Directory++import Command+import qualified Annex+import qualified AnnexQueue+import Utility+import qualified Backend+import LocationLog+import Types+import Content+import qualified Git+import qualified Git.LsFiles as LsFiles+import Messages++command :: [Command]+command = [repoCommand "unannex" paramPath seek "undo accidential add command"]++seek :: [CommandSeek]+seek = [withFilesInGit start]++{- The unannex subcommand undoes an add. -}+start :: CommandStartString+start file = isAnnexed file $ \(key, backend) -> do+	ishere <- inAnnex key+	if ishere+		then do+			force <- Annex.getState Annex.force+			unless force $ do+				g <- Annex.gitRepo+				staged <- liftIO $ LsFiles.staged g [Git.workTree g]+				unless (null staged) $+					error "This command cannot be run when there are already files staged for commit."+				Annex.changeState $ \s -> s { Annex.force = True }++			showStart "unannex" file+			next $ perform file key backend+		else stop++perform :: FilePath -> Key -> Backend Annex -> CommandPerform+perform file key backend = do+	-- force backend to always remove+	ok <- Backend.removeKey backend key (Just 0)+	if ok+		then next $ cleanup file key+		else stop++cleanup :: FilePath -> Key -> CommandCleanup+cleanup file key = do+	g <- Annex.gitRepo++	liftIO $ removeFile file+	liftIO $ Git.run g "rm" [Params "--quiet --", File file]+	-- git rm deletes empty directories; put them back+	liftIO $ createDirectoryIfMissing True (parentDir file)++	fromAnnex key file+	logStatus key InfoMissing+	+	-- Commit staged changes at end to avoid confusing the+	-- pre-commit hook if this file is later added back to+	-- git as a normal, non-annexed file.+	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] "-a"+	+	return True
+ Command/Uninit.hs view
@@ -0,0 +1,49 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Uninit where++import Control.Monad.State (liftIO)+import System.Directory++import Command+import Messages+import Types+import Utility+import qualified Git+import qualified Annex+import qualified Command.Unannex+import qualified Command.Init++command :: [Command]+command = [repoCommand "uninit" paramPath seek +        "de-initialize git-annex and clean out repository"]++seek :: [CommandSeek]+seek = [withFilesInGit Command.Unannex.start, withNothing start]++start :: CommandStartNothing+start = do+	showStart "uninit" ""+	next perform++perform :: CommandPerform+perform = do+	g <- Annex.gitRepo+	gitPreCommitHookUnWrite g+	next $ return True++gitPreCommitHookUnWrite :: Git.Repo -> Annex ()+gitPreCommitHookUnWrite repo = do+	let hook = Command.Init.preCommitHook repo+	whenM (liftIO $ doesFileExist hook) $ do+		c <- liftIO $ readFile hook+		if c == Command.Init.preCommitScript+			then liftIO $ removeFile hook+			else warning $ "pre-commit hook (" ++ hook ++ +				") contents modified; not deleting." +++				" Edit it to remove call to git annex."
+ Command/Unlock.hs view
@@ -0,0 +1,58 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Unlock where++import Control.Monad.State (liftIO)+import System.Directory hiding (copyFile)++import Command+import qualified Annex+import qualified Backend+import Types+import Messages+import Locations+import Content+import CopyFile+import Utility++command :: [Command]+command =+	[ repoCommand "unlock" paramPath seek "unlock files for modification"+	, repoCommand "edit" paramPath seek "same as unlock"+	]++seek :: [CommandSeek]+seek = [withFilesInGit start]++{- The unlock subcommand replaces the symlink with a copy of the file's+ - content. -}+start :: CommandStartString+start file = isAnnexed file $ \(key, _) -> do+	showStart "unlock" file+	next $ perform file key++perform :: FilePath -> Key -> CommandPerform+perform dest key = do+	unlessM (Backend.hasKey key) $ error "content not present"+	+	checkDiskSpace key++	g <- Annex.gitRepo+	let src = gitAnnexLocation g key+	let tmpdest = gitAnnexTmpLocation g key+	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)+	showNote "copying..."+	ok <- liftIO $ copyFile src tmpdest+        if ok+                then do+			liftIO $ do+				removeFile dest+				renameFile tmpdest dest+				allowWrite dest+			next $ return True+                else error "copy failed!"
+ Command/Untrust.hs view
@@ -0,0 +1,33 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Untrust where++import Command+import qualified Remote+import UUID+import Trust+import Messages++command :: [Command]+command = [repoCommand "untrust" (paramRepeating paramRemote) seek+	"do not trust a repository"]++seek :: [CommandSeek]+seek = [withWords start]++start :: CommandStartWords+start ws = do+	let name = unwords ws+	showStart "untrust" name+	u <- Remote.nameToUUID name+	next $ perform u++perform :: UUID -> CommandPerform+perform uuid = do+	trustSet uuid UnTrusted+	next $ return True
+ Command/Unused.hs view
@@ -0,0 +1,213 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Unused where++import Control.Monad (filterM, unless, forM_, when)+import Control.Monad.State (liftIO)+import qualified Data.Set as S+import Data.Maybe+import System.FilePath+import System.Directory++import Command+import Types+import Content+import Messages+import Locations+import Utility+import LocationLog+import qualified Annex+import qualified Git+import qualified Git.LsFiles as LsFiles+import qualified Backend+import qualified Remote++command :: [Command]+command = [repoCommand "unused" paramNothing seek+	"look for unused file content"]++seek :: [CommandSeek]+seek = [withNothing start]++{- Finds unused content in the annex. -} +start :: CommandStartNothing+start = notBareRepo $ do+	from <- Annex.getState Annex.fromremote+	let (name, action) = case from of+		Nothing -> (".", checkUnused)+		Just "." -> (".", checkUnused)+		Just n -> (n, checkRemoteUnused n)+	showStart "unused" name+	next action++checkUnused :: CommandPerform+checkUnused = do+	(unused, stalebad, staletmp) <- unusedKeys+	_ <- list "" unusedMsg unused 0 >>=+		list "bad" staleBadMsg stalebad >>=+			list "tmp" staleTmpMsg staletmp+	next $ return True+	where+		list file msg l c = do+			let unusedlist = number c l+			when (not $ null l) $ do+				showLongNote $ msg unusedlist+				showLongNote $ "\n"+			writeUnusedFile file unusedlist+			return $ c + length l++checkRemoteUnused :: String -> CommandPerform+checkRemoteUnused name = do+	checkRemoteUnused' =<< Remote.byName name+	next $ return True++checkRemoteUnused' :: Remote.Remote Annex -> Annex ()+checkRemoteUnused' r = do+	showNote $ "checking for unused data..."+	referenced <- getKeysReferenced+	remotehas <- filterM isthere =<< loggedKeys+	let remoteunused = remotehas `exclude` referenced+	let list = number 0 remoteunused+	writeUnusedFile "" list+	unless (null remoteunused) $ do+		showLongNote $ remoteUnusedMsg r list+		showLongNote $ "\n"+	where+		isthere k = do+			us <- keyLocations k+			return $ uuid `elem` us+		uuid = Remote.uuid r++writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()+writeUnusedFile prefix l = do+	g <- Annex.gitRepo+	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $+		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l++table :: [(Int, Key)] -> [String]+table l = ["  NUMBER  KEY"] ++ map cols l+	where+		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k+		pad n s = s ++ replicate (n - length s) ' '++number :: Int -> [a] -> [(Int, a)]+number _ [] = []+number n (x:xs) = (n+1, x):(number (n+1) xs)++staleTmpMsg :: [(Int, Key)] -> String+staleTmpMsg t = unlines $ +	["Some partially transferred data exists in temporary files:"]+	++ table t ++ [dropMsg Nothing]++staleBadMsg :: [(Int, Key)] -> String+staleBadMsg t = unlines $ +	["Some corrupted files have been preserved by fsck, just in case:"]+	++ table t ++ [dropMsg Nothing]++unusedMsg :: [(Int, Key)] -> String+unusedMsg u = unusedMsg' u+	["Some annexed data is no longer used by any files in the current branch:"]+	[dropMsg Nothing,+	"Please be cautious -- are you sure that another branch, or another",+	"repository does not still use this data?"]++remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String+remoteUnusedMsg r u = unusedMsg' u+	["Some annexed data on " ++ name ++ +	 " is not used by any files in the current branch:"]+	[dropMsg $ Just r,+	 "Please be cautious -- Are you sure that the remote repository",+	 "does not use this data? Or that it's not used by another branch?"]+	where+		name = Remote.name r ++unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String+unusedMsg' u header trailer = unlines $+	header +++	table u +++	["(To see where data was previously used, try: git log --stat -S'KEY')"] +++	trailer++dropMsg :: Maybe (Remote.Remote Annex) -> String+dropMsg Nothing = dropMsg' ""+dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r+dropMsg' :: String -> String+dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n"++{- Finds keys whose content is present, but that do not seem to be used+ - by any files in the git repo, or that are only present as bad or tmp+ - files. -}+unusedKeys :: Annex ([Key], [Key], [Key])+unusedKeys = do+	fast <- Annex.getState Annex.fast+	if fast+		then do+			showNote "fast mode enabled; only finding stale files"+			tmp <- staleKeys gitAnnexTmpDir+			bad <- staleKeys gitAnnexBadDir+			return ([], bad, tmp)+		else do+			showNote "checking for unused data..."+			present <- getKeysPresent+			referenced <- getKeysReferenced+			let unused = present `exclude` referenced+			staletmp <- staleKeysPrune gitAnnexTmpDir present+			stalebad <- staleKeysPrune gitAnnexBadDir present+			return (unused, stalebad, staletmp)++{- Finds items in the first, smaller list, that are not+ - present in the second, larger list.+ - + - Constructing a single set, of the list that tends to be+ - smaller, appears more efficient in both memory and CPU+ - than constructing and taking the S.difference of two sets. -}+exclude :: Ord a => [a] -> [a] -> [a]+exclude [] _ = [] -- optimisation+exclude smaller larger = S.toList $ remove larger $ S.fromList smaller+	where+		remove a b = foldl (flip S.delete) b a++{- List of keys referenced by symlinks in the git repo. -}+getKeysReferenced :: Annex [Key]+getKeysReferenced = do+	g <- Annex.gitRepo+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]+	keypairs <- mapM Backend.lookupFile files+	return $ map fst $ catMaybes keypairs++{- Looks in the specified directory for bad/tmp keys, and returns a list+ - of those that might still have value, or might be stale and removable. + - + - When a list of presently available keys is provided, stale keys+ - that no longer have value are deleted.+ -}+staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]+staleKeysPrune dirspec present = do+	contents <- staleKeys dirspec+		+	let stale = contents `exclude` present+	let dup = contents `exclude` stale++	g <- Annex.gitRepo+	let dir = dirspec g+	liftIO $ forM_ dup $ \t -> removeFile $ dir </> keyFile t++	return stale++staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]+staleKeys dirspec = do+	g <- Annex.gitRepo+	let dir = dirspec g+	exists <- liftIO $ doesDirectoryExist dir+	if not exists+		then return []+		else do+			contents <- liftIO $ getDirectoryContents dir+			files <- liftIO $ filterM doesFileExist $+				map (dir </>) contents+			return $ catMaybes $ map (fileKey . takeFileName) files
+ Command/Upgrade.hs view
@@ -0,0 +1,27 @@+{- git-annex command+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Upgrade where++import Command+import Upgrade+import Version+import Messages++command :: [Command]+command = [standaloneCommand "upgrade" paramNothing seek+	"upgrade repository layout"]++seek :: [CommandSeek]+seek = [withNothing start]++start :: CommandStartNothing+start = do+	showStart "upgrade" "."+	r <- upgrade+	setVersion+	next $ next $ return r
+ Command/Version.hs view
@@ -0,0 +1,34 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Version where++import Control.Monad.State (liftIO)+import Data.String.Utils+import Data.Maybe++import Command+import qualified SysConfig+import Version++command :: [Command]+command = [standaloneCommand "version" paramNothing seek "show version info"]++seek :: [CommandSeek]+seek = [withNothing start]++start :: CommandStartNothing+start = do+	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion+	v <- getVersion+	liftIO $ putStrLn $ "local repository version: " ++ fromMaybe "unknown" v+	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion+	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions+	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions+	stop+	where+		vs l = join " " l
+ Command/Whereis.hs view
@@ -0,0 +1,42 @@+{- git-annex command+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Command.Whereis where++import LocationLog+import Command+import Messages+import Remote+import Types++command :: [Command]+command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek+	"lists repositories that have file content"]++seek :: [CommandSeek]+seek = [withFilesInGit start]++start :: CommandStartString+start file = isAnnexed file $ \(key, _) -> do+	showStart "whereis" file+	next $ perform key++perform :: Key -> CommandPerform+perform key = do+	uuids <- keyLocations key+	let num = length uuids+	showNote $ show num ++ " " ++ copiesplural num+	if null $ uuids+		then stop+		else do+			pp <- prettyPrintUUIDs uuids+			showLongNote $ pp+			showProgress	+			next $ return True+	where+		copiesplural 1 = "copy"+		copiesplural _ = "copies"
+ Config.hs view
@@ -0,0 +1,88 @@+{- Git configuration+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Config where++import Data.Maybe+import Control.Monad.State (liftIO)++import qualified Git+import qualified Annex+import Types+import Utility++type ConfigKey = String++{- Changes a git config setting in both internal state and .git/config -}+setConfig :: ConfigKey -> String -> Annex ()+setConfig k value = do+	g <- Annex.gitRepo+	liftIO $ Git.run g "config" [Param k, Param value]+	-- re-read git config and update the repo's state+	g' <- liftIO $ Git.configRead g+	Annex.changeState $ \s -> s { Annex.repo = g' }++{- Looks up a per-remote config setting in git config.+ - Failing that, tries looking for a global config option. -}+getConfig :: Git.Repo -> ConfigKey -> String -> Annex String+getConfig r key def = do+	g <- Annex.gitRepo+	let def' = Git.configGet g ("annex." ++ key) def+	return $ Git.configGet g (remoteConfig r key) def'++remoteConfig :: Git.Repo -> ConfigKey -> String+remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key++{- Calculates cost for a remote.+ -+ - The default cost is 100 for local repositories, and 200 for remote+ - repositories; it can also be configured by remote.<name>.annex-cost+ -}+remoteCost :: Git.Repo -> Int -> Annex Int+remoteCost r def = do+	c <- getConfig r "cost" ""+	if not $ null c+		then return $ read c+		else return def++cheapRemoteCost :: Int+cheapRemoteCost = 100+semiCheapRemoteCost :: Int+semiCheapRemoteCost = 110+expensiveRemoteCost :: Int+expensiveRemoteCost = 200++{- Adjust's a remote's cost to reflect it being encrypted. -}+encryptedRemoteCostAdj :: Int+encryptedRemoteCostAdj = 50++{- Make sure the remote cost numbers work out. -}+prop_cost_sane :: Bool+prop_cost_sane = False `notElem`+	[ expensiveRemoteCost > 0+	, cheapRemoteCost < semiCheapRemoteCost+	, semiCheapRemoteCost < expensiveRemoteCost+	, cheapRemoteCost + encryptedRemoteCostAdj > semiCheapRemoteCost+	, cheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost+	, semiCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost+	]++{- Checks if a repo should be ignored, based either on annex-ignore+ - setting, or on command-line options. Allows command-line to override+ - annex-ignore. -}+remoteNotIgnored :: Git.Repo -> Annex Bool+remoteNotIgnored r = do+	ignored <- getConfig r "ignore" "false"+	to <- match Annex.toremote+	from <- match Annex.fromremote+	if to || from+		then return True+		else return $ not $ Git.configTrue ignored+	where+		match a = do+			n <- Annex.getState a+			return $ n == Git.repoRemoteName r
+ Content.hs view
@@ -0,0 +1,270 @@+{- git-annex file content managing+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Content (+	inAnnex,+	calcGitLink,+	logStatus,+	getViaTmp,+	getViaTmpUnchecked,+	withTmp,+	checkDiskSpace,+	preventWrite,+	allowWrite,+	moveAnnex,+	removeAnnex,+	fromAnnex,+	moveBad,+	getKeysPresent,+	saveState+) where++import System.IO.Error (try)+import System.Directory+import Control.Monad.State (liftIO)+import System.Path+import Control.Monad (when, filterM)+import System.Posix.Files+import System.FilePath+import Data.Maybe++import Types+import Locations+import LocationLog+import UUID+import qualified Git+import qualified Annex+import qualified AnnexQueue+import qualified Branch+import Utility+import StatFS+import Types.Key+import DataUnits+import Config++{- Checks if a given key is currently present in the gitAnnexLocation. -}+inAnnex :: Key -> Annex Bool+inAnnex key = do+	g <- Annex.gitRepo+	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"+	liftIO $ doesFileExist $ gitAnnexLocation g key++{- Calculates the relative path to use to link a file to a key. -}+calcGitLink :: FilePath -> Key -> Annex FilePath+calcGitLink file key = do+	g <- Annex.gitRepo+	cwd <- liftIO $ getCurrentDirectory+	let absfile = maybe whoops id $ absNormPath cwd file+	return $ relPathDirToFile (parentDir absfile) +			(Git.workTree g) </> ".git" </> annexLocation key+	where+		whoops = error $ "unable to normalize " ++ file++{- Updates the LocationLog when a key's presence changes in the current+ - repository. -}+logStatus :: Key -> LogStatus -> Annex ()+logStatus key status = do+	g <- Annex.gitRepo+	u <- getUUID g+	logChange g key u status++{- Runs an action, passing it a temporary filename to download,+ - and if the action succeeds, moves the temp file into + - the annex as a key's content. -}+getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool+getViaTmp key action = do+	g <- Annex.gitRepo+	let tmp = gitAnnexTmpLocation g key++	-- Check that there is enough free disk space.+	-- When the temp file already exists, count the space+	-- it is using as free.+	e <- liftIO $ doesFileExist tmp+	if e+		then do+			stat <- liftIO $ getFileStatus tmp+			checkDiskSpace' (fromIntegral $ fileSize stat) key+		else checkDiskSpace key++	when e $ liftIO $ allowWrite tmp++	getViaTmpUnchecked key action++{- Like getViaTmp, but does not check that there is enough disk space+ - for the incoming key. For use when the key content is already on disk+ - and not being copied into place. -}+getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool+getViaTmpUnchecked key action = do+	g <- Annex.gitRepo+	let tmp = gitAnnexTmpLocation g key++	liftIO $ createDirectoryIfMissing True (parentDir tmp)+	success <- action tmp+	if success+		then do+			moveAnnex key tmp+			logStatus key InfoPresent+			return True+		else do+			-- the tmp file is left behind, in case caller wants+			-- to resume its transfer+			return False++{- Creates a temp file, runs an action on it, and cleans up the temp file. -}+withTmp :: Key -> (FilePath -> Annex a) -> Annex a+withTmp key action = do+	g <- Annex.gitRepo+	let tmp = gitAnnexTmpLocation g key+	liftIO $ createDirectoryIfMissing True (parentDir tmp)+	res <- action tmp+	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp+	return res++{- Checks that there is disk space available to store a given key,+ - throwing an error if not. -}+checkDiskSpace :: Key -> Annex ()+checkDiskSpace = checkDiskSpace' 0++checkDiskSpace' :: Integer -> Key -> Annex ()+checkDiskSpace' adjustment key = do+	g <- Annex.gitRepo+	r <- getConfig g "diskreserve" ""+	let reserve = maybe megabyte id $ readSize dataUnits r+	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)+	case (stats, keySize key) of+		(Nothing, _) -> return ()+		(_, Nothing) -> return ()+		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->+			if (need + reserve > have + adjustment)+				then needmorespace (need + reserve - have - adjustment)+				else return ()+	where+		megabyte :: Integer+		megabyte = 1000000+		needmorespace n = do+			unlessM (Annex.getState Annex.force) $+				error $ "not enough free space, need " ++ +					roughSize storageUnits True n +++					" more (use --force to override this check or adjust annex.diskreserve)"++{- Removes the write bits from a file. -}+preventWrite :: FilePath -> IO ()+preventWrite f = unsetFileMode f writebits+	where+		writebits = foldl unionFileModes ownerWriteMode+					[groupWriteMode, otherWriteMode]++{- Turns a file's write bit back on. -}+allowWrite :: FilePath -> IO ()+allowWrite f = do+	s <- getFileStatus f+	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode++{- Moves a file into .git/annex/objects/+ -+ - What if the key there already has content? This could happen for+ - various reasons; perhaps the same content is being annexed again.+ - Perhaps there has been a hash collision generating the keys.+ -+ - The current strategy is to assume that in this case it's safe to delete+ - one of the two copies of the content; and the one already in the annex+ - is left there, assuming it's the original, canonical copy.+ -+ - I considered being more paranoid, and checking that both files had+ - the same content. Decided against it because A) users explicitly choose+ - a backend based on its hashing properties and so if they're dealing+ - with colliding files it's their own fault and B) adding such a check+ - would not catch all cases of colliding keys. For example, perhaps + - a remote has a key; if it's then added again with different content then+ - the overall system now has two different peices of content for that+ - key, and one of them will probably get deleted later. So, adding the+ - check here would only raise expectations that git-annex cannot truely+ - meet.+ -}+moveAnnex :: Key -> FilePath -> Annex ()+moveAnnex key src = do+	g <- Annex.gitRepo+	let dest = gitAnnexLocation g key+	let dir = parentDir dest+	e <- liftIO $ doesFileExist dest+	if e+		then liftIO $ removeFile src+		else liftIO $ do+			createDirectoryIfMissing True dir+			allowWrite dir -- in case the directory already exists+			renameFile src dest+			preventWrite dest+			preventWrite dir++{- Removes a key's file from .git/annex/objects/ -}+removeAnnex :: Key -> Annex ()+removeAnnex key = do+	g <- Annex.gitRepo+	let file = gitAnnexLocation g key+	let dir = parentDir file+	liftIO $ do+		allowWrite dir+		removeFile file+		removeDirectory dir++{- Moves a key's file out of .git/annex/objects/ -}+fromAnnex :: Key -> FilePath -> Annex ()+fromAnnex key dest = do+	g <- Annex.gitRepo+	let file = gitAnnexLocation g key+	let dir = parentDir file+	liftIO $ do+		allowWrite dir+		allowWrite file+		renameFile file dest+		removeDirectory dir++{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and+ - returns the file it was moved to. -}+moveBad :: Key -> Annex FilePath+moveBad key = do+	g <- Annex.gitRepo+	let src = gitAnnexLocation g key+	let dest = gitAnnexBadDir g </> takeFileName src+	liftIO $ do+		createDirectoryIfMissing True (parentDir dest)+		allowWrite (parentDir src)+		renameFile src dest+		removeDirectory (parentDir src)+	logStatus key InfoMissing+	return dest++{- List of keys whose content exists in .git/annex/objects/ -}+getKeysPresent :: Annex [Key]+getKeysPresent = do+	g <- Annex.gitRepo+	getKeysPresent' $ gitAnnexObjectDir g+getKeysPresent' :: FilePath -> Annex [Key]+getKeysPresent' dir = do+	exists <- liftIO $ doesDirectoryExist dir+	if (not exists)+		then return []+		else liftIO $ do+			-- 2 levels of hashing+			levela <- dirContents dir+			levelb <- mapM dirContents levela+			contents <- mapM dirContents (concat levelb)+			files <- filterM present (concat contents)+			return $ catMaybes $ map (fileKey . takeFileName) files+	where+		present d = do+			result <- try $+				getFileStatus $ d </> takeFileName d+			case result of+				Right s -> return $ isRegularFile s+				Left _ -> return False++{- Things to do to record changes to content. -}+saveState :: Annex ()+saveState = do+	AnnexQueue.flush False+	Branch.commit "update"
+ CopyFile.hs view
@@ -0,0 +1,29 @@+{- git-annex file copying+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module CopyFile (copyFile) where++import System.Directory (doesFileExist, removeFile)++import Utility+import qualified SysConfig++{- The cp command is used, because I hate reinventing the wheel,+ - and because this allows easy access to features like cp --reflink. -}+copyFile :: FilePath -> FilePath -> IO Bool+copyFile src dest = do+	whenM (doesFileExist dest) $+		removeFile dest+	boolSystem "cp" [params, File src, File dest]+	where+		params = if SysConfig.cp_reflink_auto+			then Params "--reflink=auto"+			else if SysConfig.cp_a+				then Params "-a"+				else if SysConfig.cp_p+					then Params "-p"+					else Params ""
+ Crypto.hs view
@@ -0,0 +1,253 @@+{- git-annex crypto+ -+ - Currently using gpg; could later be modified to support different+ - crypto backends if neccessary.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Crypto (+	Cipher,+	EncryptedCipher,+	genCipher,+	updateCipher,+	describeCipher,+	storeCipher,+	extractCipher,+	decryptCipher,		+	encryptKey,+	withEncryptedHandle,+	withDecryptedHandle,+	withEncryptedContent,+	withDecryptedContent,++	prop_hmacWithCipher_sane+) where++import qualified Data.ByteString.Lazy.Char8 as L+import qualified Data.Map as M+import Data.ByteString.Lazy.UTF8 (fromString)+import Data.Digest.Pure.SHA+import System.Cmd.Utils+import Data.String.Utils+import Data.List+import System.IO+import System.Posix.IO+import System.Posix.Types+import System.Posix.Process+import Control.Concurrent+import Control.Exception (finally)+import System.Exit+import System.Environment++import Types+import Types.Key+import Types.Remote+import Utility+import Base64+import Types.Crypto++{- The first half of a Cipher is used for HMAC; the remainder+ - is used as the GPG symmetric encryption passphrase.+ -+ - HMAC SHA1 needs only 64 bytes. The remainder is for expansion,+ - perhaps to HMAC SHA512, which needs 128 bytes (ideally).+ -+ - 256 is enough for gpg's symetric cipher; unlike weaker public key+ - crypto, the key does not need to be too large.+ -}+cipherHalf :: Int+cipherHalf = 256++cipherSize :: Int+cipherSize = cipherHalf * 2++cipherPassphrase :: Cipher -> String+cipherPassphrase (Cipher c) = drop cipherHalf c++cipherHmac :: Cipher -> String+cipherHmac (Cipher c) = take cipherHalf c++{- Creates a new Cipher, encrypted as specified in the remote's configuration -}+genCipher :: RemoteConfig -> IO EncryptedCipher+genCipher c = do+	ks <- configKeyIds c+	random <- genrandom+	encryptCipher (Cipher random) ks+	where+		genrandom = gpgRead+			-- Armor the random data, to avoid newlines,+			-- since gpg only reads ciphers up to the first+			-- newline.+			[ Params "--gen-random --armor"+			, Param $ show randomquality+			, Param $ show cipherSize+			]+		-- 1 is /dev/urandom; 2 is /dev/random+		randomquality = 1 :: Int++{- Updates an existing Cipher, re-encrypting it to add KeyIds specified in+ - the remote's configuration. -}+updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher+updateCipher c encipher@(EncryptedCipher _ ks) = do+	ks' <- configKeyIds c+	cipher <- decryptCipher c encipher+	encryptCipher cipher (combine ks ks')+	where+		combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b++describeCipher :: EncryptedCipher -> String+describeCipher (EncryptedCipher _ (KeyIds ks)) =+	"with gpg " ++ keys ks ++ " " ++ unwords ks+	where+		keys [_] = "key"+		keys _ = "keys"++{- Stores an EncryptedCipher in a remote's configuration. -}+storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig+storeCipher c (EncryptedCipher t ks) = +	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c++{- Extracts an EncryptedCipher from a remote's configuration. -}+extractCipher :: RemoteConfig -> Maybe EncryptedCipher+extractCipher c = +	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of+		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks)+		_ -> Nothing++{- Encrypts a Cipher to the specified KeyIds. -}+encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher+encryptCipher (Cipher c) (KeyIds ks) = do+	let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids+	encipher <- gpgPipeStrict (encrypt++recipients ks') c+	return $ EncryptedCipher encipher (KeyIds ks')+	where+		encrypt = [ Params "--encrypt" ]+		recipients l = +			-- Force gpg to only encrypt to the specified+			-- recipients, not configured defaults.+			[ Params "--no-encrypt-to --no-default-recipient"] +++			(concat $ map (\k -> [Param "--recipient", Param k]) l)++{- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}+decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher+decryptCipher _ (EncryptedCipher encipher _) = +	return . Cipher =<< gpgPipeStrict decrypt encipher+	where+		decrypt = [ Param "--decrypt" ]++{- Generates an encrypted form of a Key. The encryption does not need to be+ - reversable, nor does it need to be the same type of encryption used+ - on content. It does need to be repeatable. -}+encryptKey :: Cipher -> Key -> IO Key+encryptKey c k =+	return Key {+		keyName = hmacWithCipher c (show k),+		keyBackendName = "GPGHMACSHA1",+		keySize = Nothing, -- size and mtime omitted+		keyMtime = Nothing -- to avoid leaking data+	}++{- Runs an action, passing it a handle from which it can + - stream encrypted content. -}+withEncryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a+withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"]++{- Runs an action, passing it a handle from which it can+ - stream decrypted content. -}+withDecryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a+withDecryptedHandle = gpgCipherHandle [Param "--decrypt"]++{- Streams encrypted content to an action. -}+withEncryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a+withEncryptedContent = pass withEncryptedHandle++{- Streams decrypted content to an action. -}+withDecryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a+withDecryptedContent = pass withDecryptedHandle++pass :: (Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a) +      -> Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a+pass to c i a = to c i $ \h -> a =<< L.hGetContents h++gpgParams :: [CommandParam] -> IO [String]+gpgParams params = do+	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous+	-- gpg output about password prompts.+	e <- catch (getEnv "GPG_AGENT_INFO") (const $ return "")+	let batch = if null e then [] else ["--batch"]+	return $ batch ++ defaults ++ toCommand params+	where+		-- be quiet, even about checking the trustdb+		defaults = ["--quiet", "--trust-model", "always"]++gpgRead :: [CommandParam] -> IO String+gpgRead params = do+	params' <- gpgParams params+	pOpen ReadFromPipe "gpg" params' hGetContentsStrict++gpgPipeStrict :: [CommandParam] -> String -> IO String+gpgPipeStrict params input = do+	params' <- gpgParams params+	(pid, fromh, toh) <- hPipeBoth "gpg" params'+	_ <- forkIO $ finally (hPutStr toh input) (hClose toh)+	output <- hGetContentsStrict fromh+	forceSuccess pid+	return output++{- Runs gpg with a cipher and some parameters, feeding it an input,+ - and passing a handle to its output to an action.+ -+ - Note that to avoid deadlock with the cleanup stage,+ - the action must fully consume gpg's input before returning. -}+gpgCipherHandle :: [CommandParam] -> Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a+gpgCipherHandle params c a b = do+	-- pipe the passphrase into gpg on a fd+	(frompipe, topipe) <- createPipe+	_ <- forkIO $ do+		toh <- fdToHandle topipe+		hPutStrLn toh $ cipherPassphrase c+		hClose toh+	let Fd passphrasefd = frompipe+	let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd]++	params' <- gpgParams $ passphrase ++ params+	(pid, fromh, toh) <- hPipeBoth "gpg" params'+	_ <- forkProcess $ do+		L.hPut toh =<< a+		hClose toh+		exitSuccess+	hClose toh+	ret <- b fromh++	-- cleanup+	forceSuccess pid+	closeFd frompipe+	return ret++configKeyIds :: RemoteConfig -> IO KeyIds+configKeyIds c = do+	let k = configGet c "encryption"+	s <- gpgRead [Params "--with-colons --list-public-keys", Param k]+	return $ KeyIds $ parseWithColons s+	where+		parseWithColons s = map keyIdField $ filter pubKey $ lines s+		pubKey = isPrefixOf "pub:"+		keyIdField s = (split ":" s) !! 4++configGet :: RemoteConfig -> String -> String+configGet c key = maybe missing id $ M.lookup key c+	where missing = error $ "missing " ++ key ++ " in remote config"++hmacWithCipher :: Cipher -> String -> String+hmacWithCipher c = hmacWithCipher' (cipherHmac c) +hmacWithCipher' :: String -> String -> String+hmacWithCipher' c s = showDigest $ hmacSha1 (fromString c) (fromString s)++{- Ensure that hmacWithCipher' returns the same thing forevermore. -}+prop_hmacWithCipher_sane :: Bool+prop_hmacWithCipher_sane = known_good == hmacWithCipher' "foo" "bar"+	where+		known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51"
+ DataUnits.hs view
@@ -0,0 +1,161 @@+{- data size display and parsing+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module DataUnits (+	dataUnits,+	storageUnits,+	memoryUnits,+	bandwidthUnits,+	oldSchoolUnits,++	roughSize,+	compareSizes,+	readSize+) where++import Data.List+import Data.Char++type ByteSize = Integer+type Name = String+type Abbrev = String+data Unit = Unit ByteSize Abbrev Name+	deriving (Ord, Show, Eq)++{- And now a rant: + -+ - In the beginning, we had powers of two, and they were good.+ -+ - Disk drive manufacturers noticed that some powers of two were+ - sorta close to some powers of ten, and that rounding down to the nearest+ - power of ten allowed them to advertise their drives were bigger. This+ - was sorta annoying.+ -+ - Then drives got big. Really, really big. This was good.+ -+ - Except that the small rounding error perpretrated by the drive+ - manufacturers suffered the fate of a small error, and became a large+ - error. This was bad.+ -+ - So, a committee was formed. And it arrived at a committee-like decision,+ - which satisfied noone, confused everyone, and made the world an uglier+ - place. As with all committees, this was meh.+ -+ - And the drive manufacturers happily continued selling drives that are+ - increasingly smaller than you'd expect, if you don't count on your+ - fingers. But that are increasingly too big for anyone to much notice.+ - This caused me to need git-annex.+ -+ - Thus, I use units here that I loathe. Because if I didn't, people would+ - be confused that their drives seem the wrong size, and other people would+ - complain at me for not being standards compliant. And we call this+ - progress?+ -}++dataUnits :: [Unit]+dataUnits = storageUnits ++ memoryUnits++{- Storage units are (stupidly) powers of ten. -}+storageUnits :: [Unit]+storageUnits =+	[ Unit (p 8) "YB" "yottabyte"+	, Unit (p 7) "ZB" "zettabyte"+	, Unit (p 6) "EB" "exabyte"+	, Unit (p 5) "PB" "petabyte"+	, Unit (p 4) "TB" "terabyte"+	, Unit (p 3) "GB" "gigabyte"+	, Unit (p 2) "MB" "megabyte"+	, Unit (p 1) "kB" "kilobyte" -- weird capitalization thanks to committe+	, Unit (p 0) "B" "byte"+	]+	where+		p :: Integer -> Integer+		p n = 1000^n++{- Memory units are (stupidly named) powers of 2. -}+memoryUnits :: [Unit]+memoryUnits =+	[ Unit (p 8) "YiB" "yobibyte"+	, Unit (p 7) "ZiB" "zebibyte"+	, Unit (p 6) "EiB" "exbibyte"+	, Unit (p 5) "PiB" "pebibyte"+	, Unit (p 4) "TiB" "tebibyte"+	, Unit (p 3) "GiB" "gigabyte"+	, Unit (p 2) "MiB" "mebibyte"+	, Unit (p 1) "KiB" "kibibyte"+	, Unit (p 0) "B" "byte"+	]+	where+		p :: Integer -> Integer+		p n = 2^(n*10)++{- Bandwidth units are only measured in bits if you're some crazy telco. -}+bandwidthUnits :: [Unit]+bandwidthUnits = error "stop trying to rip people off"++{- Do you yearn for the days when men were men and megabytes were megabytes? -}+oldSchoolUnits :: [Unit]+oldSchoolUnits = map mingle $ zip storageUnits memoryUnits+	where+		mingle (Unit _ a n, Unit s' _ _) = Unit s' a n++{- approximate display of a particular number of bytes -}+roughSize :: [Unit] -> Bool -> ByteSize -> String+roughSize units abbrev i+	| i < 0 = "-" ++ findUnit units' (negate i)+	| otherwise = findUnit units' i+	where+		units' = reverse $ sort units -- largest first++		findUnit (u@(Unit s _ _):us) i'+			| i' >= s = showUnit i' u+			| otherwise = findUnit us i'+		findUnit [] i' = showUnit i' (last units') -- bytes++		showUnit i' (Unit s a n) = let num = chop i' s in+			show num ++ " " +++			(if abbrev then a else plural num n)++		chop :: Integer -> Integer -> Integer+		chop i' d = round $ (fromInteger i' :: Double) / fromInteger d++		plural n u+			| n == 1 = u+			| otherwise = u ++ "s"++{- displays comparison of two sizes -}+compareSizes :: [Unit] -> Bool -> ByteSize -> ByteSize -> String+compareSizes units abbrev old new+	| old > new = roughSize units abbrev (old - new) ++ " smaller"+	| old < new = roughSize units abbrev (new - old) ++ " larger"+	| otherwise = "same"++{- Parses strings like "10 kilobytes" or "0.5tb". -}+readSize :: [Unit] -> String -> Maybe ByteSize+readSize units input+	| null parsednum = Nothing+	| null parsedunit = Nothing+	| otherwise = Just $ round $ number * (fromIntegral multiplier)+	where+		(number, rest) = head parsednum+		multiplier = head $ parsedunit+		unitname = takeWhile isAlpha $ dropWhile isSpace rest++		parsednum = reads input :: [(Double, String)]+		parsedunit = lookupUnit units unitname++		lookupUnit _ [] = [1] -- no unit given, assume bytes+		lookupUnit [] _ = []+		lookupUnit (Unit s a n:us) v+			| a ~~ v || n ~~ v = [s]+			| plural n ~~ v || a ~~ byteabbrev v = [s]+			| otherwise = lookupUnit us v+		+		a ~~ b = map toLower a == map toLower b+		+		plural n = n ++ "s"+		byteabbrev a = a ++ "b"
+ Dot.hs view
@@ -0,0 +1,63 @@+{- a simple graphviz / dot(1) digraph description generator library+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Dot where -- import qualified++{- generates a graph description from a list of lines -}+graph :: [String] -> String+graph s = unlines $ [header] ++ map indent s ++ [footer]+	where+		header = "digraph map {"+		footer= "}"++{- a node in the graph -}+graphNode :: String -> String -> String+graphNode nodeid desc = label desc $ quote nodeid++{- an edge between two nodes -}+graphEdge :: String -> String -> Maybe String -> String+graphEdge fromid toid desc = indent $ maybe edge (\d -> label d edge) desc+	where+		edge = quote fromid ++ " -> " ++ quote toid++{- adds a label to a node or edge -}+label :: String -> String -> String+label l s = attr "label" l s++{- adds an attribute to a node or edge+ - (can be called multiple times for multiple attributes) -}+attr :: String -> String -> String -> String+attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]"++{- fills a node with a color -}+fillColor :: String -> String -> String+fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s++{- apply to graphNode to put the node in a labeled box -}+subGraph :: String -> String -> String -> String -> String+subGraph subid l color s =+	"subgraph " ++ name ++ " {\n" +++		ii setlabel +++		ii setfilled +++		ii setcolor +++		ii s +++		indent "}"+	where+		-- the "cluster_" makes dot draw a box+		name = quote ("cluster_" ++ subid)+		setlabel = "label=" ++ quote l+		setfilled = "style=" ++ quote "filled"+		setcolor = "fillcolor=" ++ quote color+		ii x = (indent $ indent x) ++ "\n"++indent ::String -> String+indent s = "\t" ++ s++quote :: String -> String+quote s = "\"" ++ s' ++ "\""+	where+		s' = filter (/= '"') s
+ GPL view
@@ -0,0 +1,674 @@+                    GNU GENERAL PUBLIC LICENSE+                       Version 3, 29 June 2007++ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>+ Everyone is permitted to copy and distribute verbatim copies+ of this license document, but changing it is not allowed.++                            Preamble++  The GNU General Public License is a free, copyleft license for+software and other kinds of works.++  The licenses for most software and other practical works are designed+to take away your freedom to share and change the works.  By contrast,+the GNU General Public License is intended to guarantee your freedom to+share and change all versions of a program--to make sure it remains free+software for all its users.  We, the Free Software Foundation, use the+GNU General Public License for most of our software; it applies also to+any other work released this way by its authors.  You can apply it to+your programs, too.++  When we speak of free software, we are referring to freedom, not+price.  Our General Public Licenses are designed to make sure that you+have the freedom to distribute copies of free software (and charge for+them if you wish), that you receive source code or can get it if you+want it, that you can change the software or use pieces of it in new+free programs, and that you know you can do these things.++  To protect your rights, we need to prevent others from denying you+these rights or asking you to surrender the rights.  Therefore, you have+certain responsibilities if you distribute copies of the software, or if+you modify it: responsibilities to respect the freedom of others.++  For example, if you distribute copies of such a program, whether+gratis or for a fee, you must pass on to the recipients the same+freedoms that you received.  You must make sure that they, too, receive+or can get the source code.  And you must show them these terms so they+know their rights.++  Developers that use the GNU GPL protect your rights with two steps:+(1) assert copyright on the software, and (2) offer you this License+giving you legal permission to copy, distribute and/or modify it.++  For the developers' and authors' protection, the GPL clearly explains+that there is no warranty for this free software.  For both users' and+authors' sake, the GPL requires that modified versions be marked as+changed, so that their problems will not be attributed erroneously to+authors of previous versions.++  Some devices are designed to deny users access to install or run+modified versions of the software inside them, although the manufacturer+can do so.  This is fundamentally incompatible with the aim of+protecting users' freedom to change the software.  The systematic+pattern of such abuse occurs in the area of products for individuals to+use, which is precisely where it is most unacceptable.  Therefore, we+have designed this version of the GPL to prohibit the practice for those+products.  If such problems arise substantially in other domains, we+stand ready to extend this provision to those domains in future versions+of the GPL, as needed to protect the freedom of users.++  Finally, every program is threatened constantly by software patents.+States should not allow patents to restrict development and use of+software on general-purpose computers, but in those that do, we wish to+avoid the special danger that patents applied to a free program could+make it effectively proprietary.  To prevent this, the GPL assures that+patents cannot be used to render the program non-free.++  The precise terms and conditions for copying, distribution and+modification follow.++                       TERMS AND CONDITIONS++  0. Definitions.++  "This License" refers to version 3 of the GNU General Public License.++  "Copyright" also means copyright-like laws that apply to other kinds of+works, such as semiconductor masks.++  "The Program" refers to any copyrightable work licensed under this+License.  Each licensee is addressed as "you".  "Licensees" and+"recipients" may be individuals or organizations.++  To "modify" a work means to copy from or adapt all or part of the work+in a fashion requiring copyright permission, other than the making of an+exact copy.  The resulting work is called a "modified version" of the+earlier work or a work "based on" the earlier work.++  A "covered work" means either the unmodified Program or a work based+on the Program.++  To "propagate" a work means to do anything with it that, without+permission, would make you directly or secondarily liable for+infringement under applicable copyright law, except executing it on a+computer or modifying a private copy.  Propagation includes copying,+distribution (with or without modification), making available to the+public, and in some countries other activities as well.++  To "convey" a work means any kind of propagation that enables other+parties to make or receive copies.  Mere interaction with a user through+a computer network, with no transfer of a copy, is not conveying.++  An interactive user interface displays "Appropriate Legal Notices"+to the extent that it includes a convenient and prominently visible+feature that (1) displays an appropriate copyright notice, and (2)+tells the user that there is no warranty for the work (except to the+extent that warranties are provided), that licensees may convey the+work under this License, and how to view a copy of this License.  If+the interface presents a list of user commands or options, such as a+menu, a prominent item in the list meets this criterion.++  1. Source Code.++  The "source code" for a work means the preferred form of the work+for making modifications to it.  "Object code" means any non-source+form of a work.++  A "Standard Interface" means an interface that either is an official+standard defined by a recognized standards body, or, in the case of+interfaces specified for a particular programming language, one that+is widely used among developers working in that language.++  The "System Libraries" of an executable work include anything, other+than the work as a whole, that (a) is included in the normal form of+packaging a Major Component, but which is not part of that Major+Component, and (b) serves only to enable use of the work with that+Major Component, or to implement a Standard Interface for which an+implementation is available to the public in source code form.  A+"Major Component", in this context, means a major essential component+(kernel, window system, and so on) of the specific operating system+(if any) on which the executable work runs, or a compiler used to+produce the work, or an object code interpreter used to run it.++  The "Corresponding Source" for a work in object code form means all+the source code needed to generate, install, and (for an executable+work) run the object code and to modify the work, including scripts to+control those activities.  However, it does not include the work's+System Libraries, or general-purpose tools or generally available free+programs which are used unmodified in performing those activities but+which are not part of the work.  For example, Corresponding Source+includes interface definition files associated with source files for+the work, and the source code for shared libraries and dynamically+linked subprograms that the work is specifically designed to require,+such as by intimate data communication or control flow between those+subprograms and other parts of the work.++  The Corresponding Source need not include anything that users+can regenerate automatically from other parts of the Corresponding+Source.++  The Corresponding Source for a work in source code form is that+same work.++  2. Basic Permissions.++  All rights granted under this License are granted for the term of+copyright on the Program, and are irrevocable provided the stated+conditions are met.  This License explicitly affirms your unlimited+permission to run the unmodified Program.  The output from running a+covered work is covered by this License only if the output, given its+content, constitutes a covered work.  This License acknowledges your+rights of fair use or other equivalent, as provided by copyright law.++  You may make, run and propagate covered works that you do not+convey, without conditions so long as your license otherwise remains+in force.  You may convey covered works to others for the sole purpose+of having them make modifications exclusively for you, or provide you+with facilities for running those works, provided that you comply with+the terms of this License in conveying all material for which you do+not control copyright.  Those thus making or running the covered works+for you must do so exclusively on your behalf, under your direction+and control, on terms that prohibit them from making any copies of+your copyrighted material outside their relationship with you.++  Conveying under any other circumstances is permitted solely under+the conditions stated below.  Sublicensing is not allowed; section 10+makes it unnecessary.++  3. Protecting Users' Legal Rights From Anti-Circumvention Law.++  No covered work shall be deemed part of an effective technological+measure under any applicable law fulfilling obligations under article+11 of the WIPO copyright treaty adopted on 20 December 1996, or+similar laws prohibiting or restricting circumvention of such+measures.++  When you convey a covered work, you waive any legal power to forbid+circumvention of technological measures to the extent such circumvention+is effected by exercising rights under this License with respect to+the covered work, and you disclaim any intention to limit operation or+modification of the work as a means of enforcing, against the work's+users, your or third parties' legal rights to forbid circumvention of+technological measures.++  4. Conveying Verbatim Copies.++  You may convey verbatim copies of the Program's source code as you+receive it, in any medium, provided that you conspicuously and+appropriately publish on each copy an appropriate copyright notice;+keep intact all notices stating that this License and any+non-permissive terms added in accord with section 7 apply to the code;+keep intact all notices of the absence of any warranty; and give all+recipients a copy of this License along with the Program.++  You may charge any price or no price for each copy that you convey,+and you may offer support or warranty protection for a fee.++  5. Conveying Modified Source Versions.++  You may convey a work based on the Program, or the modifications to+produce it from the Program, in the form of source code under the+terms of section 4, provided that you also meet all of these conditions:++    a) The work must carry prominent notices stating that you modified+    it, and giving a relevant date.++    b) The work must carry prominent notices stating that it is+    released under this License and any conditions added under section+    7.  This requirement modifies the requirement in section 4 to+    "keep intact all notices".++    c) You must license the entire work, as a whole, under this+    License to anyone who comes into possession of a copy.  This+    License will therefore apply, along with any applicable section 7+    additional terms, to the whole of the work, and all its parts,+    regardless of how they are packaged.  This License gives no+    permission to license the work in any other way, but it does not+    invalidate such permission if you have separately received it.++    d) If the work has interactive user interfaces, each must display+    Appropriate Legal Notices; however, if the Program has interactive+    interfaces that do not display Appropriate Legal Notices, your+    work need not make them do so.++  A compilation of a covered work with other separate and independent+works, which are not by their nature extensions of the covered work,+and which are not combined with it such as to form a larger program,+in or on a volume of a storage or distribution medium, is called an+"aggregate" if the compilation and its resulting copyright are not+used to limit the access or legal rights of the compilation's users+beyond what the individual works permit.  Inclusion of a covered work+in an aggregate does not cause this License to apply to the other+parts of the aggregate.++  6. Conveying Non-Source Forms.++  You may convey a covered work in object code form under the terms+of sections 4 and 5, provided that you also convey the+machine-readable Corresponding Source under the terms of this License,+in one of these ways:++    a) Convey the object code in, or embodied in, a physical product+    (including a physical distribution medium), accompanied by the+    Corresponding Source fixed on a durable physical medium+    customarily used for software interchange.++    b) Convey the object code in, or embodied in, a physical product+    (including a physical distribution medium), accompanied by a+    written offer, valid for at least three years and valid for as+    long as you offer spare parts or customer support for that product+    model, to give anyone who possesses the object code either (1) a+    copy of the Corresponding Source for all the software in the+    product that is covered by this License, on a durable physical+    medium customarily used for software interchange, for a price no+    more than your reasonable cost of physically performing this+    conveying of source, or (2) access to copy the+    Corresponding Source from a network server at no charge.++    c) Convey individual copies of the object code with a copy of the+    written offer to provide the Corresponding Source.  This+    alternative is allowed only occasionally and noncommercially, and+    only if you received the object code with such an offer, in accord+    with subsection 6b.++    d) Convey the object code by offering access from a designated+    place (gratis or for a charge), and offer equivalent access to the+    Corresponding Source in the same way through the same place at no+    further charge.  You need not require recipients to copy the+    Corresponding Source along with the object code.  If the place to+    copy the object code is a network server, the Corresponding Source+    may be on a different server (operated by you or a third party)+    that supports equivalent copying facilities, provided you maintain+    clear directions next to the object code saying where to find the+    Corresponding Source.  Regardless of what server hosts the+    Corresponding Source, you remain obligated to ensure that it is+    available for as long as needed to satisfy these requirements.++    e) Convey the object code using peer-to-peer transmission, provided+    you inform other peers where the object code and Corresponding+    Source of the work are being offered to the general public at no+    charge under subsection 6d.++  A separable portion of the object code, whose source code is excluded+from the Corresponding Source as a System Library, need not be+included in conveying the object code work.++  A "User Product" is either (1) a "consumer product", which means any+tangible personal property which is normally used for personal, family,+or household purposes, or (2) anything designed or sold for incorporation+into a dwelling.  In determining whether a product is a consumer product,+doubtful cases shall be resolved in favor of coverage.  For a particular+product received by a particular user, "normally used" refers to a+typical or common use of that class of product, regardless of the status+of the particular user or of the way in which the particular user+actually uses, or expects or is expected to use, the product.  A product+is a consumer product regardless of whether the product has substantial+commercial, industrial or non-consumer uses, unless such uses represent+the only significant mode of use of the product.++  "Installation Information" for a User Product means any methods,+procedures, authorization keys, or other information required to install+and execute modified versions of a covered work in that User Product from+a modified version of its Corresponding Source.  The information must+suffice to ensure that the continued functioning of the modified object+code is in no case prevented or interfered with solely because+modification has been made.++  If you convey an object code work under this section in, or with, or+specifically for use in, a User Product, and the conveying occurs as+part of a transaction in which the right of possession and use of the+User Product is transferred to the recipient in perpetuity or for a+fixed term (regardless of how the transaction is characterized), the+Corresponding Source conveyed under this section must be accompanied+by the Installation Information.  But this requirement does not apply+if neither you nor any third party retains the ability to install+modified object code on the User Product (for example, the work has+been installed in ROM).++  The requirement to provide Installation Information does not include a+requirement to continue to provide support service, warranty, or updates+for a work that has been modified or installed by the recipient, or for+the User Product in which it has been modified or installed.  Access to a+network may be denied when the modification itself materially and+adversely affects the operation of the network or violates the rules and+protocols for communication across the network.++  Corresponding Source conveyed, and Installation Information provided,+in accord with this section must be in a format that is publicly+documented (and with an implementation available to the public in+source code form), and must require no special password or key for+unpacking, reading or copying.++  7. Additional Terms.++  "Additional permissions" are terms that supplement the terms of this+License by making exceptions from one or more of its conditions.+Additional permissions that are applicable to the entire Program shall+be treated as though they were included in this License, to the extent+that they are valid under applicable law.  If additional permissions+apply only to part of the Program, that part may be used separately+under those permissions, but the entire Program remains governed by+this License without regard to the additional permissions.++  When you convey a copy of a covered work, you may at your option+remove any additional permissions from that copy, or from any part of+it.  (Additional permissions may be written to require their own+removal in certain cases when you modify the work.)  You may place+additional permissions on material, added by you to a covered work,+for which you have or can give appropriate copyright permission.++  Notwithstanding any other provision of this License, for material you+add to a covered work, you may (if authorized by the copyright holders of+that material) supplement the terms of this License with terms:++    a) Disclaiming warranty or limiting liability differently from the+    terms of sections 15 and 16 of this License; or++    b) Requiring preservation of specified reasonable legal notices or+    author attributions in that material or in the Appropriate Legal+    Notices displayed by works containing it; or++    c) Prohibiting misrepresentation of the origin of that material, or+    requiring that modified versions of such material be marked in+    reasonable ways as different from the original version; or++    d) Limiting the use for publicity purposes of names of licensors or+    authors of the material; or++    e) Declining to grant rights under trademark law for use of some+    trade names, trademarks, or service marks; or++    f) Requiring indemnification of licensors and authors of that+    material by anyone who conveys the material (or modified versions of+    it) with contractual assumptions of liability to the recipient, for+    any liability that these contractual assumptions directly impose on+    those licensors and authors.++  All other non-permissive additional terms are considered "further+restrictions" within the meaning of section 10.  If the Program as you+received it, or any part of it, contains a notice stating that it is+governed by this License along with a term that is a further+restriction, you may remove that term.  If a license document contains+a further restriction but permits relicensing or conveying under this+License, you may add to a covered work material governed by the terms+of that license document, provided that the further restriction does+not survive such relicensing or conveying.++  If you add terms to a covered work in accord with this section, you+must place, in the relevant source files, a statement of the+additional terms that apply to those files, or a notice indicating+where to find the applicable terms.++  Additional terms, permissive or non-permissive, may be stated in the+form of a separately written license, or stated as exceptions;+the above requirements apply either way.++  8. Termination.++  You may not propagate or modify a covered work except as expressly+provided under this License.  Any attempt otherwise to propagate or+modify it is void, and will automatically terminate your rights under+this License (including any patent licenses granted under the third+paragraph of section 11).++  However, if you cease all violation of this License, then your+license from a particular copyright holder is reinstated (a)+provisionally, unless and until the copyright holder explicitly and+finally terminates your license, and (b) permanently, if the copyright+holder fails to notify you of the violation by some reasonable means+prior to 60 days after the cessation.++  Moreover, your license from a particular copyright holder is+reinstated permanently if the copyright holder notifies you of the+violation by some reasonable means, this is the first time you have+received notice of violation of this License (for any work) from that+copyright holder, and you cure the violation prior to 30 days after+your receipt of the notice.++  Termination of your rights under this section does not terminate the+licenses of parties who have received copies or rights from you under+this License.  If your rights have been terminated and not permanently+reinstated, you do not qualify to receive new licenses for the same+material under section 10.++  9. Acceptance Not Required for Having Copies.++  You are not required to accept this License in order to receive or+run a copy of the Program.  Ancillary propagation of a covered work+occurring solely as a consequence of using peer-to-peer transmission+to receive a copy likewise does not require acceptance.  However,+nothing other than this License grants you permission to propagate or+modify any covered work.  These actions infringe copyright if you do+not accept this License.  Therefore, by modifying or propagating a+covered work, you indicate your acceptance of this License to do so.++  10. Automatic Licensing of Downstream Recipients.++  Each time you convey a covered work, the recipient automatically+receives a license from the original licensors, to run, modify and+propagate that work, subject to this License.  You are not responsible+for enforcing compliance by third parties with this License.++  An "entity transaction" is a transaction transferring control of an+organization, or substantially all assets of one, or subdividing an+organization, or merging organizations.  If propagation of a covered+work results from an entity transaction, each party to that+transaction who receives a copy of the work also receives whatever+licenses to the work the party's predecessor in interest had or could+give under the previous paragraph, plus a right to possession of the+Corresponding Source of the work from the predecessor in interest, if+the predecessor has it or can get it with reasonable efforts.++  You may not impose any further restrictions on the exercise of the+rights granted or affirmed under this License.  For example, you may+not impose a license fee, royalty, or other charge for exercise of+rights granted under this License, and you may not initiate litigation+(including a cross-claim or counterclaim in a lawsuit) alleging that+any patent claim is infringed by making, using, selling, offering for+sale, or importing the Program or any portion of it.++  11. Patents.++  A "contributor" is a copyright holder who authorizes use under this+License of the Program or a work on which the Program is based.  The+work thus licensed is called the contributor's "contributor version".++  A contributor's "essential patent claims" are all patent claims+owned or controlled by the contributor, whether already acquired or+hereafter acquired, that would be infringed by some manner, permitted+by this License, of making, using, or selling its contributor version,+but do not include claims that would be infringed only as a+consequence of further modification of the contributor version.  For+purposes of this definition, "control" includes the right to grant+patent sublicenses in a manner consistent with the requirements of+this License.++  Each contributor grants you a non-exclusive, worldwide, royalty-free+patent license under the contributor's essential patent claims, to+make, use, sell, offer for sale, import and otherwise run, modify and+propagate the contents of its contributor version.++  In the following three paragraphs, a "patent license" is any express+agreement or commitment, however denominated, not to enforce a patent+(such as an express permission to practice a patent or covenant not to+sue for patent infringement).  To "grant" such a patent license to a+party means to make such an agreement or commitment not to enforce a+patent against the party.++  If you convey a covered work, knowingly relying on a patent license,+and the Corresponding Source of the work is not available for anyone+to copy, free of charge and under the terms of this License, through a+publicly available network server or other readily accessible means,+then you must either (1) cause the Corresponding Source to be so+available, or (2) arrange to deprive yourself of the benefit of the+patent license for this particular work, or (3) arrange, in a manner+consistent with the requirements of this License, to extend the patent+license to downstream recipients.  "Knowingly relying" means you have+actual knowledge that, but for the patent license, your conveying the+covered work in a country, or your recipient's use of the covered work+in a country, would infringe one or more identifiable patents in that+country that you have reason to believe are valid.++  If, pursuant to or in connection with a single transaction or+arrangement, you convey, or propagate by procuring conveyance of, a+covered work, and grant a patent license to some of the parties+receiving the covered work authorizing them to use, propagate, modify+or convey a specific copy of the covered work, then the patent license+you grant is automatically extended to all recipients of the covered+work and works based on it.++  A patent license is "discriminatory" if it does not include within+the scope of its coverage, prohibits the exercise of, or is+conditioned on the non-exercise of one or more of the rights that are+specifically granted under this License.  You may not convey a covered+work if you are a party to an arrangement with a third party that is+in the business of distributing software, under which you make payment+to the third party based on the extent of your activity of conveying+the work, and under which the third party grants, to any of the+parties who would receive the covered work from you, a discriminatory+patent license (a) in connection with copies of the covered work+conveyed by you (or copies made from those copies), or (b) primarily+for and in connection with specific products or compilations that+contain the covered work, unless you entered into that arrangement,+or that patent license was granted, prior to 28 March 2007.++  Nothing in this License shall be construed as excluding or limiting+any implied license or other defenses to infringement that may+otherwise be available to you under applicable patent law.++  12. No Surrender of Others' Freedom.++  If conditions are imposed on you (whether by court order, agreement or+otherwise) that contradict the conditions of this License, they do not+excuse you from the conditions of this License.  If you cannot convey a+covered work so as to satisfy simultaneously your obligations under this+License and any other pertinent obligations, then as a consequence you may+not convey it at all.  For example, if you agree to terms that obligate you+to collect a royalty for further conveying from those to whom you convey+the Program, the only way you could satisfy both those terms and this+License would be to refrain entirely from conveying the Program.++  13. Use with the GNU Affero General Public License.++  Notwithstanding any other provision of this License, you have+permission to link or combine any covered work with a work licensed+under version 3 of the GNU Affero General Public License into a single+combined work, and to convey the resulting work.  The terms of this+License will continue to apply to the part which is the covered work,+but the special requirements of the GNU Affero General Public License,+section 13, concerning interaction through a network will apply to the+combination as such.++  14. Revised Versions of this License.++  The Free Software Foundation may publish revised and/or new versions of+the GNU General Public License from time to time.  Such new versions will+be similar in spirit to the present version, but may differ in detail to+address new problems or concerns.++  Each version is given a distinguishing version number.  If the+Program specifies that a certain numbered version of the GNU General+Public License "or any later version" applies to it, you have the+option of following the terms and conditions either of that numbered+version or of any later version published by the Free Software+Foundation.  If the Program does not specify a version number of the+GNU General Public License, you may choose any version ever published+by the Free Software Foundation.++  If the Program specifies that a proxy can decide which future+versions of the GNU General Public License can be used, that proxy's+public statement of acceptance of a version permanently authorizes you+to choose that version for the Program.++  Later license versions may give you additional or different+permissions.  However, no additional obligations are imposed on any+author or copyright holder as a result of your choosing to follow a+later version.++  15. Disclaimer of Warranty.++  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.++  16. Limitation of Liability.++  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF+SUCH DAMAGES.++  17. Interpretation of Sections 15 and 16.++  If the disclaimer of warranty and limitation of liability provided+above cannot be given local legal effect according to their terms,+reviewing courts shall apply local law that most closely approximates+an absolute waiver of all civil liability in connection with the+Program, unless a warranty or assumption of liability accompanies a+copy of the Program in return for a fee.++                     END OF TERMS AND CONDITIONS++            How to Apply These Terms to Your New Programs++  If you develop a new program, and you want it to be of the greatest+possible use to the public, the best way to achieve this is to make it+free software which everyone can redistribute and change under these terms.++  To do so, attach the following notices to the program.  It is safest+to attach them to the start of each source file to most effectively+state the exclusion of warranty; and each file should have at least+the "copyright" line and a pointer to where the full notice is found.++    <one line to give the program's name and a brief idea of what it does.>+    Copyright (C) <year>  <name of author>++    This program is free software: you can redistribute it and/or modify+    it under the terms of the GNU General Public License as published by+    the Free Software Foundation, either version 3 of the License, or+    (at your option) any later version.++    This program is distributed in the hope that it will be useful,+    but WITHOUT ANY WARRANTY; without even the implied warranty of+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the+    GNU General Public License for more details.++    You should have received a copy of the GNU General Public License+    along with this program.  If not, see <http://www.gnu.org/licenses/>.++Also add information on how to contact you by electronic and paper mail.++  If the program does terminal interaction, make it output a short+notice like this when it starts in an interactive mode:++    <program>  Copyright (C) <year>  <name of author>+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.+    This is free software, and you are welcome to redistribute it+    under certain conditions; type `show c' for details.++The hypothetical commands `show w' and `show c' should show the appropriate+parts of the General Public License.  Of course, your program's commands+might be different; for a GUI interface, you would use an "about box".++  You should also get your employer (if you work as a programmer) or school,+if any, to sign a "copyright disclaimer" for the program, if necessary.+For more information on this, and how to apply and follow the GNU GPL, see+<http://www.gnu.org/licenses/>.++  The GNU General Public License does not permit incorporating your program+into proprietary programs.  If your program is a subroutine library, you+may consider it more useful to permit linking proprietary applications with+the library.  If this is what you want to do, use the GNU Lesser General+Public License instead of this License.  But first, please read+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+ Git.hs view
@@ -0,0 +1,695 @@+{- git repository handling + -+ - This is written to be completely independant of git-annex and should be+ - suitable for other uses.+ -+ - Copyright 2010,2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Git (+	Repo,+	repoFromCwd,+	repoFromAbsPath,+	repoFromUnknown,+	repoFromUrl,+	localToUrl,+	repoIsUrl,+	repoIsSsh,+	repoIsLocalBare,+	repoDescribe,+	repoLocation,+	workTree,+	workTreeFile,+	gitDir,+	urlPath,+	urlHost,+	urlPort,+	urlHostUser,+	urlAuthority,+	urlScheme,+	configGet,+	configMap,+	configRead,+	hConfigRead,+	configStore,+	configTrue,+	gitCommandLine,+	run,+	runBool,+	pipeRead,+	pipeWrite,+	pipeWriteRead,+	pipeNullSplit,+	attributes,+	remotes,+	remotesAdd,+	repoRemoteName,+	repoRemoteNameSet,+	checkAttr,+	decodeGitFile,+	encodeGitFile,+	repoAbsPath,+	reap,+	useIndex,+	hashObject,+	getSha,+	shaSize,+	commit,++	prop_idempotent_deencode+) where++import Control.Monad (unless, when)+import System.Directory+import System.FilePath+import System.Posix.Directory+import System.Posix.User+import System.Posix.Process+import System.Path+import System.Cmd.Utils+import IO (bracket_)+import Data.String.Utils+import System.IO+import IO (try)+import qualified Data.Map as Map hiding (map, split)+import Network.URI+import Data.Maybe+import Data.Char+import Data.Word (Word8)+import Codec.Binary.UTF8.String (encode)+import Text.Printf+import Data.List (isInfixOf, isPrefixOf, isSuffixOf)+import System.Exit+import System.Posix.Env (setEnv, unsetEnv, getEnv)++import Utility++{- There are two types of repositories; those on local disk and those+ - accessed via an URL. -}+data RepoLocation = Dir FilePath | Url URI | Unknown+	deriving (Show, Eq)++data Repo = Repo {+	location :: RepoLocation,+	config :: Map.Map String String,+	remotes :: [Repo],+	-- remoteName holds the name used for this repo in remotes+	remoteName :: Maybe String +} deriving (Show, Eq)++newFrom :: RepoLocation -> Repo+newFrom l = +	Repo {+		location = l,+		config = Map.empty,+		remotes = [],+		remoteName = Nothing+	}++{- Local Repo constructor, requires an absolute path to the repo be+ - specified. -}+repoFromAbsPath :: FilePath -> IO Repo+repoFromAbsPath dir+	| "/" `isPrefixOf` dir = do+ 		-- Git always looks for "dir.git" in preference to+		-- to "dir", even if dir ends in a "/".+		let canondir = dropTrailingPathSeparator dir+		let dir' = canondir ++ ".git"+		e <- doesDirectoryExist dir'+		if e+			then ret dir'+			else if "/.git" `isSuffixOf` canondir+				then do+					-- When dir == "foo/.git", git looks+					-- for "foo/.git/.git", and failing+					-- that, uses "foo" as the repository.+					e' <- doesDirectoryExist $ dir </> ".git"+					if e'+						then ret dir+						else ret $ takeDirectory canondir+				else ret dir+	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"+	where+		ret = return . newFrom . Dir++{- Remote Repo constructor. Throws exception on invalid url. -}+repoFromUrl :: String -> IO Repo+repoFromUrl url+	| startswith "file://" url = repoFromAbsPath $ uriPath u+	| otherwise = return $ newFrom $ Url u+		where+			u = maybe bad id $ parseURI url+			bad = error $ "bad url " ++ url++{- Creates a repo that has an unknown location. -}+repoFromUnknown :: Repo+repoFromUnknown = newFrom Unknown++{- Converts a Local Repo into a remote repo, using the reference repo+ - which is assumed to be on the same host. -}+localToUrl :: Repo -> Repo -> Repo+localToUrl reference r+	| not $ repoIsUrl reference = error "internal error; reference repo not url"+	| repoIsUrl r = r+	| otherwise = r { location = Url $ fromJust $ parseURI absurl }+	where+		absurl =+			urlScheme reference ++ "//" +++			urlAuthority reference +++			workTree r++{- User-visible description of a git repo. -}+repoDescribe :: Repo -> String+repoDescribe Repo { remoteName = Just name } = name+repoDescribe Repo { location = Url url } = show url+repoDescribe Repo { location = Dir dir } = dir+repoDescribe Repo { location = Unknown } = "UNKNOWN"++{- Location of the repo, either as a path or url. -}+repoLocation :: Repo -> String+repoLocation Repo { location = Url url } = show url+repoLocation Repo { location = Dir dir } = dir+repoLocation Repo { location = Unknown } = undefined++{- Constructs and returns an updated version of a repo with+ - different remotes list. -}+remotesAdd :: Repo -> [Repo] -> Repo+remotesAdd repo rs = repo { remotes = rs }++{- Returns the name of the remote that corresponds to the repo, if+ - it is a remote. -}+repoRemoteName :: Repo -> Maybe String+repoRemoteName Repo { remoteName = Just name } = Just name+repoRemoteName _ = Nothing++{- Sets the name of a remote based on the git config key, such as+   "remote.foo.url". -}+repoRemoteNameSet :: Repo -> String -> Repo+repoRemoteNameSet r k = r { remoteName = Just basename }+	where+		basename = join "." $ reverse $ drop 1 $+				reverse $ drop 1 $ split "." k++{- Some code needs to vary between URL and normal repos,+ - or bare and non-bare, these functions help with that. -}+repoIsUrl :: Repo -> Bool+repoIsUrl Repo { location = Url _ } = True+repoIsUrl _ = False++repoIsSsh :: Repo -> Bool+repoIsSsh Repo { location = Url url } +	| uriScheme url == "ssh:" = True+	-- git treats these the same as ssh+	| uriScheme url == "git+ssh:" = True+	| uriScheme url == "ssh+git:" = True+	| otherwise = False+repoIsSsh _ = False++configAvail ::Repo -> Bool+configAvail Repo { config = c } = c /= Map.empty++repoIsLocalBare :: Repo -> Bool+repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r+repoIsLocalBare _ = False++assertLocal :: Repo -> a -> a+assertLocal repo action = +	if not $ repoIsUrl repo+		then action+		else error $ "acting on URL git repo " ++  repoDescribe repo ++ +				" not supported"+assertUrl :: Repo -> a -> a+assertUrl repo action = +	if repoIsUrl repo+		then action+		else error $ "acting on local git repo " ++  repoDescribe repo ++ +				" not supported"++configBare :: Repo -> Bool+configBare repo = maybe unknown configTrue $ Map.lookup "core.bare" $ config repo+	where+		unknown = error $ "it is not known if git repo " +++			repoDescribe repo +++			" is a bare repository; config not read"++{- Path to a repository's gitattributes file. -}+attributes :: Repo -> String+attributes repo+	| configBare repo = workTree repo ++ "/info/.gitattributes"+	| otherwise = workTree repo ++ "/.gitattributes"++{- Path to a repository's .git directory, relative to its workTree. -}+gitDir :: Repo -> String+gitDir repo+	| configBare repo = ""+	| otherwise = ".git"++{- Path to a repository's --work-tree, that is, its top.+ -+ - Note that for URL repositories, this is the path on the remote host. -}+workTree :: Repo -> FilePath+workTree r@(Repo { location = Url _ }) = urlPath r+workTree (Repo { location = Dir d }) = d+workTree Repo { location = Unknown } = undefined++{- Given a relative or absolute filename inside a git repository's+ - workTree, calculates the name to use to refer to that file to git.+ -+ - This is complicated because the best choice can vary depending on+ - whether the cwd is in a subdirectory of the git repository, or not.+ -+ - For example, when adding a file "/tmp/repo/foo", it's best to refer+ - to it as "foo" if the cwd is outside the repository entirely+ - (this avoids a gotcha with using the full path name when /tmp/repo+ - is itself a symlink). But, if the cwd is "/tmp/repo/subdir",+ - it's best to refer to "../foo".+ -}+workTreeFile :: Repo -> FilePath -> IO FilePath+workTreeFile repo@(Repo { location = Dir d }) file = do+	cwd <- getCurrentDirectory+	let file' = absfile cwd+	unless (inrepo file') $+		error $ file ++ " is not located inside git repository " ++ absrepo+	if (inrepo $ addTrailingPathSeparator cwd)+		then return $ relPathDirToFile cwd file'+		else return $ drop (length absrepo) file'+	where+		-- normalize both repo and file, so that repo+		-- will be substring of file+		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d+		absfile c = maybe file id $ secureAbsNormPath c file+		inrepo f = absrepo `isPrefixOf` f+		bad = error $ "bad repo" ++ repoDescribe repo+workTreeFile repo _ = assertLocal repo $ error "internal"++{- Path of an URL repo. -}+urlPath :: Repo -> String+urlPath Repo { location = Url u } = uriPath u+urlPath repo = assertUrl repo $ error "internal"++{- Scheme of an URL repo. -}+urlScheme :: Repo -> String+urlScheme Repo { location = Url u } = uriScheme u+urlScheme repo = assertUrl repo $ error "internal"++{- Work around a bug in the real uriRegName+ - <http://trac.haskell.org/network/ticket/40> -}+uriRegName' :: URIAuth -> String+uriRegName' a = fixup $ uriRegName a+	where+		fixup x@('[':rest)+			| rest !! len == ']' = take len rest+			| otherwise = x+			where+				len  = (length rest) - 1+		fixup x = x++{- Hostname of an URL repo. -}+urlHost :: Repo -> String+urlHost = urlAuthPart uriRegName'++{- Port of an URL repo, if it has a nonstandard one. -}+urlPort :: Repo -> Maybe Integer+urlPort r = +	case urlAuthPart uriPort r of+		":" -> Nothing+		(':':p) -> Just (read p)+		_ -> Nothing++{- Hostname of an URL repo, including any username (ie, "user@host") -}+urlHostUser :: Repo -> String+urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r++{- The full authority portion an URL repo. (ie, "user@host:port") -}+urlAuthority :: Repo -> String+urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName' a ++ uriPort a+	where+		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)+urlAuthority repo = assertUrl repo $ error "internal"++{- Applies a function to extract part of the uriAuthority of an URL repo. -}+urlAuthPart :: (URIAuth -> a) -> Repo -> a+urlAuthPart a Repo { location = Url u } = a auth+	where+		auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)+urlAuthPart _ repo = assertUrl repo $ error "internal"++{- Constructs a git command line operating on the specified repo. -}+gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]+gitCommandLine repo@(Repo { location = Dir d} ) params =+	-- force use of specified repo via --git-dir and --work-tree+	[ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo)+	, Param ("--work-tree=" ++ d)+	] ++ params+gitCommandLine repo _ = assertLocal repo $ error "internal"++{- Runs git in the specified repo. -}+runBool :: Repo -> String -> [CommandParam] -> IO Bool+runBool repo subcommand params = assertLocal repo $+	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))++{- Runs git in the specified repo, throwing an error if it fails. -}+run :: Repo -> String -> [CommandParam] -> IO ()+run repo subcommand params = assertLocal repo $+	runBool repo subcommand params+		>>! error $ "git " ++ show params ++ " failed"++{- Runs a git subcommand and returns its output, lazily. + -+ - Note that this leaves the git process running, and so zombies will+ - result unless reap is called.+ -}+pipeRead :: Repo -> [CommandParam] -> IO String+pipeRead repo params = assertLocal repo $ do+	(_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params+	return s++{- Runs a git subcommand, feeding it input.+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}+pipeWrite :: Repo -> [CommandParam] -> String -> IO PipeHandle+pipeWrite repo params s = assertLocal repo $+	pipeTo "git" (toCommand $ gitCommandLine repo params) s++{- Runs a git subcommand, feeding it input, and returning its output.+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}+pipeWriteRead :: Repo -> [CommandParam] -> String -> IO (PipeHandle, String)+pipeWriteRead repo params s = assertLocal repo $+	pipeBoth "git" (toCommand $ gitCommandLine repo params) s++{- Reaps any zombie git processes. -}+reap :: IO ()+reap = do+	-- throws an exception when there are no child processes+	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)+	maybe (return ()) (const reap) r++{- Forces git to use the specified index file.+ - Returns an action that will reset back to the default+ - index file. -}+useIndex :: FilePath -> IO (IO ())+useIndex index = do+	res <- try $ getEnv var+	setEnv var index True+	return $ reset res+	where+		var = "GIT_INDEX_FILE"+		reset (Right (Just v)) = setEnv var v True+		reset _ = unsetEnv var++{- Injects some content into git, returning its hash. -}+hashObject :: Repo -> String -> IO String+hashObject repo content = getSha subcmd $ do+	(h, s) <- pipeWriteRead repo (map Param params) content+	length s `seq` do+		forceSuccess h+		reap -- XXX unsure why this is needed+		return s+	where+		subcmd = "hash-object"+		params = [subcmd, "-w", "--stdin"]++{- Runs an action that causes a git subcommand to emit a sha, and strips+   any trailing newline, returning the sha. -}+getSha :: String -> IO String -> IO String+getSha subcommand a = do+	t <- a+	let t' = if last t == '\n'+		then take (length t - 1) t+		else t+	when (length t' /= shaSize) $+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"+	return t'++{- Size of a git sha. -}+shaSize :: Int+shaSize = 40++{- Commits the index into the specified branch, + - with the specified parent refs. -}+commit :: Repo -> String -> String -> [String] -> IO ()+commit g message newref parentrefs = do+	tree <- getSha "write-tree" $+		pipeRead g [Param "write-tree"]+	sha <- getSha "commit-tree" $ ignorehandle $+		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message+	run g "update-ref" [Param newref, Param sha]+	where+		ignorehandle a = return . snd =<< a+		ps = concatMap (\r -> ["-p", r]) parentrefs++{- Reads null terminated output of a git command (as enabled by the -z + - parameter), and splits it into a list of files/lines/whatever. -}+pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]+pipeNullSplit repo params = do+	fs0 <- pipeRead repo params+	return $ split0 fs0+	where+		split0 s = filter (not . null) $ split "\0" s++{- Runs git config and populates a repo with its config. -}+configRead :: Repo -> IO Repo+configRead repo@(Repo { location = Dir d }) = do+	{- Cannot use pipeRead because it relies on the config having+	   been already read. Instead, chdir to the repo. -}+	cwd <- getCurrentDirectory+	bracket_ (changeWorkingDirectory d)+		(\_ -> changeWorkingDirectory cwd) $+			pOpen ReadFromPipe "git" ["config", "--list"] $+				hConfigRead repo+configRead r = assertLocal r $ error "internal"++{- Reads git config from a handle and populates a repo with it. -}+hConfigRead :: Repo -> Handle -> IO Repo+hConfigRead repo h = do+	val <- hGetContentsStrict h+	configStore repo val++{- Parses a git config and returns a version of the repo using it. -}+configStore :: Repo -> String -> IO Repo+configStore repo s = do+	rs <- configRemotes r+	return $ r { remotes = rs }+	where+		r = repo { config = configParse s }++{- Calculates a list of a repo's configured remotes, by parsing its config. -}+configRemotes :: Repo -> IO [Repo]+configRemotes repo = mapM construct remotepairs+	where+		remotepairs = Map.toList $ filterremotes $ config repo+		filterremotes = Map.filterWithKey (\k _ -> isremote k)+		isremote k = startswith "remote." k && endswith ".url" k+		construct (k,v) = do+			r <- gen v+			return $ repoRemoteNameSet r k+		gen v	| scpstyle v = repoFromUrl $ scptourl v+			| isURI v = repoFromUrl v+			| otherwise = repoFromRemotePath v repo+		-- git remotes can be written scp style -- [user@]host:dir+		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)+		scptourl v = "ssh://" ++ host ++ slash dir+			where+				bits = split ":" v+				host = bits !! 0+				dir = join ":" $ drop 1 bits+				slash d	| d == "" = "/~/" ++ dir+					| d !! 0 == '/' = dir+					| d !! 0 == '~' = '/':dir+					| otherwise = "/~/" ++ dir++{- Checks if a string from git config is a true value. -}+configTrue :: String -> Bool+configTrue s = map toLower s == "true"++{- Parses git config --list output into a config map. -}+configParse :: String -> Map.Map String String+configParse s = Map.fromList $ map pair $ lines s+	where+		pair l = (key l, val l)+		key l = head $ keyval l+		val l = join sep $ drop 1 $ keyval l+		keyval l = split sep l :: [String]+		sep = "="++{- Returns a single git config setting, or a default value if not set. -}+configGet :: Repo -> String -> String -> String+configGet repo key defaultValue = +	Map.findWithDefault defaultValue key (config repo)++{- Access to raw config Map -}+configMap :: Repo -> Map.Map String String+configMap repo = config repo++{- Efficiently looks up a gitattributes value for each file in a list. -}+checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]+checkAttr repo attr files = do+	-- git check-attr wants files that are absolute (or relative to the+	-- top of the repo). But we're passed files relative to the current+	-- directory. Convert to absolute, and then convert the filenames+	-- in its output back to relative.+	cwd <- getCurrentDirectory+	let absfiles = map (absPathFrom cwd) files+	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)+        _ <- forkProcess $ do+		hClose fromh+                hPutStr toh $ join "\0" absfiles+                hClose toh+                exitSuccess+        hClose toh+	s <- hGetContents fromh+	return $ map (topair $ cwd++"/") $ lines s+	where+		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]+		topair cwd l = (relfile, value)+			where +				relfile +					| startswith cwd file = drop (length cwd) file+					| otherwise = file+				file = decodeGitFile $ join sep $ take end bits+				value = bits !! end+				end = length bits - 1+				bits = split sep l+				sep = ": " ++ attr ++ ": "++{- Some git commands output encoded filenames. Decode that (annoyingly+ - complex) encoding. -}+decodeGitFile :: String -> FilePath+decodeGitFile [] = []+decodeGitFile f@(c:s)+	-- encoded strings will be inside double quotes+	| c == '"' = unescape ("", middle)+	| otherwise = f+	where+		e = '\\'+		middle = take (length s - 1) s+		unescape (b, []) = b+		-- look for escapes starting with '\'+		unescape (b, v) = b ++ beginning ++ unescape (decode rest)+			where+				pair = span (/= e) v+				beginning = fst pair+				rest = snd pair+		isescape x = x == e+		-- \NNN is an octal encoded character+		decode (x:n1:n2:n3:rest)+			| isescape x && alloctal = (fromoctal, rest)+				where+					alloctal = isOctDigit n1 &&+						isOctDigit n2 &&+						isOctDigit n3+					fromoctal = [chr $ readoctal [n1, n2, n3]]+					readoctal o = read $ "0o" ++ o :: Int+		-- \C is used for a few special characters+		decode (x:nc:rest)+			| isescape x = ([echar nc], rest)+			where+				echar 'a' = '\a'+				echar 'b' = '\b'+				echar 'f' = '\f'+				echar 'n' = '\n'+				echar 'r' = '\r'+				echar 't' = '\t'+				echar 'v' = '\v'+				echar a = a+		decode n = ("", n)++{- Should not need to use this, except for testing decodeGitFile. -}+encodeGitFile :: FilePath -> String+encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""+	where+		e c = '\\' : [c]+		echar '\a' = e 'a'+		echar '\b' = e 'b'+		echar '\f' = e 'f'+		echar '\n' = e 'n'+		echar '\r' = e 'r'+		echar '\t' = e 't'+		echar '\v' = e 'v'+		echar '\\' = e '\\'+		echar '"'  = e '"'+		echar x+			| ord x < 0x20 = e_num x -- low ascii+			| ord x >= 256 = e_utf x+			| ord x > 0x7E = e_num x -- high ascii+			| otherwise = [x]        -- printable ascii+			where +				showoctal i = '\\' : printf "%03o" i+				e_num c = showoctal $ ord c+				-- unicode character is decomposed to+				-- Word8s and each is shown in octal+				e_utf c = showoctal =<< (encode [c] :: [Word8])++{- for quickcheck -}+prop_idempotent_deencode :: String -> Bool+prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)++{- Constructs a Repo from the path specified in the git remotes of+ - another Repo. -}+repoFromRemotePath :: FilePath -> Repo -> IO Repo+repoFromRemotePath dir repo = do+	dir' <- expandTilde dir+	repoFromAbsPath $ workTree repo </> dir'++{- Git remotes can have a directory that is specified relative+ - to the user's home directory, or that contains tilde expansions.+ - This converts such a directory to an absolute path.+ - Note that it has to run on the system where the remote is.+ -}+repoAbsPath :: FilePath -> IO FilePath+repoAbsPath d = do+	d' <- expandTilde d+	h <- myHomeDir+	return $ h </> d'++expandTilde :: FilePath -> IO FilePath+expandTilde = expandt True+	where+		expandt _ [] = return ""+		expandt _ ('/':cs) = do+			v <- expandt True cs+			return ('/':v)+		expandt True ('~':'/':cs) = do+			h <- myHomeDir+			return $ h </> cs+		expandt True ('~':cs) = do+			let (name, rest) = findname "" cs+			u <- getUserEntryForName name+			return $ homeDirectory u </> rest+		expandt _ (c:cs) = do+			v <- expandt False cs+			return (c:v)+		findname n [] = (n, "")+		findname n (c:cs)+			| c == '/' = (n, cs)+			| otherwise = findname (n++[c]) cs++{- Finds the current git repository, which may be in a parent directory. -}+repoFromCwd :: IO Repo+repoFromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo+	where+		makerepo = return . newFrom . Dir+		norepo = error "Not in a git repository."++seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)+seekUp want dir = do+	ok <- want dir+	if ok+		then return (Just dir)+		else case (parentDir dir) of+			"" -> return Nothing+			d -> seekUp want d++isRepoTop :: FilePath -> IO Bool+isRepoTop dir = do+	r <- isRepo+	b <- isBareRepo+	return (r || b)+	where+		isRepo = gitSignature ".git" ".git/config"+		isBareRepo = gitSignature "objects" "config"+		gitSignature subdir file = do+			s <- (doesDirectoryExist (dir ++ "/" ++ subdir))+			f <- (doesFileExist (dir ++ "/" ++ file))+			return (s && f)
+ Git/LsFiles.hs view
@@ -0,0 +1,68 @@+{- git ls-files interface+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Git.LsFiles (+	inRepo,+	notInRepo,+	staged,+	stagedNotDeleted,+	changedUnstaged,+	typeChanged,+	typeChangedStaged,+) where++import Git+import Utility++{- Scans for files that are checked into git at the specified locations. -}+inRepo :: Repo -> [FilePath] -> IO [FilePath]+inRepo repo l = pipeNullSplit repo $+	[Params "ls-files --cached -z --"] ++ map File l++{- Scans for files at the specified locations that are not checked into+ - git. -}+notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]+notInRepo repo include_ignored l =+	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l+	where+		exclude = if include_ignored then [] else [Param "--exclude-standard"]++{- Returns a list of all files that are staged for commit. -}+staged :: Repo -> [FilePath] -> IO [FilePath]+staged repo l = staged' repo l []++{- Returns a list of the files, staged for commit, that are being added,+ - moved, or changed (but not deleted), from the specified locations. -}+stagedNotDeleted :: Repo -> [FilePath] -> IO [FilePath]+stagedNotDeleted repo l = staged' repo l [Param "--diff-filter=ACMRT"]++staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]+staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end+	where+		start = [Params "diff --cached --name-only -z"]+		end = [Param "--"] ++ map File l++{- Returns a list of files that have unstaged changes. -}+changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]+changedUnstaged repo l = pipeNullSplit repo $+	[Params "diff --name-only -z --"] ++ map File l++{- Returns a list of the files in the specified locations that are staged+ - for commit, and whose type has changed. -}+typeChangedStaged :: Repo -> [FilePath] -> IO [FilePath]+typeChangedStaged repo l = typeChanged' repo l [Param "--cached"]++{- Returns a list of the files in the specified locations whose type has+ - changed.  Files only staged for commit will not be included. -}+typeChanged :: Repo -> [FilePath] -> IO [FilePath]+typeChanged repo l = typeChanged' repo l []++typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]+typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end+	where+		start = [Params "diff --name-only --diff-filter=T -z"]+		end = [Param "--"] ++ map File l
+ Git/Queue.hs view
@@ -0,0 +1,89 @@+{- git repository command queue+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Git.Queue (+	Queue,+	empty,+	add,+	size,+	full,+	flush+) where++import qualified Data.Map as M+import System.IO+import System.Cmd.Utils+import Data.String.Utils+import Control.Monad (unless, forM_)+import Utility++import Git++{- An action to perform in a git repository. The file to act on+ - is not included, and must be able to be appended after the params. -}+data Action = Action {+		getSubcommand :: String,+		getParams :: [CommandParam]+	} deriving (Show, Eq, Ord)++{- A queue of actions to perform (in any order) on a git repository,+ - with lists of files to perform them on. This allows coalescing + - similar git commands. -}+data Queue = Queue Int (M.Map Action [FilePath])+	deriving (Show, Eq)++{- A recommended maximum size for the queue, after which it should be+ - run.+ -+ - 10240 is semi-arbitrary. If we assume git filenames are between 10 and+ - 255 characters long, then the queue will build up between 100kb and+ - 2550kb long commands. The max command line length on linux is somewhere+ - above 20k, so this is a fairly good balance -- the queue will buffer+ - only a few megabytes of stuff and a minimal number of commands will be+ - run by xargs. -}+maxSize :: Int+maxSize = 10240++{- Constructor for empty queue. -}+empty :: Queue+empty = Queue 0 M.empty++{- Adds an action to a queue. -}+add :: Queue -> String -> [CommandParam] -> FilePath -> Queue+add (Queue n m) subcommand params file = Queue (n + 1) m'+	where+		action = Action subcommand params+		-- There are probably few items in the map, but there+		-- can be a lot of files per item. So, optimise adding+		-- files.+		m' = M.insertWith' const action files m+		files = file:(M.findWithDefault [] action m)++{- Number of items in a queue. -}+size :: Queue -> Int+size (Queue n _) = n++{- Is a queue large enough that it should be flushed? -}+full :: Queue -> Bool+full (Queue n _) = n > maxSize++{- Runs a queue on a git repository. -}+flush :: Repo -> Queue -> IO Queue+flush repo (Queue _ m) = do+	forM_ (M.toList m) $ uncurry $ runAction repo+	return empty++{- Runs an Action on a list of files in a git repository.+ -+ - Complicated by commandline length limits. -}+runAction :: Repo -> Action -> [FilePath] -> IO ()+runAction repo action files = unless (null files) runxargs+	where+		runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs+		params = toCommand $ gitCommandLine repo+			(Param (getSubcommand action):getParams action)+		feedxargs h = hPutStr h $ join "\0" files
+ Git/UnionMerge.hs view
@@ -0,0 +1,95 @@+{- git-union-merge library+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Git.UnionMerge (+	merge,+	update_index,+	update_index_line,+	ls_tree+) where++import System.Cmd.Utils+import Data.List+import Data.Maybe+import Data.String.Utils++import Git+import Utility++{- Performs a union merge between two branches, staging it in the index.+ - Any previously staged changes in the index will be lost.+ -+ - When only one branch is specified, it is merged into the index.+ - In this case, previously staged changes in the index are preserved.+ -+ - Should be run with a temporary index file configured by Git.useIndex.+ -}+merge :: Repo -> [String] -> IO ()+merge g (x:y:[]) = do+	a <- ls_tree g x+	b <- merge_trees g x y+	update_index g (a++b)+merge g [x] = merge_tree_index g x >>= update_index g+merge _ _ = error "wrong number of branches to merge"++{- Feeds a list into update-index. Later items in the list can override+ - earlier ones, so the list can be generated from any combination of+ - ls_tree, merge_trees, and merge_tree_index. -}+update_index :: Repo -> [String] -> IO ()+update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)+	where+		togit ps content = pipeWrite g (map Param ps) content+			>>= forceSuccess++{- Generates a line suitable to be fed into update-index, to add+ - a given file with a given sha. -}+update_index_line :: String -> FilePath -> String+update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file++{- Gets the contents of a tree in a format suitable for update_index. -}+ls_tree :: Repo -> String -> IO [String]+ls_tree g x = pipeNullSplit g $ +	map Param ["ls-tree", "-z", "-r", "--full-tree", x]++{- For merging two trees. -}+merge_trees :: Repo -> String -> String -> IO [String]+merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]++{- For merging a single tree into the index. -}+merge_tree_index :: Repo -> String -> IO [String]+merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]++diff_opts :: [String]+diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]++{- Calculates how to perform a merge, using git to get a raw diff,+ - and returning a list suitable for update_index. -}+calc_merge :: Repo -> [String] -> IO [String]+calc_merge g differ = do+	diff <- pipeNullSplit g $ map Param differ+	l <- mapM (mergeFile g) (pairs diff)+	return $ catMaybes l+	where+		pairs [] = []+		pairs (_:[]) = error "calc_merge parse error"+		pairs (a:b:rest) = (a,b):pairs rest++{- Given an info line from a git raw diff, and the filename, generates+ - a line suitable for update_index that union merges the two sides of the+ - diff. -}+mergeFile :: Repo -> (String, FilePath) -> IO (Maybe String)+mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of+	[] -> return Nothing+	(sha:[]) -> return $ Just $ update_index_line sha file+	shas -> do+		content <- pipeRead g $ map Param ("show":shas)+		sha <- hashObject g $ unionmerge content+		return $ Just $ update_index_line sha file+	where+		[_colonamode, _bmode, asha, bsha, _status] = words info+		nullsha = take shaSize $ repeat '0'+		unionmerge = unlines . nub . lines
+ GitAnnex.hs view
@@ -0,0 +1,120 @@+{- git-annex main program+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module GitAnnex where++import System.Console.GetOpt++import qualified Git+import CmdLine+import Command+import Options+import Utility+import Types.TrustLevel+import qualified Annex+import qualified Remote++import qualified Command.Add+import qualified Command.Unannex+import qualified Command.Drop+import qualified Command.Move+import qualified Command.Copy+import qualified Command.Get+import qualified Command.FromKey+import qualified Command.DropKey+import qualified Command.SetKey+import qualified Command.Fix+import qualified Command.Init+import qualified Command.Describe+import qualified Command.InitRemote+import qualified Command.Fsck+import qualified Command.Unused+import qualified Command.DropUnused+import qualified Command.Unlock+import qualified Command.Lock+import qualified Command.PreCommit+import qualified Command.Find+import qualified Command.Whereis+import qualified Command.Merge+import qualified Command.Status+import qualified Command.Migrate+import qualified Command.Uninit+import qualified Command.Trust+import qualified Command.Untrust+import qualified Command.Semitrust+import qualified Command.AddUrl+import qualified Command.Map+import qualified Command.Upgrade+import qualified Command.Version++cmds :: [Command]+cmds = concat+	[ Command.Add.command+	, Command.Get.command+	, Command.Drop.command+	, Command.Move.command+	, Command.Copy.command+	, Command.Unlock.command+	, Command.Lock.command+	, Command.Init.command+	, Command.Describe.command+	, Command.InitRemote.command+	, Command.Unannex.command+	, Command.Uninit.command+	, Command.PreCommit.command+	, Command.Trust.command+	, Command.Untrust.command+	, Command.Semitrust.command+	, Command.AddUrl.command+	, Command.FromKey.command+	, Command.DropKey.command+	, Command.SetKey.command+	, Command.Fix.command+	, Command.Fsck.command+	, Command.Unused.command+	, Command.DropUnused.command+	, Command.Find.command+	, Command.Whereis.command+	, Command.Merge.command+	, Command.Status.command+	, Command.Migrate.command+	, Command.Map.command+	, Command.Upgrade.command+	, Command.Version.command+	]++options :: [Option]+options = commonOptions +++	[ Option ['k'] ["key"] (ReqArg setkey paramKey)+		"specify a key to use"+	, Option ['t'] ["to"] (ReqArg setto paramRemote)+		"specify to where to transfer content"+	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)+		"specify from where to transfer content"+	, Option ['x'] ["exclude"] (ReqArg addexclude paramGlob)+		"skip files matching the glob pattern"+	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)+		"override default number of copies"+	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)+		"override trust setting"+	, Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote)+		"override trust setting back to default value"+	, Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote)+		"override trust setting to untrusted"+	]+	where+		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }+		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }+		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }+		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }+		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }++header :: String+header = "Usage: git-annex command [option ..]"++run :: [String] -> IO ()+run args = dispatch args cmds options header =<< Git.repoFromCwd
+ LocationLog.hs view
@@ -0,0 +1,66 @@+{- git-annex location log+ -+ - git-annex keeps track of which repositories have the contents of annexed+ - files.+ -+ - Repositories record their UUID and the date when they --get or --drop+ - a value.+ - + - Copyright 2010-2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module LocationLog (+	LogStatus(..),+	logChange,+	readLog,+	writeLog,+	keyLocations,+	loggedKeys,+	logFile,+	logFileKey	+) where++import System.FilePath+import Control.Monad (when)+import Data.Maybe++import qualified Git+import qualified Branch+import UUID+import Types+import Types.Key+import Locations+import PresenceLog++{- Log a change in the presence of a key's value in a repository. -}+logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()+logChange repo key u s = do+	when (null u) $+		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ +			" (have you run git annex init there?)"+	addLog (logFile key) =<< logNow s u++{- Returns a list of repository UUIDs that, according to the log, have+ - the value of a key. -}+keyLocations :: Key -> Annex [UUID]+keyLocations key = currentLog $ logFile key++{- Finds all keys that have location log information.+ - (There may be duplicate keys in the list.) -}+loggedKeys :: Annex [Key]+loggedKeys =+	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files++{- The filename of the log file for a given key. -}+logFile :: Key -> String+logFile key = hashDirLower key ++ keyFile key ++ ".log"++{- Converts a log filename into a key. -}+logFileKey :: FilePath -> Maybe Key+logFileKey file+	| end == ".log" = readKey beginning+	| otherwise = Nothing+	where+		(beginning, end) = splitAt (length file - 4) file
+ Locations.hs view
@@ -0,0 +1,174 @@+{- git-annex file locations+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Locations (+	keyFile,+	fileKey,+	gitAnnexLocation,+	annexLocation,+	gitAnnexDir,+	gitAnnexObjectDir,+	gitAnnexTmpDir,+	gitAnnexTmpLocation,+	gitAnnexBadDir,+	gitAnnexBadLocation,+	gitAnnexUnusedLog,+	gitAnnexJournalDir,+	isLinkToAnnex,+	hashDirMixed,+	hashDirLower,++	prop_idempotent_fileKey+) where++import System.FilePath+import Data.String.Utils+import Data.List+import Bits+import Word+import Data.Hash.MD5++import Types+import Types.Key+import qualified Git++{- Conventions:+ -+ - Functions ending in "Dir" should always return values ending with a+ - trailing path separator. Most code does not rely on that, but a few+ - things do. + -+ - Everything else should not end in a trailing path sepatator. + -+ - Only functions (with names starting with "git") that build a path+ - based on a git repository should return an absolute path.+ - Everything else should use relative paths.+ -}++{- The directory git annex uses for local state, relative to the .git+ - directory -}+annexDir :: FilePath+annexDir = addTrailingPathSeparator $ "annex"++{- The directory git annex uses for locally available object content,+ - relative to the .git directory -}+objectDir :: FilePath+objectDir = addTrailingPathSeparator $ annexDir </> "objects"++{- Annexed file's location relative to the .git directory. -}+annexLocation :: Key -> FilePath+annexLocation key = objectDir </> hashDirMixed key </> f </> f+	where+		f = keyFile key++{- Annexed file's absolute location in a repository. -}+gitAnnexLocation :: Git.Repo -> Key -> FilePath+gitAnnexLocation r key+	| Git.repoIsLocalBare r = Git.workTree r </> annexLocation key+	| otherwise = Git.workTree r </> ".git" </> annexLocation key++{- The annex directory of a repository. -}+gitAnnexDir :: Git.Repo -> FilePath+gitAnnexDir r+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r </> annexDir+	| otherwise = addTrailingPathSeparator $ Git.workTree r </> ".git" </> annexDir++{- The part of the annex directory where file contents are stored.+ -}+gitAnnexObjectDir :: Git.Repo -> FilePath+gitAnnexObjectDir r+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r </> objectDir+	| otherwise = addTrailingPathSeparator $ Git.workTree r </> ".git" </> objectDir++{- .git/annex/tmp/ is used for temp files -}+gitAnnexTmpDir :: Git.Repo -> FilePath+gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"++{- The temp file to use for a given key. -}+gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath+gitAnnexTmpLocation r key = gitAnnexTmpDir r </> keyFile key++{- .git/annex/bad/ is used for bad files found during fsck -}+gitAnnexBadDir :: Git.Repo -> FilePath+gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r </> "bad"++{- The bad file to use for a given key. -}+gitAnnexBadLocation :: Git.Repo -> Key -> FilePath+gitAnnexBadLocation r key = gitAnnexBadDir r </> keyFile key++{- .git/annex/*unused is used to number possibly unused keys -}+gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath+gitAnnexUnusedLog prefix r = gitAnnexDir r </> (prefix ++ "unused")++{- .git/annex/journal/ is used to journal changes made to the git-annex+ - branch -}+gitAnnexJournalDir :: Git.Repo -> FilePath+gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r </> "journal"++{- Checks a symlink target to see if it appears to point to annexed content. -}+isLinkToAnnex :: FilePath -> Bool+isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s++{- Converts a key into a filename fragment.+ -+ - Escape "/" in the key name, to keep a flat tree of files and avoid+ - issues with keys containing "/../" or ending with "/" etc. + -+ - "/" is escaped to "%" because it's short and rarely used, and resembles+ -     a slash+ - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping+ -     is one to one.+ - ":" is escaped to "&c", because despite it being 2011, people still care+ -     about FAT.+ - -}+keyFile :: Key -> FilePath+keyFile key = replace "/" "%" $ replace ":" "&c" $+	replace "%" "&s" $ replace "&" "&a"  $ show key++{- Reverses keyFile, converting a filename fragment (ie, the basename of+ - the symlink target) into a key. -}+fileKey :: FilePath -> Maybe Key+fileKey file = readKey $+	replace "&a" "&" $ replace "&s" "%" $+		replace "&c" ":" $ replace "%" "/" file++{- for quickcheck -}+prop_idempotent_fileKey :: String -> Bool+prop_idempotent_fileKey s = Just k == fileKey (keyFile k)+	where k = stubKey { keyName = s, keyBackendName = "test" }++{- Given a key, generates a short directory name to put it in,+ - to do hashing to protect against filesystems that dislike having+ - many items in a single directory. -}+hashDirMixed :: Key -> FilePath+hashDirMixed k = addTrailingPathSeparator $ take 2 dir </> drop 2 dir+	where+		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]+		ABCD (a,b,c,d) = md5 $ Str $ show k++{- Generates a hash directory that is all lower case. -}+hashDirLower :: Key -> FilePath+hashDirLower k = addTrailingPathSeparator $ take 3 dir </> drop 3 dir+	where+		dir = take 6 $ md5s $ Str $ show k++{- modified version of display_32bits_as_hex from Data.Hash.MD5+ -   Copyright (C) 2001 Ian Lynagh + -   License: Either BSD or GPL+ -}+display_32bits_as_dir :: Word32 -> String+display_32bits_as_dir w = trim $ swap_pairs cs+	where +		-- Need 32 characters to use. To avoid inaverdently making+		-- a real word, use letters that appear less frequently.+		chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF"+		cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7]+		getc n = chars !! (fromIntegral n)+		swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs+		swap_pairs _ = []+		-- Last 2 will always be 00, so omit.+		trim s = take 6 s
+ Makefile view
@@ -0,0 +1,98 @@+PREFIX=/usr+IGNORE=-ignore-package monads-fd+GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5+ifdef PROFILE+GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)+endif+GHCMAKE=ghc $(GHCFLAGS) --make++bins=git-annex git-annex-shell git-union-merge+mans=git-annex.1 git-annex-shell.1 git-union-merge.1++all: $(bins) $(mans) docs++sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs++SysConfig.hs: configure.hs TestConfig.hs+	$(GHCMAKE) configure+	./configure++%.hs: %.hsc+	hsc2hs $<+	perl -i -pe 's/^{-# INCLUDE.*//' $@++Remote/S3.hs:+	@ln -sf S3real.hs Remote/S3.hs++Remote/S3.o: Remote/S3.hs+	@if ! $(GHCMAKE) Remote/S3.hs; then \+		ln -sf S3stub.hs Remote/S3.hs; \+		echo "** building without S3 support"; \+	fi++$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o+	$(GHCMAKE) $@++git-annex.1: doc/git-annex.mdwn+	./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1+git-annex-shell.1: doc/git-annex-shell.mdwn+	./mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1+git-union-merge.1: doc/git-union-merge.mdwn+	./mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1++install: all+	install -d $(DESTDIR)$(PREFIX)/bin+	install $(bins) $(DESTDIR)$(PREFIX)/bin+	install -d $(DESTDIR)$(PREFIX)/share/man/man1+	install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1+	install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex+	if [ -d html ]; then \+		rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \+	fi++test: $(bins)+	@if ! $(GHCMAKE) -O0 test; then \+		echo "** not running test suite" >&2; \+	else \+		./test; \+	fi++testcoverage: $(bins)+	rm -f test.tix test+	ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test+	./test+	@echo ""+	@hpc report test --exclude=Main --exclude=QC+	@hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null++# If ikiwiki is available, build static html docs suitable for being+# shipped in the software package.+ifeq ($(shell which ikiwiki),)+IKIWIKI=@echo "** ikiwiki not found, skipping building docs" >&2; true+else+IKIWIKI=ikiwiki+endif++docs: $(mans)+	$(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \+		--no-usedirs --disable-plugin=openid --plugin=sidebar \+		--underlaydir=/dev/null --disable-plugin=shortcut \+		--disable-plugin=smiley \+		--plugin=comments --set comments_pagespec="*" \+		--exclude='news/.*'++clean:+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \+		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs+	rm -rf doc/.ikiwiki html dist+	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;++# Workaround for cabal sdist not running Setup hooks, so I cannot+# generate a file list there.+sdist: clean+	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -type f -print)!i" < git-annex.cabal.orig > git-annex.cabal+	@cabal sdist+	@mv git-annex.cabal.orig git-annex.cabal++.PHONY: $(bins) test install
+ Messages.hs view
@@ -0,0 +1,75 @@+{- git-annex output messages+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Messages where++import Control.Monad.State (liftIO)+import System.IO+import Control.Monad (unless)+import Data.String.Utils++import Types+import qualified Annex++verbose :: Annex () -> Annex ()+verbose a = do+	q <- Annex.getState Annex.quiet+	unless q a++showSideAction :: String -> Annex ()+showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")"++showStart :: String -> String -> Annex ()+showStart command file = verbose $ do+	liftIO $ putStr $ command ++ " " ++ file ++ " "+	liftIO $ hFlush stdout++showNote :: String -> Annex ()+showNote s = verbose $ do+	liftIO $ putStr $ "(" ++ s ++ ") "+	liftIO $ hFlush stdout++showProgress :: Annex ()+showProgress = verbose $ liftIO $ putStr "\n"++showLongNote :: String -> Annex ()+showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indent s++showEndOk :: Annex ()+showEndOk = verbose $ liftIO $ putStrLn "ok"++showEndFail :: Annex ()+showEndFail = verbose $ liftIO $ putStrLn "\nfailed"++showEndResult :: Bool -> Annex ()+showEndResult True = showEndOk+showEndResult False = showEndFail++showErr :: (Show a) => a -> Annex ()+showErr e = warning $ "git-annex: " ++ show e++warning :: String -> Annex ()+warning w = do+	verbose $ liftIO $ putStr "\n"+	liftIO $ hFlush stdout+	liftIO $ hPutStrLn stderr $ indent w++indent :: String -> String+indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s++{- By default, haskell honors the user's locale in its output to stdout+ - and stderr. While that's great for proper unicode support, for git-annex+ - all that's really needed is the ability to display simple messages+ - (currently untranslated), and importantly, to display filenames exactly+ - as they are written on disk, no matter what their encoding. So, force+ - raw mode. + -+ - NB: Once git-annex gets localized, this will need a rethink. -}+setupConsole :: IO ()+setupConsole = do+	hSetBinaryMode stdout True+	hSetBinaryMode stderr True
+ Options.hs view
@@ -0,0 +1,44 @@+{- git-annex dashed options+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Options where++import System.Console.GetOpt+import System.Log.Logger+import Control.Monad.State (liftIO)++import qualified Annex+import Types+import Command++{- Each dashed command-line option results in generation of an action+ - in the Annex monad that performs the necessary setting.+ -}+type Option = OptDescr (Annex ())++commonOptions :: [Option]+commonOptions =+	[ Option [] ["force"] (NoArg (setforce True))+		"allow actions that may lose annexed data"+	, Option ['F'] ["fast"] (NoArg (setfast True))+		"avoid slow operations"+	, Option ['q'] ["quiet"] (NoArg (setquiet True))+		"avoid verbose output"+	, Option ['v'] ["verbose"] (NoArg (setquiet False))+		"allow verbose output (default)"+	, Option ['d'] ["debug"] (NoArg (setdebug))+		"show debug messages"+	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)+		"specify key-value backend to use"+	]+	where+		setforce v = Annex.changeState $ \s -> s { Annex.force = v }+		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }+		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }+		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }+		setdebug = liftIO $ updateGlobalLogger rootLoggerName $+			setLevel DEBUG
+ PresenceLog.hs view
@@ -0,0 +1,129 @@+{- git-annex presence log+ -+ - This is used to store presence information in the git-annex branch in+ - a way that can be union merged.+ -+ - A line of the log will look like: "date N INFO"+ - Where N=1 when the INFO is present, and 0 otherwise.+ - + - Copyright 2010-2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module PresenceLog (+	LogStatus(..),+	addLog,+	readLog,+	writeLog,+	logNow,+	compactLog,+	currentLog+) where++import Data.Time.Clock.POSIX+import Data.Time+import System.Locale+import qualified Data.Map as Map+import Control.Monad.State (liftIO)++import qualified Branch+import Types++data LogLine = LogLine {+	date :: POSIXTime,+	status :: LogStatus,+	info :: String+} deriving (Eq)++data LogStatus = InfoPresent | InfoMissing | Undefined+	deriving (Eq)++instance Show LogStatus where+	show InfoPresent = "1"+	show InfoMissing = "0"+	show Undefined = "undefined"++instance Read LogStatus where+	readsPrec _ "1" = [(InfoPresent, "")]+	readsPrec _ "0" = [(InfoMissing, "")]+	readsPrec _ _   = [(Undefined, "")]++instance Show LogLine where+	show (LogLine d s i) = unwords [show d, show s, i]++instance Read LogLine where+	-- This parser is robust in that even unparsable log lines are+	-- read without an exception being thrown.+	-- Such lines have a status of Undefined.+	readsPrec _ string = +		if length w >= 3+			then maybe bad good pdate+			else bad+		where+			w = words string+			s = read $ w !! 1+			i = w !! 2+			pdate :: Maybe UTCTime+			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w++			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s i+			bad = ret $ LogLine 0 Undefined ""+			ret v = [(v, "")]++addLog :: FilePath -> LogLine -> Annex ()+addLog file line = do+	ls <- readLog file+	writeLog file (compactLog $ line:ls)++{- Reads a log file.+ - Note that the LogLines returned may be in any order. -}+readLog :: FilePath -> Annex [LogLine]+readLog file = return . parseLog =<< Branch.get file++parseLog :: String -> [LogLine]+parseLog s = filter parsable $ map read $ lines s+	where+		-- some lines may be unparseable, avoid them+		parsable l = status l /= Undefined++{- Stores a set of lines in a log file -}+writeLog :: FilePath -> [LogLine] -> Annex ()+writeLog file ls = Branch.change file (unlines $ map show ls)++{- Generates a new LogLine with the current date. -}+logNow :: LogStatus -> String -> Annex LogLine+logNow s i = do+	now <- liftIO $ getPOSIXTime+	return $ LogLine now s i++{- Reads a log and returns only the info that is still in effect. -}+currentLog :: FilePath -> Annex [String]+currentLog file = do+	ls <- readLog file+	return $ map info $ filterPresent ls++{- Returns the info from LogLines that are in effect. -}+filterPresent :: [LogLine] -> [LogLine]+filterPresent ls = filter (\l -> InfoPresent == status l) $ compactLog ls++type LogMap = Map.Map String LogLine++{- Compacts a set of logs, returning a subset that contains the current+ - status. -}+compactLog :: [LogLine] -> [LogLine]+compactLog ls = compactLog' Map.empty ls+compactLog' :: LogMap -> [LogLine] -> [LogLine]+compactLog' m [] = Map.elems m+compactLog' m (l:ls) = compactLog' (mapLog m l) ls++{- Inserts a log into a map of logs, if the log has better (ie, newer)+ - information than the other logs in the map -}+mapLog :: LogMap -> LogLine -> LogMap+mapLog m l = +	if better+		then Map.insert i l m+		else m+	where+		better = maybe True (\l' -> date l' <= date l) $ Map.lookup i m+		i = info l
+ README view
@@ -0,0 +1,6 @@+git-annex allows managing files with git, without checking the file+contents into git. While that may seem paradoxical, it is useful when+dealing with files larger than git can currently easily handle, whether due+to limitations in memory, checksumming time, or disk space.++For documentation, see doc/ or <http://git-annex.branchable.com/>
+ Remote.hs view
@@ -0,0 +1,259 @@+{- git-annex remotes+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote (+	Remote,+	uuid,+	name,+	storeKey,+	retrieveKeyFile,+	removeKey,+	hasKey,+	hasKeyCheap,+	keyPossibilities,+	keyPossibilitiesTrusted,+	forceTrust,++	remoteTypes,+	genList,+	byName,+	nameToUUID,+	remotesWithUUID,+	remotesWithoutUUID,+	prettyPrintUUIDs,++	remoteLog,+	readRemoteLog,+	configSet,+	keyValToConfig,+	configToKeyVal,+	+	prop_idempotent_configEscape+) where++import Control.Monad (filterM, liftM2)+import Data.List+import qualified Data.Map as M+import Data.Maybe+import Data.Char++import qualified Branch+import Types+import Types.Remote+import UUID+import qualified Annex+import Config+import Trust+import LocationLog++import qualified Remote.Git+import qualified Remote.S3+import qualified Remote.Bup+import qualified Remote.Directory+import qualified Remote.Rsync+import qualified Remote.Web+import qualified Remote.Hook++remoteTypes :: [RemoteType Annex]+remoteTypes =+	[ Remote.Git.remote+	, Remote.S3.remote+	, Remote.Bup.remote+	, Remote.Directory.remote+	, Remote.Rsync.remote+	, Remote.Web.remote+	, Remote.Hook.remote+	]++{- Builds a list of all available Remotes.+ - Since doing so can be expensive, the list is cached. -}+genList :: Annex [Remote Annex]+genList = do+	rs <- Annex.getState Annex.remotes+	if null rs+		then do+			m <- readRemoteLog+			l <- mapM (process m) remoteTypes+			let rs' = concat l+			Annex.changeState $ \s -> s { Annex.remotes = rs' }+			return rs'+		else return rs+	where+		process m t = +			enumerate t >>=+			filterM remoteNotIgnored >>=+			mapM (gen m t)+		gen m t r = do+			u <- getUUID r+			generate t r u (M.lookup u m)++{- Looks up a remote by name. (Or by UUID.) -}+byName :: String -> Annex (Remote Annex)+byName n = do+	res <- byName' n+	case res of+		Left e -> error e+		Right r -> return r+byName' :: String -> Annex (Either String (Remote Annex))+byName' "" = return $ Left "no remote specified"+byName' n = do+	allremotes <- genList+	let match = filter matching allremotes+	if (null match)+		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""+		else return $ Right $ head match+	where+		matching r = n == name r || n == uuid r++{- Looks up a remote by name (or by UUID, or even by description),+ - and returns its UUID. -}+nameToUUID :: String -> Annex UUID+nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo+nameToUUID n = do+	res <- byName' n+	case res of+		Left e -> return . (maybe (error e) id) =<< byDescription+		Right r -> return $ uuid r+	where+		byDescription = return . M.lookup n . invertMap =<< uuidMap+		invertMap = M.fromList . map swap . M.toList+		swap (a, b) = (b, a)++{- Pretty-prints a list of UUIDs of remotes. -}+prettyPrintUUIDs :: [UUID] -> Annex String+prettyPrintUUIDs uuids = do+	here <- getUUID =<< Annex.gitRepo+	-- Show descriptions from the uuid log, falling back to remote names,+	-- as some remotes may not be in the uuid log.+	m <- liftM2 M.union uuidMap $+		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList+	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids+	where+		prettify m u here = base ++ ishere+			where+				base = if not $ null $ findlog m u+					then u ++ "  -- " ++ findlog m u+					else u+				ishere = if here == u then " <-- here" else ""+		findlog m u = M.findWithDefault "" u m++{- Filters a list of remotes to ones that have the listed uuids. -}+remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]+remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs++{- Filters a list of remotes to ones that do not have the listed uuids. -}+remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]+remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs++{- Cost ordered lists of remotes that the LocationLog indicate may have a key.+ -}+keyPossibilities :: Key -> Annex [Remote Annex]+keyPossibilities key = return . fst =<< keyPossibilities' False key++{- Cost ordered lists of remotes that the LocationLog indicate may have a key.+ -+ - Also returns a list of UUIDs that are trusted to have the key+ - (some may not have configured remotes).+ -}+keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])+keyPossibilitiesTrusted = keyPossibilities' True++keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])+keyPossibilities' withtrusted key = do+	g <- Annex.gitRepo+	u <- getUUID g+	trusted <- if withtrusted then trustGet Trusted else return []++	-- get uuids of all remotes that are recorded to have the key+	uuids <- keyLocations key+	let validuuids = filter (/= u) uuids++	-- note that validuuids is assumed to not have dups+	let validtrusteduuids = intersect validuuids trusted++	-- remotes that match uuids that have the key+	allremotes <- genList+	let validremotes = remotesWithUUID allremotes validuuids++	return (sort validremotes, validtrusteduuids)++forceTrust :: TrustLevel -> String -> Annex ()+forceTrust level remotename = do+	r <- Remote.nameToUUID remotename+	Annex.changeState $ \s ->+		s { Annex.forcetrust = (r, level):Annex.forcetrust s }++{- Filename of remote.log. -}+remoteLog :: FilePath+remoteLog = "remote.log"++{- Adds or updates a remote's config in the log. -}+configSet :: UUID -> RemoteConfig -> Annex ()+configSet u c = do+	m <- readRemoteLog+	Branch.change remoteLog $ unlines $ sort $+		map toline $ M.toList $ M.insert u c m+	where+		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')++{- Map of remotes by uuid containing key/value config maps. -}+readRemoteLog :: Annex (M.Map UUID RemoteConfig)+readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog++remoteLogParse :: String -> M.Map UUID RemoteConfig+remoteLogParse s =+	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s+	where+		parseline l+			| length w > 2 = Just (u, c)+			| otherwise = Nothing+			where+				w = words l+				u = w !! 0+				c = keyValToConfig $ tail w++{- Given Strings like "key=value", generates a RemoteConfig. -}+keyValToConfig :: [String] -> RemoteConfig+keyValToConfig ws = M.fromList $ map (/=/) ws+	where+		(/=/) s = (k, v)+			where+				k = takeWhile (/= '=') s+				v = configUnEscape $ drop (1 + length k) s++configToKeyVal :: M.Map String String -> [String]+configToKeyVal m = map toword $ sort $ M.toList m+	where+		toword (k, v) = k ++ "=" ++ configEscape v++configEscape :: String -> String+configEscape = (>>= escape)+	where+		escape c+			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"+			| otherwise = [c]++configUnEscape :: String -> String+configUnEscape = unescape+	where+		unescape [] = []+		unescape (c:rest)+			| c == '&' = entity rest+			| otherwise = c : unescape rest+		entity s = if ok+				then chr (read num) : unescape rest+				else '&' : unescape s+			where+				num = takeWhile isNumber s+				r = drop (length num) s+				rest = drop 1 r+				ok = not (null num) && +					not (null r) && r !! 0 == ';'++{- for quickcheck -}+prop_idempotent_configEscape :: String -> Bool+prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
+ Remote/Bup.hs view
@@ -0,0 +1,235 @@+{- Using bup as a remote.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Bup (remote) where++import qualified Data.ByteString.Lazy.Char8 as L+import IO+import Control.Exception.Extensible (IOException)+import qualified Data.Map as M+import Control.Monad (when)+import Control.Monad.State (liftIO)+import System.Process+import System.Exit+import System.FilePath+import Data.List.Utils+import System.Cmd.Utils++import Types+import Types.Remote+import qualified Git+import qualified Annex+import UUID+import Locations+import Config+import Utility+import Messages+import Ssh+import Remote.Special+import Remote.Encryptable+import Crypto++type BupRepo = String++remote :: RemoteType Annex+remote = RemoteType {+	typename = "bup",+	enumerate = findSpecialRemotes "buprepo",+	generate = gen,+	setup = bupSetup+}++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u c = do+	buprepo <- getConfig r "buprepo" (error "missing buprepo")+	cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost)+	bupr <- liftIO $ bup2GitRemote buprepo+	(u', bupr') <- getBupUUID bupr u+	+	return $ encryptableRemote c+		(storeEncrypted r buprepo)+		(retrieveEncrypted buprepo)+		Remote {+			uuid = u',+			cost = cst,+			name = Git.repoDescribe r,+ 			storeKey = store r buprepo,+			retrieveKeyFile = retrieve buprepo,+			removeKey = remove,+			hasKey = checkPresent r bupr',+			hasKeyCheap = bupLocal buprepo,+			config = c+		}++bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig+bupSetup u c = do+	-- verify configuration is sane+	let buprepo = maybe (error "Specify buprepo=") id $+		M.lookup "buprepo" c+	c' <- encryptionSetup c++	-- bup init will create the repository.+	-- (If the repository already exists, bup init again appears safe.)+	showNote "bup init"+	bup "init" buprepo [] >>! error "bup init failed"++	storeBupUUID u buprepo++	-- The buprepo is stored in git config, as well as this repo's+	-- persistant state, so it can vary between hosts.+	gitConfigSpecialRemote u c' "buprepo" buprepo++	return c'++bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam]+bupParams command buprepo params = +	(Param command) : [Param "-r", Param buprepo] ++ params++bup :: String -> BupRepo -> [CommandParam] -> Annex Bool+bup command buprepo params = do+	showProgress -- make way for bup output+	liftIO $ boolSystem "bup" $ bupParams command buprepo params++pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool+pipeBup params inh outh = do+	p <- runProcess "bup" (toCommand params)+		Nothing Nothing inh outh Nothing+	ok <- waitForProcess p+	case ok of+		ExitSuccess -> return True+		_ -> return False++bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam]+bupSplitParams r buprepo k src = do+	o <- getConfig r "bup-split-options" ""+	let os = map Param $ words o+	showProgress -- make way for bup output+	return $ bupParams "split" buprepo +		(os ++ [Param "-n", Param (show k), src])++store :: Git.Repo -> BupRepo -> Key -> Annex Bool+store r buprepo k = do+	g <- Annex.gitRepo+	let src = gitAnnexLocation g k+	params <- bupSplitParams r buprepo k (File src)+	liftIO $ boolSystem "bup" params++storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool+storeEncrypted r buprepo (cipher, enck) k = do+	g <- Annex.gitRepo+	let src = gitAnnexLocation g k+	params <- bupSplitParams r buprepo enck (Param "-")+	liftIO $ catchBool $ do+		withEncryptedHandle cipher (L.readFile src) $ \h -> do+			pipeBup params (Just h) Nothing++retrieve :: BupRepo -> Key -> FilePath -> Annex Bool+retrieve buprepo k f = do+	let params = bupParams "join" buprepo [Param $ show k]+	liftIO $ catchBool $ do+		tofile <- openFile f WriteMode+		pipeBup params Nothing (Just tofile)++retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool+retrieveEncrypted buprepo (cipher, enck) f = do+	let params = bupParams "join" buprepo [Param $ show enck]+	liftIO $ catchBool $ do+		(pid, h) <- hPipeFrom "bup" $ toCommand params+		withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f+		forceSuccess pid+		return True++remove :: Key -> Annex Bool+remove _ = do+	warning "content cannot be removed from bup remote"+	return False++{- Bup does not provide a way to tell if a given dataset is present+ - in a bup repository. One way it to check if the git repository has+ - a branch matching the name (as created by bup split -n).+ -}+checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool)+checkPresent r bupr k+	| Git.repoIsUrl bupr = do+		showNote ("checking " ++ Git.repoDescribe r ++ "...")+		ok <- onBupRemote bupr boolSystem "git" params+		return $ Right ok+	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params+	where+		params = +			[ Params "show-ref --quiet --verify"+			, Param $ "refs/heads/" ++ show k]++{- Store UUID in the annex.uuid setting of the bup repository. -}+storeBupUUID :: UUID -> BupRepo -> Annex ()+storeBupUUID u buprepo = do+	r <- liftIO $ bup2GitRemote buprepo+	if Git.repoIsUrl r+		then do+			showNote "storing uuid"+			onBupRemote r boolSystem "git"+				[Params $ "config annex.uuid " ++ u]+					>>! error "ssh failed"+		else liftIO $ do+			r' <- Git.configRead r+			let olduuid = Git.configGet r' "annex.uuid" ""+			when (olduuid == "") $+				Git.run r' "config" [Param "annex.uuid", Param u]++onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a+onBupRemote r a command params = do+	let dir = shellEscape (Git.workTree r)+	sshparams <- sshToRepo r [Param $+			"cd " ++ dir ++ " && " ++ (unwords $ command : toCommand params)]+	liftIO $ a "ssh" sshparams++{- Allow for bup repositories on removable media by checking+ - local bup repositories to see if they are available, and getting their+ - uuid (which may be different from the stored uuid for the bup remote).+ -+ - If a bup repository is not available, returns a dummy uuid of "".+ - This will cause checkPresent to indicate nothing from the bup remote+ - is known to be present.+ -+ - Also, returns a version of the repo with config read, if it is local.+ -}+getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo)+getBupUUID r u+	| Git.repoIsUrl r = return (u, r)+	| otherwise = liftIO $ do+		ret <- try $ Git.configRead r+		case ret of+			Right r' -> return (Git.configGet r' "annex.uuid" "", r')+			Left _ -> return ("", r)++{- Converts a bup remote path spec into a Git.Repo. There are some+ - differences in path representation between git and bup. -}+bup2GitRemote :: BupRepo -> IO Git.Repo+bup2GitRemote "" = do+	-- bup -r "" operates on ~/.bup+	h <- myHomeDir+	Git.repoFromAbsPath $ h </> ".bup"+bup2GitRemote r+	| bupLocal r = +		if r !! 0 == '/'+			then Git.repoFromAbsPath r+			else error "please specify an absolute path"+	| otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir+		where+			bits = split ":" r+			host = bits !! 0+			dir = join ":" $ drop 1 bits+			-- "host:~user/dir" is not supported specially by bup;+			-- "host:dir" is relative to the home directory;+			-- "host:" goes in ~/.bup+			slash d+				| d == "" = "/~/.bup"+				| d !! 0 == '/' = d+				| otherwise = "/~/" ++ d++bupLocal :: BupRepo -> Bool+bupLocal = notElem ':'
+ Remote/Directory.hs view
@@ -0,0 +1,128 @@+{- A "remote" that is just a filesystem directory.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Directory (remote) where++import qualified Data.ByteString.Lazy.Char8 as L+import IO+import Control.Exception.Extensible (IOException)+import qualified Data.Map as M+import Control.Monad (when)+import Control.Monad.State (liftIO)+import System.Directory hiding (copyFile)+import System.FilePath++import Types+import Types.Remote+import qualified Git+import qualified Annex+import UUID+import Locations+import CopyFile+import Config+import Content+import Utility+import Remote.Special+import Remote.Encryptable+import Crypto++remote :: RemoteType Annex+remote = RemoteType {+	typename = "directory",+	enumerate = findSpecialRemotes "directory",+	generate = gen,+	setup = directorySetup+}++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u c = do+	dir <- getConfig r "directory" (error "missing directory")+	cst <- remoteCost r cheapRemoteCost+	return $ encryptableRemote c+		(storeEncrypted dir)+		(retrieveEncrypted dir)+		Remote {+			uuid = u,+			cost = cst,+			name = Git.repoDescribe r,+ 			storeKey = store dir,+			retrieveKeyFile = retrieve dir,+			removeKey = remove dir,+			hasKey = checkPresent dir,+			hasKeyCheap = True,+			config = Nothing+		}++directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig+directorySetup u c = do+	-- verify configuration is sane+	let dir = maybe (error "Specify directory=") id $+		M.lookup "directory" c+	liftIO $ doesDirectoryExist dir+		>>! error $ "Directory does not exist: " ++ dir+	c' <- encryptionSetup c++	-- The directory is stored in git config, not in this remote's+	-- persistant state, so it can vary between hosts.+	gitConfigSpecialRemote u c' "directory" dir+	return $ M.delete "directory" c'++dirKey :: FilePath -> Key -> FilePath+dirKey d k = d </> hashDirMixed k </> f </> f+	where+		f = keyFile k++store :: FilePath -> Key -> Annex Bool+store d k = do+	g <- Annex.gitRepo+	let src = gitAnnexLocation g k+	let dest = dirKey d k+	liftIO $ catchBool $ storeHelper dest $ copyFile src dest++storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool+storeEncrypted d (cipher, enck) k = do+	g <- Annex.gitRepo+	let src = gitAnnexLocation g k+	let dest = dirKey d enck+	liftIO $ catchBool $ storeHelper dest $ encrypt src dest+	where+		encrypt src dest = do+			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest+			return True++storeHelper :: FilePath -> IO Bool -> IO Bool+storeHelper dest a = do+	let dir = parentDir dest+	createDirectoryIfMissing True dir+	allowWrite dir	+	ok <- a+	when ok $ do+		preventWrite dest+		preventWrite dir+	return ok++retrieve :: FilePath -> Key -> FilePath -> Annex Bool+retrieve d k f = liftIO $ copyFile (dirKey d k) f++retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool+retrieveEncrypted d (cipher, enck) f =+	liftIO $ catchBool $ do+		withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f+		return True++remove :: FilePath -> Key -> Annex Bool+remove d k = liftIO $ catchBool $ do+	allowWrite dir+	removeFile file+	removeDirectory dir+	return True+	where+		file = dirKey d k+		dir = parentDir file++checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)+checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)
+ Remote/Encryptable.hs view
@@ -0,0 +1,87 @@+{- common functions for encryptable remotes+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Encryptable where++import qualified Data.Map as M+import Control.Monad.State (liftIO)++import Types+import Types.Remote+import Crypto+import qualified Annex+import Messages+import Config++{- Encryption setup for a remote. The user must specify whether to use+ - an encryption key, or not encrypt. An encrypted cipher is created, or is+ - updated to be accessible to an additional encryption key. -}+encryptionSetup :: RemoteConfig -> Annex RemoteConfig+encryptionSetup c =+	case (M.lookup "encryption" c, extractCipher c) of+		(Nothing, Nothing) -> error "Specify encryption=key or encryption=none"+		(Just "none", Nothing) -> return c+		(Just "none", Just _) -> error "Cannot change encryption type of existing remote."+		(Nothing, Just _) -> return c+		(Just _, Nothing) -> use "encryption setup" $ genCipher c+		(Just _, Just v) -> use "encryption updated" $ updateCipher c v+	where+		use m a = do+			cipher <- liftIO a+			showNote $ m ++ " " ++ describeCipher cipher+			return $ M.delete "encryption" $ storeCipher c cipher++{- Modifies a Remote to support encryption.+ -+ - Two additional functions must be provided by the remote,+ - to support storing and retrieving encrypted content. -}+encryptableRemote+	:: Maybe RemoteConfig+	-> ((Cipher, Key) -> Key -> Annex Bool)+	-> ((Cipher, Key) -> FilePath -> Annex Bool)+	-> Remote Annex +	-> Remote Annex+encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = +	r {+		storeKey = store,+		retrieveKeyFile = retrieve,+		removeKey = withkey $ removeKey r,+		hasKey = withkey $ hasKey r,+		cost = cost r + encryptedRemoteCostAdj+	}+	where+		store k = cip k >>= maybe+			(storeKey r k)+			(\x -> storeKeyEncrypted x k)+		retrieve k f = cip k >>= maybe+			(retrieveKeyFile r k f)+			(\x -> retrieveKeyFileEncrypted x f)+		withkey a k = cip k >>= maybe (a k) (a . snd)+		cip = cipherKey c++{- Gets encryption Cipher. The decrypted Cipher is cached in the Annex+ - state. -}+remoteCipher :: RemoteConfig -> Annex (Maybe Cipher)+remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher+	where+		cached cipher = return $ Just cipher+		expensive = case extractCipher c of+			Nothing -> return Nothing+			Just encipher -> do+				showNote "gpg"+				cipher <- liftIO $ decryptCipher c encipher+				Annex.changeState (\s -> s { Annex.cipher = Just cipher })+				return $ Just cipher++{- Gets encryption Cipher, and encrypted version of Key. -}+cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))+cipherKey Nothing _ = return Nothing+cipherKey (Just c) k = remoteCipher c >>= maybe (return Nothing) encrypt+	where+		encrypt ciphertext = do+			k' <- liftIO $ encryptKey ciphertext k+			return $ Just (ciphertext, k')
+ Remote/Git.hs view
@@ -0,0 +1,209 @@+{- Standard git remotes.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Git (remote) where++import Control.Exception.Extensible+import Control.Monad.State (liftIO)+import qualified Data.Map as M+import System.Cmd.Utils+import System.Posix.Files++import Types+import Types.Remote+import qualified Git+import qualified Annex+import Locations+import UUID+import Utility+import qualified Content+import Messages+import CopyFile+import RsyncFile+import Ssh+import Config++remote :: RemoteType Annex+remote = RemoteType {+	typename = "git",+	enumerate = list,+	generate = gen,+	setup = error "not supported"+}++list :: Annex [Git.Repo]+list = do+	g <- Annex.gitRepo+	return $ Git.remotes g++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u _ = do+ 	{- It's assumed to be cheap to read the config of non-URL remotes,+	 - so this is done each time git-annex is run. Conversely,+	 - the config of an URL remote is only read when there is no+	 - cached UUID value. -}+	let cheap = not $ Git.repoIsUrl r+	r' <- case (cheap, u) of+		(True, _) -> tryGitConfigRead r+		(False, "") -> tryGitConfigRead r+		_ -> return r++	u' <- getUUID r'++	let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost+	cst <- remoteCost r' defcst++	return $ Remote {+		uuid = u',+		cost = cst,+		name = Git.repoDescribe r',+		storeKey = copyToRemote r',+		retrieveKeyFile = copyFromRemote r',+		removeKey = dropKey r',+		hasKey = inAnnex r',+		hasKeyCheap = cheap,+		config = Nothing+	}++{- Tries to read the config for a specified remote, updates state, and+ - returns the updated repo. -}+tryGitConfigRead :: Git.Repo -> Annex Git.Repo+tryGitConfigRead r +	| not $ M.null $ Git.configMap r = return r -- already read+	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []+	| Git.repoIsUrl r = return r+	| otherwise = store $ safely $ Git.configRead r+	where+		-- Reading config can fail due to IO error or+		-- for other reasons; catch all possible exceptions.+		safely a = do+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))+			case result of+				Left _ -> return r+				Right r' -> return r'+		pipedconfig cmd params = safely $+			pOpen ReadFromPipe cmd (toCommand params) $+				Git.hConfigRead r+		store a = do+			r' <- a+			g <- Annex.gitRepo+			let l = Git.remotes g+			let g' = Git.remotesAdd g $ exchange l r'+			Annex.changeState $ \s -> s { Annex.repo = g' }+			return r'+		exchange [] _ = []+		exchange (old:ls) new =+			if Git.repoRemoteName old == Git.repoRemoteName new+				then new : exchange ls new+				else old : exchange ls new++{- Checks if a given remote has the content for a key inAnnex.+ - If the remote cannot be accessed, returns a Left error.+ -}+inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)+inAnnex r key = if Git.repoIsUrl r+		then checkremote+		else liftIO (try checklocal ::IO (Either IOException Bool))+	where+		checklocal = do+			-- run a local check inexpensively,+			-- by making an Annex monad using the remote+			a <- Annex.new r []+			Annex.eval a (Content.inAnnex key)+		checkremote = do+			showNote ("checking " ++ Git.repoDescribe r ++ "...")+			inannex <- onRemote r (boolSystem, False) "inannex" +				[Param (show key)]+			return $ Right inannex+	+dropKey :: Git.Repo -> Key -> Annex Bool+dropKey r key = +	onRemote r (boolSystem, False) "dropkey"+		[ Params "--quiet --force"+		, Param $ show key+		]++{- Tries to copy a key's content from a remote's annex to a file. -}+copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool+copyFromRemote r key file+	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file+	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file+	| otherwise = error "copying from non-ssh repo not supported"+		+{- Tries to copy a key's content to a remote's annex. -}+copyToRemote :: Git.Repo -> Key -> Annex Bool+copyToRemote r key+	| not $ Git.repoIsUrl r = do+		g <- Annex.gitRepo+		let keysrc = gitAnnexLocation g key+		-- run copy from perspective of remote+		liftIO $ do+			a <- Annex.new r []+			Annex.eval a $ do+				ok <- Content.getViaTmp key $+					rsyncOrCopyFile r keysrc+				Content.saveState+				return ok+	| Git.repoIsSsh r = do+		g <- Annex.gitRepo+		let keysrc = gitAnnexLocation g key+		rsyncHelper =<< rsyncParamsRemote r False key keysrc+	| otherwise = error "copying to non-ssh repo not supported"++rsyncHelper :: [CommandParam] -> Annex (Bool)+rsyncHelper p = do+	showProgress -- make way for progress bar+	res <- liftIO $ rsync p+	if res+		then return res+		else do+			showLongNote "rsync failed -- run git annex again to resume file transfer"+			return res++{- Copys a file with rsync unless both locations are on the same+ - filesystem. Then cp could be faster. -}+rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool+rsyncOrCopyFile r src dest = do+	ss <- liftIO $ getFileStatus $ parentDir src+	ds <- liftIO $ getFileStatus $ parentDir dest+	if deviceID ss == deviceID ds+		then liftIO $ copyFile src dest+		else do+			params <- rsyncParams r+			rsyncHelper $ params ++ [Param src, Param dest]++{- Generates rsync parameters that ssh to the remote and asks it+ - to either receive or send the key's content. -}+rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]+rsyncParamsRemote r sending key file = do+	Just (shellcmd, shellparams) <- git_annex_shell r+		(if sending then "sendkey" else "recvkey")+		[ Param $ show key+		-- Command is terminated with "--", because+		-- rsync will tack on its own options afterwards,+		-- and they need to be ignored.+		, Param "--"+		]+	-- Convert the ssh command into rsync command line.+	let eparam = rsyncShell (Param shellcmd:shellparams)+	o <- rsyncParams r+	if sending+		then return $ o ++ eparam ++ [dummy, File file]+		else return $ o ++ eparam ++ [File file, dummy]+	where+		-- the rsync shell parameter controls where rsync+		-- goes, so the source/dest parameter can be a dummy value,+		-- that just enables remote rsync mode.+		dummy = Param ":"++rsyncParams :: Git.Repo -> Annex [CommandParam]+rsyncParams r = do+	o <- getConfig r "rsync-options" ""+	return $ options ++ map Param (words o)+	where+ 		-- --inplace to resume partial files+		options = [Params "-p --progress --inplace"]
+ Remote/Hook.hs view
@@ -0,0 +1,154 @@+{- A remote that provides hooks to run shell commands.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Hook (remote) where++import qualified Data.ByteString.Lazy.Char8 as L+import Control.Exception.Extensible (IOException)+import qualified Data.Map as M+import Control.Monad.State (liftIO)+import System.FilePath+import System.Posix.Process hiding (executeFile)+import System.Posix.IO+import System.IO+import System.IO.Error (try)+import System.Exit++import Types+import Types.Remote+import qualified Git+import qualified Annex+import UUID+import Locations+import Config+import Content+import Utility+import Remote.Special+import Remote.Encryptable+import Crypto+import Messages++remote :: RemoteType Annex+remote = RemoteType {+	typename = "hook",+	enumerate = findSpecialRemotes "hooktype",+	generate = gen,+	setup = hookSetup+}++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u c = do+	hooktype <- getConfig r "hooktype" (error "missing hooktype")+	cst <- remoteCost r expensiveRemoteCost+	return $ encryptableRemote c+		(storeEncrypted hooktype)+		(retrieveEncrypted hooktype)+		Remote {+			uuid = u,+			cost = cst,+			name = Git.repoDescribe r,+ 			storeKey = store hooktype,+			retrieveKeyFile = retrieve hooktype,+			removeKey = remove hooktype,+			hasKey = checkPresent r hooktype,+			hasKeyCheap = False,+			config = Nothing+		}++hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig+hookSetup u c = do+	let hooktype = maybe (error "Specify hooktype=") id $+		M.lookup "hooktype" c+	c' <- encryptionSetup c+	gitConfigSpecialRemote u c' "hooktype" hooktype+	return c'++hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)]+hookEnv k f = Just $ fileenv f ++ keyenv+	where+		env s v = ("ANNEX_" ++ s, v)+		keyenv =+			[ env "KEY" (show k)+			, env "HASH_1" (hashbits !! 0)+			, env "HASH_2" (hashbits !! 1)+			]+		fileenv Nothing = []+		fileenv (Just file) =  [env "FILE" file]+		hashbits = map takeDirectory $ splitPath $ hashDirMixed k++lookupHook :: String -> String -> Annex (Maybe String)+lookupHook hooktype hook =do+	g <- Annex.gitRepo+	command <- getConfig g hookname ""+	if null command+		then do+			warning $ "missing configuration for " ++ hookname+			return Nothing+		else return $ Just command+	where+		hookname =  hooktype ++ "-" ++ hook ++ "-hook"++runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool+runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook+	where+		run command = do+			showProgress -- make way for hook output+			res <- liftIO $ boolSystemEnv+				"sh" [Param "-c", Param command] $ hookEnv k f+			if res+				then a+				else do+					warning $ hook ++ " hook exited nonzero!"+					return res++store :: String -> Key -> Annex Bool+store h k = do+	g <- Annex.gitRepo+	runHook h "store" k (Just $ gitAnnexLocation g k) $ return True++storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool+storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do+	g <- Annex.gitRepo+	let f = gitAnnexLocation g k+	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s+	runHook h "store" enck (Just tmp) $ return True++retrieve :: String -> Key -> FilePath -> Annex Bool+retrieve h k f = runHook h "retrieve" k (Just f) $ return True++retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool+retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->+	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBool $ do+		withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f+		return True++remove :: String -> Key -> Annex Bool+remove h k = runHook h "remove" k Nothing $ do return True++checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)+checkPresent r h k = do+	showNote ("checking " ++ Git.repoDescribe r ++ "...")+	v <- lookupHook h "checkpresent"+	liftIO (try (check v) ::IO (Either IOException Bool))+	where+		findkey s = (show k) `elem` (lines s)+		env = hookEnv k Nothing+		check Nothing = error "checkpresent hook misconfigured"+		check (Just hook) = do+			(frompipe, topipe) <- createPipe+			pid <- forkProcess $ do+				_ <- dupTo topipe stdOutput+				closeFd frompipe+				executeFile "sh" True ["-c", hook] env+			closeFd topipe+			fromh <- fdToHandle frompipe+			reply <- hGetContentsStrict fromh+			hClose fromh+			s <- getProcessStatus True False pid+			case s of+				Just (Exited (ExitSuccess)) -> return $ findkey reply+				_ -> error "checkpresent hook failed"
+ Remote/Rsync.hs view
@@ -0,0 +1,203 @@+{- A remote that is only accessible by rsync.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Rsync (remote) where++import qualified Data.ByteString.Lazy.Char8 as L+import Control.Exception.Extensible (IOException)+import qualified Data.Map as M+import Control.Monad.State (liftIO)+import System.FilePath+import System.Directory+import System.Posix.Files+import System.Posix.Process++import Types+import Types.Remote+import qualified Git+import qualified Annex+import UUID+import Locations+import Config+import Content+import Utility+import Remote.Special+import Remote.Encryptable+import Crypto+import Messages+import RsyncFile++type RsyncUrl = String++data RsyncOpts = RsyncOpts {+	rsyncUrl :: RsyncUrl,+	rsyncOptions :: [CommandParam]+}++remote :: RemoteType Annex+remote = RemoteType {+	typename = "rsync",+	enumerate = findSpecialRemotes "rsyncurl",+	generate = gen,+	setup = rsyncSetup+}++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u c = do+	o <- genRsyncOpts r+	cst <- remoteCost r expensiveRemoteCost+	return $ encryptableRemote c+		(storeEncrypted o)+		(retrieveEncrypted o)+		Remote {+			uuid = u,+			cost = cst,+			name = Git.repoDescribe r,+ 			storeKey = store o,+			retrieveKeyFile = retrieve o,+			removeKey = remove o,+			hasKey = checkPresent r o,+			hasKeyCheap = False,+			config = Nothing+		}++genRsyncOpts :: Git.Repo -> Annex RsyncOpts+genRsyncOpts r = do+	url <- getConfig r "rsyncurl" (error "missing rsyncurl")+	opts <- getConfig r "rsync-options" ""+	return $ RsyncOpts url $ map Param $ filter safe $ words opts+	where+		safe o+			-- Don't allow user to pass --delete to rsync;+			-- that could cause it to delete other keys+			-- in the same hash bucket as a key it sends.+			| o == "--delete" = False+			| o == "--delete-excluded" = False+			| otherwise = True++rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig+rsyncSetup u c = do+	-- verify configuration is sane+	let url = maybe (error "Specify rsyncurl=") id $+		M.lookup "rsyncurl" c+	c' <- encryptionSetup c++	-- The rsyncurl is stored in git config, not only in this remote's+	-- persistant state, so it can vary between hosts.+	gitConfigSpecialRemote u c' "rsyncurl" url+	return c'++rsyncKey :: RsyncOpts -> Key -> String+rsyncKey o k = rsyncUrl o </> hashDirMixed k </> f </> f+        where+                f = keyFile k++store :: RsyncOpts -> Key -> Annex Bool+store o k = do+	g <- Annex.gitRepo+	rsyncSend o k (gitAnnexLocation g k)++storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool+storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do+	g <- Annex.gitRepo+	let f = gitAnnexLocation g k+	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s+	rsyncSend o enck tmp++retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool+retrieve o k f = rsyncRemote o+	-- use inplace when retrieving to support resuming+	[ Param "--inplace"+	, Param $ rsyncKey o k+	, Param f+	]++retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool+retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do+	res <- retrieve o enck tmp+	if res+		then liftIO $ catchBool $ do+			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f+			return True+		else return res++remove :: RsyncOpts -> Key -> Annex Bool+remove o k = withRsyncScratchDir $ \tmp -> do+	{- Send an empty directory to rysnc as the parent directory+         - of the file to remove. -}+	let dummy = tmp </> keyFile k+	liftIO $ createDirectoryIfMissing True dummy+	liftIO $ rsync $ rsyncOptions o +++		[ Params "--delete --recursive"+		, partialParams+		, Param $ addTrailingPathSeparator dummy+		, Param $ parentDir $ rsyncKey o k+		]++checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)+checkPresent r o k = do+	showNote ("checking " ++ Git.repoDescribe r ++ "...")+	-- note: Does not currently differnetiate between rsync failing+	-- to connect, and the file not being present.+	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]+	return $ Right res+	where+		cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null"+		testfile = shellEscape $ rsyncKey o k++{- Rsync params to enable resumes of sending files safely,+ - ensure that files are only moved into place once complete+ -}+partialParams :: CommandParam+partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"++{- Runs an action in an empty scratch directory that can be used to build+ - up trees for rsync. -}+withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool+withRsyncScratchDir a = do+	g <- Annex.gitRepo+	pid <- liftIO $ getProcessID+	let tmp = gitAnnexTmpDir g </> "rsynctmp" </> show pid+	nuke tmp+	liftIO $ createDirectoryIfMissing True $ tmp+	res <- a tmp+	nuke tmp+	return res+	where+		nuke d = liftIO $ +			doesDirectoryExist d >>? removeDirectoryRecursive d++rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool+rsyncRemote o params = do+	showProgress -- make way for progress bar+	res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params+	if res+		then return res+		else do+			showLongNote "rsync failed -- run git annex again to resume file transfer"+			return res+	where+		defaultParams = [Params "--progress"]++{- To send a single key is slightly tricky; need to build up a temporary+   directory structure to pass to rsync so it can create the hash+   directories. -}+rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool+rsyncSend o k src = withRsyncScratchDir $ \tmp -> do+	let dest = tmp </> hashDirMixed k </> f </> f+	liftIO $ createDirectoryIfMissing True $ parentDir $ dest+	liftIO $ createLink src dest+	res <- rsyncRemote o+		[ Param "--recursive"+		, partialParams+ 		  -- tmp/ to send contents of tmp dir+		, Param $ addTrailingPathSeparator tmp+		, Param $ rsyncUrl o+		]+	return res+	where+		f = keyFile k
+ Remote/S3real.hs view
@@ -0,0 +1,325 @@+{- Amazon S3 remotes.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.S3 (remote) where++import Control.Exception.Extensible (IOException)+import Network.AWS.AWSConnection+import Network.AWS.S3Object+import Network.AWS.S3Bucket hiding (size)+import Network.AWS.AWSResult+import qualified Data.ByteString.Lazy.Char8 as L+import qualified Data.Map as M+import Data.Maybe+import Data.List+import Data.Char+import Data.String.Utils+import Control.Monad (when)+import Control.Monad.State (liftIO)+import System.Environment+import System.Posix.Files+import System.Posix.Env (setEnv)++import Types+import Types.Remote+import Types.Key+import qualified Git+import qualified Annex+import UUID+import Messages+import Locations+import Config+import Remote.Special+import Remote.Encryptable+import Crypto+import Content+import Base64++remote :: RemoteType Annex+remote = RemoteType {+	typename = "S3",+	enumerate = findSpecialRemotes "s3",+	generate = gen,+	setup = s3Setup+}++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r u c = do+	cst <- remoteCost r expensiveRemoteCost+	return $ gen' r u c cst+gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex+gen' r u c cst = do+	encryptableRemote c+		(storeEncrypted this)+		(retrieveEncrypted this)+		this+	where+		this = Remote {+			uuid = u,+			cost = cst,+			name = Git.repoDescribe r,+	 		storeKey = store this,+			retrieveKeyFile = retrieve this,+			removeKey = remove this,+			hasKey = checkPresent this,+			hasKeyCheap = False,+			config = c+		}++s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig+s3Setup u c = handlehost $ M.lookup "host" c+	where+		remotename = fromJust (M.lookup "name" c)+		defbucket = remotename ++ "-" ++ u+		defaults = M.fromList+			[ ("datacenter", "US")+			, ("storageclass", "STANDARD")+			, ("host", defaultAmazonS3Host)+			, ("port", show defaultAmazonS3Port)+			, ("bucket", defbucket)+			]+		+		handlehost Nothing = defaulthost+		handlehost (Just h)+			| ".archive.org" `isSuffixOf` (map toLower h) = archiveorg+			| otherwise = defaulthost++		use fullconfig = do+			gitConfigSpecialRemote u fullconfig "s3" "true"+			s3SetCreds fullconfig	++		defaulthost = do+			c' <- encryptionSetup c+			let fullconfig = M.union c' defaults+			genBucket fullconfig+			use fullconfig++		archiveorg = do+			showNote $ "Internet Archive mode"+			maybe (error "specify bucket=") (const $ return ()) $+				M.lookup "bucket" archiveconfig+			use archiveconfig+			where+				archiveconfig =+					-- hS3 does not pass through+					-- x-archive-* headers+					M.mapKeys (replace "x-archive-" "x-amz-") $+					-- encryption does not make sense here+					M.insert "encryption" "none" $+					M.union c $+					-- special constraints on key names+					M.insert "mungekeys" "ia" $+					-- bucket created only when files+					-- are uploaded+					M.insert "x-amz-auto-make-bucket" "1" $+					-- no default bucket name; should+					-- be human-readable+					M.delete "bucket" defaults++store :: Remote Annex -> Key -> Annex Bool+store r k = s3Action r False $ \(conn, bucket) -> do+	g <- Annex.gitRepo+	res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k+	s3Bool res++storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool+storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> +	-- To get file size of the encrypted content, have to use a temp file.+	-- (An alternative would be chunking to to a constant size.)+	withTmp enck $ \tmp -> do+		g <- Annex.gitRepo+		let f = gitAnnexLocation g k+		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s+		res <- liftIO $ storeHelper (conn, bucket) r enck tmp+		s3Bool res++storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ())+storeHelper (conn, bucket) r k file = do+	content <- liftIO $ L.readFile file+	-- size is provided to S3 so the whole content does not need to be+	-- buffered to calculate it+	size <- maybe getsize (return . fromIntegral) $ keySize k+	let object = setStorageClass storageclass $ +		S3Object bucket (bucketFile r k) ""+			(("Content-Length", show size) : xheaders) content+	sendObject conn object+	where+		storageclass =+			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of+				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY+				_ -> STANDARD+		getsize = do+			s <- liftIO $ getFileStatus file+			return $ fileSize s+		+		xheaders = filter isxheader $ M.assocs $ fromJust $ config r+		isxheader (h, _) = "x-amz-" `isPrefixOf` h++retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool+retrieve r k f = s3Action r False $ \(conn, bucket) -> do+	res <- liftIO $ getObject conn $ bucketKey r bucket k+	case res of+		Right o -> do+			liftIO $ L.writeFile f $ obj_data o+			return True+		Left e -> s3Warning e++retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool+retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do+	res <- liftIO $ getObject conn $ bucketKey r bucket enck+	case res of+		Right o -> liftIO $ +			withDecryptedContent cipher (return $ obj_data o) $ \content -> do+				L.writeFile f content+				return True+		Left e -> s3Warning e++remove :: Remote Annex -> Key -> Annex Bool+remove r k = s3Action r False $ \(conn, bucket) -> do+	res <- liftIO $ deleteObject conn $ bucketKey r bucket k+	s3Bool res++checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)+checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do+	showNote ("checking " ++ name r ++ "...")+	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k+	case res of+		Right _ -> return $ Right True+		Left (AWSError _ _) -> return $ Right False+		Left e -> return $ Left (s3Error e)+	where+		noconn = Left $ error "S3 not configured"+			+s3Warning :: ReqError -> Annex Bool+s3Warning e = do+	warning $ prettyReqError e+	return False++s3Error :: ReqError -> a+s3Error e = error $ prettyReqError e++s3Bool :: AWSResult () -> Annex Bool+s3Bool res = do+	case res of+		Right _ -> return True+		Left e -> s3Warning e++s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a+s3Action r noconn action = do+	when (config r == Nothing) $+		error $ "Missing configuration for special remote " ++ name r+	let bucket = M.lookup "bucket" $ fromJust $ config r+	conn <- s3Connection $ fromJust $ config r+	case (bucket, conn) of+		(Just b, Just c) -> action (c, b)+		_ -> return noconn++bucketFile :: Remote Annex -> Key -> FilePath+bucketFile r k = (munge $ show k)+	where+		munge s = case M.lookup "mungekeys" $ fromJust $ config r of+			Just "ia" -> iaMunge s+			_ -> s++bucketKey :: Remote Annex -> String -> Key -> S3Object+bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty++{- Internet Archive limits filenames to a subset of ascii,+ - with no whitespace. Other characters are xml entity+ - encoded. -}+iaMunge :: String -> String+iaMunge = (>>= munge)+	where+		munge c+			| isAsciiUpper c || isAsciiLower c || isNumber c = [c]+			| c `elem` "_-.\"" = [c]+			| isSpace c = []+			| otherwise = "&" ++ show (ord c) ++ ";"++genBucket :: RemoteConfig -> Annex ()+genBucket c = do+	conn <- s3ConnectionRequired c+	showNote "checking bucket"+	loc <- liftIO $ getBucketLocation conn bucket +	case loc of+		Right _ -> return ()+		Left err@(NetworkError _) -> s3Error err+		Left (AWSError _ _) -> do+			showNote $ "creating bucket in " ++ datacenter+			res <- liftIO $ createBucketIn conn bucket datacenter+			case res of+				Right _ -> return ()+				Left err -> s3Error err+	where+		bucket = fromJust $ M.lookup "bucket" c+		datacenter = fromJust $ M.lookup "datacenter" c++s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection+s3ConnectionRequired c =+	maybe (error "Cannot connect to S3") return =<< s3Connection c++s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection)+s3Connection c = do+	creds <- s3GetCreds c+	case creds of+		Just (ak, sk) -> return $ Just $ AWSConnection host port ak sk+		_ -> do+			warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey  ++ " to use S3"+			return Nothing+	where+		host = fromJust $ (M.lookup "host" c)+		port = let s = fromJust $ (M.lookup "port" c) in+			case reads s of+			[(p, _)] -> p+			_ -> error $ "bad S3 port value: " ++ s++{- S3 creds come from the environment if set. + - Otherwise, might be stored encrypted in the remote's config. -}+s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String))+s3GetCreds c = do+	ak <- getEnvKey s3AccessKey+	sk <- getEnvKey s3SecretKey+	if (null ak || null sk)+		then do+			mcipher <- remoteCipher c+			case (M.lookup "s3creds" c, mcipher) of+				(Just encrypted, Just cipher) -> do+					s <- liftIO $ withDecryptedContent cipher+						(return $ L.pack $ fromB64 encrypted)+						(return . L.unpack)+					let line = lines s+					let ak' = line !! 0+					let sk' = line !! 1+					liftIO $ do+						setEnv s3AccessKey ak True+						setEnv s3SecretKey sk True+					return $ Just (ak', sk')+				_ -> return Nothing+		else return $ Just (ak, sk)+	where+		getEnvKey s = liftIO $ catch (getEnv s) (const $ return "")++{- Stores S3 creds encrypted in the remote's config if possible. -}+s3SetCreds :: RemoteConfig -> Annex RemoteConfig+s3SetCreds c = do+	creds <- s3GetCreds c+	case creds of+		Just (ak, sk) -> do+			mcipher <- remoteCipher c+			case mcipher of+				Just cipher -> do+					s <- liftIO $ withEncryptedContent cipher+						(return $ L.pack $ unlines [ak, sk])+						(return . L.unpack)+					return $ M.insert "s3creds" (toB64 s) c+				Nothing -> return c+		_ -> return c++s3AccessKey :: String+s3AccessKey = "AWS_ACCESS_KEY_ID"+s3SecretKey :: String+s3SecretKey = "AWS_SECRET_ACCESS_KEY"
+ Remote/S3stub.hs view
@@ -0,0 +1,13 @@+-- stub for when hS3 is not available+module Remote.S3 (remote) where++import Types.Remote+import Types++remote :: RemoteType Annex+remote = RemoteType {+	typename = "S3",+	enumerate = return [],+	generate = error "S3 not enabled",+	setup = error "S3 not enabled"+}
+ Remote/Special.hs view
@@ -0,0 +1,44 @@+{- common functions for special remotes+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Special where++import qualified Data.Map as M+import Data.Maybe+import Data.String.Utils+import Control.Monad.State (liftIO)++import Types+import Types.Remote+import qualified Git+import qualified Annex+import UUID+import Utility++{- Special remotes don't have a configured url, so Git.Repo does not+ - automatically generate remotes for them. This looks for a different+ - configuration key instead.+ -}+findSpecialRemotes :: String -> Annex [Git.Repo]+findSpecialRemotes s = do+	g <- Annex.gitRepo+	return $ map construct $ remotepairs g+	where+		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k+		match k _ = startswith "remote." k && endswith (".annex-"++s) k++{- Sets up configuration for a special remote in .git/config. -}+gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex ()+gitConfigSpecialRemote u c k v = do+	g <- Annex.gitRepo+	liftIO $ do+		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]+		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]+	where+		remotename = fromJust (M.lookup "name" c)+		configsetting s = "remote." ++ remotename ++ "." ++ s
+ Remote/Web.hs view
@@ -0,0 +1,127 @@+{- Web remotes.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Remote.Web (+	remote,+	setUrl,+	download+) where++import Control.Monad.State (liftIO)+import Control.Exception+import System.FilePath+import Network.Curl.Easy+import Network.Curl.Opts+import Network.Curl.Types+import Network.Curl.Code++import Types+import Types.Remote+import qualified Git+import qualified Annex+import Messages+import Utility+import UUID+import Config+import PresenceLog+import LocationLog+import Locations++remote :: RemoteType Annex+remote = RemoteType {+	typename = "web",+	enumerate = list,+	generate = gen,+	setup = error "not supported"+}++-- There is only one web remote, and it always exists.+-- (If the web should cease to exist, remove this module and redistribute+-- a new release to the survivors by carrier pigeon.)+list :: Annex [Git.Repo]+list = return [Git.repoRemoteNameSet Git.repoFromUnknown "remote.web.dummy"]++-- Dummy uuid for the whole web. Do not alter.+webUUID :: UUID+webUUID = "00000000-0000-0000-0000-000000000001"++gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)+gen r _ _ = +	return $ Remote {+		uuid = webUUID,+		cost = expensiveRemoteCost,+		name = Git.repoDescribe r,+		storeKey = uploadKey,+		retrieveKeyFile = downloadKey,+		removeKey = dropKey,+		hasKey = checkKey,+		hasKeyCheap = False,+		config = Nothing+	}++{- The urls for a key are stored in remote/web/hash/key.log + - in the git-annex branch. -}+urlLog :: Key -> FilePath+urlLog key = "remote/web" </> hashDirLower key </> show key ++ ".log"++getUrls :: Key -> Annex [URLString]+getUrls key = currentLog (urlLog key)++{- Records a change in an url for a key. -}+setUrl :: Key -> URLString -> LogStatus -> Annex ()+setUrl key url status = do+	g <- Annex.gitRepo+	addLog (urlLog key) =<< logNow status url++	-- update location log to indicate that the web has the key, or not+	us <- getUrls key+	logChange g key webUUID (if null us then InfoMissing else InfoPresent)++downloadKey :: Key -> FilePath -> Annex Bool+downloadKey key file = do+	us <- getUrls key+	download us file++uploadKey :: Key -> Annex Bool+uploadKey _ = do+	warning "upload to web not supported"+	return False++dropKey :: Key -> Annex Bool+dropKey _ = do+	warning "removal from web not supported"+	return False++checkKey :: Key -> Annex (Either IOException Bool)+checkKey key = do+	us <- getUrls key+	if null us+		then return $ Right False+		else return . Right =<< checkKey' us+checkKey' :: [URLString] -> Annex Bool+checkKey' [] = return False+checkKey' (u:us) = do+	showNote ("checking " ++ u)+	e <- liftIO $ urlexists u+	if e then return e else checkKey' us++urlexists :: URLString -> IO Bool+urlexists url = do+	curl <- initialize+	_ <- setopt curl (CurlURL url)+	_ <- setopt curl (CurlNoBody True)+	_ <- setopt curl (CurlFailOnError True)+	_ <- setopt curl (CurlFollowLocation True)+	res <- perform curl+	return $ res == CurlOK++download :: [URLString] -> FilePath -> Annex Bool+download [] _ = return False+download (url:us) file = do+	showProgress -- make way for curl progress bar+	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]+	if ok then return ok else download us file
+ RsyncFile.hs view
@@ -0,0 +1,48 @@+{- git-annex file copying with rsync+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module RsyncFile where++import Data.String.Utils++import Utility++{- Generates parameters to make rsync use a specified command as its remote+ - shell. -}+rsyncShell :: [CommandParam] -> [CommandParam]+rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)]+	where+		{- rsync requires some weird, non-shell like quoting in+                 - here. A doubled single quote inside the single quoted+                 - string is a single quote. -}+		escape s = "'" ++  (join "''" $ split "'" s) ++ "'"++{- Runs rsync in server mode to send a file, and exits. -}+rsyncServerSend :: FilePath -> IO ()+rsyncServerSend file = rsyncExec $+	rsyncServerParams ++ [Param "--sender", File file]++{- Runs rsync in server mode to receive a file. -}+rsyncServerReceive :: FilePath -> IO Bool+rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file]++rsyncServerParams :: [CommandParam]+rsyncServerParams =+	[ Param "--server"+	-- preserve permissions+	, Param "-p"+	-- allow resuming of transfers of big files+	, Param "--inplace"+	-- other options rsync normally uses in server mode+	, Params "-e.Lsf ."+	]++rsync :: [CommandParam] -> IO Bool+rsync = boolSystem "rsync"++rsyncExec :: [CommandParam] -> IO ()+rsyncExec params = executeFile "rsync" True (toCommand params) Nothing
+ Setup.hs view
@@ -0,0 +1,17 @@+{- cabal setup file -}++import Distribution.Simple+import System.Cmd++main = defaultMainWithHooks simpleUserHooks {+	preConf = makeSources,+	postClean = makeClean+}++makeSources _ _ = do+	system "make sources"+	return (Nothing, [])++makeClean _ _ _ _ = do+	system "make clean"+	return ()
+ Ssh.hs view
@@ -0,0 +1,61 @@+{- git-annex repository access with ssh+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Ssh where++import Control.Monad.State (liftIO)++import qualified Git+import Utility+import Types+import Config++{- Generates parameters to ssh to a repository's host and run a command.+ - Caller is responsible for doing any neccessary shellEscaping of the+ - passed command. -}+sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]+sshToRepo repo sshcmd = do+	s <- getConfig repo "ssh-options" ""+	let sshoptions = map Param (words s)+	let sshport = case Git.urlPort repo of+		Nothing -> []+		Just p -> [Param "-p", Param (show p)]+	let sshhost = Param $ Git.urlHostUser repo+	return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd++{- Generates parameters to run a git-annex-shell command on a remote+ - repository. -}+git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))+git_annex_shell r command params+	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)+	| Git.repoIsSsh r = do+		sshparams <- sshToRepo r [Param sshcmd]+		return $ Just ("ssh", sshparams)+	| otherwise = return Nothing+	where+		dir = Git.workTree r+		shellcmd = "git-annex-shell"+		shellopts = (Param command):(File dir):params+		sshcmd = shellcmd ++ " " ++ +			unwords (map shellEscape $ toCommand shellopts)++{- Uses a supplied function (such as boolSystem) to run a git-annex-shell+ - command on a remote.+ -+ - Or, if the remote does not support running remote commands, returns+ - a specified error value. -}+onRemote +	:: Git.Repo+	-> (FilePath -> [CommandParam] -> IO a, a)+	-> String+	-> [CommandParam]+	-> Annex a+onRemote r (with, errorval) command params = do+	s <- git_annex_shell r command params+	case s of+		Just (c, ps) -> liftIO $ with c ps+		Nothing -> return errorval
+ StatFS.hsc view
@@ -0,0 +1,125 @@+-----------------------------------------------------------------------------+-- |+--+-- (This code originally comes from xmobar)+-- +-- Module      :  StatFS+-- Copyright   :  (c) Jose A Ortega Ruiz+-- License     :  BSD-3-clause+--+-- All rights reserved.+-- +-- Redistribution and use in source and binary forms, with or without+-- modification, are permitted provided that the following conditions+-- are met:+-- +-- 1. Redistributions of source code must retain the above copyright+--    notice, this list of conditions and the following disclaimer.+-- 2. Redistributions in binary form must reproduce the above copyright+--    notice, this list of conditions and the following disclaimer in the+--    documentation and/or other materials provided with the distribution.+-- 3. Neither the name of the author nor the names of his contributors+--    may be used to endorse or promote products derived from this software+--    without specific prior written permission.+-- +-- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND+-- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+-- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE+-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS+-- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)+-- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+-- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY+-- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+-- SUCH DAMAGE.+--+-- Maintainer  :  Jose A Ortega Ruiz <jao@gnu.org>+-- Stability   :  unstable+-- Portability :  unportable+--+--  A binding to C's statvfs(2)+--+-----------------------------------------------------------------------------++{-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}+++module StatFS ( FileSystemStats(..), getFileSystemStats ) where++import Foreign+import Foreign.C.Types+import Foreign.C.String+import Data.ByteString (useAsCString)+import Data.ByteString.Char8 (pack)++#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)+# include <sys/param.h>+# include <sys/mount.h>+#else+#if defined (__linux__)+#include <sys/vfs.h>+#else+#define UNKNOWN+#endif+#endif++data FileSystemStats = FileSystemStats {+  fsStatBlockSize :: Integer+  -- ^ Optimal transfer block size.+  , fsStatBlockCount :: Integer+  -- ^ Total data blocks in file system.+  , fsStatByteCount :: Integer+  -- ^ Total bytes in file system.+  , fsStatBytesFree :: Integer+  -- ^ Free bytes in file system.+  , fsStatBytesAvailable :: Integer+  -- ^ Free bytes available to non-superusers.+  , fsStatBytesUsed :: Integer+  -- ^ Bytes used.+  } deriving (Show, Eq)++data CStatfs++#ifdef UNKNOWN+#warning free space checking code not available for this OS+#else+#if defined(__APPLE__)+foreign import ccall unsafe "sys/mount.h statfs64"+#else+#if defined(__FreeBSD__) || defined (__FreeBSD_kernel__)+foreign import ccall unsafe "sys/mount.h statfs"+#else+foreign import ccall unsafe "sys/vfs.h statfs64"+#endif+#endif+  c_statfs :: CString -> Ptr CStatfs -> IO CInt+#endif++toI :: CLong -> Integer+toI = toInteger++getFileSystemStats :: String -> IO (Maybe FileSystemStats)+getFileSystemStats path =+#ifdef UNKNOWN+  return Nothing+#else+  allocaBytes (#size struct statfs) $ \vfs ->+  useAsCString (pack path) $ \cpath -> do+    res <- c_statfs cpath vfs+    if res == -1 then return Nothing+      else do+        bsize <- (#peek struct statfs, f_bsize) vfs+        bcount <- (#peek struct statfs, f_blocks) vfs+        bfree <- (#peek struct statfs, f_bfree) vfs+        bavail <- (#peek struct statfs, f_bavail) vfs+        let bpb = toI bsize+        return $ Just FileSystemStats+                       { fsStatBlockSize = bpb+                       , fsStatBlockCount = toI bcount+                       , fsStatByteCount = toI bcount * bpb+                       , fsStatBytesFree = toI bfree * bpb+                       , fsStatBytesAvailable = toI bavail * bpb+                       , fsStatBytesUsed = toI (bcount - bfree) * bpb+                       }+#endif
+ TestConfig.hs view
@@ -0,0 +1,111 @@+{- Tests the system and generates SysConfig.hs. -}++module TestConfig where++import System.IO+import System.Cmd+import System.Exit++type ConfigKey = String+data ConfigValue =+	BoolConfig Bool |+	StringConfig String |+	MaybeStringConfig (Maybe String)+data Config = Config ConfigKey ConfigValue++type Test = IO Config+type TestName = String+data TestCase = TestCase TestName Test++instance Show ConfigValue where+	show (BoolConfig b) = show b+	show (StringConfig s) = show s+	show (MaybeStringConfig s) = show s++instance Show Config where+	show (Config key value) = unlines+		[ key ++ " :: " ++ valuetype value+		, key ++ " = " ++ show value+		]+		where+			valuetype (BoolConfig _) = "Bool"+			valuetype (StringConfig _) = "String"+			valuetype (MaybeStringConfig _) = "Maybe String"++writeSysConfig :: [Config] -> IO ()+writeSysConfig config = writeFile "SysConfig.hs" body+	where+		body = unlines $ header ++ map show config ++ footer+		header = [+			  "{- Automatically generated. -}"+			, "module SysConfig where"+			, ""+			]+		footer = []++runTests :: [TestCase] -> IO [Config]+runTests [] = return []+runTests ((TestCase tname t):ts) = do+	testStart tname+	c <- t+	testEnd c+	rest <- runTests ts+	return $ c:rest++{- Tests that a command is available, aborting if not. -}+requireCmd :: ConfigKey -> String -> Test+requireCmd k cmdline = do+	ret <- testCmd k cmdline+	handle ret+	where+		handle r@(Config _ (BoolConfig True)) = return r+		handle r = do+			testEnd r+			error $ "** the " ++ c ++ " command is required"+		c = (words cmdline) !! 0++{- Checks if a command is available by running a command line. -}+testCmd :: ConfigKey -> String -> Test+testCmd k cmdline = do+	ret <- system $ quiet cmdline+	return $ Config k (BoolConfig $ ret == ExitSuccess)++{- Ensures that one of a set of commands is available by running each in+ - turn. The Config is set to the first one found. -}+selectCmd :: ConfigKey -> [String] -> String -> Test+selectCmd k = searchCmd+		(\match -> return $ Config k $ StringConfig match)+		(\cmds -> do+			testEnd $ Config k $ BoolConfig False+			error $ "* need one of these commands, but none are available: " ++ show cmds+		)++maybeSelectCmd :: ConfigKey -> [String] -> String -> Test+maybeSelectCmd k = searchCmd+		(\match -> return $ Config k $ MaybeStringConfig $ Just match)+		(\_ -> return $ Config k $ MaybeStringConfig Nothing)++searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test+searchCmd success failure cmds param = search cmds+	where+		search [] = failure cmds+		search (c:cs) = do+			ret <- system $ quiet c ++ " " ++ param+			if (ret == ExitSuccess)+				then success c+				else search cs++quiet :: String -> String+quiet s = s ++ " >/dev/null 2>&1"++testStart :: TestName -> IO ()+testStart s = do+	putStr $ "  checking " ++ s ++ "..."+	hFlush stdout++testEnd :: Config -> IO ()+testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes"+testEnd (Config _ (BoolConfig False)) = putStrLn $ " no"+testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s+testEnd (Config _ (MaybeStringConfig (Just s))) = putStrLn $ " " ++ s+testEnd (Config _ (MaybeStringConfig Nothing)) = putStrLn $ " not available"
+ Touch.hsc view
@@ -0,0 +1,120 @@+{- More control over touching a file.+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++{-# LANGUAGE ForeignFunctionInterface #-}++module Touch (+	TimeSpec(..),+	touchBoth,+	touch+) where++import Foreign+import Foreign.C++newtype TimeSpec = TimeSpec CTime++{- Changes the access and modification times of an existing file.+   Can follow symlinks, or not. Throws IO error on failure. -}+touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()++touch :: FilePath -> TimeSpec -> Bool -> IO ()+touch file mtime follow = touchBoth file mtime mtime follow++#include <sys/types.h>+#include <sys/stat.h>+#include <fcntl.h>+#include <sys/time.h>++#ifndef _BSD_SOURCE+#define _BSD_SOURCE+#endif++#if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW)++at_fdcwd :: CInt+at_fdcwd = #const AT_FDCWD++at_symlink_nofollow :: CInt+at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW++instance Storable TimeSpec where+	-- use the larger alignment of the two types in the struct+	alignment _ = max sec_alignment nsec_alignment+		where+			sec_alignment = alignment (undefined::CTime)+			nsec_alignment = alignment (undefined::CLong)+	sizeOf _ = #{size struct timespec}+	peek ptr = do+		sec <- #{peek struct timespec, tv_sec} ptr+		return $ TimeSpec sec+	poke ptr (TimeSpec sec) = do+		#{poke struct timespec, tv_sec} ptr sec+		#{poke struct timespec, tv_nsec} ptr (0 :: CLong)++{- While its interface is beastly, utimensat is in recent+   POSIX standards, unlike lutimes. -}+foreign import ccall "utimensat" +	c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt++touchBoth file atime mtime follow = +	allocaArray 2 $ \ptr ->+	withCString file $ \f -> do+		pokeArray ptr [atime, mtime]+		r <- c_utimensat at_fdcwd f ptr flags+		if (r /= 0)+			then throwErrno "touchBoth"+			else return ()+	where+		flags = if follow+			then 0+			else at_symlink_nofollow ++#else+#if 0+{- Using lutimes is needed for BSD.+ - + - TODO: test if lutimes is available. May have to do it in configure.+ - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation+ - dependent whether these are the same; need to find a cast that works.+ - (Without the cast it works on linux i386, but+ - maybe not elsewhere.)+ -}++instance Storable TimeSpec where+	alignment _ = alignment (undefined::CLong)+	sizeOf _ = #{size struct timeval}+	peek ptr = do+		sec <- #{peek struct timeval, tv_sec} ptr+		return $ TimeSpec sec+	poke ptr (TimeSpec sec) = do+		#{poke struct timeval, tv_sec} ptr sec+		#{poke struct timeval, tv_usec} ptr (0 :: CLong) ++foreign import ccall "utimes" +	c_utimes :: CString -> Ptr TimeSpec -> IO CInt+foreign import ccall "lutimes" +	c_lutimes :: CString -> Ptr TimeSpec -> IO CInt++touchBoth file atime mtime follow = +	allocaArray 2 $ \ptr ->+	withCString file $ \f -> do+		pokeArray ptr [atime, mtime]+		r <- syscall f ptr+		if (r /= 0)+			then throwErrno "touchBoth"+			else return ()+	where+		syscall = if follow+			then c_lutimes+			else c_utimes++#else+#warning "utimensat and lutimes not available; building without symlink timestamp preservation support"+touchBoth _ _ _ _ = return ()+#endif+#endif
+ Trust.hs view
@@ -0,0 +1,72 @@+{- git-annex trust+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Trust (+	TrustLevel(..),+	trustLog,+	trustGet,+	trustSet+) where++import Control.Monad.State+import qualified Data.Map as M++import Types.TrustLevel+import qualified Branch+import Types+import UUID+import qualified Annex++{- Filename of trust.log. -}+trustLog :: FilePath+trustLog = "trust.log"++{- Returns a list of UUIDs at the specified trust level. -}+trustGet :: TrustLevel -> Annex [UUID]+trustGet level = do+	m <- trustMap+	return $ M.keys $ M.filter (== level) m++{- Read the trustLog into a map, overriding with any+ - values from forcetrust -}+trustMap :: Annex TrustMap+trustMap = do+	cached <- Annex.getState Annex.trustmap+	case cached of+		Just m -> return m+		Nothing -> do+			overrides <- Annex.getState Annex.forcetrust+			l <- Branch.get trustLog+			let m = M.fromList $ trustMapParse l ++ overrides+			Annex.changeState $ \s -> s { Annex.trustmap = Just m }+			return m++{- Trust map parser. -}+trustMapParse :: String -> [(UUID, TrustLevel)]+trustMapParse s = map pair $ filter (not . null) $ lines s+	where+		pair l+			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)+			-- for back-compat; the trust log used to only+			-- list trusted uuids+			| otherwise = (w !! 0, Trusted)+			where+				w = words l++{- Changes the trust level for a uuid in the trustLog. -}+trustSet :: UUID -> TrustLevel -> Annex ()+trustSet uuid level = do+	when (null uuid) $+		error "unknown UUID; cannot modify trust level"+        m <- trustMap+	when (M.lookup uuid m /= Just level) $ do+		let m' = M.insert uuid level m+		Branch.change trustLog (serialize m')+		Annex.changeState $ \s -> s { Annex.trustmap = Just m' }+        where+                serialize m = unlines $ map showpair $ M.toList m+		showpair (u, t) = u ++ " " ++ show t
+ Types.hs view
@@ -0,0 +1,16 @@+{- git-annex abstract data types+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types (+	Annex,+	Backend,+	Key+) where++import Annex+import Types.Backend+import Types.Key
+ Types/Backend.hs view
@@ -0,0 +1,41 @@+{- git-annex key/value backend data type+ -+ - Most things should not need this, using Types instead+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.Backend where++import Types.Key++data Backend a = Backend {+	-- name of this backend+	name :: String,+	-- converts a filename to a key+	getKey :: FilePath -> a (Maybe Key),+	-- stores a file's contents to a key+	storeFileKey :: FilePath -> Key -> a Bool,+	-- retrieves a key's contents to a file+	retrieveKeyFile :: Key -> FilePath -> a Bool,+	-- removes a key, optionally checking that enough copies are stored+	-- elsewhere+	removeKey :: Key -> Maybe Int -> a Bool,+	-- checks if a backend is storing the content of a key+	hasKey :: Key -> a Bool,+	-- called during fsck to check a key+	-- (second parameter may be the filename associated with it)+	-- (third parameter may be the number of copies that there should+	-- be of the key)+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,+	-- Is a newer repesentation possible for a key?+	upgradableKey :: Key -> a Bool+}++instance Show (Backend a) where+	show backend = "Backend { name =\"" ++ name backend ++ "\" }"++instance Eq (Backend a) where+	a == b = name a == name b
+ Types/BranchState.hs view
@@ -0,0 +1,24 @@+{- git-annex BranchState data type+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.BranchState where++import System.IO++data BranchState = BranchState {+	branchUpdated :: Bool, -- has the branch been updated this run?++	-- (from, to) handles used to talk to a git-cat-file process+	catFileHandles :: Maybe (Handle, Handle),++	-- the content of one file is cached+	cachedFile :: Maybe FilePath,+	cachedContent :: String+}++startBranchState :: BranchState+startBranchState = BranchState False Nothing Nothing ""
+ Types/Crypto.hs view
@@ -0,0 +1,23 @@+{- git-annex crypto types+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.Crypto where++import Data.String.Utils++-- XXX ideally, this would be a locked memory region+newtype Cipher = Cipher String++data EncryptedCipher = EncryptedCipher String KeyIds++newtype KeyIds = KeyIds [String]++instance Show KeyIds where+	show (KeyIds ks) = join "," ks++instance Read KeyIds where+	readsPrec _ s = [(KeyIds (split "," s), "")]
+ Types/Key.hs view
@@ -0,0 +1,76 @@+{- git-annex Key data type+ - + - Most things should not need this, using Types instead+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.Key (+	Key(..),+	stubKey,+	readKey,++	prop_idempotent_key_read_show+) where++import Utility+import System.Posix.Types++{- A Key has a unique name, is associated with a key/value backend,+ - and may contain other optional metadata. -}+data Key = Key {+	keyName :: String,+	keyBackendName :: String,+	keySize :: Maybe Integer,+	keyMtime :: Maybe EpochTime+} deriving (Eq, Ord)++stubKey :: Key+stubKey = Key {+	keyName = "",+	keyBackendName = "",+	keySize = Nothing,+	keyMtime = Nothing+}++fieldSep :: Char+fieldSep = '-'++{- Keys show as strings that are suitable for use as filenames.+ - The name field is always shown last, separated by doubled fieldSeps,+ - and is the only field allowed to contain the fieldSep. -}+instance Show Key where+	show Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } =+		b +++ ('s' ?: s) +++ ('m' ?: m) +++ (fieldSep : n)+		where+			"" +++ y = y+			x +++ "" = x+			x +++ y = x ++ fieldSep:y+			c ?: (Just v) = c:(show v)+			_ ?: _ = ""++readKey :: String -> Maybe Key+readKey s = if key == Just stubKey then Nothing else key+	where+		key = startbackend stubKey s++		startbackend k v = sepfield k v addbackend+		+		sepfield k v a = case span (/= fieldSep) v of+			(v', _:r) -> findfields r $ a k v'+			_ -> Nothing++		findfields (c:v) (Just k)+			| c == fieldSep = Just $ k { keyName = v }+			| otherwise = sepfield k v $ addfield c+		findfields _ v = v++		addbackend k v = Just k { keyBackendName = v }+		addfield 's' k v = Just k { keySize = readMaybe v }+		addfield 'm' k v = Just k { keyMtime = readMaybe v }+		addfield _ _ _ = Nothing++prop_idempotent_key_read_show :: Key -> Bool+prop_idempotent_key_read_show k = Just k == (readKey $ show k)
+ Types/Remote.hs view
@@ -0,0 +1,65 @@+{- git-annex remotes types+ -+ - Most things should not need this, using Remote instead+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.Remote where++import Control.Exception+import Data.Map as M++import qualified Git+import Types.Key++type RemoteConfig = M.Map String String++{- There are different types of remotes. -}+data RemoteType a = RemoteType {+	-- human visible type name+	typename :: String,+	-- enumerates remotes of this type+	enumerate :: a [Git.Repo],+	-- generates a remote of this type+	generate :: Git.Repo -> String -> Maybe RemoteConfig -> a (Remote a),+	-- initializes or changes a remote+	setup :: String -> RemoteConfig -> a RemoteConfig+}++{- An individual remote. -}+data Remote a = Remote {+	-- each Remote has a unique uuid+	uuid :: String,+	-- each Remote has a human visible name+	name :: String,+	-- Remotes have a use cost; higher is more expensive+	cost :: Int,+	-- Transfers a key to the remote.+	storeKey :: Key -> a Bool,+	-- retrieves a key's contents to a file+	retrieveKeyFile :: Key -> FilePath -> a Bool,+	-- removes a key's contents+	removeKey :: Key -> a Bool,+	-- Checks if a key is present in the remote; if the remote+	-- cannot be accessed returns a Left error.+	hasKey :: Key -> a (Either IOException Bool),+	-- Some remotes can check hasKey without an expensive network+	-- operation.+	hasKeyCheap :: Bool,+	-- a Remote can have a persistent configuration store+	config :: Maybe RemoteConfig+}++instance Show (Remote a) where+	show remote = "Remote { name =\"" ++ name remote ++ "\" }"++-- two remotes are the same if they have the same uuid+instance Eq (Remote a) where+	x == y = uuid x == uuid y++-- order remotes by cost+instance Ord (Remote a) where+	compare x y = compare (cost x) (cost y)
+ Types/TrustLevel.hs view
@@ -0,0 +1,30 @@+{- git-annex trust levels+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.TrustLevel (+	TrustLevel(..),+	TrustMap+) where++import qualified Data.Map as M++import Types.UUID++data TrustLevel = SemiTrusted | UnTrusted | Trusted+	deriving Eq++instance Show TrustLevel where+        show SemiTrusted = "?"+        show UnTrusted = "0"+        show Trusted = "1"++instance Read TrustLevel where+        readsPrec _ "1" = [(Trusted, "")]+        readsPrec _ "0" = [(UnTrusted, "")]+	readsPrec _ _ = [(SemiTrusted, "")]++type TrustMap = M.Map UUID TrustLevel
+ Types/UUID.hs view
@@ -0,0 +1,11 @@+{- git-annex UUID type+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Types.UUID where++-- might be nice to have a newtype, but lots of stuff treats uuids as strings+type UUID = String
+ UUID.hs view
@@ -0,0 +1,106 @@+{- git-annex uuids+ -+ - Each git repository used by git-annex has an annex.uuid setting that+ - uniquely identifies that repository.+ -+ - UUIDs of remotes are cached in git config, using keys named+ - remote.<name>.annex-uuid+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module UUID (+	UUID,+	getUUID,+	getUncachedUUID,+	prepUUID,+	genUUID,+	describeUUID,+	uuidMap,+	uuidLog+) where++import Control.Monad.State+import System.Cmd.Utils+import System.IO+import qualified Data.Map as M+import Data.Maybe++import qualified Git+import qualified Branch+import Types+import Types.UUID+import qualified Annex+import qualified SysConfig+import Config++configkey :: String+configkey = "annex.uuid"++{- Filename of uuid.log. -}+uuidLog :: FilePath+uuidLog = "uuid.log"++{- Generates a UUID. There is a library for this, but it's not packaged,+ - so use the command line tool. -}+genUUID :: IO UUID+genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h+	where+		command = SysConfig.uuid+		params = if (command == "uuid")+			-- request a random uuid be generated+			then ["-m"]+			-- uuidgen generates random uuid by default+			else []++{- Looks up a repo's UUID. May return "" if none is known.+ -}+getUUID :: Git.Repo -> Annex UUID+getUUID r = do+	g <- Annex.gitRepo++	let c = cached g+	let u = getUncachedUUID r+	+	if c /= u && u /= ""+		then do+			updatecache g u+			return u+		else return c+	where+		cached g = Git.configGet g cachekey ""+		updatecache g u = when (g /= r) $ setConfig cachekey u+		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"++getUncachedUUID :: Git.Repo -> UUID+getUncachedUUID r = Git.configGet r configkey ""++{- Make sure that the repo has an annex.uuid setting. -}+prepUUID :: Annex ()+prepUUID = do+	u <- getUUID =<< Annex.gitRepo+	when ("" == u) $ do+		uuid <- liftIO $ genUUID+		setConfig configkey uuid++{- Records a description for a uuid in the uuidLog. -}+describeUUID :: UUID -> String -> Annex ()+describeUUID uuid desc = do+	m <- uuidMap+	let m' = M.insert uuid desc m+	Branch.change uuidLog (serialize m')+	where+		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m++{- Read and parse the uuidLog into a Map -}+uuidMap :: Annex (M.Map UUID String)+uuidMap = do+	s <- Branch.get uuidLog+	return $ M.fromList $ map pair $ lines s+	where+		pair l =+			if 1 < length (words l)+				then (head $ words l, unwords $ drop 1 $ words l)+				else ("", "")
+ Upgrade.hs view
@@ -0,0 +1,24 @@+{- git-annex upgrade support+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Upgrade where++import Types+import Version+import qualified Upgrade.V0+import qualified Upgrade.V1+import qualified Upgrade.V2++{- Uses the annex.version git config setting to automate upgrades. -}+upgrade :: Annex Bool+upgrade = do+	version <- getVersion+	case version of+		Just "0" -> Upgrade.V0.upgrade+		Just "1" -> Upgrade.V1.upgrade+		Just "2" -> Upgrade.V2.upgrade+		_ -> return True
+ Upgrade/V0.hs view
@@ -0,0 +1,63 @@+{- git-annex v0 -> v1 upgrade support+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Upgrade.V0 where++import System.IO.Error (try)+import System.Directory+import Control.Monad.State (liftIO)+import Control.Monad (filterM, forM_)+import System.Posix.Files+import System.FilePath++import Content+import Types+import Locations+import qualified Annex+import Messages+import qualified Upgrade.V1++upgrade :: Annex Bool+upgrade = do+	showNote "v0 to v1..."+	g <- Annex.gitRepo++	-- do the reorganisation of the key files+	let olddir = gitAnnexDir g+	keys <- getKeysPresent0 olddir+	forM_ keys $ \k -> moveAnnex k $ olddir </> keyFile0 k++	-- update the symlinks to the key files+	-- No longer needed here; V1.upgrade does the same thing++	-- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2+	Upgrade.V1.upgrade++-- these stayed unchanged between v0 and v1+keyFile0 :: Key -> FilePath+keyFile0 = Upgrade.V1.keyFile1+fileKey0 :: FilePath -> Key+fileKey0 = Upgrade.V1.fileKey1+lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend Annex))+lookupFile0 = Upgrade.V1.lookupFile1++getKeysPresent0 :: FilePath -> Annex [Key]+getKeysPresent0 dir = do+	exists <- liftIO $ doesDirectoryExist dir+	if (not exists)+		then return []+		else do+			contents <- liftIO $ getDirectoryContents dir+			files <- liftIO $ filterM present contents+			return $ map fileKey0 files+	where+		present d = do+			result <- try $+				getFileStatus $ dir ++ "/" ++ takeFileName d+			case result of+				Right s -> return $ isRegularFile s+				Left _ -> return False
+ Upgrade/V1.hs view
@@ -0,0 +1,236 @@+{- git-annex v1 -> v2 upgrade support+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Upgrade.V1 where++import System.IO.Error (try)+import System.Directory+import Control.Monad.State (liftIO)+import Control.Monad (filterM, forM_, unless)+import System.Posix.Files+import System.FilePath+import Data.String.Utils+import System.Posix.Types+import Data.Maybe+import Data.Char++import Types.Key+import Content+import Types+import Locations+import LocationLog+import qualified Annex+import qualified AnnexQueue+import qualified Git+import qualified Git.LsFiles as LsFiles+import Backend+import Messages+import Version+import Utility+import qualified Upgrade.V2++-- v2 adds hashing of filenames of content and location log files.+-- Key information is encoded in filenames differently, so+-- both content and location log files move around, and symlinks+-- to content need to be changed.+-- +-- When upgrading a v1 key to v2, file size metadata ought to be+-- added to the key (unless it is a WORM key, which encoded+-- mtime:size in v1). This can only be done when the file content+-- is present. Since upgrades need to happen consistently, +-- (so that two repos get changed the same way by the upgrade, and+-- will merge), that metadata cannot be added on upgrade.+--+-- Note that file size metadata+-- will only be used for detecting situations where git-annex+-- would run out of disk space, so if some keys don't have it,+-- the impact is minor. At least initially. It could be used in the+-- future by smart auto-repo balancing code, etc.+--+-- Anyway, since v2 plans ahead for other metadata being included+-- in keys, there should probably be a way to update a key.+-- Something similar to the migrate subcommand could be used,+-- and users could then run that at their leisure.++upgrade :: Annex Bool+upgrade = do+	showNote "v1 to v2"++	g <- Annex.gitRepo+	if Git.repoIsLocalBare g+		then do+			moveContent+			setVersion+		else do+			moveContent+			updateSymlinks+			moveLocationLogs+	+			AnnexQueue.flush True+			setVersion+	+	Upgrade.V2.upgrade++moveContent :: Annex ()+moveContent = do+	showNote "moving content..."+	files <- getKeyFilesPresent1+	forM_ files move+	where+		move f = do+			let k = fileKey1 (takeFileName f)+			let d = parentDir f+			liftIO $ allowWrite d+			liftIO $ allowWrite f+			moveAnnex k f+			liftIO $ removeDirectory d++updateSymlinks :: Annex ()+updateSymlinks = do+	showNote "updating symlinks..."+	g <- Annex.gitRepo+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]+	forM_ files $ fixlink+	where+		fixlink f = do+			r <- lookupFile1 f+			case r of+				Nothing -> return ()+				Just (k, _) -> do+					link <- calcGitLink f k+					liftIO $ removeFile f+					liftIO $ createSymbolicLink link f+					AnnexQueue.add "add" [Param "--"] f++moveLocationLogs :: Annex ()+moveLocationLogs = do+	showNote "moving location logs..."+	logkeys <- oldlocationlogs+	forM_ logkeys move+		where+			oldlocationlogs = do+				g <- Annex.gitRepo+				let dir = Upgrade.V2.gitStateDir g+				exists <- liftIO $ doesDirectoryExist dir+				if exists+					then do+						contents <- liftIO $ getDirectoryContents dir+						return $ catMaybes $ map oldlog2key contents+					else return []+			move (l, k) = do+				g <- Annex.gitRepo+				let dest = logFile k+				let dir = Upgrade.V2.gitStateDir g+				let f = dir </> l+				liftIO $ createDirectoryIfMissing True (parentDir dest)+				-- could just git mv, but this way deals with+				-- log files that are not checked into git,+				-- as well as merging with already upgraded+				-- logs that have been pulled from elsewhere+				old <- readLog f+				new <- readLog dest+				writeLog dest (old++new)+				AnnexQueue.add "add" [Param "--"] dest+				AnnexQueue.add "add" [Param "--"] f+				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f+		+oldlog2key :: FilePath -> Maybe (FilePath, Key)+oldlog2key l = +	let len = length l - 4 in+		if drop len l == ".log"+		then let k = readKey1 (take len l) in+			if null (keyName k) || null (keyBackendName k)+			then Nothing+			else Just (l, k)+		else Nothing++-- WORM backend keys: "WORM:mtime:size:filename"+-- all the rest: "backend:key"+--+-- If the file looks like "WORM:XXX-...", then it was created by mixing+-- v2 and v1; that infelicity is worked around by treating the value+-- as the v2 key that it is.+readKey1 :: String -> Key+readKey1 v = +	if mixup+		then fromJust $ readKey $ join ":" $ tail bits+		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }+	where+		bits = split ":" v+		b = head bits+		n = join ":" $ drop (if wormy then 3 else 1) bits+		t = if wormy+			then Just (read (bits !! 1) :: EpochTime)+			else Nothing+		s = if wormy+			then Just (read (bits !! 2) :: Integer)+			else Nothing+		wormy = head bits == "WORM"+		mixup = wormy && (isUpper $ head $ bits !! 1)++showKey1 :: Key -> String+showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =+	join ":" $ filter (not . null) [b, showifhere t, showifhere s, n]+		where+			showifhere Nothing = ""+			showifhere (Just v) = show v++keyFile1 :: Key -> FilePath+keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a"  $ showKey1 key++fileKey1 :: FilePath -> Key+fileKey1 file = readKey1 $+	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file++logFile1 :: Git.Repo -> Key -> String+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"++lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))+lookupFile1 file = do+	bs <- Annex.getState Annex.supportedBackends+	tl <- liftIO $ try getsymlink+	case tl of+		Left _ -> return Nothing+		Right l -> makekey bs l+	where+		getsymlink = do+			l <- readSymbolicLink file+			return $ takeFileName l+		makekey bs l = do+			case maybeLookupBackendName bs bname of+				Nothing -> do+					unless (null kname || null bname ||+					        not (isLinkToAnnex l)) $+						warning skip+					return Nothing+				Just backend -> return $ Just (k, backend)+			where+				k = fileKey1 l+				bname = keyBackendName k+				kname = keyName k+				skip = "skipping " ++ file ++ +					" (unknown backend " ++ bname ++ ")"++getKeyFilesPresent1 :: Annex [FilePath]+getKeyFilesPresent1  = do+	g <- Annex.gitRepo+	getKeyFilesPresent1' $ gitAnnexObjectDir g+getKeyFilesPresent1' :: FilePath -> Annex [FilePath]+getKeyFilesPresent1' dir = do+	exists <- liftIO $ doesDirectoryExist dir+	if (not exists)+		then return []+		else do+			dirs <- liftIO $ getDirectoryContents dir+			let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs+			liftIO $ filterM present files+	where+		present f = do+			result <- try $ getFileStatus f+			case result of+				Right s -> return $ isRegularFile s+				Left _ -> return False
+ Upgrade/V2.hs view
@@ -0,0 +1,138 @@+{- git-annex v2 -> v3 upgrade support+ -+ - Copyright 2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Upgrade.V2 where++import System.Directory+import System.FilePath+import Control.Monad.State (unless, when, liftIO)+import List+import Data.Maybe++import Types.Key+import Types+import qualified Annex+import qualified Git+import qualified Branch+import Messages+import Utility+import LocationLog+import Content++olddir :: Git.Repo -> FilePath+olddir g+	| Git.repoIsLocalBare g = ""+	| otherwise = ".git-annex"++{- .git-annex/ moved to a git-annex branch.+ - + - Strategy:+ - + - * Create the git-annex branch.+ - * Find each location log file in .git-annex/, and inject its content+ -   into the git-annex branch, unioning with any content already in+ -   there. (in passing, this deals with the semi transition that left+ -   some location logs hashed two different ways; both are found and+ -   merged).+ - * Also inject remote.log, trust.log, and uuid.log.+ - * git rm -rf .git-annex+ - * Remove stuff that used to be needed in .gitattributes.+ - * Commit changes.+ -}+upgrade :: Annex Bool+upgrade = do+	showNote "v2 to v3"+	g <- Annex.gitRepo+	let bare = Git.repoIsLocalBare g++	Branch.create+	e <- liftIO $ doesDirectoryExist (olddir g)+	when e $ do+		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g+		mapM_ (\f -> inject f f) =<< logFiles (olddir g)+		liftIO $ do+			Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]+			unless bare $ gitAttributesUnWrite g++	saveState+	unless bare $ push++	return True++locationLogs :: Git.Repo -> Annex [(Key, FilePath)]+locationLogs repo = liftIO $ do+	levela <- dirContents dir+	levelb <- mapM tryDirContents levela+	files <- mapM tryDirContents (concat levelb)+	return $ catMaybes $ map islogfile (concat files)+	where+		tryDirContents d = catch (dirContents d) (return . const [])+		dir = gitStateDir repo+		islogfile f = maybe Nothing (\k -> Just $ (k, f)) $+				logFileKey $ takeFileName f++inject :: FilePath -> FilePath -> Annex ()+inject source dest = do+	g <- Annex.gitRepo+	new <- liftIO (readFile $ olddir g </> source)+	prev <- Branch.get dest+	Branch.change dest $ unlines $ nub $ lines prev ++ lines new++logFiles :: FilePath -> Annex [FilePath]+logFiles dir = return . filter (".log" `isSuffixOf`)+		=<< liftIO (getDirectoryContents dir)++push :: Annex ()+push = do+	origin_master <- Branch.refExists "origin/master"+	origin_gitannex <- Branch.hasOrigin+	case (origin_master, origin_gitannex) of+		(_, True) -> do+			-- Merge in the origin's git-annex branch,+			-- so that pushing the git-annex branch+			-- will immediately work. Not pushed here,+			-- because it's less obnoxious to let the user+			-- push.+			Branch.update+		(True, False) -> do+			-- push git-annex to origin, so that+			-- "git push" will from then on+			-- automatically push it+			Branch.update -- just in case+			showNote "pushing new git-annex branch to origin"+			showProgress+			g <- Annex.gitRepo+			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]+		_ -> do+			-- no origin exists, so just let the user+			-- know about the new branch+			Branch.update+			showLongNote $+				"git-annex branch created\n" +++				"Be sure to push this branch when pushing to remotes.\n"+			showProgress++{- Old .gitattributes contents, not needed anymore. -}+attrLines :: [String]+attrLines =+	[ stateDir </> "*.log merge=union"+	, stateDir </> "*/*/*.log merge=union"+	]++gitAttributesUnWrite :: Git.Repo -> IO ()+gitAttributesUnWrite repo = do+	let attributes = Git.attributes repo+	whenM (doesFileExist attributes) $ do+		c <- readFileStrict attributes+		liftIO $ viaTmp writeFile attributes $ unlines $+			filter (\l -> not $ l `elem` attrLines) $ lines c+		Git.run repo "add" [File attributes]++stateDir :: FilePath+stateDir = addTrailingPathSeparator $ ".git-annex"+gitStateDir :: Git.Repo -> FilePath+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo </> stateDir
+ Utility.hs view
@@ -0,0 +1,293 @@+{- general purpose utility functions+ -+ - Copyright 2010-2011 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Utility (+	CommandParam(..),+	toCommand,+	hGetContentsStrict,+	readFileStrict,+	parentDir,+	absPath,+	absPathFrom,+	relPathCwdToFile,+	relPathDirToFile,+	boolSystem,+	boolSystemEnv,+	executeFile,+	shellEscape,+	shellUnEscape,+	unsetFileMode,+	readMaybe,+	viaTmp,+	dirContains,+	dirContents,+	myHomeDir,+	catchBool,+	whenM,+	(>>?),+	unlessM,+	(>>!),+	+	prop_idempotent_shellEscape,+	prop_idempotent_shellEscape_multiword,+	prop_parentDir_basics,+	prop_relPathDirToFile_basics+) where++import System.IO+import System.Exit+import qualified System.Posix.Process+import System.Posix.Process hiding (executeFile)+import System.Posix.Signals+import System.Posix.Files+import System.Posix.Types+import System.Posix.User+import Data.String.Utils+import System.Path+import System.FilePath+import System.Directory+import Foreign (complement)+import Data.List+import Data.Maybe+import Control.Monad (liftM2, when, unless)+import System.Log.Logger++{- A type for parameters passed to a shell command. A command can+ - be passed either some Params (multiple parameters can be included,+ - whitespace-separated, or a single Param (for when parameters contain+ - whitespace), or a File.+ -}+data CommandParam = Params String | Param String | File FilePath+	deriving (Eq, Show, Ord)++{- Used to pass a list of CommandParams to a function that runs+ - a command and expects Strings. -}+toCommand :: [CommandParam] -> [String]+toCommand = (>>= unwrap)+	where+		unwrap (Param s) = [s]+		unwrap (Params s) = filter (not . null) (split " " s)+		-- Files that start with a dash are modified to avoid+		-- the command interpreting them as options.+		unwrap (File ('-':s)) = ["./-" ++ s]+		unwrap (File s) = [s]++{- Run a system command, and returns True or False+ - if it succeeded or failed.+ -+ - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.+ -}+boolSystem :: FilePath -> [CommandParam] -> IO Bool+boolSystem command params = boolSystemEnv command params Nothing++boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool+boolSystemEnv command params env = do+	-- Going low-level because all the high-level system functions+	-- block SIGINT etc. We need to block SIGCHLD, but allow+	-- SIGINT to do its default program termination.+	let sigset = addSignal sigCHLD emptySignalSet+	oldint <- installHandler sigINT Default Nothing+	oldset <- getSignalMask+	blockSignals sigset+	childpid <- forkProcess $ childaction oldint oldset+	mps <- getProcessStatus True False childpid+	restoresignals oldint oldset+	case mps of+		Just (Exited ExitSuccess) -> return True+		_ -> return False+	where+		restoresignals oldint oldset = do+			_ <- installHandler sigINT oldint Nothing+			setSignalMask oldset+		childaction oldint oldset = do+			restoresignals oldint oldset+			executeFile command True (toCommand params) env++{- executeFile with debug logging -}+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO a+executeFile c path p e = do+	debugM "Utility.executeFile" $+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e+	System.Posix.Process.executeFile c path p e++{- Escapes a filename or other parameter to be safely able to be exposed to+ - the shell. -}+shellEscape :: String -> String+shellEscape f = "'" ++ escaped ++ "'"+	where+		-- replace ' with '"'"'+		escaped = join "'\"'\"'" $ split "'" f++{- Unescapes a set of shellEscaped words or filenames. -}+shellUnEscape :: String -> [String]+shellUnEscape [] = []+shellUnEscape s = word : shellUnEscape rest+	where+		(word, rest) = findword "" s+		findword w [] = (w, "")+		findword w (c:cs)+			| c == ' ' = (w, cs)+			| c == '\'' = inquote c w cs+			| c == '"' = inquote c w cs+			| otherwise = findword (w++[c]) cs+		inquote _ w [] = (w, "")+		inquote q w (c:cs)+			| c == q = findword w cs+			| otherwise = inquote q (w++[c]) cs++{- For quickcheck. -}+prop_idempotent_shellEscape :: String -> Bool+prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s)+prop_idempotent_shellEscape_multiword :: [String] -> Bool+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s)++{- A version of hgetContents that is not lazy. Ensures file is + - all read before it gets closed. -}+hGetContentsStrict :: Handle -> IO String+hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s++{- A version of readFile that is not lazy. -}+readFileStrict :: FilePath -> IO String+readFileStrict f = readFile f >>= \s -> length s `seq` return s++{- Returns the parent directory of a path. Parent of / is "" -}+parentDir :: FilePath -> FilePath+parentDir dir =+	if not $ null dirs+	then slash ++ join s (take (length dirs - 1) dirs)+	else ""+		where+			dirs = filter (not . null) $ split s dir+			slash = if isAbsolute dir then s else ""+			s = [pathSeparator]++prop_parentDir_basics :: FilePath -> Bool+prop_parentDir_basics dir+	| null dir = True+	| dir == "/" = parentDir dir == ""+	| otherwise = p /= dir+	where+		p = parentDir dir++{- Checks if the first FilePath is, or could be said to contain the second.+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc+ - are all equivilant.+ -}+dirContains :: FilePath -> FilePath -> Bool+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'+	where+		norm p = fromMaybe "" $ absNormPath p "."+		a' = norm a+		b' = norm b++{- Converts a filename into a normalized, absolute path. -}+absPath :: FilePath -> IO FilePath+absPath file = do+	cwd <- getCurrentDirectory+	return $ absPathFrom cwd file++{- Converts a filename into a normalized, absolute path+ - from the specified cwd. -}+absPathFrom :: FilePath -> FilePath -> FilePath+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file+	where+		bad = error $ "unable to normalize " ++ file++{- Constructs a relative path from the CWD to a file.+ -+ - For example, assuming CWD is /tmp/foo/bar:+ -    relPathCwdToFile "/tmp/foo" == ".."+ -    relPathCwdToFile "/tmp/foo/bar" == "" + -}+relPathCwdToFile :: FilePath -> IO FilePath+relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)++{- Constructs a relative path from a directory to a file.+ -+ - Both must be absolute, and normalized (eg with absNormpath).+ -}+relPathDirToFile :: FilePath -> FilePath -> FilePath+relPathDirToFile from to = path+	where+		s = [pathSeparator]+		pfrom = split s from+		pto = split s to+		common = map fst $ filter same $ zip pfrom pto+		same (c,d) = c == d+		uncommon = drop numcommon pto+		dotdots = replicate (length pfrom - numcommon) ".."+		numcommon = length common+		path = join s $ dotdots ++ uncommon++prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool+prop_relPathDirToFile_basics from to+	| from == to = null r+	| otherwise = not (null r)+	where+		r = relPathDirToFile from to ++{- Removes a FileMode from a file.+ - For example, call with otherWriteMode to chmod o-w -}+unsetFileMode :: FilePath -> FileMode -> IO ()+unsetFileMode f m = do+	s <- getFileStatus f+	setFileMode f $ fileMode s `intersectFileModes` complement m++{- Attempts to read a value from a String. -}+readMaybe :: (Read a) => String -> Maybe a+readMaybe s = case reads s of+	((x,_):_) -> Just x+	_ -> Nothing++{- Runs an action like writeFile, writing to a tmp file first and+ - then moving it into place. -}+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()+viaTmp a file content = do+	pid <- getProcessID+        let tmpfile = file ++ ".tmp" ++ show pid+	createDirectoryIfMissing True (parentDir file)+	a tmpfile content+	renameFile tmpfile file++{- Lists the contents of a directory.+ - Unlike getDirectoryContents, paths are not relative to the directory. -}+dirContents :: FilePath -> IO [FilePath]+dirContents d = do+	c <- getDirectoryContents d+	return $ map (d </>) $ filter notcruft c+	where+		notcruft "." = False+		notcruft ".." = False+		notcruft _ = True++{- Current user's home directory. -}+myHomeDir :: IO FilePath+myHomeDir = do+	uid <- getEffectiveUserID+	u <- getUserEntryForID uid+	return $ homeDirectory u++{- Catches IO errors and returns a Bool -}+catchBool :: IO Bool -> IO Bool+catchBool = flip catch (const $ return False)++{- when with a monadic conditional -}+whenM :: Monad m => m Bool -> m () -> m ()+whenM c a = c >>= flip when a++unlessM :: Monad m => m Bool -> m () -> m ()+unlessM c a = c >>= flip unless a++(>>?) :: Monad m => m Bool -> m () -> m ()+(>>?) = whenM++(>>!) :: Monad m => m Bool -> m () -> m ()+(>>!) = unlessM++-- low fixity allows eg, foo bar <|> error $ "failed " ++ meep+infixr 0 >>?+infixr 0 >>!
+ Version.hs view
@@ -0,0 +1,53 @@+{- git-annex repository versioning+ -+ - Copyright 2010 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Version where++import Control.Monad (unless)++import Types+import qualified Annex+import qualified Git+import Config++type Version = String++defaultVersion :: Version+defaultVersion = "3"++supportedVersions :: [Version]+supportedVersions = [defaultVersion]++upgradableVersions :: [Version]+upgradableVersions = ["0", "1", "2"]++versionField :: String+versionField = "annex.version"++getVersion :: Annex (Maybe Version)+getVersion = do+	g <- Annex.gitRepo+	let v = Git.configGet g versionField ""+	if not $ null v+		then return $ Just v+		else return Nothing++setVersion :: Annex ()+setVersion = setConfig versionField defaultVersion++checkVersion :: Annex ()+checkVersion = getVersion >>= handle+	where+		handle Nothing = error "First run: git-annex init"+		handle (Just v) = do+			unless (v `elem` supportedVersions) $ do+			error $ "Repository version " ++ v ++ +				" is not supported. " +++				msg v+		msg v+			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"+			| otherwise = "Upgrade git-annex."
+ configure.hs view
@@ -0,0 +1,89 @@+{- Checks system configuration and generates SysConfig.hs. -}++import System.Directory+import Data.List++import TestConfig++tests :: [TestCase]+tests =+	[ TestCase "version" $ getVersion+	, testCp "cp_a" "-a"+	, testCp "cp_p" "-p"+	, testCp "cp_reflink_auto" "--reflink=auto"+	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] ""+	, TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 </dev/null"+	, TestCase "rsync" $ requireCmd "rsync" "rsync --version >/dev/null"+	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"+	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"+	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"+	] ++ shaTestCases [1, 256, 512, 224, 384]++shaTestCases :: [Int] -> [TestCase]+shaTestCases l = map make l+	where make n =+		let+			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]+			key = "sha" ++ show n+		in TestCase key $ maybeSelectCmd key cmds "</dev/null"++tmpDir :: String+tmpDir = "tmp"++testFile :: String+testFile = tmpDir ++ "/testfile"++testCp :: ConfigKey -> String -> TestCase+testCp k option = TestCase cmd $ testCmd k run+	where+		cmd = "cp " ++ option+		run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new"++{- Pulls package version out of the changelog. -}+getVersion :: Test+getVersion = do+	version <- getVersionString+	return $ Config "packageversion" (StringConfig version)+	+getVersionString :: IO String+getVersionString = do+	changelog <- readFile "CHANGELOG"+	let verline = head $ lines changelog+	return $ middle (words verline !! 1)+	where+		middle s = drop 1 $ take (length s - 1) s++{- Set up cabal file with version. -}+cabalSetup :: IO ()+cabalSetup = do+	version <- getVersionString+	cabal <- readFile cabalfile+	writeFile tmpcabalfile $ unlines $ +		map (setfield "Version" version) $+		lines cabal+	renameFile tmpcabalfile cabalfile+	where+		cabalfile = "git-annex.cabal"+		tmpcabalfile = cabalfile++".tmp"+		setfield field value s+			| fullfield `isPrefixOf` s = fullfield ++ value+			| otherwise = s+			where+				fullfield = field ++ ": "++setup :: IO ()+setup = do+	createDirectoryIfMissing True tmpDir+	writeFile testFile "test file contents"++cleanup :: IO ()+cleanup = do+	removeDirectoryRecursive tmpDir++main :: IO ()+main = do+	setup+	config <- runTests tests+	writeSysConfig config+	cleanup+	cabalSetup
+ debian/NEWS view
@@ -0,0 +1,31 @@+git-annex (3.20110702) unstable; urgency=low++  The URL backend has been removed. Instead the new web remote can be used.++ -- Joey Hess <joeyh@debian.org>  Fri, 01 Jul 2011 15:40:51 -0400++git-annex (3.20110624) exerimental; urgency=low++  There has been another change to the git-annex data store.+  Use `git annex upgrade` to migrate your repositories to the new+  layout. See <http://git-annex.branchable.com/upgrades/> or+  /usr/share/doc/git-annex/html/upgrades.html++  The significant change this time is that the .git-annex/ directory+  is gone; instead there is a git-annex branch that is automatically+  maintained by git-annex, and encapsulates all its state nicely out+  of your way.++  You should make sure you include the git-annex branch when+  git pushing and pulling.++ -- Joey Hess <joeyh@debian.org>  Tue, 21 Jun 2011 20:18:00 -0400++git-annex (0.20110316) experimental; urgency=low++  This version reorganises the layout of git-annex's files in your repository.+  There is an upgrade process to convert a repository from the old git-annex+  to this version. See <http://git-annex.branchable.com/upgrades/> or+  /usr/share/doc/git-annex/html/upgrades.html++ -- Joey Hess <joeyh@debian.org>  Wed, 16 Mar 2011 15:49:15 -0400
+ debian/changelog view
@@ -0,0 +1,583 @@+git-annex (3.20110702) unstable; urgency=low++  * Now the web can be used as a special remote. +    This feature replaces the old URL backend.+  * addurl: New command to download an url and store it in the annex.+  * Sped back up fsck, copy --from, and other commands that often+    have to read a lot of information from the git-annex branch. Such+    commands are now faster than they were before introduction of the+    git-annex branch.+  * Always ensure git-annex branch exists.+  * Modify location log parser to allow future expansion.+  * --force will cause add, etc, to operate on ignored files.+  * Avoid mangling encoding when storing the description of repository+    and other content.+  * cabal can now be used to build git-annex. This is substantially+    slower than using make, does not build or install documentation,+    does not run the test suite, and is not particularly recommended,+    but could be useful to some.++ -- Joey Hess <joeyh@debian.org>  Sat, 02 Jul 2011 15:00:18 -0400++git-annex (3.20110624) experimental; urgency=low++  * New repository format, annex.version=3. Use `git annex upgrade` to migrate.+  * git-annex now stores its logs in a git-annex branch.+  * merge: New subcommand. Auto-merges the new git-annex branch.+  * Improved handling of bare git repos with annexes. Many more commands will+    work in them.+  * git-annex is now more robust; it will never leave state files+    uncommitted when some other git process comes along and locks the index+    at an inconvenient time.+  * rsync is now used when copying files from repos on other filesystems.+    cp is still used when copying file from repos on the same filesystem,+    since --reflink=auto can make it significantly faster on filesystems+    such as btrfs.+  * Allow --trust etc to specify a repository by name, for temporarily +    trusting repositories that are not configured remotes.+  * unlock: Made atomic.+  * git-union-merge: New git subcommand, that does a generic union merge+    operation, and operates efficiently without touching the working tree.++ -- Joey Hess <joeyh@debian.org>  Fri, 24 Jun 2011 14:32:18 -0400++git-annex (0.20110610) unstable; urgency=low++  * Add --numcopies option.+  * Add --trust, --untrust, and --semitrust options.+  * get --from is the same as copy --from+  * Bugfix: Fix fsck to not think all SHAnE keys are bad.++ -- Joey Hess <joeyh@debian.org>  Fri, 10 Jun 2011 11:48:40 -0400++git-annex (0.20110601) unstable; urgency=low++  * Minor bugfixes and error message improvements.+  * Massively sped up `git annex lock` by avoiding use of the uber-slow+    `git reset`, and only running `git checkout` once, even when many files+    are being locked.+  * Fix locking of files with staged changes.+  * Somewhat sped up `git commit` of modifications to unlocked files.+  * Build fix for older ghc.++ -- Joey Hess <joeyh@debian.org>  Wed, 01 Jun 2011 11:50:47 -0400++git-annex (0.20110522) unstable; urgency=low++  * Closer emulation of git's behavior when told to use "foo/.git" as a+    git repository instead of just "foo". Closes: #627563+  * Fix bug in --exclude introduced in 0.20110516.++ -- Joey Hess <joeyh@debian.org>  Fri, 27 May 2011 20:20:41 -0400++git-annex (0.20110521) unstable; urgency=low++  * status: New subcommand to show info about an annex, including its size.+  * --backend now overrides any backend configured in .gitattributes files.+  * Add --debug option. Closes: #627499++ -- Joey Hess <joeyh@debian.org>  Sat, 21 May 2011 11:52:53 -0400++git-annex (0.20110516) unstable; urgency=low++  * Add a few tweaks to make it easy to use the Internet Archive's variant+    of S3. In particular, munge key filenames to comply with the IA's filename+    limits, disable encryption, support their nonstandard way of creating+    buckets, and allow x-archive-* headers to be specified in initremote to+    set item metadata.+  * Added filename extension preserving variant backends SHA1E, SHA256E, etc.+  * migrate: Use current filename when generating new key, for backends+    where the filename affects the key name.+  * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.++ -- Joey Hess <joeyh@debian.org>  Mon, 16 May 2011 14:16:52 -0400++git-annex (0.20110503) unstable; urgency=low++  * Fix hasKeyCheap setting for bup and rsync special remotes.+  * Add hook special remotes.+  * Avoid crashing when an existing key is readded to the annex.+  * unused: Now also lists files fsck places in .git/annex/bad/+  * S3: When encryption is enabled, the Amazon S3 login credentials+    are stored, encrypted, in .git-annex/remotes.log, so environment+    variables need not be set after the remote is initialized.++ -- Joey Hess <joeyh@debian.org>  Tue, 03 May 2011 20:56:01 -0400++git-annex (0.20110427) unstable; urgency=low++  * Switch back to haskell SHA library, so git-annex remains buildable on+    Debian stable.+  * Added rsync special remotes. This could be used, for example, to +    store annexed content on rsync.net (encrypted naturally). Or anywhere else.+  * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a+    lot of files. Possibly only occured with ghc 7.++ -- Joey Hess <joeyh@debian.org>  Wed, 27 Apr 2011 22:50:26 -0400++git-annex (0.20110425) unstable; urgency=low++  * Use haskell Crypto library instead of haskell SHA library.+  * Remove testpack from build depends for non x86 architectures where it+    is not available. The test suite will not be run if it cannot be compiled.+  * Avoid using absolute paths when staging location log, as that can+    confuse git when a remote's path contains a symlink. Closes: #621386++ -- Joey Hess <joeyh@debian.org>  Mon, 25 Apr 2011 15:47:00 -0400++git-annex (0.20110420) unstable; urgency=low++  * Update Debian build dependencies for ghc 7.+  * Debian package is now built with S3 support.+    Thanks Joachim Breitner for making this possible.+  * Somewhat improved memory usage of S3, still work to do.+    Thanks Greg Heartsfield for ongoing work to improve the hS3 library+    for git-annex.++ -- Joey Hess <joeyh@debian.org>  Thu, 21 Apr 2011 15:00:48 -0400++git-annex (0.20110419) unstable; urgency=low++  * Don't run gpg in batch mode, so it can prompt for passphrase when+    there is no agent.+  * Add missing build dep on dataenc.+  * S3: Fix stalls when transferring encrypted data.+  * bup: Avoid memory leak when transferring encrypted data.++ -- Joey Hess <joeyh@debian.org>  Tue, 19 Apr 2011 21:26:51 -0400++git-annex (0.20110417) unstable; urgency=low++  * bup is now supported as a special type of remote.+  * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted+    using GPG for privacy.+  * Use lowercase hash directories for locationlog files, to avoid+    some issues with git on OSX with the mixed-case directories.+    No migration is needed; the old mixed case hash directories are still+    read; new information is written to the new directories.+  * Unused files on remotes, particulary special remotes, can now be+    identified and dropped, by using "--from remote" with git annex unused+    and git annex dropunused.+  * Clear up short option confusion between --from and --force (-f is now+    --from, and there is no short option for --force).+  * Add build depend on perlmagick so docs are consistently built.+    Closes: #621410+  * Add doc-base file. Closes: #621408+  * Periodically flush git command queue, to avoid boating memory usage+    too much.+  * Support "sha1" and "sha512" commands on FreeBSD, and allow building+    if any/all SHA commands are not available. Thanks, Fraser Tweedale++ -- Joey Hess <joeyh@debian.org>  Sun, 17 Apr 2011 12:00:24 -0400++git-annex (0.20110401) experimental; urgency=low++  * Amazon S3 is now supported as a special type of remote.+    Warning: Encrypting data before sending it to S3 is not yet supported.+  * Note that Amazon S3 support is not built in by default on Debian yet,+    as hS3 is not packaged.+  * fsck: Ensure that files and directories in .git/annex/objects+    have proper permissions.+  * Added a special type of remote called a directory remote, which+    simply stores files in an arbitrary local directory.+  * Bugfix: copy --to --fast never really copied, fixed.++ -- Joey Hess <joeyh@debian.org>  Fri, 01 Apr 2011 21:27:22 -0400++git-annex (0.20110328) experimental; urgency=low++  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")+  * Generalized remotes handling, laying groundwork for remotes that are+    not regular git remotes. (Think Amazon S3.)+  * Provide a less expensive version of `git annex copy --to`, enabled+    via --fast. This assumes that location tracking information is correct,+    rather than contacting the remote for every file.+  * Bugfix: Keys could be received into v1 annexes from v2 annexes, via+    v1 git-annex-shell. This results in some oddly named keys in the v1+    annex. Recognise and fix those keys when upgrading, instead of crashing.++ -- Joey Hess <joeyh@debian.org>  Mon, 28 Mar 2011 10:47:29 -0400++git-annex (0.20110325) experimental; urgency=low++  * Free space checking is now done, for transfers of data for keys+    that have free space metadata. (Notably, not for SHA* keys generated+    with git-annex 0.2x or earlier.) The code is believed to work on+    Linux, FreeBSD, and OSX; check compile-time messages to see if it+    is not enabled for your OS.+  * Add annex.diskreserve config setting, to control how much free space+    to reserve for other purposes and avoid using (defaults to 1 mb).+  * Add --fast flag, that can enable less expensive, but also less thorough+    versions of some commands.+  * fsck: In fast mode, avoid checking checksums.+  * unused: In fast mode, just show all existing temp files as unused,+    and avoid expensive scan for other unused content.+  * migrate: Support migrating v1 SHA keys to v2 SHA keys with+    size information that can be used for free space checking.+  * Fix space leak in fsck and drop commands.+  * migrate: Bugfix for case when migrating a file results in a key that+    is already present in .git/annex/objects.+  * dropunused: Significantly sped up; only read unused log file once.++ -- Joey Hess <joeyh@debian.org>  Fri, 25 Mar 2011 00:47:37 -0400++git-annex (0.20110320) experimental; urgency=low++  * Fix dropping of files using the URL backend.+  * Fix support for remotes with '.' in their names.+  * Add version command to show git-annex version as well as repository+    version information.+  * No longer auto-upgrade to repository format 2, to avoid accidental+    upgrades, etc. Use git-annex upgrade when you're ready to run this+    version.++ -- Joey Hess <joeyh@debian.org>  Sun, 20 Mar 2011 16:36:33 -0400++git-annex (0.20110316) experimental; urgency=low++  * New repository format, annex.version=2.+  * The first time git-annex is run in an old format repository, it+    will automatically upgrade it to the new format, staging all+    necessary changes to git. Also added a "git annex upgrade" command.+  * Colons are now avoided in filenames, so bare clones of git repos+    can be put on USB thumb drives formatted with vFAT or similar+    filesystems.+  * Added two levels of hashing to object directory and .git-annex logs,+    to improve scalability with enormous numbers of annexed+    objects. (With one hundred million annexed objects, each+    directory would contain fewer than 1024 files.)+  * The setkey, fromkey, and dropkey subcommands have changed how+    the key is specified. --backend is no longer used with these.++ -- Joey Hess <joeyh@debian.org>  Wed, 16 Mar 2011 16:20:23 -0400++git-annex (0.24) unstable; urgency=low++  Branched the 0.24 series, which will be maintained for a while to+  support v1 git-annex repos, while main development moves to the 0.2011+  series, with v2 git-annex repos.++  * Add Suggests on graphviz. Closes: #618039+  * When adding files to the annex, the symlinks pointing at the annexed+    content are made to have the same mtime as the original file.+    While git does not preserve that information, this allows a tool+    like metastore to be used with annexed files.+    (Currently this is only done on systems supporting POSIX 200809.)++ -- Joey Hess <joeyh@debian.org>  Wed, 16 Mar 2011 18:35:13 -0400++git-annex (0.23) unstable; urgency=low++  * Support ssh remotes with a port specified.+  * whereis: New subcommand to show where a file's content has gotten to.+  * Rethink filename encoding handling for display. Since filename encoding+    may or may not match locale settings, any attempt to decode filenames +    will fail for some files. So instead, do all output in binary mode.++ -- Joey Hess <joeyh@debian.org>  Sat, 12 Mar 2011 15:02:49 -0400++git-annex (0.22) unstable; urgency=low++  * Git annexes can now be attached to bare git repositories.+    (Both the local and remote host must have this version of git-annex+    installed for it to work.)+  * Support filenames that start with a dash; when such a file is passed+    to a utility it will be escaped to avoid it being interpreted as an+    option. (I went a little overboard and got the type checker involved+    in this, so such files are rather comprehensively supported now.)+  * New backends: SHA512 SHA384 SHA256 SHA224+    (Supported on systems where corresponding shaNsum commands are available.)+  * describe: New subcommand that can set or change the description of+    a repository.+  * Fix test suite to reap zombies.+    (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang+    for his help eliminating the infestation... for now.)+  * Make test suite not rely on a working cp -pr.+    (The Unix wars are still ON!)+  * Look for dir.git directories the same as git does.+  * Support remote urls specified as relative paths.+  * Support non-ssh remote paths that contain tilde expansions.+  * fsck: Check for and repair location log damage.+  * Bugfix: When fsck detected and moved away corrupt file content, it did+    not update the location log.++ -- Joey Hess <joeyh@debian.org>  Fri, 04 Mar 2011 15:10:57 -0400++git-annex (0.21) unstable; urgency=low++  * test: Don't rely on chmod -R working.+  * unannex: Fix recently introduced bug when attempting to unannex more+    than one file at a time.+  * test: Set git user name and email in case git can't guess values.+  * Fix display of unicode filenames.++ -- Joey Hess <joeyh@debian.org>  Fri, 11 Feb 2011 23:21:08 -0400++git-annex (0.20) unstable; urgency=low++  * Preserve specified file ordering when instructed to act on multiple+    files or directories. For example, "git annex get a b" will now always+    get "a" before "b". Previously it could operate in either order.+  * unannex: Commit staged changes at end, to avoid some confusing behavior+    with the pre-commit hook, which would see some types of commits after+    an unannex as checking in of an unlocked file.+  * map: New subcommand that uses graphviz to display a nice map of+    the git repository network.+  * Deal with the mtl/monads-fd conflict.+  * configure: Check for sha1sum.++ -- Joey Hess <joeyh@debian.org>  Tue, 08 Feb 2011 18:57:24 -0400++git-annex (0.19) unstable; urgency=low++  * configure: Support using the uuidgen command if the uuid command is+    not available.+  * Allow --exclude to be specified more than once.+  * There are now three levels of repository trust.+  * untrust: Now marks the current repository as untrusted.+  * semitrust: Now restores the default trust level. (What untrust used to do.)+  * fsck, drop: Take untrusted repositories into account.+  * Bugfix: Files were copied from trusted remotes first even if their+    annex.cost was higher than other remotes.+  * Improved temp file handling. Transfers of content can now be resumed+    from temp files later; the resume does not have to be the immediate+    next git-annex run.+  * unused: Include partially transferred content in the list.+  * Bugfix: Running a second git-annex while a first has a transfer in+    progress no longer deletes the first processes's temp file.++ -- Joey Hess <joeyh@debian.org>  Fri, 28 Jan 2011 14:31:37 -0400++git-annex (0.18) unstable; urgency=low++  * Bugfix: `copy --to` and `move --to` forgot to stage location log changes+    after transferring the file to the remote repository.+    (Did not affect ssh remotes.)+  * fsck: Fix bug in moving of corrupted files to .git/annex/bad/+  * migrate: Fix support for --backend option.+  * unlock: Fix behavior when file content is not present.+  * Test suite improvements. Current top-level test coverage: 80%++ -- Joey Hess <joeyh@debian.org>  Fri, 14 Jan 2011 14:17:44 -0400++git-annex (0.17) unstable; urgency=low++  * unannex: Now skips files whose content is not present, rather than+    it being an error.+  * New migrate subcommand can be used to switch files to using a different+    backend, safely and with no duplication of content.+  * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.)++ -- Joey Hess <joeyh@debian.org>  Sun, 09 Jan 2011 10:04:11 -0400++git-annex (0.16) unstable; urgency=low++  * git-annex-shell: Avoid exposing any git repo config except for the+    annex.uuid when doing configlist.+  * bugfix: Running `move --to` with a remote whose UUID was not yet known+    could result in git-annex not recording on the local side where the+    file was moved to. This could not result in data loss, or even a+    significant problem, since the remote *did* record that it had the file.+  * Also, add a general guard to detect attempts to record information+    about repositories with missing UUIDs.+  * bugfix: Running `move --to` with a non-ssh remote failed.+  * bugfix: Running `copy --to` with a non-ssh remote actually did a move.+  * Many test suite improvements. Current top-level test coverage: 65%++ -- Joey Hess <joeyh@debian.org>  Fri, 07 Jan 2011 14:33:13 -0400++git-annex (0.15) unstable; urgency=low++  * Support scp-style urls for remotes (host:path).+  * Support ssh urls containing "~".+  * Add trust and untrust subcommands, to allow configuring repositories+    that are trusted to retain files without explicit checking.+  * Fix bug in numcopies handling when multiple remotes pointed to the+    same repository.+  * Introduce the git-annex-shell command. It's now possible to make+    a user have it as a restricted login shell, similar to git-shell.+  * Note that git-annex will always use git-annex-shell when accessing+    a ssh remote, so all of your remotes need to be upgraded to this+    version of git-annex at the same time.+  * Now rsync is exclusively used for copying files to and from remotes.+    scp is not longer supported.++ -- Joey Hess <joeyh@debian.org>  Fri, 31 Dec 2010 22:00:52 -0400++git-annex (0.14) unstable; urgency=low++  * Bugfix to git annex unused in a repository with nothing yet annexed.+  * Support upgrading from a v0 annex with nothing in it.+  * Avoid multiple calls to git ls-files when passed eg, "*".++ -- Joey Hess <joeyh@debian.org>  Fri, 24 Dec 2010 17:38:48 -0400++git-annex (0.13) unstable; urgency=low++  * Makefile: Install man page and html (when built).+  * Makefile: Add GHCFLAGS variable.+  * Fix upgrade from 0.03.+  * Support remotes using git+ssh and ssh+git as protocol.+    Closes: #607056++ -- Joey Hess <joeyh@debian.org>  Tue, 14 Dec 2010 13:05:10 -0400++git-annex (0.12) unstable; urgency=low++  * Add --exclude option to exclude files from processing.+  * mwdn2man: Fix a bug in newline supression. Closes: #606578+  * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579+  * Makefile: Add PREFIX variable.++ -- Joey Hess <joeyh@debian.org>  Sat, 11 Dec 2010 17:32:00 -0400++git-annex (0.11) unstable; urgency=low++  * If available, rsync will be used for file transfers from remote+    repositories. This allows resuming interrupted transfers.+  * Added remote.annex-rsync-options.+  * Avoid deleting temp files when rsync fails.+  * Improve detection of version 0 repos.+  * Add uninit subcommand. Closes: #605749++ -- Joey Hess <joeyh@debian.org>  Sat, 04 Dec 2010 17:27:42 -0400++git-annex (0.10) unstable; urgency=low++  * In .gitattributes, the annex.numcopies attribute can be used+    to control the number of copies to retain of different types of files.+  * Bugfix: Always correctly handle gitattributes when in a subdirectory of+    the repository. (Had worked ok for ones like "*.mp3", but failed for+    ones like "dir/*".)+  * fsck: Fix warning about not enough copies of a file, when locations+    are known, but are not available in currently configured remotes.+  * precommit: Optimise to avoid calling git-check-attr more than once.+  * The git-annex-backend attribute has been renamed to annex.backend.++ -- Joey Hess <joeyh@debian.org>  Sun, 28 Nov 2010 19:28:05 -0400++git-annex (0.09) unstable; urgency=low++  * Add copy subcommand.+  * Fix bug in setkey subcommand triggered by move --to.++ -- Joey Hess <joeyh@debian.org>  Sat, 27 Nov 2010 17:14:59 -0400++git-annex (0.08) unstable; urgency=low++  * Fix `git annex add ../foo` (when ran in a subdir of the repo).+  * Add configure step to build process.+  * Only use cp -a if it is supported, falling back to cp -p or plain cp+    as needed for portability.+  * cp --reflink=auto is used if supported, and will make git annex unlock+    much faster on filesystems like btrfs that support copy on write.++ -- Joey Hess <joeyh@debian.org>  Sun, 21 Nov 2010 13:45:44 -0400++git-annex (0.07) unstable; urgency=low++  * find: New subcommand.+  * unused: New subcommand, finds unused data. (Split out from fsck.)+  * dropunused: New subcommand, provides for easy dropping of unused keys+    by number, as listed by the unused subcommand.+  * fsck: Print warnings to stderr; --quiet can now be used to only see+    problems.++ -- Joey Hess <joeyh@debian.org>  Mon, 15 Nov 2010 18:41:50 -0400++git-annex (0.06) unstable; urgency=low++  * fsck: Check if annex.numcopies is satisfied.+  * fsck: Verify the sha1 of files when the SHA1 backend is used.+  * fsck: Verify the size of files when the WORM backend is used.+  * fsck: Allow specifying individual files if fscking everything+    is not desired.+  * fsck: Fix bug, introduced in 0.04, in detection of unused data.++ -- Joey Hess <joeyh@debian.org>  Sat, 13 Nov 2010 16:24:29 -0400++git-annex (0.05) unstable; urgency=low++  * Optimize both pre-commit and lock subcommands to not call git diff+    on every file being committed/locked.+    (This actually also works around a bug in ghc, that caused+    git-annex 0.04 pre-commit to sometimes corrupt filename being read+    from git ls-files and fail. +    See <http://hackage.haskell.org/trac/ghc/ticket/4493>+    The excessive number of calls made by pre-commit exposed the ghc bug.+    Thanks Josh Triplett for the debugging.)+  * Build with -O2.++ -- Joey Hess <joeyh@debian.org>  Thu, 11 Nov 2010 18:31:09 -0400++git-annex (0.04) unstable; urgency=low++  * Add unlock subcommand, which replaces the symlink with a copy of+    the file's content in preparation of changing it. The "edit" subcommand+    is an alias for unlock.+  * Add lock subcommand.+  * Unlocked files will now automatically be added back into the annex when+    committed (and the updated symlink committed), by some magic in the+    pre-commit hook.+  * The SHA1 backend is now fully usable.+  * Add annex.version, which will be used to automate upgrades+    between incompatible versions.+  * Reorganised the layout of .git/annex/+  * The new layout will be automatically upgraded to the first time+    git-annex is used in a repository with the old layout.+  * Note that git-annex 0.04 cannot transfer content from old repositories+    that have not yet been upgraded.+  * Annexed file contents are now made unwritable and put in unwriteable+    directories, to avoid them accidentially being removed or modified.+    (Thanks Josh Triplett for the idea.)+  * Add build dep on libghc6-testpack-dev. Closes: #603016+  * Avoid using runghc to run test suite as it is not available on all+    architectures. Closes: #603006++ -- Joey Hess <joeyh@debian.org>  Wed, 10 Nov 2010 14:23:23 -0400++git-annex (0.03) unstable; urgency=low++  * Fix support for file:// remotes.+  * Add --verbose+  * Fix SIGINT handling.+  * Fix handling of files with unusual characters in their name.+  * Fixed memory leak; git-annex no longer reads the whole file list+    from git before starting, and will be much faster with large repos.+  * Fix crash on unknown symlinks.+  * Added remote.annex-scp-options and remote.annex-ssh-options.+  * The backends to use when adding different sets of files can be configured+    via gitattributes.+  * In .gitattributes, the git-annex-backend attribute can be set to the+    names of backends to use when adding different types of files.+  * Add fsck subcommand. (For now it only finds unused key contents in the+    annex.)++ -- Joey Hess <joeyh@debian.org>  Sun, 07 Nov 2010 18:26:04 -0400++git-annex (0.02) unstable; urgency=low++  * Can scp annexed files from remote hosts, and check remote hosts for+    file content when dropping files.+  * New move subcommand, that makes it easy to move file contents from+    or to a remote.+  * New fromkey subcommand, for registering urls, etc.+  * git-annex init will now set up a pre-commit hook that fixes up symlinks+    before they are committed, to ensure that moving symlinks around does not+    break them.+  * More intelligent and fast staging of modified files; git add coalescing.+  * Add remote.annex-ignore git config setting to allow completly disabling+    a given remote.+  * --from/--to can be used to control the remote repository that git-annex+    uses.+  * --quiet can be used to avoid verbose output+  * New plumbing-level dropkey and addkey subcommands.+  * Lots of bug fixes.++ -- Joey Hess <joeyh@debian.org>  Wed, 27 Oct 2010 16:39:29 -0400++git-annex (0.01) unstable; urgency=low++  * First prerelease.++ -- Joey Hess <joeyh@debian.org>  Wed, 20 Oct 2010 12:54:24 -0400
+ debian/compat view
@@ -0,0 +1,1 @@+7
+ debian/control view
@@ -0,0 +1,47 @@+Source: git-annex+Section: utils+Priority: optional+Build-Depends: +	debhelper (>= 7.0.50),+	ghc,+	libghc-missingh-dev,+	libghc-hslogger-dev,+	libghc-pcre-light-dev,+	libghc-sha-dev,+	libghc-dataenc-dev,+	libghc-utf8-string-dev,+	libghc-curl-dev,+	libghc-hs3-dev (>= 0.5.6),+	libghc-testpack-dev [any-i386 any-amd64],+	ikiwiki,+	perlmagick,+	git | git-core,+	uuid,+	rsync,+Maintainer: Joey Hess <joeyh@debian.org>+Standards-Version: 3.9.2+Vcs-Git: git://git.kitenet.net/git-annex+Homepage: http://git-annex.branchable.com/++Package: git-annex+Architecture: any+Section: utils+Depends: ${misc:Depends}, ${shlibs:Depends},+	git | git-core,+	uuid,+	rsync,+	openssh-client+Suggests: graphviz, bup, gnupg+Description: manage files with git, without checking their contents into git+ git-annex allows managing files with git, without checking the file+ contents into git. While that may seem paradoxical, it is useful when+ dealing with files larger than git can currently easily handle, whether due+ to limitations in memory, checksumming time, or disk space.+ .+ Even without file content tracking, being able to manage files with git,+ move files around and delete files with versioned directory trees, and use+ branches and distributed clones, are all very handy reasons to use git. And+ annexed files can co-exist in the same git repository with regularly+ versioned files, which is convenient for maintaining documents, Makefiles,+ etc that are associated with annexed files but that benefit from full+ revision control.
+ debian/copyright view
@@ -0,0 +1,39 @@+Format: http://dep.debian.net/deps/dep5/+Source: native package++Files: *+Copyright: © 2010-2011 Joey Hess <joey@kitenet.net>+License: GPL-3++ The full text of version 3 of the GPL is distributed as doc/GPL in+ this package's source, or in /usr/share/common-licenses/GPL-3 on+ Debian systems.++Files: StatFS.hsc+Copyright: Jose A Ortega Ruiz <jao@gnu.org>+License: BSD-3-clause+ -- All rights reserved.+ -- + -- Redistribution and use in source and binary forms, with or without+ -- modification, are permitted provided that the following conditions+ -- are met:+ -- + -- 1. Redistributions of source code must retain the above copyright+ --    notice, this list of conditions and the following disclaimer.+ -- 2. Redistributions in binary form must reproduce the above copyright+ --    notice, this list of conditions and the following disclaimer in the+ --    documentation and/or other materials provided with the distribution.+ -- 3. Neither the name of the author nor the names of his contributors+ --    may be used to endorse or promote products derived from this software+ --    without specific prior written permission.+ -- + -- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND+ -- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ -- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE+ -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+ -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS+ -- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)+ -- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+ -- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY+ -- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+ -- SUCH DAMAGE.
+ debian/doc-base view
@@ -0,0 +1,9 @@+Document: git-annex+Title: git-annex documentation+Author: Joey Hess+Abstract: All the documentation from git-annex's website.+Section: File Management++Format: HTML+Index: /usr/share/doc/git-annex/html/index.html+Files: /usr/share/doc/git-annex/html/*.html
+ debian/manpages view
@@ -0,0 +1,1 @@+git-annex.1
+ debian/rules view
@@ -0,0 +1,7 @@+#!/usr/bin/make -f+%:+	dh $@++# Not intended for use by anyone except the author.+announcedir:+	@echo ${HOME}/src/git-annex/doc/news
+ doc/GPL view
@@ -0,0 +1,674 @@+                    GNU GENERAL PUBLIC LICENSE+                       Version 3, 29 June 2007++ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>+ Everyone is permitted to copy and distribute verbatim copies+ of this license document, but changing it is not allowed.++                            Preamble++  The GNU General Public License is a free, copyleft license for+software and other kinds of works.++  The licenses for most software and other practical works are designed+to take away your freedom to share and change the works.  By contrast,+the GNU General Public License is intended to guarantee your freedom to+share and change all versions of a program--to make sure it remains free+software for all its users.  We, the Free Software Foundation, use the+GNU General Public License for most of our software; it applies also to+any other work released this way by its authors.  You can apply it to+your programs, too.++  When we speak of free software, we are referring to freedom, not+price.  Our General Public Licenses are designed to make sure that you+have the freedom to distribute copies of free software (and charge for+them if you wish), that you receive source code or can get it if you+want it, that you can change the software or use pieces of it in new+free programs, and that you know you can do these things.++  To protect your rights, we need to prevent others from denying you+these rights or asking you to surrender the rights.  Therefore, you have+certain responsibilities if you distribute copies of the software, or if+you modify it: responsibilities to respect the freedom of others.++  For example, if you distribute copies of such a program, whether+gratis or for a fee, you must pass on to the recipients the same+freedoms that you received.  You must make sure that they, too, receive+or can get the source code.  And you must show them these terms so they+know their rights.++  Developers that use the GNU GPL protect your rights with two steps:+(1) assert copyright on the software, and (2) offer you this License+giving you legal permission to copy, distribute and/or modify it.++  For the developers' and authors' protection, the GPL clearly explains+that there is no warranty for this free software.  For both users' and+authors' sake, the GPL requires that modified versions be marked as+changed, so that their problems will not be attributed erroneously to+authors of previous versions.++  Some devices are designed to deny users access to install or run+modified versions of the software inside them, although the manufacturer+can do so.  This is fundamentally incompatible with the aim of+protecting users' freedom to change the software.  The systematic+pattern of such abuse occurs in the area of products for individuals to+use, which is precisely where it is most unacceptable.  Therefore, we+have designed this version of the GPL to prohibit the practice for those+products.  If such problems arise substantially in other domains, we+stand ready to extend this provision to those domains in future versions+of the GPL, as needed to protect the freedom of users.++  Finally, every program is threatened constantly by software patents.+States should not allow patents to restrict development and use of+software on general-purpose computers, but in those that do, we wish to+avoid the special danger that patents applied to a free program could+make it effectively proprietary.  To prevent this, the GPL assures that+patents cannot be used to render the program non-free.++  The precise terms and conditions for copying, distribution and+modification follow.++                       TERMS AND CONDITIONS++  0. Definitions.++  "This License" refers to version 3 of the GNU General Public License.++  "Copyright" also means copyright-like laws that apply to other kinds of+works, such as semiconductor masks.++  "The Program" refers to any copyrightable work licensed under this+License.  Each licensee is addressed as "you".  "Licensees" and+"recipients" may be individuals or organizations.++  To "modify" a work means to copy from or adapt all or part of the work+in a fashion requiring copyright permission, other than the making of an+exact copy.  The resulting work is called a "modified version" of the+earlier work or a work "based on" the earlier work.++  A "covered work" means either the unmodified Program or a work based+on the Program.++  To "propagate" a work means to do anything with it that, without+permission, would make you directly or secondarily liable for+infringement under applicable copyright law, except executing it on a+computer or modifying a private copy.  Propagation includes copying,+distribution (with or without modification), making available to the+public, and in some countries other activities as well.++  To "convey" a work means any kind of propagation that enables other+parties to make or receive copies.  Mere interaction with a user through+a computer network, with no transfer of a copy, is not conveying.++  An interactive user interface displays "Appropriate Legal Notices"+to the extent that it includes a convenient and prominently visible+feature that (1) displays an appropriate copyright notice, and (2)+tells the user that there is no warranty for the work (except to the+extent that warranties are provided), that licensees may convey the+work under this License, and how to view a copy of this License.  If+the interface presents a list of user commands or options, such as a+menu, a prominent item in the list meets this criterion.++  1. Source Code.++  The "source code" for a work means the preferred form of the work+for making modifications to it.  "Object code" means any non-source+form of a work.++  A "Standard Interface" means an interface that either is an official+standard defined by a recognized standards body, or, in the case of+interfaces specified for a particular programming language, one that+is widely used among developers working in that language.++  The "System Libraries" of an executable work include anything, other+than the work as a whole, that (a) is included in the normal form of+packaging a Major Component, but which is not part of that Major+Component, and (b) serves only to enable use of the work with that+Major Component, or to implement a Standard Interface for which an+implementation is available to the public in source code form.  A+"Major Component", in this context, means a major essential component+(kernel, window system, and so on) of the specific operating system+(if any) on which the executable work runs, or a compiler used to+produce the work, or an object code interpreter used to run it.++  The "Corresponding Source" for a work in object code form means all+the source code needed to generate, install, and (for an executable+work) run the object code and to modify the work, including scripts to+control those activities.  However, it does not include the work's+System Libraries, or general-purpose tools or generally available free+programs which are used unmodified in performing those activities but+which are not part of the work.  For example, Corresponding Source+includes interface definition files associated with source files for+the work, and the source code for shared libraries and dynamically+linked subprograms that the work is specifically designed to require,+such as by intimate data communication or control flow between those+subprograms and other parts of the work.++  The Corresponding Source need not include anything that users+can regenerate automatically from other parts of the Corresponding+Source.++  The Corresponding Source for a work in source code form is that+same work.++  2. Basic Permissions.++  All rights granted under this License are granted for the term of+copyright on the Program, and are irrevocable provided the stated+conditions are met.  This License explicitly affirms your unlimited+permission to run the unmodified Program.  The output from running a+covered work is covered by this License only if the output, given its+content, constitutes a covered work.  This License acknowledges your+rights of fair use or other equivalent, as provided by copyright law.++  You may make, run and propagate covered works that you do not+convey, without conditions so long as your license otherwise remains+in force.  You may convey covered works to others for the sole purpose+of having them make modifications exclusively for you, or provide you+with facilities for running those works, provided that you comply with+the terms of this License in conveying all material for which you do+not control copyright.  Those thus making or running the covered works+for you must do so exclusively on your behalf, under your direction+and control, on terms that prohibit them from making any copies of+your copyrighted material outside their relationship with you.++  Conveying under any other circumstances is permitted solely under+the conditions stated below.  Sublicensing is not allowed; section 10+makes it unnecessary.++  3. Protecting Users' Legal Rights From Anti-Circumvention Law.++  No covered work shall be deemed part of an effective technological+measure under any applicable law fulfilling obligations under article+11 of the WIPO copyright treaty adopted on 20 December 1996, or+similar laws prohibiting or restricting circumvention of such+measures.++  When you convey a covered work, you waive any legal power to forbid+circumvention of technological measures to the extent such circumvention+is effected by exercising rights under this License with respect to+the covered work, and you disclaim any intention to limit operation or+modification of the work as a means of enforcing, against the work's+users, your or third parties' legal rights to forbid circumvention of+technological measures.++  4. Conveying Verbatim Copies.++  You may convey verbatim copies of the Program's source code as you+receive it, in any medium, provided that you conspicuously and+appropriately publish on each copy an appropriate copyright notice;+keep intact all notices stating that this License and any+non-permissive terms added in accord with section 7 apply to the code;+keep intact all notices of the absence of any warranty; and give all+recipients a copy of this License along with the Program.++  You may charge any price or no price for each copy that you convey,+and you may offer support or warranty protection for a fee.++  5. Conveying Modified Source Versions.++  You may convey a work based on the Program, or the modifications to+produce it from the Program, in the form of source code under the+terms of section 4, provided that you also meet all of these conditions:++    a) The work must carry prominent notices stating that you modified+    it, and giving a relevant date.++    b) The work must carry prominent notices stating that it is+    released under this License and any conditions added under section+    7.  This requirement modifies the requirement in section 4 to+    "keep intact all notices".++    c) You must license the entire work, as a whole, under this+    License to anyone who comes into possession of a copy.  This+    License will therefore apply, along with any applicable section 7+    additional terms, to the whole of the work, and all its parts,+    regardless of how they are packaged.  This License gives no+    permission to license the work in any other way, but it does not+    invalidate such permission if you have separately received it.++    d) If the work has interactive user interfaces, each must display+    Appropriate Legal Notices; however, if the Program has interactive+    interfaces that do not display Appropriate Legal Notices, your+    work need not make them do so.++  A compilation of a covered work with other separate and independent+works, which are not by their nature extensions of the covered work,+and which are not combined with it such as to form a larger program,+in or on a volume of a storage or distribution medium, is called an+"aggregate" if the compilation and its resulting copyright are not+used to limit the access or legal rights of the compilation's users+beyond what the individual works permit.  Inclusion of a covered work+in an aggregate does not cause this License to apply to the other+parts of the aggregate.++  6. Conveying Non-Source Forms.++  You may convey a covered work in object code form under the terms+of sections 4 and 5, provided that you also convey the+machine-readable Corresponding Source under the terms of this License,+in one of these ways:++    a) Convey the object code in, or embodied in, a physical product+    (including a physical distribution medium), accompanied by the+    Corresponding Source fixed on a durable physical medium+    customarily used for software interchange.++    b) Convey the object code in, or embodied in, a physical product+    (including a physical distribution medium), accompanied by a+    written offer, valid for at least three years and valid for as+    long as you offer spare parts or customer support for that product+    model, to give anyone who possesses the object code either (1) a+    copy of the Corresponding Source for all the software in the+    product that is covered by this License, on a durable physical+    medium customarily used for software interchange, for a price no+    more than your reasonable cost of physically performing this+    conveying of source, or (2) access to copy the+    Corresponding Source from a network server at no charge.++    c) Convey individual copies of the object code with a copy of the+    written offer to provide the Corresponding Source.  This+    alternative is allowed only occasionally and noncommercially, and+    only if you received the object code with such an offer, in accord+    with subsection 6b.++    d) Convey the object code by offering access from a designated+    place (gratis or for a charge), and offer equivalent access to the+    Corresponding Source in the same way through the same place at no+    further charge.  You need not require recipients to copy the+    Corresponding Source along with the object code.  If the place to+    copy the object code is a network server, the Corresponding Source+    may be on a different server (operated by you or a third party)+    that supports equivalent copying facilities, provided you maintain+    clear directions next to the object code saying where to find the+    Corresponding Source.  Regardless of what server hosts the+    Corresponding Source, you remain obligated to ensure that it is+    available for as long as needed to satisfy these requirements.++    e) Convey the object code using peer-to-peer transmission, provided+    you inform other peers where the object code and Corresponding+    Source of the work are being offered to the general public at no+    charge under subsection 6d.++  A separable portion of the object code, whose source code is excluded+from the Corresponding Source as a System Library, need not be+included in conveying the object code work.++  A "User Product" is either (1) a "consumer product", which means any+tangible personal property which is normally used for personal, family,+or household purposes, or (2) anything designed or sold for incorporation+into a dwelling.  In determining whether a product is a consumer product,+doubtful cases shall be resolved in favor of coverage.  For a particular+product received by a particular user, "normally used" refers to a+typical or common use of that class of product, regardless of the status+of the particular user or of the way in which the particular user+actually uses, or expects or is expected to use, the product.  A product+is a consumer product regardless of whether the product has substantial+commercial, industrial or non-consumer uses, unless such uses represent+the only significant mode of use of the product.++  "Installation Information" for a User Product means any methods,+procedures, authorization keys, or other information required to install+and execute modified versions of a covered work in that User Product from+a modified version of its Corresponding Source.  The information must+suffice to ensure that the continued functioning of the modified object+code is in no case prevented or interfered with solely because+modification has been made.++  If you convey an object code work under this section in, or with, or+specifically for use in, a User Product, and the conveying occurs as+part of a transaction in which the right of possession and use of the+User Product is transferred to the recipient in perpetuity or for a+fixed term (regardless of how the transaction is characterized), the+Corresponding Source conveyed under this section must be accompanied+by the Installation Information.  But this requirement does not apply+if neither you nor any third party retains the ability to install+modified object code on the User Product (for example, the work has+been installed in ROM).++  The requirement to provide Installation Information does not include a+requirement to continue to provide support service, warranty, or updates+for a work that has been modified or installed by the recipient, or for+the User Product in which it has been modified or installed.  Access to a+network may be denied when the modification itself materially and+adversely affects the operation of the network or violates the rules and+protocols for communication across the network.++  Corresponding Source conveyed, and Installation Information provided,+in accord with this section must be in a format that is publicly+documented (and with an implementation available to the public in+source code form), and must require no special password or key for+unpacking, reading or copying.++  7. Additional Terms.++  "Additional permissions" are terms that supplement the terms of this+License by making exceptions from one or more of its conditions.+Additional permissions that are applicable to the entire Program shall+be treated as though they were included in this License, to the extent+that they are valid under applicable law.  If additional permissions+apply only to part of the Program, that part may be used separately+under those permissions, but the entire Program remains governed by+this License without regard to the additional permissions.++  When you convey a copy of a covered work, you may at your option+remove any additional permissions from that copy, or from any part of+it.  (Additional permissions may be written to require their own+removal in certain cases when you modify the work.)  You may place+additional permissions on material, added by you to a covered work,+for which you have or can give appropriate copyright permission.++  Notwithstanding any other provision of this License, for material you+add to a covered work, you may (if authorized by the copyright holders of+that material) supplement the terms of this License with terms:++    a) Disclaiming warranty or limiting liability differently from the+    terms of sections 15 and 16 of this License; or++    b) Requiring preservation of specified reasonable legal notices or+    author attributions in that material or in the Appropriate Legal+    Notices displayed by works containing it; or++    c) Prohibiting misrepresentation of the origin of that material, or+    requiring that modified versions of such material be marked in+    reasonable ways as different from the original version; or++    d) Limiting the use for publicity purposes of names of licensors or+    authors of the material; or++    e) Declining to grant rights under trademark law for use of some+    trade names, trademarks, or service marks; or++    f) Requiring indemnification of licensors and authors of that+    material by anyone who conveys the material (or modified versions of+    it) with contractual assumptions of liability to the recipient, for+    any liability that these contractual assumptions directly impose on+    those licensors and authors.++  All other non-permissive additional terms are considered "further+restrictions" within the meaning of section 10.  If the Program as you+received it, or any part of it, contains a notice stating that it is+governed by this License along with a term that is a further+restriction, you may remove that term.  If a license document contains+a further restriction but permits relicensing or conveying under this+License, you may add to a covered work material governed by the terms+of that license document, provided that the further restriction does+not survive such relicensing or conveying.++  If you add terms to a covered work in accord with this section, you+must place, in the relevant source files, a statement of the+additional terms that apply to those files, or a notice indicating+where to find the applicable terms.++  Additional terms, permissive or non-permissive, may be stated in the+form of a separately written license, or stated as exceptions;+the above requirements apply either way.++  8. Termination.++  You may not propagate or modify a covered work except as expressly+provided under this License.  Any attempt otherwise to propagate or+modify it is void, and will automatically terminate your rights under+this License (including any patent licenses granted under the third+paragraph of section 11).++  However, if you cease all violation of this License, then your+license from a particular copyright holder is reinstated (a)+provisionally, unless and until the copyright holder explicitly and+finally terminates your license, and (b) permanently, if the copyright+holder fails to notify you of the violation by some reasonable means+prior to 60 days after the cessation.++  Moreover, your license from a particular copyright holder is+reinstated permanently if the copyright holder notifies you of the+violation by some reasonable means, this is the first time you have+received notice of violation of this License (for any work) from that+copyright holder, and you cure the violation prior to 30 days after+your receipt of the notice.++  Termination of your rights under this section does not terminate the+licenses of parties who have received copies or rights from you under+this License.  If your rights have been terminated and not permanently+reinstated, you do not qualify to receive new licenses for the same+material under section 10.++  9. Acceptance Not Required for Having Copies.++  You are not required to accept this License in order to receive or+run a copy of the Program.  Ancillary propagation of a covered work+occurring solely as a consequence of using peer-to-peer transmission+to receive a copy likewise does not require acceptance.  However,+nothing other than this License grants you permission to propagate or+modify any covered work.  These actions infringe copyright if you do+not accept this License.  Therefore, by modifying or propagating a+covered work, you indicate your acceptance of this License to do so.++  10. Automatic Licensing of Downstream Recipients.++  Each time you convey a covered work, the recipient automatically+receives a license from the original licensors, to run, modify and+propagate that work, subject to this License.  You are not responsible+for enforcing compliance by third parties with this License.++  An "entity transaction" is a transaction transferring control of an+organization, or substantially all assets of one, or subdividing an+organization, or merging organizations.  If propagation of a covered+work results from an entity transaction, each party to that+transaction who receives a copy of the work also receives whatever+licenses to the work the party's predecessor in interest had or could+give under the previous paragraph, plus a right to possession of the+Corresponding Source of the work from the predecessor in interest, if+the predecessor has it or can get it with reasonable efforts.++  You may not impose any further restrictions on the exercise of the+rights granted or affirmed under this License.  For example, you may+not impose a license fee, royalty, or other charge for exercise of+rights granted under this License, and you may not initiate litigation+(including a cross-claim or counterclaim in a lawsuit) alleging that+any patent claim is infringed by making, using, selling, offering for+sale, or importing the Program or any portion of it.++  11. Patents.++  A "contributor" is a copyright holder who authorizes use under this+License of the Program or a work on which the Program is based.  The+work thus licensed is called the contributor's "contributor version".++  A contributor's "essential patent claims" are all patent claims+owned or controlled by the contributor, whether already acquired or+hereafter acquired, that would be infringed by some manner, permitted+by this License, of making, using, or selling its contributor version,+but do not include claims that would be infringed only as a+consequence of further modification of the contributor version.  For+purposes of this definition, "control" includes the right to grant+patent sublicenses in a manner consistent with the requirements of+this License.++  Each contributor grants you a non-exclusive, worldwide, royalty-free+patent license under the contributor's essential patent claims, to+make, use, sell, offer for sale, import and otherwise run, modify and+propagate the contents of its contributor version.++  In the following three paragraphs, a "patent license" is any express+agreement or commitment, however denominated, not to enforce a patent+(such as an express permission to practice a patent or covenant not to+sue for patent infringement).  To "grant" such a patent license to a+party means to make such an agreement or commitment not to enforce a+patent against the party.++  If you convey a covered work, knowingly relying on a patent license,+and the Corresponding Source of the work is not available for anyone+to copy, free of charge and under the terms of this License, through a+publicly available network server or other readily accessible means,+then you must either (1) cause the Corresponding Source to be so+available, or (2) arrange to deprive yourself of the benefit of the+patent license for this particular work, or (3) arrange, in a manner+consistent with the requirements of this License, to extend the patent+license to downstream recipients.  "Knowingly relying" means you have+actual knowledge that, but for the patent license, your conveying the+covered work in a country, or your recipient's use of the covered work+in a country, would infringe one or more identifiable patents in that+country that you have reason to believe are valid.++  If, pursuant to or in connection with a single transaction or+arrangement, you convey, or propagate by procuring conveyance of, a+covered work, and grant a patent license to some of the parties+receiving the covered work authorizing them to use, propagate, modify+or convey a specific copy of the covered work, then the patent license+you grant is automatically extended to all recipients of the covered+work and works based on it.++  A patent license is "discriminatory" if it does not include within+the scope of its coverage, prohibits the exercise of, or is+conditioned on the non-exercise of one or more of the rights that are+specifically granted under this License.  You may not convey a covered+work if you are a party to an arrangement with a third party that is+in the business of distributing software, under which you make payment+to the third party based on the extent of your activity of conveying+the work, and under which the third party grants, to any of the+parties who would receive the covered work from you, a discriminatory+patent license (a) in connection with copies of the covered work+conveyed by you (or copies made from those copies), or (b) primarily+for and in connection with specific products or compilations that+contain the covered work, unless you entered into that arrangement,+or that patent license was granted, prior to 28 March 2007.++  Nothing in this License shall be construed as excluding or limiting+any implied license or other defenses to infringement that may+otherwise be available to you under applicable patent law.++  12. No Surrender of Others' Freedom.++  If conditions are imposed on you (whether by court order, agreement or+otherwise) that contradict the conditions of this License, they do not+excuse you from the conditions of this License.  If you cannot convey a+covered work so as to satisfy simultaneously your obligations under this+License and any other pertinent obligations, then as a consequence you may+not convey it at all.  For example, if you agree to terms that obligate you+to collect a royalty for further conveying from those to whom you convey+the Program, the only way you could satisfy both those terms and this+License would be to refrain entirely from conveying the Program.++  13. Use with the GNU Affero General Public License.++  Notwithstanding any other provision of this License, you have+permission to link or combine any covered work with a work licensed+under version 3 of the GNU Affero General Public License into a single+combined work, and to convey the resulting work.  The terms of this+License will continue to apply to the part which is the covered work,+but the special requirements of the GNU Affero General Public License,+section 13, concerning interaction through a network will apply to the+combination as such.++  14. Revised Versions of this License.++  The Free Software Foundation may publish revised and/or new versions of+the GNU General Public License from time to time.  Such new versions will+be similar in spirit to the present version, but may differ in detail to+address new problems or concerns.++  Each version is given a distinguishing version number.  If the+Program specifies that a certain numbered version of the GNU General+Public License "or any later version" applies to it, you have the+option of following the terms and conditions either of that numbered+version or of any later version published by the Free Software+Foundation.  If the Program does not specify a version number of the+GNU General Public License, you may choose any version ever published+by the Free Software Foundation.++  If the Program specifies that a proxy can decide which future+versions of the GNU General Public License can be used, that proxy's+public statement of acceptance of a version permanently authorizes you+to choose that version for the Program.++  Later license versions may give you additional or different+permissions.  However, no additional obligations are imposed on any+author or copyright holder as a result of your choosing to follow a+later version.++  15. Disclaimer of Warranty.++  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.++  16. Limitation of Liability.++  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF+SUCH DAMAGES.++  17. Interpretation of Sections 15 and 16.++  If the disclaimer of warranty and limitation of liability provided+above cannot be given local legal effect according to their terms,+reviewing courts shall apply local law that most closely approximates+an absolute waiver of all civil liability in connection with the+Program, unless a warranty or assumption of liability accompanies a+copy of the Program in return for a fee.++                     END OF TERMS AND CONDITIONS++            How to Apply These Terms to Your New Programs++  If you develop a new program, and you want it to be of the greatest+possible use to the public, the best way to achieve this is to make it+free software which everyone can redistribute and change under these terms.++  To do so, attach the following notices to the program.  It is safest+to attach them to the start of each source file to most effectively+state the exclusion of warranty; and each file should have at least+the "copyright" line and a pointer to where the full notice is found.++    <one line to give the program's name and a brief idea of what it does.>+    Copyright (C) <year>  <name of author>++    This program is free software: you can redistribute it and/or modify+    it under the terms of the GNU General Public License as published by+    the Free Software Foundation, either version 3 of the License, or+    (at your option) any later version.++    This program is distributed in the hope that it will be useful,+    but WITHOUT ANY WARRANTY; without even the implied warranty of+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the+    GNU General Public License for more details.++    You should have received a copy of the GNU General Public License+    along with this program.  If not, see <http://www.gnu.org/licenses/>.++Also add information on how to contact you by electronic and paper mail.++  If the program does terminal interaction, make it output a short+notice like this when it starts in an interactive mode:++    <program>  Copyright (C) <year>  <name of author>+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.+    This is free software, and you are welcome to redistribute it+    under certain conditions; type `show c' for details.++The hypothetical commands `show w' and `show c' should show the appropriate+parts of the General Public License.  Of course, your program's commands+might be different; for a GUI interface, you would use an "about box".++  You should also get your employer (if you work as a programmer) or school,+if any, to sign a "copyright disclaimer" for the program, if necessary.+For more information on this, and how to apply and follow the GNU GPL, see+<http://www.gnu.org/licenses/>.++  The GNU General Public License does not permit incorporating your program+into proprietary programs.  If your program is a subroutine library, you+may consider it more useful to permit linking proprietary applications with+the library.  If this is what you want to do, use the GNU Lesser General+Public License instead of this License.  But first, please read+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+ doc/backends.mdwn view
@@ -0,0 +1,43 @@+git-annex uses a key-value abstraction layer to allow file contents to be+stored in different ways. In theory, any key-value storage system could be+used to store file contents.++When a file is annexed, a key is generated from its content and/or metadata.+The file checked into git symlinks to the key. This key can later be used+to retrieve the file's content (its value).++Multiple pluggable backends are supported, and a single repository+can use different backends for different files.++These backends can transfer file contents between configured git remotes.+It's also possible to use [[special_remotes]], such as Amazon S3 with+these backends.++* `WORM` ("Write Once, Read Many") This backend assumes that any file with+  the same basename, size, and modification time has the same content. So with+  this backend, files can be moved around, but should never be added to+  or changed. This is the default, and the least expensive backend.+* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend+  allows modifications of files to be tracked. Its need to generate checksums+  can make it slower for large files.+* `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger+  checksums. Mostly useful for the very paranoid, or anyone who is+  researching checksum collisions and wants to annex their colliding data. ;)+* `SHA1E`, `SHA512E`, etc -- Variants that preserve filename extension as+  part of the key. Useful for archival tasks where the filename extension+  contains metadata that should be preserved.++The `annex.backends` git-config setting can be used to list the backends+git-annex should use. The first one listed will be used by default when+new files are added.++For finer control of what backend is used when adding different types of+files, the `.gitattributes` file can be used. The `annex.backend`+attribute can be set to the name of the backend to use for matching files.++For example, to use the SHA1 backend for sound files, which tend to be+smallish and might be modified over time, you could set in+`.gitattributes`:++	*.mp3 annex.backend=SHA1+	*.ogg annex.backend=SHA1
+ doc/bare_repositories.mdwn view
@@ -0,0 +1,22 @@+Due to popular demand, git-annex can now be used with bare repositories.++So, for example, you can stash a file away in the origin:+`git annex move mybigfile --to origin`++Of course, for that to work, the bare repository has to be on a system with+[[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't+use git-annex to store stuff there.++Known to work ok:++* `git annex move --to` and `--from`, when pointed at a bare repository.+* `git annex copy` ditto.+* `git annex drop` can check that a bare repository has a copy of data+  that is being dropped.+* `git annex get` can transfer data from a bare repository.+* Most other stuff (ie, init, describe, trust, etc.)++There are a few caveats to keep in mind when using bare repositories:++* A few subcommands, like `unused` cannot be run in a bare repository.+  Those subcommands will refuse to do anything.
+ doc/bugs.mdwn view
@@ -0,0 +1,4 @@+This is git-annex's bug list. Link bugs to [[bugs/done]] when done.++[[!inline pages="./bugs/* and !./bugs/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]]
+ doc/bugs/Displayed_copy_speed_is_wrong.mdwn view
@@ -0,0 +1,8 @@+When copying data to my remote, I regularly see speeds in excess of 100 MB/s on my home DSL line.++    2073939 100%  176.96MB/s    0:00:00 (xfer#1, to-check=0/1)++This is definitely not correct.++> Closing, as rsync does this to show you when it's making your life+> faster than it would be w/o rsync. [[done]] --[[Joey]]
+ doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-03T01:37:29Z"+ content="""+That is displayed by rsync. It's not unheard of for rsync to resume a transfer and display extremely high speeds.+"""]]
+ doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-03T08:56:48Z"+ content="""+Pity. Mark as done/upstream (or similar) for house-keeping?+"""]]
+ doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn view
@@ -0,0 +1,46 @@+I'm importing a directory where some files are hard links of each other.++This is confusing git-annex. Here's a small test of that:++<pre>+paulproteus@pathi:/tmp$ mkdir annex-test+paulproteus@pathi:/tmp$ cd annex-test+paulproteus@pathi:/tmp/annex-test$ git init+Initialized empty Git repository in /tmp/annex-test/.git/+paulproteus@pathi:/tmp/annex-test$ git annex init testing+init testing ok+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes +paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'+[master dd54b41] Default to sha1+ 1 files changed, 1 insertions(+), 0 deletions(-)+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2+paulproteus@pathi:/tmp/annex-test$ git annex add file1+add file1 (checksum...) ok+(Recording state in git...)+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'+[master eb43084] So far, so good+ 2 files changed, 2 insertions(+), 0 deletions(-)+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log+ create mode 120000 file1+paulproteus@pathi:/tmp/annex-test$ git annex add file2+add file2 (checksum...) +  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)+git-annex: 1 failed+paulproteus@pathi:/tmp/annex-test$ +</pre>++When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine.++I think the right behavior here is to annex file2 just fine, as if they weren't hard links before.+++-- Asheesh.++> The same thing happens anytime the key for a file collides with a key+> already in the annex, AFAICS. (Including when the files have the same+> content but are not hard links... unless you're using WORM backend.)+> +> I've fixed this bug. The first file in wins. See commit for some+> interesting discussion about why it should not check for hash collisions+> in this situation. [[done]] --[[Joey]]
+ doc/bugs/Makefile_is_missing_dependancies.mdwn view
@@ -0,0 +1,47 @@+<pre>+From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001+From: Jimmy Tang <jtang@tchpc.tcd.ie>+Date: Tue, 22 Mar 2011 22:24:07 +0000+Subject: [PATCH] Touch up Makefile to depend on StatFS.hs++---+ Makefile |    2 +-+ 1 files changed, 1 insertions(+), 1 deletions(-)++diff --git a/Makefile b/Makefile+index 08e2f59..4ae8392 100644+--- a/Makefile++++ b/Makefile+@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs+        hsc2hs $<+        perl -i -pe 's/^{-# INCLUDE.*//' $@+ +-$(bins): SysConfig.hs Touch.hs++$(bins): SysConfig.hs Touch.hs StatFS.hs+        $(GHCMAKE) $@+ + git-annex.1: doc/git-annex.mdwn+-- +1.7.4.1++</pre>+++StatFS.hs never gets depended on and compiled, the makefile was just missing something++> Thanks, [[done]]! Interested to hear if StatFS.hs works on OSX (no warning) or+> is a no-op (with warning). --[[Joey]] ++>> +>> for now it gives a warning, it looks like it should be easy enough to add OSX+>> support, I guess it's a case of just digging around documentation to find the equivalent+>> calls/headers. I'll give it a go at making this feature work on OSX and get back to you.+>> ++<pre>+jtang@exia:~/develop/git-annex $ make+hsc2hs StatFS.hsc+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS+</pre>
+ doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment view
@@ -0,0 +1,47 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 1"+ date="2011-03-23T08:21:30Z"+ content="""+Just did some minor digging around and checking, this seems to satisfy the compilers etc... I have yet to confirm that it *really* is working as expected. Also it might be better to check for a darwin operating system instead of apple I think, though I don't know of any one really using a pure darwin OS. But for now it works (I think)++<pre>+From fbfe27c2e19906ac02e3673b91bffa920f6dae5d Mon Sep 17 00:00:00 2001+From: Jimmy Tang <jtang@tchpc.tcd.ie>+Date: Wed, 23 Mar 2011 08:15:39 +0000+Subject: [PATCH] Define (__APPLE__) in StatFS++At least on OSX 10.6.6 it appears to have the same defintions as+FreeBSD. The build process doesn't complain and the code is enabled,+this needs to be tested and checked more.+---+ StatFS.hsc |    4 ++--+ 1 files changed, 2 insertions(+), 2 deletions(-)++diff --git a/StatFS.hsc b/StatFS.hsc+index 8b453dc..45fd7e4 100644+--- a/StatFS.hsc++++ b/StatFS.hsc+@@ -53,7 +53,7 @@ import Foreign.C.String+ import Data.ByteString (useAsCString)+ import Data.ByteString.Char8 (pack)+ +-#if defined (__FreeBSD__)++#if defined (__FreeBSD__) || defined(__APPLE__)+ # include <sys/param.h>+ # include <sys/mount.h>+ #else+@@ -84,7 +84,7 @@ data CStatfs+ #ifdef UNKNOWN+ #warning free space checking code not available for this OS+ #else+-#if defined(__FreeBSD__)++#if defined(__FreeBSD__) || defined(__APPLE__)+ foreign import ccall unsafe \"sys/mount.h statfs\"+ #else+ foreign import ccall unsafe \"sys/vfs.h statfs64\"+-- +1.7.4.1+</pre>+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-03-23T15:05:12Z"+ content="""+There's a simple test -- just configure annex.diskreserve to be say, 10 megabytes less than the total free space on your disk. Then try to git annex get a 11 mb file, and a 9 mb file. :)+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment view
@@ -0,0 +1,25 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-23T15:13:33Z"+ content="""+Alternatively, you can just load it up in ghci and see if it reports numbers that make sense:++<pre>+joey@gnu:~/src/git-annex>make StatFS.hs+hsc2hs StatFS.hsc+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs+joey@gnu:~/src/git-annex>ghci StatFS.hs+GHCi, version 6.12.1: http://www.haskell.org/ghc/  :? for help+Loading package ghc-prim ... linking ... done.+Loading package integer-gmp ... linking ... done.+Loading package base ... linking ... done.+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )+Ok, modules loaded: StatFS.+*StatFS> s <- getFileSystemStats \".\"+Loading package bytestring-0.9.1.5 ... linking ... done.+*StatFS> s+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 7427989, fsStatByteCount = 30425042944, fsStatBytesFree = 2528489472, fsStatBytesAvailable = 2219384832, fsStatBytesUsed = 27896553472})+</pre>+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment view
@@ -0,0 +1,30 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 4"+ date="2011-03-23T16:02:34Z"+ content="""+Ok, well it looks like it isn't doing anything useful at all.++<pre>+jtang@x00:~/develop/git-annex $ make StatFS.hs                                                                                                                                    +hsc2hs StatFS.hsc+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    +GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help+Loading package ghc-prim ... linking ... done.+Loading package integer-gmp ... linking ... done.+Loading package base ... linking ... done.+Loading package ffi-1.0 ... linking ... done.+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )+Ok, modules loaded: StatFS.+*StatFS> s <- getFileSystemStats \".\"+Loading package bytestring-0.9.1.7 ... linking ... done.+*StatFS> s+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})+*StatFS> s <- getFileSystemStats \"/\"+*StatFS> s+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})+*StatFS> +</pre>+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment view
@@ -0,0 +1,59 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 5"+ date="2011-03-23T16:14:22Z"+ content="""+Actually I may have just been stupid and should have read the man page on statfs...++<pre>+jtang@x00:~/develop/git-annex $ git diff+diff --git a/StatFS.hsc b/StatFS.hsc+index 8b453dc..e10b2dd 100644+--- a/StatFS.hsc++++ b/StatFS.hsc+@@ -53,7 +53,7 @@ import Foreign.C.String+ import Data.ByteString (useAsCString)+ import Data.ByteString.Char8 (pack)+ +-#if defined (__FreeBSD__)++#if defined (__FreeBSD__) || defined (__APPLE__)+ # include <sys/param.h>+ # include <sys/mount.h>+ #else+@@ -84,8 +84,8 @@ data CStatfs+ #ifdef UNKNOWN+ #warning free space checking code not available for this OS+ #else+-#if defined(__FreeBSD__)+-foreign import ccall unsafe \"sys/mount.h statfs\"++#if defined(__FreeBSD__) || defined (__APPLE__)++foreign import ccall unsafe \"sys/mount.h statfs64\"+ #else+ foreign import ccall unsafe \"sys/vfs.h statfs64\"+ #endif+</pre>++yields this...++<pre>+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    +GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help+Loading package ghc-prim ... linking ... done.+Loading package integer-gmp ... linking ... done.+Loading package base ... linking ... done.+Loading package ffi-1.0 ... linking ... done.+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )+Ok, modules loaded: StatFS.+*StatFS> s <- getFileSystemStats \".\"+Loading package bytestring-0.9.1.7 ... linking ... done.+*StatFS> s+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 244106668, fsStatByteCount = 999860912128, fsStatBytesFree = 423097798656, fsStatBytesAvailable = 422835654656, fsStatBytesUsed = 576763113472})+*StatFS> +</pre>+++we could just stick another if defined (__APPLE__) instead of what I previously had and it looks like it will do the right thing on OSX.+++"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 6"+ date="2011-03-23T16:23:56Z"+ content="""+I forgot to mention that the statfs64 stuff in OSX seems to be deprecated, see http://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man2/statfs64.2.html++on a slightly different note, is anonymous pushing to the \"wiki\" over git allowed? I'd prefer to be able to edit stuff inline for updating some of my own comments if I can :P+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 7"+ date="2011-03-23T16:57:56Z"+ content="""+Try the changes I've pushed to use statfs64 on apple.++There is actually a standardized statvfs that I'd rather use, but after the last time that I tried going with the POSIX option first only to find it was not broadly implemented, I was happy to find some already existing code that worked for some OSs.++(While ikiwiki supports anonymous git push, it's a feature we have not rolled out on Branchable.com yet, and anyway, ikiwiki disallows editing existing comments that way. I would, however, be happy to git pull changes from somewhere.)+"""]]
+ doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 8"+ date="2011-03-23T17:03:51Z"+ content="""+The latest change looks good, it seems to be returning sensible numbers for me. Just tried it out on a few different mount points and it appears to be working.+"""]]
+ doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn view
@@ -0,0 +1,31 @@+I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves.++> What do you mean by "an annex remote"? git-annex uses the same+> remotes configuration as does git. If you put invalid+> stuff in .git/config it might handle it slightly different than +> git, I don't know. Examples needed. --[[Joey]] ++>> What I mean is this:++    % cd 1+    % git init+    % git annex init "my:colon"+    % [...]+    % cd ../2+    % git init+    % git annex init "second"+    % git remote add "my:colon" ../1+    fatal: 'my:colon' is not a valid remote name++>> -- RichiH++>>> I see.. Git annex init does not specifiy a remote's name, it specifies+>>> an arbitrary human-readable description of the repository, which will+>>> be displayed when there is no configured remote corresponding to the+>>> repository. So this is not a bug unless some documentation of that is+>>> unclear. --[[Joey]] ++>>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :)+>>>> [[done]] --[[Joey]] ++>>>>> I still think git-annex should follow the same rules as git in this regard, but if your design decision is different, I won't try to argue the point :) -- RichiH
+ doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn view
@@ -0,0 +1,12 @@+My local git index got corrupted and I needed to clone and annex get all data from my main repo.++Some files were never copied anywhere so I am stuck with symlinks to nowhere.++I tried to copy over the symlink with a copy of the actual file, which did not work. Trying to unlock, copying over the symlink, and relock did not work, either.++Then, I copied the annex object to the correct place in .git/annex/objects/..., set all modes, re-ran fsck and the file re-appeared.+++Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force.++> [[closing|done]], seems addressed --[[Joey]]
+ doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-03T01:46:16Z"+ content="""+Have you seen [[walkthrough/recover_data_from_lost+found]]? The method described there will also work in this scenario.+"""]]
+ doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-03T09:00:17Z"+ content="""+I did not. Thanks :)++This still means that you can't re-inject a new version of a file unless you have the old one if you are using a SHA* backend, but that might be a corner case anyway.+"""]]
+ doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-05-14T16:28:36Z"+ content="""+To re-inject new content for a file, you really want to get a new key for the file. Otherwise, other repos that have the old file will never get the new content. So:++<pre>+git rm file+mv ~/newcontent file+git annex add file+</pre>+"""]]
+ doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-05-14T16:29:35Z"+ content="""+Although, if you really do want to shoot yourself in the foot, or know you have the old content, you can use `git-annex setkey`.+"""]]
+ doc/bugs/No_version_information_from_cli.mdwn view
@@ -0,0 +1,18 @@+git-annex does not listen to -v, --version or version.++At the very least, it should return both the version of the binary and the version of the object store it supports.+If it supports several annex versions, they should be listed in a comma-separated fashion.+If git-annex is called from within an annex, it should print the version of the local object store.++Sample:++    % git annex version+    git-annex version               : 0.24+    default object store version    : 3+    supported object store versions : 2,3+    local object store version      : 2+    % ++The above might look like overkill, but it's in a form that will, most likely, never need to be extended.++> Great idea, [[done]] --[[Joey]]
+ doc/bugs/Problems_running_make_on_osx.mdwn view
@@ -0,0 +1,49 @@+Followed the instructions over here: http://git-annex.branchable.com/forum/git-annex_on_OSX/++and had to install the following extra packages to be able to get make to start:++[realizes pcre-light is needed but pcre not installed on my mac]  +sudo port install pcre  +sudo cabal install pcre-light  ++> Ah right, that is a new dependency. I've updated the forum page+> with this info.+> --[[Joey]] ++But then I got the following error:  ++<pre>+ghc -O2 -Wall --make git-annex  +[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   ++BackendTypes.hs:71:17:  +    No instance for (Arbitrary Char)  +      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25  +    Possible fix: add an instance declaration for (Arbitrary Char)  +    In a stmt of a 'do' expression: backendname <- arbitrary  +    In the expression:  +        do backendname <- arbitrary  +           keyname <- arbitrary  +             return $ Key (backendname, keyname)  +    In the definition of `arbitrary':  +        arbitrary = do backendname <- arbitrary  +                       keyname <- arbitrary  +                         return $ Key (backendname, keyname)  +make: *** [git-annex] Error 1  +</pre>++My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.++> Hmm, it seems you may be missing part of the quickcheck haskell+> library, or have a different version than me.+> +> The easy fix is probably to just edit BackendTypes.hs and delete the+> entire end of the file from line 68, "for quickcheck" down. This code+> is only used by the test suite (so "make test" will fail), +> but it should get it to build. --[[Joey]]++---++Closing this bug because the above problem now has a solution documented on+the install page, and the below test suite failure problems should all be+resolved on OSX. [[done]] --[[Joey]]
+ doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 10"+ date="2011-02-09T15:04:50Z"+ content="""+I don't know what these problems forking could be. Can you strace it?+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 11"+ date="2011-02-09T19:35:47Z"+ content="""+I got dtruss to give me a trace, the output is quite big to post here (~560kb gzip'd), do you mind if I emailed it or posted it somewhere else for you?+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 12"+ date="2011-02-09T19:47:30Z"+ content="""+joey@kitenet.net (hope I can make sense of dtruss output)+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment view
@@ -0,0 +1,16 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 13"+ date="2011-02-09T21:59:47Z"+ content="""+The dtrace puzzlingly does not have the same errors shown above, but a set of mostly new errors. I don't know what to make of that.++> git-annex: git-annex/.t/repo/.git/hooks/pre-commit: fileAccess: permission denied (Operation not permitted)++This seems to be caused by it setting the execute bit on the file. I don't know why that would fail; it's just written the file and renamed it into place so clearly should be able to write to it.++> was able to modify annexed file's sha1foo content++This also suggests something breaking with permissions.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 14"+ date="2011-02-12T21:19:24Z"+ content="""+I've been trying to dig around the trace and code, and used google to see if the forkProcess issue was a haskell thing or an OSX thing. It seems that <http://hackage.haskell.org/trac/ghc/ticket/4493> someone may have ran into a similar issue, though I am not sure if its related.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment view
@@ -0,0 +1,71 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 15"+ date="2011-02-13T02:45:51Z"+ content="""+It may be possible that OSX has some low resource limits, for user processes (266 per user I think) doing a ++    sudo sysctl -w kern.maxproc=2048+    sudo sysctl -w kern.maxprocperuid=1024+    sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf+    sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf++seems to change the behaviour of the tests abit...++<pre>+Testing 1:blackbox:3:git-annex unannex:1:with content                         +### Failure in: 1:blackbox:3:git-annex unannex:1:with content+foo is not a symlink+Testing 1:blackbox:4:git-annex drop:0:no remotes                              +### Failure in: 1:blackbox:4:git-annex drop:0:no remotes+drop wrongly succeeded with no known copy of file+Testing 1:blackbox:4:git-annex drop:1:with remote                             +Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        +Testing 1:blackbox:5:git-annex get                                            +Testing 1:blackbox:6:git-annex move                                           +Testing 1:blackbox:7:git-annex copy                                           +Testing 1:blackbox:8:git-annex unlock/lock                                    +Testing 1:blackbox:9:git-annex edit/commit:0                                  +Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok+ok+Testing 1:blackbox:9:git-annex edit/commit:1                                  +Testing 1:blackbox:10:git-annex fix                                           +Testing 1:blackbox:11:git-annex trust/untrust/semitrust                       +Testing 1:blackbox:12:git-annex fsck:0                                        +Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.+  Back it up with git-annex copy.+  Only 1 of 2 trustworthy copies of sha1foo exist.+  Back it up with git-annex copy.+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297565141:20:foo+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa+Testing 1:blackbox:12:git-annex fsck:1                                        +### Failure in: 1:blackbox:12:git-annex fsck:1+fsck failed to fail with content only available in untrusted (current) repository+Testing 1:blackbox:12:git-annex fsck:2                                        +Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.+  Back it up with git-annex copy.+  The following untrusted locations may also have copies: +  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo+  Only 1 of 2 trustworthy copies of sha1foo exist.+  Back it up with git-annex copy.+  The following untrusted locations may also have copies: +  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo+Testing 1:blackbox:13:git-annex migrate:0                                     +Cases: 30  Tried: 27  Errors: 0  Failures: 3  git-annex: user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))+### Failure in: 1:blackbox:13:git-annex migrate:0+migrate annexedfile failed+Testing 1:blackbox:13:git-annex migrate:1                                     +### Error in:   1:blackbox:13:git-annex migrate:1+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:14:git-annex unused/dropunused                             +### Error in:   1:blackbox:14:git-annex unused/dropunused+forkProcess: resource exhausted (Resource temporarily unavailable)+Cases: 30  Tried: 30  Errors: 2  Failures: 4+test: failed+</pre>+++the number of failures vary as I change the values of the maxprocs, I think I  have narrowed it down to OSX just being stupid with limits thus causing the tests to fail.++"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 16"+ date="2011-02-13T04:52:26Z"+ content="""+I've fixed the test suite to not accumulate all those zombie processes. Now only 2 or 3 processes should run max. Am curious to see if that clears up all the problems.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment view
@@ -0,0 +1,43 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 17"+ date="2011-02-13T10:46:54Z"+ content="""+Yeap, that did the trick. I just tested a few separate OSX 10.6.6 systems and the tests are better behaved now, only 3 failures now.++So the tests behave better (at least we don't get resource fork errors any more)+ + * after the commit c319a3 without modifying the system limits (of 266 procs per user)+ * without the commit c319a3 and when I increase the system process limits to as much as OSX allows++On all the systems I tested on, I'm down to 3 failures now.++<pre>+### Failure in: 1:blackbox:3:git-annex unannex:1:with content+foo is not a symlink+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes+drop wrongly succeeded with no known copy of file+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok+ok+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.+  Back it up with git-annex copy.+  Only 1 of 2 trustworthy copies of sha1foo exist.+  Back it up with git-annex copy.+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297594011:20:foo+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa+### Failure in: 1:blackbox:12:git-annex fsck:1+fsck failed to fail with content only available in untrusted (current) repository+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.+  Back it up with git-annex copy.+  The following untrusted locations may also have copies: +  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo+  Only 1 of 2 trustworthy copies of sha1foo exist.+  Back it up with git-annex copy.+  The following untrusted locations may also have copies: +  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo+Cases: 30  Tried: 30  Errors: 0  Failures: 3+</pre>++It's the same set of failures across all the OSX systems that I have tested on. Now I just need to figure out why there are still these three failures.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment view
@@ -0,0 +1,24 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="maybe killed another osx bug in the test."+ date="2011-02-13T15:12:10Z"+ content="""+I think I have figured out why ++    ### Failure in: 1:blackbox:3:git-annex unannex:1:with content+    foo is not a symlink++It goes back to the this piece of code (in test.hs)++    copyrepo :: FilePath -> FilePath -> IO FilePath+    copyrepo old new = do+            cleanup new+            ensuretmpdir+            Utility.boolSystem \"cp\" [\"-pr\", old, new] @? \"cp -pr failed\"++It seems that on OSX it does not preserve the symbolic link information, basically cp is not gnu cp on OSX, doing a \"cp -a SOURCE DEST\" seem's to the right thing on OSX. I tried it out on my archlinux workstation by replacing *-pr* with just *-a* and all the tests passed on archlinux.++I'm not sure what the implications would be with changing the test with changing the cp command.++"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 19"+ date="2011-02-13T15:55:47Z"+ content="""+On second thought and after some messing (trying most of the options and combinations of options on OSX for).... I tried replacing cp with gnu cp from coreutils on my OSX install, and all the tests passed. *sigh* cp -a is preserving some permissions and attributes but not all, its not behaving in the same way as the gnu cp does... the closet thing that I have found on OSX that behaves in the same way as gnu \"cp -pr\" is to use \"ditto\".++Just doing a \"ditto SOURCE DEST\" in the tests passes everything. I'm not sure if its a good idea to use this even though it works. Though this is just the tests, does it affect CopyFile.hs where \"cp\" is called?++"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment view
@@ -0,0 +1,17 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"+ nickname="Antoine"+ subject="Got it going!"+ date="2011-02-06T06:02:57Z"+ content="""+Thanks to your feedback, I got it going.  ++Maybe those two should be added to the 'OSX how-to' in the forum   ++[realizes pcre-light is needed but pcre not installed on my mac]  +sudo port install pcre  +sudo cabal install pcre-light  ++[tests are failing, need haskell's quickcheck]  +sudo cabal install quickcheck  +"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 20"+ date="2011-02-13T17:54:09Z"+ content="""+Outside the test suite, git-annex's actual use of cp puts fairly low demands on it. It tries to use cp -a or cp -p if available just to preserve whatever attributes it can preserve, but the worst case if that you have a symlink pointing to a file that doesn't have the original timestamp or whatever. And there's little expectation git preserves that stuff anyway.++I will probably try to make the test suite entirely use git clone rather than cp.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-02-06T17:39:52Z"+ content="""+Yes, I've moved it to [[install/OSX]] page where anyone can update it in this wiki, and added your improvements.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment view
@@ -0,0 +1,57 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="tests fail with more recent installs of haskell platform"+ date="2011-02-07T12:43:43Z"+ content="""+I'm running ghc 6.12.3 with the corresponding haskell-platform package from the HP site which I installed in preference to the macports version of haskell-platform (it's quite old). it seems when you install quickcheck, the version that is installed is of version 2.4.0.1 and not 1.2.0 which git-annex depends on for its tests.++<pre>+jtang@x00:~ $ cabal install quickcheck --reinstall               +Resolving dependencies...+Configuring QuickCheck-2.4.0.1...+Preprocessing library QuickCheck-2.4.0.1...++..+and so on..+..++</pre>++it fails with this++<pre>+[54 of 54] Compiling Main             ( test.hs, test.o )++test.hs:56:3:+    No instance for (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)+      arising from a use of `qctest' at test.hs:56:3-64+    Possible fix:+      add an instance declaration for+      (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)+    In the expression:+        qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode+    In the first argument of `TestList', namely+        `[qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,+          qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,+          qctest+            \"prop_idempotent_key_read_show\"+            BackendTypes.prop_idempotent_key_read_show,+          qctest+            \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,+          ....]'+    In the second argument of `($)', namely+        `TestList+           [qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,+            qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,+            qctest+              \"prop_idempotent_key_read_show\"+              BackendTypes.prop_idempotent_key_read_show,+            qctest+              \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,+            ....]'+</pre>++I'd imagine if I could downgrade, it would compile and pass the tests (I hope)++"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-02-08T19:00:14Z"+ content="""+I doubt that git-annex can be used with QuickCheck 1.2.0. The QuickCheck I've tested it with is 2.1.0.3 actually.++I suspect you have an old version of the TestPack haskell library on your system, that is linked against QuickCheck 1.2.0. Git-annex has been tested with TestPack 2.0.0, which uses QuickCheck 2.x.++In any case, you don't have to run 'make test' to build git-annex, and my comments above should make the main program compile, I expect.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment view
@@ -0,0 +1,107 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 5"+ date="2011-02-08T19:56:55Z"+ content="""+Ah, that gave me a good clue, my system just got pretty confused with a mixture of quickcheck and testpack installs. Would it be possible to put up a list of versions of the software you are using on your development environment? (at least the minimum tested version) ++I guess it shouldn't matter to most users who are going to rely on packagers to sort these dependancy issues, but it's nice to know. ++Anyway, the tests build now, and they seem to fail on my (rather messy) install of haskell platform + ghc 6.12 on osx 10.6.6.++<pre>+< output that passed some tests >+Testing 1:blackbox:0:git-annex init+Testing 1:blackbox:1:git-annex add:0+Testing 1:blackbox:1:git-annex add:1+Cases: 30  Tried: 9  Errors: 0  Failures: 0test: sha1sum: executeFile: does not exist (No such file or directory)+  git-annex: <file descriptor: 6>: hGetLine: end of file+### Failure in: 1:blackbox:1:git-annex add:1+add with SHA1 failed+Testing 1:blackbox:2:git-annex setkey/fromkey+Cases: 30  Tried: 10  Errors: 0  Failures: 1(checksum...) test: sha1sum: executeFile: does not exist (No such file or directory)+### Error in:   1:blackbox:2:git-annex setkey/fromkey+<file descriptor: 3>: hGetLine: end of file+Testing 1:blackbox:3:git-annex unannex:0:no content+Cases: 30  Tried: 11  Errors: 1  Failures: 1chmod: -R: No such file or directory+chmod: -R: No such file or directory+Testing 1:blackbox:3:git-annex unannex:1:with content+### Failure in: 1:blackbox:3:git-annex unannex:1:with content+foo is not a symlink+Testing 1:blackbox:4:git-annex drop:0:no remotes+Cases: 30  Tried: 13  Errors: 1  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:4:git-annex drop:0:no remotes+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:4:git-annex drop:1:with remote+Cases: 30  Tried: 14  Errors: 2  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:4:git-annex drop:1:with remote+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:4:git-annex drop:2:untrusted remote+Cases: 30  Tried: 15  Errors: 3  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:4:git-annex drop:2:untrusted remote+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:5:git-annex get+Cases: 30  Tried: 16  Errors: 4  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:5:git-annex get+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:6:git-annex move+Cases: 30  Tried: 17  Errors: 5  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:6:git-annex move+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:7:git-annex copy+Cases: 30  Tried: 18  Errors: 6  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:7:git-annex copy+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:8:git-annex unlock/lock+Cases: 30  Tried: 19  Errors: 7  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:8:git-annex unlock/lock+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:9:git-annex edit/commit:0+Cases: 30  Tried: 20  Errors: 8  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:9:git-annex edit/commit:0+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:9:git-annex edit/commit:1+Cases: 30  Tried: 21  Errors: 9  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:9:git-annex edit/commit:1+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:10:git-annex fix+Cases: 30  Tried: 22  Errors: 10  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:10:git-annex fix+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:11:git-annex trust/untrust/semitrust+Cases: 30  Tried: 23  Errors: 11  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:12:git-annex fsck:0+Cases: 30  Tried: 24  Errors: 12  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:12:git-annex fsck:0+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:12:git-annex fsck:1+Cases: 30  Tried: 25  Errors: 13  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:12:git-annex fsck:1+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:12:git-annex fsck:2+Cases: 30  Tried: 26  Errors: 14  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:12:git-annex fsck:2+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:13:git-annex migrate:0+Cases: 30  Tried: 27  Errors: 15  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:13:git-annex migrate:0+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:13:git-annex migrate:1+Cases: 30  Tried: 28  Errors: 16  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:13:git-annex migrate:1+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Testing 1:blackbox:14:git-annex unused/dropunused+Cases: 30  Tried: 29  Errors: 17  Failures: 2chmod: -R: No such file or directory+### Error in:   1:blackbox:14:git-annex unused/dropunused+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+Cases: 30  Tried: 30  Errors: 18  Failures: 2+chmod: -R: No such file or directory+test: .t/repo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)+make: *** [test] Error 1+</pre>++I assumed that since the tests built, then running them shouldn't be a problem. It looks like some argument isn't being passed about for the location of the .t directory that gets created. I will check the dependancies on my system again.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment view
@@ -0,0 +1,9 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 6"+ date="2011-02-08T23:20:08Z"+ content="""+You're missing the sha1sum command, everything else is a followon error from that. Added a hint about this to [[install]],+and in the next version configure will check for sha1sum.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment view
@@ -0,0 +1,22 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 7"+ date="2011-02-09T00:45:31Z"+ content="""+That's odd, I have the md5sha1sum package installed and it still fails with pretty much the same error++<pre>+Testing 1:blackbox:0:git-annex init+Cases: 30  Tried: 7  Errors: 0  Failures: 0chmod: -R: No such file or directory+### Error in:   1:blackbox:0:git-annex init+.t/repo/.git/annex/objects/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa: removeLink: permission denied (Permission denied)+Testing 1:blackbox:1:git-annex add:0+### Error in:   1:blackbox:1:git-annex add:0+foo: openFile: permission denied (Permission denied)++< and so on >+</pre>++the configure script finds sha1sum, builds and starts to run.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 8"+ date="2011-02-09T04:10:27Z"+ content="""+The chmod errors are because your chmod does not understand the -R argument. Only the test suite uses chmod -R. I've fixed it to modify modes manually.+"""]]
+ doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment view
@@ -0,0 +1,66 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 9"+ date="2011-02-09T09:12:52Z"+ content="""+[a0826293][] fixed the last problem, there is coreutils available in macports, if they are installed you get the gnu equivalents but they are prefixed with a g (e.g. gchmod instead of chmod), I guess not everyone will have these install or prefer these on [[install/OSX]]++Some more tests fail now...++<pre>+Testing 1:blackbox:3:git-annex unannex:1:with content+### Failure in: 1:blackbox:3:git-annex unannex:1:with content+foo is not a symlink+Testing 1:blackbox:4:git-annex drop:0:no remotes+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes+drop wrongly succeeded with no known copy of file+Testing 1:blackbox:4:git-annex drop:1:with remote+Testing 1:blackbox:4:git-annex drop:2:untrusted remote+Testing 1:blackbox:5:git-annex get+Testing 1:blackbox:6:git-annex move+Testing 1:blackbox:7:git-annex copy+### Failure in: 1:blackbox:7:git-annex copy+move --to of file already there failed+Testing 1:blackbox:8:git-annex unlock/lock+### Error in:   1:blackbox:8:git-annex unlock/lock+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:9:git-annex edit/commit:0+### Error in:   1:blackbox:9:git-annex edit/commit:0+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:9:git-annex edit/commit:1+### Error in:   1:blackbox:9:git-annex edit/commit:1+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:10:git-annex fix+### Error in:   1:blackbox:10:git-annex fix+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:11:git-annex trust/untrust/semitrust+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:12:git-annex fsck:0+### Error in:   1:blackbox:12:git-annex fsck:0+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:12:git-annex fsck:1+### Error in:   1:blackbox:12:git-annex fsck:1+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:12:git-annex fsck:2+### Error in:   1:blackbox:12:git-annex fsck:2+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:13:git-annex migrate:0+### Error in:   1:blackbox:13:git-annex migrate:0+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:13:git-annex migrate:1+### Error in:   1:blackbox:13:git-annex migrate:1+forkProcess: resource exhausted (Resource temporarily unavailable)+Testing 1:blackbox:14:git-annex unused/dropunused+### Error in:   1:blackbox:14:git-annex unused/dropunused+forkProcess: resource exhausted (Resource temporarily unavailable)+Cases: 30  Tried: 30  Errors: 11  Failures: 3+test: failed+make: *** [test] Error 1+</pre>++On a side note, I think I found another bug in the testing. I had tested in a virtual machine in archlinux (a very recent updated version) Please see the report here [[tests fail when there is no global .gitconfig for the user]]++[a0826293]: http://git.kitenet.net/?p=git-annex;a=commit;h=7a0826293e0ac6c0000f49a1618c1c613b909aa1+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn view
@@ -0,0 +1,10 @@+While using HMAC instead of "plain" hash functions is inherently more secure, it's still a bad idea to re-use keys for different purposes.++Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice?++Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing.++> Bug was filed prematurely, but was a good bit of paranoia, and gpg and+> hmac are given different secret keys [[done]] --[[Joey]] ++>> Thanks :) -- RIchiH
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-30T14:32:34Z"+ content="""+S3 doesn't support encryption at all, yet.++It certainly makes sense to use a different portion of the encrypted secret key for HMAC than is uses as the gpg symmetric encryption key.++The two keys used in HMAC would be the secret key and the key/value key for the content being stored.++There is a difficult problem with encrypting filenames in S3 buckets, and that is determining when some data in the bucket is unused for dropunused. I've considered two choices:++1. gpg encrypt the filenames. This would allow dropunused to recover the original filenames, and is probably more robust encryption. But it would double the number of times gpg is run when moving content in/out, and to check for unused content, gpg would have to be run once for every item in the bucket, which just feels way excessive, even though it would not be prompting for a passphrase. Still, haven't ruled this out.++2. HMAC or other hash. To determine what data was unused the same hash and secret key would have to be used to hash all filenames currently used, and then that set of hashes could be interested with the set in the bucket. But then git-annex could only say \"here are some opaque hashes of content that appears unused by anything in your current git repository, but there's no way, short of downloading it and examining it to tell what it is\". (This could be improved by keeping a local mapping between filenames and S3 keys, but maintaining and committing that would bring pain of its own.)+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-03-30T17:01:40Z"+ content="""+After mulling this over, I think actually encrypting the filenames is preferable.++Did you consider encrypting the symmetric key with an asymmetric one? That's what TrueCrypt etc are using to allow different people access to a shared volume. This has the added benefit that you could, potentially, add new keys for data that new people should have access to while making access to old data impossible. Or keys per subdirectory, or, or, or.++As an aside, could the same mechanism be extended to transparently encrypt data for a remote annex repo? A friend of mine is interested to host his data with me, but he wants to encrypt his data for obvious reasons.+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-30T18:15:18Z"+ content="""+Yes, encrypting the symmetric key with users' regular gpg keys is the plan.++I don't think that encryption of content in a git annex remote makes much sense; the filenames obviously cannot be encrypted there. It's more likely that the same encryption would get used for a bup remote, or with the [[special_remotes/directory]] remote I threw in today.+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 4"+ date="2011-03-30T18:20:56Z"+ content="""+Picking up the automagic encryption idea for annex remotes, this would allow you to host a branchable-esque git-annex hosting service. (Nexenta with ZFS is a cheap and reliable option until btrfs becomes stable in a year or five).+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 5"+ date="2011-03-30T18:59:19Z"+ content="""+This is brain-storming only so the idea might be crap, but a branch could keep encrypted filenames while master keeps the real deal. This might fit into the whole scheme just nicely or break future stuff in a dozen places, I am not really sure yet. But at least I can't forget the idea, now.+"""]]
+ doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 6"+ date="2011-03-30T19:02:20Z"+ content="""+OTOH, if encryption makes a bup backend more likely disregard the idea above ;)+"""]]
+ doc/bugs/S3_memory_leaks.mdwn view
@@ -0,0 +1,10 @@+S3 has memory leaks++Sending a file to S3 causes a slow memory increase toward the file size.++Copying the file back from S3 causes a slow memory increase toward the+file size.++The author of hS3 is aware of the problem, and working on it. I think I+have identified the root cause of the buffering; it's done by hS3 so it can+resend the data if S3 sends it a 307 redirect. --[[Joey]]
+ doc/bugs/Unfortunate_interaction_with_Calibre.mdwn view
@@ -0,0 +1,21 @@+# Calibre ++Calibre is a somewhat popular eBook management package that's also free software.  <http://calibre-ebook.com/>  ++Install via+    # apt-get install calibre++There is a somewhat unfortunate interaction between Calibre and git-annex...++* git-annex makes its files become read-only.  By the way, that's not quite obvious from the documentation; I suggest making that more prominent.+* Calibre modifies files (not quite sure of semantics, how, or why) when doing various operations, notably such as when copying a book from one's library to one's portable reading device.++These don't play well together, sadly.++I'd expect most of the issue to sit on the Calibre side, and have reported it as a bug.+[Calibre bug #739045](https://bugs.launchpad.net/calibre/+bug/739045)+Preliminary indication is that they're treating it as a functionality change they'll decline to fix.  Which isn't entirely unreasonable - I anticipated as much, and I don't want to treat that as a bad/wrong decision.++However, I think it's:+* Unfortunate, as fitting Calibre together with git-annex seems like a neat idea.+* Useful to make sure that this kind of "doesn't play well together" condition is documented, even if only as a bug report.
+ doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-03-21T13:15:03Z"+ content="""+Maybe I will run into issues myself somewhere down the road, but generally speaking, I really really like the fact that files are immutable by default.+"""]]
+ doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-03-31T19:32:25Z"+ content="""+One option would be to use the new [[news/sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`+"""]]
+ doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn view
@@ -0,0 +1,1 @@+I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters.
+ doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-08T17:14:25Z"+ content="""+Seems like you probably have files in git with nearly as long filenames as the key files. Course, you can rename those yourself.++This couldn't be changed directly in WORM without some ugly transition, but it would be possible to implement it as a WORM100 or so. OTOH, if you're going to git annex migrate, you might as well use SHA1.+"""]]
+ doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-08T22:02:41Z"+ content="""+What if your files have the same prefix and it happens to be 100 chars long? This can not be solved within WORM, but as Joey pointed out, SHA* exists.+"""]]
+ doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-04-09T20:11:59Z"+ content="""+I wouldn't say it's completly impossible for a WORM100 to work. It would just have the contract that the pair of mtime+100chars has to be unique for each unique piece of data.++But, I have yet to be convinced there's any point, since SHA1 exists.+"""]]
+ doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 4"+ date="2011-04-09T23:45:28Z"+ content="""+mtime+100chars can still get collisions and a _lot_ easier than even SHA1. This introduces more problems that it solves, imo.+"""]]
+ doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn view
@@ -0,0 +1,18 @@+The command `git annex dropunused` currently takes a number, as referenced in output of last `git annex unused` command.++When you want to drop all, or a range, this may be annoying, as you have to specify each number on the command line. ++A range argument, such as `1-1845`, possibly combined with other argument types (Cf. many print dialogues: `1,3,5-7,9`) would be great.++I work around this lack as I want to drop all unused files anyway by something like this:+       +    git annex unused | grep -o -P "^    [0-9]+" | xargs git annex dropunused++> It's designed to be used with `seq`. There's an example in the+> [[walkthrough|walkthrough/unused_data]], and of course multiple seq calls can be used to+> specifiy multiple ranges. So:++	git annex dropunused `seq 1 9` `seq 11 1845`++> I don't see adding my own range operations to be an improvement worth+> making; it'd arguably only be a complication. --[[Joey]] [[done]]
+ doc/bugs/annex_add_in_annex.mdwn view
@@ -0,0 +1,6 @@+I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.++> There is a guard against `git annex add .git-annex/foo`, but it doesn't+> notice `cd .git-annex; git annex add foo`. --[[Joey]]++> Now fixed, by removing the .git-annex directory. [[done]] --[[Joey]]
+ doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn view
@@ -0,0 +1,72 @@+foo is a local repo, bar is a bare remote.++I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend+to version 2. I then ran `git annex copy . --to bar` and checked the+remote. This created WORM:SHA512--123123 files in annex/objects.+Understandable but unwanted. So I upgraded git-annex on bar's machine, as+well.++    % git annex copy . --to bar+    copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar)+    git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]+    rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.7]+    +      rsync failed -- run git annex again to resume file transfer+    failed++Running `git annex upgrade` on bar's machine I get:++    % git annex upgrade+    upgrade  (v1 to v2) (moving content...) git-annex: Prelude.read: no parse++Again, bar is a bare repo.+Running the copy job again, I am still getting the same error as above (as expected). Partial contents of annex/objects on bar:++    [...]+    SHA512:123+    WORM:SHA512--234+    [...]+++-- RichiH++> Upgrading bare repos to v2 generally works fine, so I actually need+> to see the full content of annex/, not a fragment, in order to debug this.+> (Filename contents I don't need to see.) Feel free to email me the details at+> joey@kitenet.net if you don't want to post them here. --[[Joey]]++>> Sent. -- RichiH++>>> Ok, I'm going to go work on my reading comprehension. I see now+>>> that you+>>> explained the problem pretty well. The problem is caused by these+>>> few weird v1 mixed with v2 keys in the annex.+>>> Ones like "annex/objects/WORM:SHA512--$sha512".+>>>+>>> That's a v1 key, but a corrupt form of the key; it's missing the +>>> size and mtime fields that all WORM keys have in v1. And +>>> the filename is itself a key, a v2 SHA512 key. These were+>>> created when you did the `git annex copy to the v1 bare repo.+>>> In v2, git-annex-shell takes a full key object, while in v1,+>>> it takes a key name and a backend name. This incompatability+>>> leads to the weird behavior seen.+>>>+>>> I had suggested you delete data.. don't. On second thought,+>>> you shouldn't delete anything. I'll simply make the v2 upgrade+>>> detect and work around this bug.+>>> --[[Joey]]++>>>> This should be fixed in current git. The scambled keys will be +>>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]]++>>>>> I should stop reading your answers via git; by the time I got to+>>>>> "second thoughts", I had already deleted the files & directories+>>>>> in question, upgraded the bare repo and was busy uploading from my+>>>>> local repo. I agree that taking care of this in the upgrade code+>>>>> is the cleanest approach, by the way.+>>>>> No need to thank me for my patience; thank you for your quickness!+>>>>> RichiH+>>>>> +>>>>> PS: If I get a handle on the mtime issue in the SHA backend, git+>>>>> annex will be pretty much perfect :)
+ doc/bugs/bare_git_repos.mdwn view
@@ -0,0 +1,29 @@+It would be nice if git-annex could be used in bare git repos.+However, that is not currently supported. Problems include:++* git-annex often does not read a git repo's config before touching it,+  so it doesn't know if the repo is bare or not+  (reading the config when operating on ssh repos would be a pain and SLOW;+  I had some of that code in as of 1aa19422ac8748eeff219ac4f46df166dae783c5,+  but ripped it all out)+* .. which results in creating `.git/annex` in a bare repo, which mightily+  confuses git (so it will complain that the bare repo is not+  a git repo at all!)+* `.git-annex/` needs to have state recorded to it and committed, and that+  is not possible with a bare repo. (If [[todo/branching]] were done,+  that might be fixed.) (now fixed)++----++Update: Now that git-annex-shell is used for accessing remote repos,+it would be possible to add smarts about bare repos there, and avoid+some of the above problems. Probably only the state recording problem+remains.++A possible other approach to the state recording repo is to not+record state changes on the remote in that case. Git-annex already+records remote state changes locally whenever it modifies the state of a+remote. --[[Joey]]++> And... [[done]]! See [[/bare_repositories]] for current status+> and gotchas. --[[Joey]]
+ doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn view
@@ -0,0 +1,14 @@+A recent checkout of git-annex fails to build for me (I've installed the new dependancies as well)++<pre>+[70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )+[71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )++Command/Status.hs:133:37: Not in scope: `swap'+make: *** [git-annex] Error 1+</pre>++it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though.++> Probably `swap` appeared only in a newer GHC. I've reverted to avoid a+> versioned build dependency. [[done]] --[[Joey]]
+ doc/bugs/building_on_lenny.mdwn view
@@ -0,0 +1,80 @@+hi,++I am trying to build git annex on lenny.++I checked out the latest from git c88d4939453845efee04da811d64aa41046f9c11,+installed all the packages (some from backports) as required by dpkg-buildpackage++Then I get this:++	...+	mkdir -p build+	ghc -odir build -hidir build --make git-annex+	[ 1 of 19] Compiling Utility          ( Utility.hs, build/Utility.o )+	[ 2 of 19] Compiling GitRepo          ( GitRepo.hs, build/GitRepo.o )+	[ 3 of 19] Compiling GitQueue         ( GitQueue.hs, build/GitQueue.o )+	[ 4 of 19] Compiling TypeInternals    ( TypeInternals.hs, build/TypeInternals.o )+	[ 5 of 19] Compiling Types            ( Types.hs, build/Types.o )+	[ 6 of 19] Compiling Annex            ( Annex.hs, build/Annex.o )+	[ 7 of 19] Compiling Locations        ( Locations.hs, build/Locations.o )+	[ 8 of 19] Compiling UUID             ( UUID.hs, build/UUID.o )+	[ 9 of 19] Compiling LocationLog      ( LocationLog.hs, build/LocationLog.o )+	[10 of 19] Compiling Core             ( Core.hs, build/Core.o )+	[11 of 19] Compiling Backend.URL      ( Backend/URL.hs, build/Backend/URL.o )+	[12 of 19] Compiling Backend          ( Backend.hs, build/Backend.o )++	Backend.hs:114:50:+	    Not in scope: type constructor or class `SomeException'+	make[1]: *** [git-annex] Error 1+	make[1]: Leaving directory `/home/cstamas/tmp/git-annex'+	dh_auto_build: make -j1 returned exit code 2+	make: *** [build] Error 2+	dpkg-buildpackage: failure: debian/rules build gave error exit status 2++I will try to check the mentioned file for error, but I do not know how to program in haskell.++Thanks for your help! --[[cstamas]]++> Newer versions of ghc changed their exception handling types, and+> I coded git-annex to use the new style and not the old. gch6 6.12 will+> work. I do not think there is a backport available though. --[[Joey]]+>+> Ok, found and deployed a workaround. It is not tested. Let me know how it+> works for you. --[[Joey]]++>> I did a git pull and now I get:++	mkdir -p build+	ghc -cpp -odir build -hidir build --make git-annex+	[ 1 of 20] Compiling Portability      ( Portability.hs, build/Portability.o )++	Portability.hs:13:21:+	    Not in scope: type constructor or class `Exception'+	make[1]: *** [git-annex] Error 1+	make[1]: Leaving directory `/home/cstamas/tmp/git-annex'+	dh_auto_build: make -j1 returned exit code 2+	make: *** [build] Error 2+	dpkg-buildpackage: failure: debian/rules build gave error exit status 2++>> --[[cstamas]]++>>> Ok well, I'm not going to try to reimplement all of+>>> Control.Exception.Extensible so I've made it use it. You will have to+>>> figure out how to install that library yourself though, I don't know+>>> how to use cabal with such an old ghc. Library is here:+>>> <http://hackage.haskell.org/package/extensible-exceptions> +>>> and I asked how to get it on stable here:+>>> <http://ask.debian.net/questions/how-to-get-haskell-extensible-extceptions-on-stable> --[[Joey]] ++>>>> I made some effort with cabal on lenny. I can install (and I did it) cabal+>>>> from squeeze as dependencies are ok. Then I installed extensible+>>>> exceptions, but it places it in some local dir that git-annex's installer+>>>> (or ghc itself) does not know about.+>>>>+>>>> Later I realized that *only* for the compilation ghc6 and its friends are+>>>> needed. So I built the package on my other machine running squeeze. Then+>>>> resulting deb packages cleanly installs on lenny+>>>> +>>>> For me this is OK. Thanks! --[[cstamas]]++[[done]]
+ doc/bugs/check_for_curl_in_configure.hs.mdwn view
@@ -0,0 +1,92 @@+[[!meta title="arbitrary/configurable backends"]]++(Retitling as this has drifted..)++---++I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence.++<pre>+diff --git a/configure.hs b/configure.hs+index 772ba54..1a563e0 100644+--- a/configure.hs++++ b/configure.hs+@@ -13,6 +13,7 @@ tests = [+        , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]+        , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 </dev/null"+        , TestCase "rsync" $ requireCmd "rsync" "rsync --version >/dev/null"++       , TestCase "curl" $ requireCmd "curl" "curl --version >/dev/null"+        , TestCase "unicode FilePath support" $ unicodeFilePath+        ] ++ shaTestCases [1, 256, 512, 224, 384]+</pre>++> Well, curl is an optional extra, so requireCmd is too strong. Changed+> to testCmd and applied, thank you!+>+> I thought about actually *using* the resulting SysConfig.curl+> to disable the URL backend if False.. but probably it's better+> to just let it fail if curl is not available. Although, if we wanted+> to add a check for wget or something and use it when curl was not+> available, that might be worth doing. --[[Joey]] ++>> I was thinking that is it worth doing a generic "stat", "delete", "get" +>> and "put" options, I do like the idea of having the possibility of +>> being about to use completely arbitrary storage systems or arbitrary +>> transfer systems. If there was the capability of doing so it would be +>> interesting to see possibilities of using aria2 for using something +>> like bittorrent as backend, or using something like irods or some +>> grid storage system as the storage archive. It's just an idea as +>> I have seen it implemented quite well in irods.++>>> I'm unsure about the idea of having a backend where that is+>>> parameterized. It would mean that one annex's GENERIC-foo key+>>> might be entirely different from another's key with the same backend+>>> and details. And a misconfiguration could get data the wrong+>>> way and get the wrong data, etc.+>>>+>>> I mostly look at the URL backend as an example that can be modified to+>>> make this kind of custom backend. You already probably know enough to+>>> make a TORRENT backend where keys are the urls to torrents to download+>>> with `aria2c --follow-torrent=mem`.+>>>+>>> I am also interested in doing backends that use eg, cloud storage.+>>> A S3 backend that could upload files to S3 in addition to downloading+>>> them, for example, would be handy. --[[Joey]]++>>>> So, rather than use backends to do this, it instead made more sense+>>>> to make them [[special_remotes]]. The URL backend remains a bit+>>>> of a special case, and a bittorrent backend that downloaded a file+>>>> from a bittorrent url would still be a good use of backend, but for+>>>> storing files in external data stores like S3, making it a remote+>>>> makes better sense. I think I can close this bug now, [[done]]+>>>> --[[Joey]] ++also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)++> It's correct, typewise, but I don't see any real reason to bother+> with the change. But I do appreciate patches, which have been rare+> so far, probaby because of Haskell.. :) --[[Joey]] ++>> heh agreed++<pre>+diff --git a/Backend/URL.hs b/Backend/URL.hs+index 29dc8fe..4afcf86 100644+--- a/Backend/URL.hs++++ b/Backend/URL.hs+@@ -50,10 +50,13 @@ dummyFsck _ _ _ = return True+ dummyOk :: Key -> Annex Bool+ dummyOk _ = return True+ ++curl :: [CommandParam] -> IO Bool++curl = boolSystem "curl"+++ downloadUrl :: Key -> FilePath -> Annex Bool+ downloadUrl key file = do+        showNote "downloading"+        showProgress -- make way for curl progress bar+-       liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]++       liftIO $ curl [Params "-# -o", File file, File url]+        where+                url = join ":" $ drop 1 $ split ":" $ show key +</pre>
+ doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn view
@@ -0,0 +1,6 @@+On RHEL5 (and clones) systems uuidgen is available as an alternative to+uuid, the configure script fails, it should probably detect either uuid or+uuidgen, or let the user decide? - also uuidgen behaves differently from+uuid on debian.++> uuidgen is now supported. --[[Joey]] [[done]]
+ doc/bugs/conflicting_haskell_packages.mdwn view
@@ -0,0 +1,17 @@+The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them.++This can be done by the flags -hide-packages and then -package foo++> My ghc does not have a `--hide-packages` option.+> +> Could you just show the build problem that you are suggesting I work+> around? --[[Joey]]+++> Thanks npouillard, I see the problem now.+> <http://stackoverflow.com/questions/2048953/control-monad-state-found-in-multiple-packages-haskell>+> +> I've added "-ignore-package monads-fd" to GHCFLAGS. I hope I don't+> really have to hide all packages and individually turn them back on;+> surely this monads-fd/mtl conflict is an exception, and Haskell's module+> system is not a mess of conflicting modules? --[[Joey]] [[done]]
+ doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment view
@@ -0,0 +1,24 @@+[[!comment format=mdwn+ username="http://ertai.myopenid.com/"+ nickname="npouillard"+ subject="how to reproduce the package conflict issue"+ date="2011-02-07T14:12:43Z"+ content="""+If you install the monads-fd package (with cabal install for instance), then you can no longer build git-annex:++<pre>+./configure+  checking cp -a... yes+  checking cp -p... yes+  checking cp --reflink=auto... yes+  checking uuid generator... uuid+  checking xargs -0... yes+  checking rsync... yes+ghc -O2 -Wall --make git-annex++Annex.hs:22:7:+    Ambiguous module name `Control.Monad.State':+      it was found in multiple packages: monads-fd-0.2.0.0 mtl-2.0.1.0+make: *** [git-annex] Error 1+</pre>+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn view
@@ -0,0 +1,6 @@+Conversation moved from [[walkthrough/recover_data_from_lost+found]]+to a proper bug. --[[Joey]]++(Unfortunatly that scrambled the comment creation times and thus order.)++> Added a message [[done]] --[[Joey]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 10"+ date="2011-05-15T16:47:53Z"+ content="""+The key is the basename of the symlink target.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment view
@@ -0,0 +1,24 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 11"+ date="2011-05-15T18:53:26Z"+ content="""+It seems the objects are in the remote after all, but the remote is unaware of this fact. No idea where/why the remote lost that info, but.. Anyway, with the SHA backends, wouldn't it make sense to simply return \"OK\" and update the annex logs accordingly, no?++Local:++    % ls -l foo+    lrwxrwxrwx 1 richih richih 312 Apr  3 01:18 foo -> .git/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491+    % ++Remote:++    % git-annex-shell recvkey <remote> SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491+    git-annex-shell: key is already present in annex+    % strace git-annex-shell recvkey /base/git-annex/fun SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 2>&1 | grep SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491+    stat64(\"/base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491\", {st_mode=S_IFREG|0444, st_size=80781, ...}) = 0+    % ls -l /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491+    -r--r--r-- 1 richih richih 80781 2011-04-01 12:44 /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491+    % +"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 12"+ date="2011-05-15T19:40:47Z"+ content="""+So, it appears that you're using git annex copy --fast. As documented that assumes the location log is correct. So it avoids directly checking if the bare repo contains the file, and tries to upload it, and the bare repo is all like \"but I've already got this file!\". The only way to improve that behavior might be to let rsync go ahead and retransfer the file, which, with recovery, should require sending little data etc. But I can't say I like the idea much, as the repo already has the content, so unlocking it and letting rsync mess with it is an unnecessary risk. I think it's ok for --force to blow up+if its assumptions turn out to be wrong.++If you use git annex copy without --fast in this situation, it will do the right thing.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment view
@@ -0,0 +1,35 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 13"+ date="2011-05-15T20:25:25Z"+ content="""+Yes, makes sense. I am so used to using --fast, I forgot a non-fast mode existed. I still think it would be a good idea to fall back to non-fast mode if --fast runs into an error from the remote, but as that is well without my abilities how about this patch?+++    From 4855510c7a84eb5d28fdada429580a8a42b7112a Mon Sep 17 00:00:00 2001+    From: Richard Hartmann <richih.mailinglist@gmail.com>+    Date: Sun, 15 May 2011 22:20:42 +0200+    Subject: [PATCH] Make error in RecvKey.hs suggest possible solution+    +    ---+     Command/RecvKey.hs |    2 +-+     1 files changed, 1 insertions(+), 1 deletions(-)+    +    diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs+    index 126608f..b917a1c 100644+    --- a/Command/RecvKey.hs+    +++ b/Command/RecvKey.hs+    @@ -27,7 +27,7 @@ start :: CommandStartKey+     start key = do+        present <- inAnnex key+        when present $+    -       error \"key is already present in annex\"+    +       error \"key is already present in annex. If you are running copy, try without '--fast'\"+        +        ok <- getViaTmp key (liftIO . rsyncServerReceive)+        if ok+    -- +    1.7.4.4++"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 14"+ date="2011-05-15T20:50:26Z"+ content="""+Or, even better, wouldn't it make sense to have SHA backends always default to --fast and only use non-fast when any snags are hit, use non-fast mode for that file.++Though if we continue here, we should probably move this to its own page.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 15"+ date="2011-05-15T21:38:47Z"+ content="""+PS: Just to make this clear, I am using a custom alias for all my copying needs and thus didn't even see that I used --fast. :p+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 16"+ date="2011-05-16T20:01:28Z"+ content="""+Thanks.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-05-12T00:07:29Z"+ content="""+I followed this to re-inject files which git annex fsck listed as missing.++For everyone of those files, I get ++    git-annex-shell: key is already present in annex+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]+    rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.8]++when trying to copy the files to the remote.++-- Richard+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-05-12T01:01:34Z"+ content="""+Sounds like you probably didn't commit after the fsck, or didn't push so the other repository did not know the first had the content again -- but I'm not 100% sure.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 3"+ date="2011-05-14T09:06:54Z"+ content="""+As my comment from work is stuck in moderation:++I ran this twice:++    git pull && git annex add . && git annex copy . --to <remote> --fast --quiet && git commit -a -m \"$HOST $(date +%F--%H-%M-%S-%Z)\" && git push++but nothing changed+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-05-14T16:13:58Z"+ content="""+Hmm. Old versions may have forgotten to git add a .git-annex location log file when recovering content with fsck. That could be another reason things are out of sync.++But I'm not clear on which repo is trying to copy files to which.++(NB: If the files were recovered on a bare git repo, fsck cannot update the location log there, which could also explain this.)+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 5"+ date="2011-05-14T19:03:43Z"+ content="""+Version: 0.20110503++My local non-bare repo is copying to a remote bare repo.++I have been recovering in a non-bare repo.++If there is anything I can send you to help... If I removed said files and went through http://git-annex.branchable.com/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/ -- would that help?+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 6"+ date="2011-05-14T19:23:45Z"+ content="""+Well, focus on a specific file that exhibits the problem. What does `git annex whereis` say about it? Is the content actually present in annex/objects/ on the bare repository? Does that contradict whereis?+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 7"+ date="2011-05-14T23:13:15Z"+ content="""+It exists locally, whereis tells me it exists locally and locally, only.++The object is _not_ in the bare repo.++The file _might_ have gone missing before I upgraded my annex backend version to 2. Could this be a factor?+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 8"+ date="2011-05-15T00:09:34Z"+ content="""+What you're describing should be impossible; the error message shown can only occur if the object is present in the annex where `git-annex-shell recvkey` is run. So something strange is going on.++Try reproducing it by running on the remote system, `git-annex-shell recvkey /remote/repo.git $key` .. if you can reproduce it, I guess the next thing to do will be to strace the command and see why it's thinking the object is there.+"""]]
+ doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 9"+ date="2011-05-15T09:16:49Z"+ content="""+Just to make sure: How do I get $key? What I did was look at the path in the object store of the local repo and see if that exact same path & file existed in the remote.+"""]]
+ doc/bugs/done.mdwn view
@@ -0,0 +1,4 @@+recently fixed [[bugs]]++[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10+archive=yes]]
+ doc/bugs/dotdot_problem.mdwn view
@@ -0,0 +1,4 @@+cannot "git annex ../foo"   (GitRepo.relative is buggy and+git-ls-files also refuses w/o --full-name, which would need other changes)++[[done]]
+ doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn view
@@ -0,0 +1,13 @@+I was trying out the example with the walkthrough [[walkthrough/using_the_URL_backend]]. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with+++<pre>+[jtang@lenny gc]$ git annex drop -v curl-7.21.4.tar.gz+drop curl-7.21.4.tar.gz+failed+git-annex: 1 failed+</pre>++At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough.++> Whoops, got some logic backwards. [[fixed|done]]! --[[Joey]]
+ doc/bugs/encrypted_S3_stalls.mdwn view
@@ -0,0 +1,9 @@+Sending large-ish (few megabytes) files to encrypted S3 remotes stalls out.+It works for the tiny files I was using to test while developing it, on+dialup.++There was a similar issue with bup, which I fixed by forking a process+rather than using a thread to do some IO. Probably need the same here.+--[[Joey]]++[[done]] --[[Joey]]
+ doc/bugs/error_propigation.mdwn view
@@ -0,0 +1,3 @@+If a subcommand fails w/o throwing an error, no error is propigated to the+git-annex exit code. With --quiet, this makes it look like the command+succeeded. [[done]]
+ doc/bugs/error_with_file_names_starting_with_dash.mdwn view
@@ -0,0 +1,15 @@+git annex add has problems if items start with dashes, example:++-wut-a-directory-name-/file1++leads to++[[!format bash """+add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u'+„sha1sum --help“ gibt weitere Informationen.++  git-annex: <file descriptor: 15>: hGetLine: end of file+"""]]++> This is fixed in git, at least I think I've found all cases where+> filenames are passed to programs and escaped them. --[[Joey]] [[done]]
+ doc/bugs/fat_support.mdwn view
@@ -0,0 +1,15 @@+Klaus pointed out that there are two problems that keep+git-annex from being used on USB keys, that would typically+be VFAT formatted:++- Use of symlinks, which VFAT does not support. Very hard to fix.+  Instead, just use [[/bare_repositories]] on the key,+  they're supported now.+- Use of ":" in filenames of object files, also not supported.+  Could easily be fixed by reorganizing the object directory.++[[!tag wishlist]]++[[Done]]; in annex.version 2 repos, colons are entirely avoided in+filenames. So a bare git clone can be put on VFAT, and git-annex+used to move stuff --to and --from it, for sneakernet.
+ doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="fmarier"+ ip="121.73.248.43"+ subject="Exporting to a FAT filesystem?"+ date="2011-04-04T07:40:41Z"+ content="""+I'm using git-annex to keep my music in sync between all of my different machines. What I'd love to be able to do is to also keep it in sync with my iRiver player. Unfortunately, the firmware, Rockbox, doesn't support ext3, so I'm stuck with a FAT filesystem.++I can see how the design of git-annex makes it rather difficult to get rid of the symlinks, so how about taking a different approach: something like a \"git annex export DEST\" which would take a destination (not a git remote) and rsync the content over to there as regular files.++Maybe \"git annex sync DEST\" or \"git annex rsync DEST\" would be better names if we want to convey the idea that the destination will be made to look like the source repo, including performing the necessary deletions.+"""]]
+ doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-04-04T18:20:45Z"+ content="""+Hey @fmarier. Well, this bug report is closed because you can already get rid of the symlinks. Just put a bare git repo on your fat filesystem, and use git-annex copy --to/--from there.++Now, that puts all the files that are on the device in .git/annex/objects/xx/yy/blah.mp3 -- how well rockbox would support that I don't know. And if it tries to modify or delete those files, git annex also can't help you manage those changes.++Another recent option is the [[special_remotes/directory]] special remote type, which again uses \"xx/yy/blah.mp3\" and can't track changes made to the files. This could perhaps be extended in the direction you suggest, although trying to fit this into the special remote infrastructure might not be a good fit really.++The most likely way this has to get dealt with is really by using [[todo/smudge]] filters, which would eliminate the symlinks and allow copying a non-bare git repo onto vfat. +"""]]
+ doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="fmarier"+ subject="comment 3"+ date="2011-04-05T10:00:21Z"+ content="""+Thanks for the reply @joey.++While it would certainly be possible for a bare repo to exist on my iRiver, the problem is that the music player uses the filesystem to organize files into directories like \"Artist/Album/Track.ogg\". So replacing that with \"..../xx/yy/Track.ogg\" would make it fairly difficult to browse my music collection and select the album/track I want to listen to :)++So unless I have the files physically organized like the symlinks, then it's probably not going to work very for that particular workflow. Smudge filters are interesting though. In the meantime, I'll look into rsyncing from another box which has the right filesystem layout onto my iRiver directly.+"""]]
+ doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://ethan.betacantrips.com/"+ nickname="ethan.glasser.camp"+ subject="no symlinks"+ date="2011-06-08T20:59:38Z"+ content="""+If you try to clone a git repo that has a symlink over to a VFAT filesystem, you get (in its place) a regular file that contains the name of the symlink target.  So why can't git-annex use that?  I could still do git annex get on this file, git annex would still \"know\" that it's a symlink, and could replace it with a copy of the real file (instead of putting it in .git/annex).++I know if it were that simple, someone would have done it already, so what am I missing?  I guess trying to get the file FROM the repository would fail because it wouldn't find the file in .git/annex?  Couldn't you store a reverse mapping?  You wouldn't be able to move the file around, but you already lose that once you give up symlinks.  It would also be a little harder to tell which symlinks were \"dangling\"; I don't see an easy way to get around that.  It would still be better than a bare repo..+"""]]
+ doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 5"+ date="2011-06-10T16:41:43Z"+ content="""+@ethan the reason that wouldn't work is because git would then see a file that was checked in and had its one line symlinkish content replaced with a huge binary blob. And git commit would try to commit that etc. The potential for foot-shooting is too high.+"""]]
+ doc/bugs/free_space_checking.mdwn view
@@ -0,0 +1,21 @@+Should check that there is enough free space before trying to copy a+file around.++* Need a way to tell how much free space is available on the disk containing+  a given repository.++* And, need a way to tell the size of a file before copying it from+  a remote, to check local disk space.++  As of annex.version 2, this metadata can be available for any type+  of backend. Newly added files will always have file size metadata,+  while files that used a SHA backend and were added before the upgrade+  won't.++  So, need a migration process from eg SHA1 to SHA1+filesize. It will+  find files that lack size info, and rename their keys to add the size+  info. Users with old repos can run this on them, to get the missing+  info recorded.++> [[done]]; no migtation process for old SHA1 keys from v1 repo though.+> --[[Joey]]
+ doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-03-15T14:11:27Z"+ content="""+Keep in mind that lots of small files may have significant overhead, so a warning that it's not possible to make sure there's enough space would make sense for certain corner cases. Actually finding out the exact overhead is beyond git-annex' scope and, given transparent compression etc, ability, but a warning, optionally with a \"do you want to continue\" prompt can't hurt.++-- RichiH+"""]]
+ doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-03-16T03:04:50Z"+ content="""+Right. You probably don't want git-annex to fill up your entire drive anyway, so if it tries to reseve 10 mb or 1% or whatever (probably configurable) for overhead, that should be good enough.+"""]]
+ doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 3"+ date="2011-03-16T15:40:56Z"+ content="""+Sometimes, I might want to fill up the disk as much as possible. Thus, a warning is preferable to erroring out too early, imo -- Richard+"""]]
+ doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn view
@@ -0,0 +1,8 @@+git annex carefully setup restrictive permissions of .git/annex directories and files.++The fsck command should check that they are still correct.+The fix command should fix them.++PS: Thanks for this nice tool!++> Good idea, [[done]] (actually, fsck just fixes them too)! --[[Joey]]
+ doc/bugs/fsck_output.mdwn view
@@ -0,0 +1,36 @@+When you check several files and the fsck fails, you get confusing output:++<pre>+O fsck test1 (checksum...) +E  Only 1 of 2 trustworthy copies of test1 exist.+E  Back it up with git-annex copy.+O+O failed+O fsck test2 (checksum...) +E  Only 1 of 2 trustworthy copies of test2 exist.+E  Back it up with git-annex copy.+O +O failed+</pre>++The newline is in the wrong place and confuses the user. It should be printed _after_ "failed".++> This is a consequence of part of the output being printed to stderr, and+> part to stdout. I've marked the lines above with E and O.+> +> Normally a "failed" is preceeded by a message output to stdout desribing+> the problem; such a message will not be "\n" terminated, so a newline+> is always displayed before "failed". In this case, since the message+> is sent to stderr, it is newline terminated.+> +> Fixing this properly would involve storing state, or rethinking +> when git-annex displays newlines (and I rather like its behavior+> otherwise).+> +> A related problem occurs if an error message is unexpetedly printed.+> Dummying up an example:+> +> O get test1 (from foo...) E git-annex: failed to run ssh+> failed+> +> --[[Joey]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn view
@@ -0,0 +1,75 @@+I ran git-annex (git version) on three machines with ghc-7.0.2 for about a month, but recently (no more than a week ago) I've started getting this error for every file on "git annex get":++    git-annex-shell: internal error: evacuate(static): strange closure type 30799+        (GHC version 7.0.2 for i386_unknown_linux)+        Please report this as a GHC bug:  http://www.haskell.org/ghc/reportabug++There were no changes to ghc or it's modules, so I assume something has changed in git-annex itself.++strace shows "git annnex get" (on "host1") performing following exec's:++    [pid  9481] execve("/usr/bin/rsync", ["rsync", "-p", "--progress", "--inplace", "-e", "'ssh' 'user@host2' 'git-annex-shell ''sendkey'' ''/remote/path'' ''SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b'' ''--'''", ":", "/local/path/.git/annex/tmp/SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b"], [/* 41 vars */]) = 0+    [pid  9482] execve("/usr/bin/ssh", ["ssh", "user@host2", "git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--'", "", "rsync", "--server", "--sender", "-vpe.Lsf", "--inplace", ".", ""], [/* 41 vars */] <unfinished ...>++I've tried running the second command directly from the shell and got the same error message from a remote GHC.+Adding strace before git-annex-shell to remote command yielded something like this in the end:++    stat64("/local/path.git", 0xb727d610) = -1 ENOENT (No such file or directory)+    stat64("/local/path.git", 0xb727d6b0) = -1 ENOENT (No such file or directory)+    waitpid(7525, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0) = 7525+    chdir("/home/user")                  = 0+    rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0+    write(2, "git-annex-shell: internal error: ", 33git-annex-shell: internal error: ) = 33+    ...++Note that "/local/path" here is not what's specified in rsync arguments at all, and git repo with files-to-be-fetched on "host2" is in "/remote/path", but "/local/path" is present in git remotes there since I mount it via nfs from "host1" (yes, to the same path as it's there):++    [remote "nfs"]+      url = /local/path+      fetch = +refs/heads/*:refs/remotes/nfs/*+      push = refs/heads/*:refs/remotes/host2/*+      annex-uuid = 0a4e14ba-5236-11e0-9004-7f24452c0f05++If I comment that remote out from "/remote/path/.git/config", "git annex get" works fine.+The only git-command git-annex-shell seem to exec there (on "host2") is "git config --list", so it's shouldn't be git trying to do something with it's remotes - it's git-annex itself, right?++Anyways, looks like a simple path-joining error, if "/local/path.git" should be "/local/path/.git" there.++I'm actually quite confused about what it's trying to do with that path.+Connect from "host1" to "host2" just to connect back to "host1"?+What for, when it should just fetch files from "host2"?++> git-annex (and git-annex shell) always start up by learning what git+> remotes are locally configured, and this includes checking them to+> try to look up their annex.uuid setting.+> +> Since git will, given a remote like "url = /foo", first look in+> "/foo.git" for a bare git repository, so too does git-annex.+> I do not think this is a path joining error. That seems likely to+> be a red herring. --[[Joey]]++Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess.+Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing.++Thanks!++> I've never seen anything like this error message. I don't know if the+> problem is caused by building with GHC 7, or what. You didn't say what+> OS you're using. Searching for the error message, it seems to involve+> Mac OS X. ++> For example: <http://hackage.haskell.org/trac/ghc/ticket/3771>+>> The error "strange closure type" indicates some kind of memory corruption, which can have many different causes, from bugs in the GC to hardware failures.+> +> You said that you'd been using git-annex built with that version of GHC+> successfully before. Perhaps you could use `git bisect` to see if you can+> identify a point in git-annex's history where this started happening?+> Since you  can reproduce the problem by just running git-annex-shell at+> the command line with the right parameters, it should be easy to bisect it.+> +> Probably your best bet will be changing to a different version or build of+> GHC.. --[[Joey]] ++---++forwarded to GHC upstream; closing [[done]] --[[Joey]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/"+ subject="Bisect it is, then"+ date="2011-04-03T04:45:49Z"+ content="""+Hm, if path's ok, guess there's no way around git-bisect indeed. Wonder if there's some kind of ccache for haskell...++OS is linux, amd64 on \"host1\" and i386 on \"host2\" where git-annex-shell is crashing.+I'll try to come up with a commit, thanks for clarifications.+"""]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment view
@@ -0,0 +1,66 @@+[[!comment format=mdwn+ username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/"+ subject="Bisect results"+ date="2011-04-03T06:22:15Z"+ content="""+Completed git-bisect twice, getting roughly the same results:++    828a84ba3341d4b7a84292d8b9002a8095dd2382 is the first bad commit+    commit 828a84ba3341d4b7a84292d8b9002a8095dd2382+    Author: Joey Hess <joey@kitenet.net>+    Date:   Sat Mar 19 14:33:24 2011 -0400++        Add version command to show git-annex version as well as repository version information.++    :040000 040000 ed849b7b6e9b177d6887ecebd6a0f146357824f3 1c98699dfd3fc3a3e2ce6b55150c4ef917de96e9 M      Command+    :100644 100644 b9c22bdfb403b0bdb1999411ccfd34e934f45f5c adf07e5b3e6260b296c982a01a73116b8a9a023c M      GitAnnex.hs+    :100644 100644 76dd156f83f3d757e1c20c80d689d24d0c533e16 d201cc73edb31f833b6d00edcbe4cf3f48eaecb0 M      Upgrade.hs+    :100644 100644 5f414e93b84589473af5b093381694090c278e50 d4a58d77a29a6a02daf13cec0df08b5aab74f65e M      Version.hs+    :100644 100644 f5c2956488a7afafd20374873d79579fb09b1677 f8cd577e992d38c7ec1438ce5c141eb0eb410243 M      configure.hs+    :040000 040000 f9b7295e997c0a5b1dda352f151417564458bd6e a30008475c1889f4fd8d60d4d9c982563380a692 M      debian+    :040000 040000 9d87a5d8b9b9fe7b722df303252ffd5760d66f75 08834f61a10d36651b3cdcc38389f45991acdf5e M      doc++contents of final refs/bisect:++    bad (828a84ba3341d4b7a84292d8b9002a8095dd2382)+    good-33cb114be5135ce02671d8ce80440d40e97ca824+    good-942480c47f69e13cf053b8f50c98c2ce4eaa256e+    good-ca48255495e1b8ef4bda5f7f019c482d2a59b431++\"roughly\" because second bisect gave two commits as a result, failing to build one of them (missing .o file on link, guess it's because of -j4 and bad deps in that version's build system):++    There are only 'skip'ped commits left to test.+    The first bad commit could be any of:+    828a84ba3341d4b7a84292d8b9002a8095dd2382+    5022a69e45a073046a2b14b6a4e798910c920ee9+    We cannot bisect more!++Also noticed that \"git-annex-shell ...\" command succeeds if ran as root user, while failing from unprivileged one.+There are no permission/access errors in \"strace -f git-annex-shell ...\", so I guess it could be some bug in the GHC indeed.++JIC, logged a whole second bisect operation.+Resulting log: [http://fraggod.net/static/share/git-annex-bisect.log](http://fraggod.net/static/share/git-annex-bisect.log)++Bisect script I've used (git-annex-shell dies with error code 134 - SIGABRT on GHC error):++    res=+    while true; do+      if [[ -n \"$res\" ]]; then+        cd /var/tmp/paludis/build/dev-scm-git-annex-scm.bak/work/git-annex-scm+        echo \"---=== BISECT ($res) ===---\"; git bisect \"$res\" 2>&1; echo '---=== /BISECT ===---'+        cd+        rm -Rf /var/tmp/paludis/build/dev-scm-git-annex-scm+        cp -a --reflink=auto /var/tmp/paludis/build/dev-scm-git-annex-scm{.bak,}+        chown -R paludisbuild: /var/tmp/paludis/build/dev-scm-git-annex-scm+      fi+      res=+      cave resolve -zx1 git-annex --skip-until-phase configure || res=skip+      if [[ -z \"$res\" ]]; then+        cd /remote/path+        sudo -u user git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--' rsync --server --sender -vpe.Lsf --inplace . ''+        if [[ $? -eq 134 ]]; then res=bad; else res=good; fi+        cd+      fi+    done 2>&1 | tee ~/git-annex-bisect.log++"""]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment view
@@ -0,0 +1,38 @@+[[!comment format=mdwn+ username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/"+ subject="comment 3"+ date="2011-04-03T06:57:02Z"+ content="""+Repeated bisect with -j1, just to be sure it's not a random error, and it gave me 828a84ba3341d4b7a84292d8b9002a8095dd2382 again.+Guess I'll look through the changes there a bit later and try to revert these until it works.++Not sure if it's repeatable by anyone but me  (and hence worth fixing), but here's a bit more of info about the system:++    Exherbo linux+    Linux sacrilege 2.6.38.2-fg.roam #4 SMP PREEMPT Mon Mar 28 21:08:47 YEKST 2011 i686 GNU/Linux++    dev-lang/ghc-7.0.2:7.0.2::installed+    dev-haskell/HUnit-1.2.2.3:1.2.2.3::installed+    dev-haskell/MissingH-1.1.0.3:1.1.0.3::installed+    dev-haskell/QuickCheck-2.4.0.1:2.4.0.1::installed+    dev-haskell/array-0.3.0.2:0.3.0.2::installed+    dev-haskell/bytestring-0.9.1.7:0.9.1.7::installed+    dev-haskell/containers-0.4.0.0:0.4.0.0::installed+    dev-haskell/extensible-exceptions-0.1.1.2:0.1.1.2::installed+    dev-haskell/filepath-1.2.0.0:1.2.0.0::installed+    dev-haskell/hslogger-1.1.3:0::installed+    dev-haskell/mtl-2.0.1.0:2.0.1.0::installed+    dev-haskell/network-2.3.0.1:2.3.0.1::installed+    dev-haskell/old-locale-1.0.0.2:1.0.0.2::installed+    dev-haskell/parsec-3.1.0:3.1.0::installed+    dev-haskell/pcre-light-0.4:0::installed+    dev-haskell/regex-base-0.93.2:0.93.2::installed+    dev-haskell/regex-compat-0.93.1:0.93.1::installed+    dev-haskell/regex-posix-0.94.4:0.94.4::installed+    dev-haskell/syb-0.3:0.3::installed+    dev-haskell/transformers-0.2.2.0:0.2.2.0::installed+    dev-haskell/utf8-string-0.3.6:0.3.6::installed++(some stuff listed here as ::installed, but contains no files, since these packages detect whether ghc-7.0.2 already comes with the same/newer package version)++"""]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-04-03T16:06:34Z"+ content="""+Nice work on the bisection. It's obviously a compiler bug. Having two test cases that differ in only as trivial and innocous a commit as 828a84ba3341d4b7a84292d8b9002a8095dd2382 might help a GHC developer track it down.++We should probably forward this as a GHC bug. I hope you can find a different version or build of GHC to build git-annex with.+"""]]
+ doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/"+ subject="Reported the issue to GHC"+ date="2011-04-07T13:44:36Z"+ content="""+Finally got around to [report the issue to GHC tracker](http://hackage.haskell.org/trac/ghc/ticket/5085#comment:7).++Looks quite alike (at least to the haskell-illiterate person like me) to a highest-priority issue that's hanging right at the top of the list.+There are other similar reports, but they seem to be either related to PowerPC Macs, closed as invalid or due to needinfo inactivity.++Guess any further discussion belongs there, unless ghc developers will bounce it back.+Thanks a lot for your help, Joey, and for sharing a great thing that git-annex is.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn view
@@ -0,0 +1,100 @@+Currently the hashed directories in .git-annex allow for upper and lower case directory names... on linux (or any case sensitive filesystem) the directory names such as 'Gg' and 'GG' are different and unique. However on systems like OSX (and probably windows if it is ever supported) the directory names 'Gg' is the same as 'GG'++In one of the annex'd repos that I have this has occured...++<pre>+$ git add -i                                                                                          +           staged     unstaged path+  1:    unchanged        +1/-1 .git-annex/GM/GV/WORM-s183630166-m1301072171--somefile.log+  2:    unchanged        +1/-1 .git-annex/Gm/GV/WORM-s183630166-m1301072171--somefile.log+</pre>+++this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.++> I feel a bit stupid to not have considered case-insensative filesystems.+> They are just so far from where I have lived for 20 years that it's hard+> to keep them in mind.+> +> I guess that+> [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]] is+> somehow a consequence (or cause?) of this, but I don't quite understand+> how this is causing git to fail to stage files, or stage the same file+> twice under different capitalizations. git-annex always will run git add+> on the path with the "correct" capitalization. So unless something else+> has added the path with the other capitalization (perhaps git add+> .git-annex manually?) I don't understand how you get to this state.+> --[[Joey]]++>> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific.++>>> If you copied `.git/` over, perhaps you got a git repo without+>>> core.ignorecase set right for the filesystem it landed on?++>>>> I usually git clone or do a fresh repository and pull things in, I was also unaware of this ignorecase setting as well.++>>> +>>> Something like this might reproduce it:++<pre>+# mkdir test; cd test; git init+# git config core.ignorecase false+# mkdir Foo+# touch Foo/bar+# git add Foo/bar+# git add foo/bar+# git add fOo/bar+# git status+# touch foo/other+# git add fOo/other+# git status+</pre>++>>>> And then either git commit or git clone would probably get confused+>>>> if it thought 3 distinct files had been committed.+>>>> --[[Joey]]++>>>>> Doing the above test on a HFS+ partition yields this++<pre>+## with ignorecase=false+commit bb024c6fd7482b2d10f60ae899cb7a949aca1ad8+Author: Jimmy Tang <jtang@exia>+Date:   Sun Mar 27 18:40:24 2011 +0100++    commit++diff --git a/Foo/bar b/Foo/bar+new file mode 100644+index 0000000..e69de29+diff --git a/fOo/bar b/fOo/bar+new file mode 100644+index 0000000..e69de29+diff --git a/fOo/other b/fOo/other+new file mode 100644+index 0000000..e69de29+diff --git a/foo/bar b/foo/bar+new file mode 100644+index 0000000..e69de29+</pre>++>>>>> and without changing ignorecase++<pre>+commit 909a089158ffb98f8e91f98905e2bfdc7234666f+Author: Jimmy Tang <jtang@exia>+Date:   Sun Mar 27 18:46:57 2011 +0100++    commit++diff --git a/Foo/bar b/Foo/bar+new file mode 100644+index 0000000..e69de29+diff --git a/Foo/other b/Foo/other+new file mode 100644+index 0000000..e69de29+</pre>++> Closing this bug, as it seems I have dealt with it adequately now.+> [[done]]+> --[[Joey]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 10"+ date="2011-04-01T16:11:52Z"+ content="""+No, I don't need a copy of your repo now.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 11"+ date="2011-04-02T17:53:58Z"+ content="""+I have pushed out a preliminary fix. The old mixed-case directories will be left where they are, and still read from by git-annex. New data will be written to new, lower-case directories. I think that once git stops seeing changes being made+to mixed-case, colliding directories, the bugs you ran into won't manifest any more.++You will need to find a way to get your git repository out of the state where it complains about uncommitted files (and won't let you commit them). I have not found a reliable way to do that; git reset --hard worked in one case but not in another. May need to clone a fresh git repository.++Let me know how it works out.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 12"+ date="2011-04-02T17:58:24Z"+ content="""+Also, you can delete `.git-annex/??` if you want to, then running `git annex fsck --fast` in each of your clones would regenerate the data using only the lower-case hash directories.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 13"+ date="2011-04-03T07:43:37Z"+ content="""+Ok, thanks for the fix. It seems the fix isn't too reliable with my repos, I get different numbers of  \"** No known copies of...\" in the various cloned repos that I have. After all the \"messing\" that I have done to my repos I think git-annex has gotten very confused. I will just leave things as they are and let git-annex slowly migrate over to the new format or re-clone from a linux source and see how things go. I will report back on this issue in abit after I use it more to see.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 14"+ date="2011-04-03T08:24:17Z"+ content="""+I meant to say in it wasn't reliable when I was following the instructions for \"Comment 12\". I did find that just doing a  \"git annex copy -t externalusb .\"  then a \"git annex drop .\" from the root of my cloned and \"none trusted\" annexed repos to be more reliable, it just means I temporarily need a load of space to get myself out of my earlier mess.++On testing this bug fix, I found a minor behavioural issue with [[git annex copy -f REMOTE . doesn't work as expected]]+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment view
@@ -0,0 +1,17 @@+[[!comment format=mdwn+ username="gernot"+ ip="213.168.117.192"+ subject="comment 15"+ date="2011-04-03T15:41:00Z"+ content="""+I also ran into problems on a case-insensitive HFS+ file system, it seems.  I+tried following the instructions in comment 12:++	1. Remove everything in .git-annex besides uuid.log and trust.log+	2. git annex fsck --fast+	3. Commit++However, I still see upper and lower case directories in .git-annex.  Did I+misunderstand that they should all be lower case now?++"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment view
@@ -0,0 +1,22 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 16"+ date="2011-04-03T16:02:33Z"+ content="""+I think the correct steps should be, make a backup first :) then ...++1. git pull  # update your clone, and commit everything so you don't lose anything+2. git annex fsck --fast # check the repo first, just in case+3. rm -rf .git-annex/?? # remove the old metadata+4. git annex fsck --fast # get git annex to regenerate it all+5. push your changes out to your other repos, you will need to make sure git-annex is updated everywhere if there are remotes in your setup.++I eventually migrated all of my own annex'd repos and I no longer have the old hashed directories but the new ones in the form++    .git/annex/aaa/bbb/foo.log++I did lose some tracking information but not data (as far as I can see for now), but that was quickly fixed by pushing and pulling to my bare repo which tracks most of my data.++I also found that it worked a bit more reliably for me on the copies of repos that were located on case sensitive filesystems, but I guess that was expected.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment view
@@ -0,0 +1,9 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 17"+ date="2011-04-03T16:53:51Z"+ content="""+@gernot step 0 is to upgrade git-annex to current git, on all systems where you use it, in case that wasn't clear.++"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="gernot"+ ip="213.168.117.192"+ subject="comment 18"+ date="2011-04-03T19:46:16Z"+ content="""+Joey, sorry, I got it wrong.  I thought upgrading git didn't help and you+adjusted things in git-annex instead.++Anyway, can I get around upgrading on all hosts by reformatting the drive to+case-sensitive HFS+? Or will I have to upgrade git (currently version 1.7.2.5)+eventually anyway?++"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 19"+ date="2011-04-03T19:53:44Z"+ content="""+Git does not need to be upgraded. Git-annex needs to be upgraded to git rev 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 or newer, on all machines.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 1"+ date="2011-03-28T07:23:41Z"+ content="""+One possible work around is to just create a loopback file system with a case sensitive filesystem. I think I might do that for anything that I really care about for now.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment view
@@ -0,0 +1,19 @@+[[!comment format=mdwn+ username="ssqq"+ ip="208.70.196.4"+ subject="Still a problem on 0.20110523"+ date="2011-06-02T20:31:55Z"+ content="""+Hi,++(I'm new to git and git annex, so please forgive any mistakes I make...)++My repo is messed up right now.  The fact that I copied the repo with rsync -a back and forth from a case insensitive filesystem to a case sensitive one, probably didn't help.++I believe the annexed files in .git/annex/objects/ are still using a mixed case directory hashing scheme.  That's the problem I'm having.  The symlinks point to the wrong case and are now broken.  I don't think the latest versions of git-annex changed that (it only changed the hashing under .git-annex, right?).++Even if I clean up my repo, I think I'm still going to have a problem because I have one repo on an OS X case insensitive filesystem and my other repos on case sensitive Linux filesystems.  Potentially the directory name under .git/annex/objects will have a different case.  Then the symlink might have a different case than my Linux FS.  Does git-annex track changes in git by the contents of the symlink?  In which case the case difference would show up as a change even though there is no change?++Is it possible to change the directory hashing scheme under .git/annex/objects to use lowercase names?++"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 21"+ date="2011-06-10T16:46:03Z"+ content="""+@seqq git-annex always uses the same case when creating and accessing the files pointed to by the symlinks. So it will not matter if it's used on a case-insensative, or case-insensative but preserving system like OSX.++You need to fix up the cases of the files in .git/annex/objects to what it expects. I'm not sure what would be the best way to do that. The method described in [[walkthrough/recover_data_from_lost+found]] might work well.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment view
@@ -0,0 +1,185 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-03-28T15:09:45Z"+ content="""+I think I know how I got myself into this mess... I was on my mac workstation and I had just pulled in a change set from another repo on a linux workstation after I had a made a bunch of moves. here's a bit of a log of what happened...+++<pre>+jtang@x00:~/sources $ git pull cports-devel master+Warning: untrusted X11 forwarding setup failed: xauth key data not generated+Warning: No xauth data; using fake authentication data for X11 forwarding.+remote: Counting objects: 4195, done.+remote: Compressing objects: 100% (1135/1135), done.+remote: Total 2582 (delta 866), reused 2576 (delta 860)+Receiving objects: 100% (2582/2582), 229.42 KiB | 111 KiB/s, done.+Resolving deltas: 100% (866/866), completed with 9 local objects.+From cports-devel:/home/people/jtang/sources+ * branch            master     -> FETCH_HEAD+Updating 319df99..ab0a98c+error: Your local changes to the following files would be overwritten by merge:+	.git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log+	.git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log+	.git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log+	.git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log+	.git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log+	.git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log+	.git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log+	.git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log+	.git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log+	.git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log+	.git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log+	.git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log+	.git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log+	.git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log+	.git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log+	.git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log+Please, commit your changes or stash them before you can merge.+error: Your local changes to the following files would be overwritten by merge:+	.git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log+	.git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log+	.git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log+	.git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log+	.git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log+	.git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log+	.git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log+	.git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log+	.git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log+	.git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log+	.git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log+	.git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log+	.git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log+	.git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log+	.git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log+	.git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log+	.git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log+	.git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log+	.git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log+	.git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log+	.git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log+	.git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log+	.git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log+	.git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log+	.git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log+	.git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log+	.git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log+	.git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log+	.git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log+	.git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log+Please, commit your changes or stash them before you can merge.+error: The following untracked working tree files would be overwritten by merge:+	.git-annex/1g/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log+	.git-annex/3w/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log+	.git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log+	.git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log+	.git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log+	.git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log+	.git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log+	.git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log+	.git-annex/jK/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log+	.git-annex/vK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log+	.git-annex/vw/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log+	.git-annex/zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log+Please move or remove them before you can merge.+Aborting+1|jtang@x00:~/sources $ git status+# On branch master+# Your branch is ahead of 'origin/master' by 2 commits.+#+# Changes to be committed:+#   (use \"git reset HEAD <file>...\" to unstage)+#+#	modified:   .git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log+#	modified:   .git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log+#	modified:   .git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log+#	modified:   .git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log+#	modified:   .git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log+#	modified:   .git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log+#	modified:   .git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log+#	modified:   .git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log+#	modified:   .git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log+#	modified:   .git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log+#	modified:   .git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log+#	modified:   .git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log+#	modified:   .git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log+#	modified:   .git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log+#	modified:   .git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log+#	modified:   .git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log+#+# Changes not staged for commit:+#   (use \"git add <file>...\" to update what will be committed)+#   (use \"git checkout -- <file>...\" to discard changes in working directory)+#+#	modified:   .git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log+#	modified:   .git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log+#	modified:   .git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log+#	modified:   .git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log+#	modified:   .git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log+#	modified:   .git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log+#	modified:   .git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log+#	modified:   .git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log+#	modified:   .git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log+#	modified:   .git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log+#	modified:   .git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log+#	modified:   .git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log+#	modified:   .git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log+#	modified:   .git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log+#	modified:   .git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log+#	modified:   .git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log+#	modified:   .git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log+#	modified:   .git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log+#	modified:   .git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log+#	modified:   .git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log+#	modified:   .git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log+#	modified:   .git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log+#	modified:   .git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log+#	modified:   .git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log+#	modified:   .git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log+#	modified:   .git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log+#	modified:   .git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log+#	modified:   .git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log+#	modified:   .git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log+#	modified:   .git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log+#+# Untracked files:+#   (use \"git add <file>...\" to include in what will be committed)+#+#	.git-annex/1G/X3/+#	.git-annex/3W/Xf/+#	.git-annex/9q/Wz/+#	.git-annex/Fq/4z/+#	.git-annex/Jk/zK/+#	.git-annex/Kx/w1/+#	.git-annex/VK/kv/+#	.git-annex/fv/0P/+#	.git-annex/jX/qM/+#	.git-annex/vW/jK/+#	.git-annex/vW/v1/+jtang@x00:~/sources $ git commit -a -m \"snap\"+[master 45f254a] snap+ 47 files changed, 64 insertions(+), 30 deletions(-)+jtang@x00:~/sources $ git status+# On branch master+# Your branch is ahead of 'origin/master' by 3 commits.+#+# Untracked files:+#   (use \"git add <file>...\" to include in what will be committed)+#+#	.git-annex/1G/X3/+#	.git-annex/3W/Xf/+#	.git-annex/9q/Wz/+#	.git-annex/Fq/4z/+#	.git-annex/Jk/zK/+#	.git-annex/Kx/w1/+#	.git-annex/VK/kv/+#	.git-annex/fv/0P/+#	.git-annex/jX/qM/+#	.git-annex/vW/jK/+#	.git-annex/vW/v1/+nothing added to commit but untracked files present (use \"git add\" to track)+jtang@x00:~/sources $ git pull+</pre>+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-28T15:25:18Z"+ content="""+So, there is evidence here of a circumstance caused by the [[other_bug|git-annex_has_issues_with_git_when_staging__47__commiting_logs]], as I suspected.++I don't think that manual `git commit -a` caused the problem. I suspect it was a subsequent `git add` that caused git to follow the wrong case paths and add the files in the wrong place. Ie, when you run \"git add .git-annex\", it recurses into `.git-annex/Gm/`, and adds files using that case, that were previously added from `.git-annex/GM/`.++For completeness, can you verify this repo's core.ignorecase setting?++---++I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a loop filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.++It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd be sad -- upgrading all my offline disks from v1 to v2 took me many days.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 4"+ date="2011-03-28T15:41:56Z"+ content="""+In my \"sources\" repo on x00, the current setting is this \"ignorecase = true\" it was the first repo that I created before I clone it elsewhere and pull my changes back, it is on a HFS+ partition which is case insensitive and it is replicated on a portable hdd with a bare repo on a exfat partition. I wonder if my portable disk has a partially borked repo :P+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment view
@@ -0,0 +1,37 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 5"+ date="2011-03-28T15:51:11Z"+ content="""+I also failed to mention, that in the case when i have stray log files after what has happened in comment 2, I get this left over after a commit when git is confused...+++<pre>+jtang@x00:~/sources $ git status+# On branch master+# Your branch is ahead of 'origin/master' by 1 commit.+#+# Changes not staged for commit:+#   (use \"git add <file>...\" to update what will be committed)+#   (use \"git checkout -- <file>...\" to discard changes in working directory)+#+#	modified:   .git-annex/1G/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log+#	modified:   .git-annex/3W/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log+#	modified:   .git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log+#	modified:   .git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log+#	modified:   .git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log+#	modified:   .git-annex/Jk/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log+#	modified:   .git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log+#	modified:   .git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log+#	modified:   .git-annex/VK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log+#	modified:   .git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log+#	modified:   .git-annex/vW/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log+#+no changes added to commit (use \"git add\" and/or \"git commit -a\")+</pre>+++Up until now I have just been updating the status of the staged files by hand and commiting it on my mac x00, this probably isn't helping. I'd rather not lose the tracking information.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 6"+ date="2011-03-31T18:02:42Z"+ content="""+Alright, I have created a case-insensative HFS+ filesystem here on my linux laptop. ++I have not been able to trick git into staging the same file with 2 different capitalizations yet.++It might be helpful if you can send me a copy of a git repository where 'git add -i' shows the same file staged with two capitalizations. Leaving out .git/annex of course. (joey@kitenet.net; a tarball would probably work)++It seems that `git add` only started properly working on case insensative filesystems quite recently. The commit in question is 5e738ae820ec53c45895b029baa3a1f63e654b1b, \"Support case folding for git add when core.ignorecase=true\", which was first released in git 1.7.4, January 30, 2011. If you don't yet have that version, that could explain the problem entirely. In about half an hour (dialup!) I will have downloaded an older git and will see if I can reproduce the problem with it.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment view
@@ -0,0 +1,68 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 7"+ date="2011-03-31T19:08:01Z"+ content="""+git 1.7.4 does not make things better. With it, if I add first \"X/foo\" and then \"x/bar\", it commits \"X/bar\".++That will *certianly* cause problems when interoperating with a repo clone on a case-sensative filesystem, since+git-annex there will not see the location log that git committed to the wrong case directory.++It's possible there is some interoperability problem when pulling from linux like you did, onto HFS+, too. I am not quite sure. Ah, I did find one.. if I clone the repo with \"X/foo\" in it to a case-sensative filesystem, and add a \"x/foo\" there,+and pull that commit back to HFS+, git says:++<pre>+ * branch            master     -> FETCH_HEAD+Updating 8754149..e3d4640+Fast-forward+ x/foo |    1 ++ 1 files changed, 1 insertions(+), 0 deletions(-)+ create mode 100644 x/foo+joey@gnu:/mnt/r4>ls+X/+joey@gnu:/mnt/r4>git st+# On branch master+# Changes not staged for commit:+#   (use \"git add <file>...\" to update what will be committed)+#   (use \"git checkout -- <file>...\" to discard changes in working directory++#	modified:   X/foo+</pre>++Aha -- that lets me reproduce your problem with the same file being staged twice with different capitalizations, too:++<pre>+joey@gnu:/mnt/r4>echo haaai >| x/foo+joey@gnu:/mnt/r4>git st+# On branch master+# Changes not staged for commit:+#   (use \"git add <file>...\" to update what will be committed)+#   (use \"git checkout -- <file>...\" to discard changes in working directory)+#+#	modified:   X/bar+#	modified:   X/foo+#	modified:   x/foo+#+joey@gnu:/mnt/r4>git commit -a+fatal: Will not add file alias 'X/Bar' ('x/Bar' already exists in index)+</pre>++And modified files that git refuses to commit, which entirely explains [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]].++<pre>+joey@gnu:/mnt/r4>git add X/foo+joey@gnu:/mnt/r4>git commit X/foo+# On branch master+# Changes not staged for commit:+#   (use \"git add <file>...\" to update what will be committed)+#   (use \"git checkout -- <file>...\" to discard changes in working directory)+#+#	modified:   X/bar+#	modified:   X/foo+#+no changes added to commit (use \"git add\" and/or \"git commit -a\")+</pre>++I think git is frankly, buggy. It seems I will need to work around this by stopping using mixed case hashing for location logs.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 8"+ date="2011-03-31T19:28:02Z"+ content="""+I've posted about this on the git mailing list. It's possible that these bugs, which can be shown to affect things other than just git-annex, will be fixed in git.++I will wait a while to see. But am considering making git-annex use all-lowercase hash dirs for the log files. Maybe it could first look for .git-annex/aaaa/bbbb/foo.log, but also look for, read, and merge in any info from +.git-annex/Aa/Bb/foo.log. And always write to the new style filenames. This would avoid confusing git with changes to+mixed-case files, and avoid another massive transition.+"""]]
+ doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 9"+ date="2011-03-31T21:32:10Z"+ content="""+I'm was running git 1.7.4.1 at the time when I came across it, I have just upgraded to 1.7.4.2. I've also just moved to using a loopback fs for the stuff i care about. Do you still want a repo that exhibits the problem (excluding the .git/annex data) ??? I'm also not sure if 1.7.4.2 has corrected the problem yet as I haven't done much with my repos since. I suspect just making all the .git-annex hashed directories seems to be lower case might be better in the long run. +"""]]
+ doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn view
@@ -0,0 +1,34 @@+After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit.++I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.++For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.++> What do you mean when you say that git "got wedged"? It hung somehow?+>+> If git-annex runs concurrently with another git command that locks+> the repository, its git add of log files can fail.+> +> Update: Also, of course, if you are running a "got annex get" or+> similar, and ctrl-c it after it has gotten some files, it can+> end up with unstaged or in some cases un-added log files that git-annex+> wrote -- since git-annex only stages log files in git on shutdown, and+> ctrl-c bypasses that.+> --[[Joey]] ++>> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.++>>> I've never seen git refuse to commit staged files. There would have to+>>> be some error message? --[[Joey]] ++>>>> there were no error messages at all++>>>>> Can I see a transcript? I'm having difficulty getting my head around+>>>>> what git is doing. Sounds like the files could just not be `git+>>>>> added` yet, but I get the impression from other things that you say+>>>>> that it's not so simple. --[[Joey]] ++This turns out to be a bug in git, and I have posted a bug report on the mailing list.+The git-annex behavior that causes this situation is being handled as+another bug, [[git-annex directory hashing problems on osx]].+So, closing this bug report. [[done]] --[[Joey]]
+ doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn view
@@ -0,0 +1,59 @@+I have a git remote in a git-annex-enabled repository. Here's what it looks like in .git/config:++<pre>+[remote "renaissance"]+        url = ssh://[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/home/paulproteus/Music/annex/+        fetch = +refs/heads/*:refs/remotes/renaissance/*+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64+</pre>++I wanted to "git annex get" some data. git-annex appears to pass incorrectly-formatted IPv6 addresses to rsync:++<pre>+get primary/emusiq/Arab Strap/Monday At The Hug And Pint/01-The Shy Retirer.mp3 (copying from renaissance...) +ssh: Could not resolve hostname [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]: Name or service not known+rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]+rsync error: unexplained error (code 255) at io.c(601) [Receiver=3.0.7]++  rsync failed -- run git annex again to resume file transfer+  Unable to access these remotes: renaissance+  Try making some of these repositories available:+  	2992752e-1a13-11e0-ba68-57d3c800da64+failed+</pre>++In this case, the square brackets should not be there.++I tried changing the .git/config syntax slightly, and got a different, also-incorrect behavior:++<pre>+[remote "renaissance"]+        url = [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/+        fetch = +refs/heads/*:refs/remotes/renaissance/*+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64+</pre>++<pre>+paulproteus@pathi:~/Music/annex$ git annex get+git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/+</pre>++(Note that both these .git/config entries work fine with "git fetch".)++-- Asheesh.++> Technically, this seems to be a bug in the haskell URI library; it honors+> the `[]` in parsing, but does not remove them when the URI is queried for+> the host part. ++<pre>+Prelude Network.URI> let (Just u) = parseURI "http://foo@[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/bar"+Prelude Network.URI> let (Just a) = uriAuthority u+Prelude Network.URI> uriRegName a+"[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]"+Prelude Network.URI> isIPv6address $ uriRegName a+False+</pre>++> I have filed a [bug upstream](http://trac.haskell.org/network/ticket/40), and put a workaround in git-annex. [[done]]+> --[[Joey]]
+ doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn view
@@ -0,0 +1,22 @@+Workflow:++    % git annex add+      # list new files+    % git commit -a -m "foo"+      # commit summary+    % git annex copy . --to remote --fast+      # all files listed with "ok"+    % git annex copy . --to remote+      # again, lists all files, _but the new ones are actually copied, this time_.++This happens no matter if I++    % git push++before copy or not.++PS: Arguably, a copy should push automagically.++> Whups, not supposed to be that fast! [[Fixed|done]], and+> you should run `git annex fsck --fast` on the repo you ran the+> copy in. --[[Joey]]
+ doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn view
@@ -0,0 +1,18 @@+I was testing out the fix/workaround for [[git-annex directory hashing problems on osx]] and I tried using the short forms of some of the commands i.e.++    git annex copy -f externalusb .++which gives me++    git-annex: user error (option `-f' is ambiguous; could be one of:+      -f         --force        allow actions that may lose annexed data+      -f REMOTE  --from=REMOTE  specify from where to transfer content+++I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options.++I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk.++> Yeah, -f needs to be from; -F was already --fast. I have made --force not+> have any short option abbreviation, I think it's entirely reasonable to+> avoid fat-fingering an option that can lose data. [[done]] --[[Joey]]
+ doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn view
@@ -0,0 +1,15 @@+What is says on the tin:++git annex fsck is a no-op in bare repos++See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html++> Thinking about this some more, it would be difficult to do anything+> when bad content is found, since it also cannot update the location log.+> +> So this may be another thing blocked by [[todo/branching]], assuming+> that is fixed in a way that makes `.git-annex` available to bare repos.+> --[[Joey]] ++>> Even if there is nothing it can _do_, knowing that the data is intact,+>> or not, is valuable in and as of itself. -- RichiH
+ doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-06-13T16:58:52Z"+ content="""+And, maybe, a way to start a fsck from remote? At least when the other side is a ssh or git annex shell, this would work.+"""]]
+ doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn view
@@ -0,0 +1,13 @@+when i want to++    git annex get file++on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command+ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port, it doesn't support host:port notation.+this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url.++temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning.++if i had more time, i would have learned haskell and provided a patch ;)++> Fixed in git! --[[Joey]] [[done]]
+ doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn view
@@ -0,0 +1,34 @@+For test.com//test, I get this:++    % git annex copy . --to test.com//test+    (getting UUID for test...) git-annex: there is no git remote named "test.com//test"++And my .git/config changes from++    [remote "test.com//test"]+    	url = richih@test.com:/test+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*++to++    [remote "test.com//test"]+    	url = richih@test.com:/test+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*+    	annex-uuid = xyz+    [remote "test"]+    	annex-uuid = xyz+++Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others.++> I've fixed this bug. [[done]]+> +> However, using "/" in a remote name seems likely to me to confuse +> git's own remote branch handling. Although I've never tried it.+> --[[Joey]] ++>> From what I can see, git handles / just fine, but would get upset about : which is why it's not allowed in a remote's name.+>> My naming scheme is host//path/to/annex. It sorts nicely and gives all important information left to right with the most specific parts at the beginning and end.+>> If you have any other ideas or scheme, I am all ears :)+>> Either way, thanks for fixing this so quickly.+>> -- RichiH
+ doc/bugs/git_annex_initremote_walks_.git-annex.mdwn view
@@ -0,0 +1,19 @@+a [[!taglink minor]] <!-- (a suggestion for introducing severity tags on bugs,+feel free to discard) --> issue: `git annex initremote` (in particular, adding+a key as described in [[encryption]] -- `git annex initremote my_remote+encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files+with lstat (tested using strace).++in a 50k key git-annex on a slow disk, this takes quite a while, while not+seeming necessary (it's just re-encrypting the shared secret, is it?).++could you verify the observed behavior?++> This is due to `git commit` being called. `git commit` exposes git's +> rather innefficient handling of the index; in order to make a commit+> it has to write a new index file, and it does this by scanning every+> file in the repository. I think that git generally needs its index+> file handleing overhauled, particularly to deal with repositories with+> large numbers of files. git-annex is seems to already be running+> `git commit` in its most efficient mode, by specifying exactly what file+> to commit. [[done]] --[[Joey]]
+ doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn view
@@ -0,0 +1,19 @@+`git annex migrate` leaves old, unlinked backend versions lying around. It+would be great if these were purged automatically somehow. ++> Yes, this is an issue mentioned in the+> [[walkthrough|walkthrough/migrating_data_to_a_new_backend]].+> +> Since multiple files can point to the same content, it could be that+> only one file has been migrated, and the content is still used. So+> the content either has to be retained, or an operation as expensive+> as `git annex unused` used to find if something else still uses it. +> +> Rather than adding such an+> expensive operation to each call to migrate, I focused on hard-linking+> the values for the old and new keys, so that the old keys don't actually+> use any additional resources (beyond an extra inode).+> +> This way a lot of migrations can be done, and only when you're done you+> can do the more expensive cleanup pass if you want to. --[[Joey]]+> [[done]]
+ doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn view
@@ -0,0 +1,11 @@+I have this line in the .gitignore file of one of my repos:+*log++So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files.++> This is avoided on the v3 branch, which does not store these files in the+> same branch as your repository.++Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it.++> Reasonable, [[done]] --[[Joey]]
+ doc/bugs/git_annex_unlock_is_not_atomic.mdwn view
@@ -0,0 +1,7 @@+Running a command like++git annex unlock myfile++is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex.++> [[fixed|done]] --[[Joey]]
+ doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn view
@@ -0,0 +1,15 @@+[[!meta title="`git annex unused` fails on empty repository"]]++The ``git annex unused`` command fails on a git-annex repository, if there are no objects yet:++    $ git annex unused+    unused  (checking for unused data...) +    git-annex: /tmp/annextest/other_annex/.git/annex/objects: getDirectoryContents: does not exist (No such file or directory) +    git-annex: 1 failed+    $++This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be).++This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it).++> [[fixed|done]] --[[Joey]]
+ doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn view
@@ -0,0 +1,37 @@+When I run `git annex unused` from my repository's root it shows everything ok:++    ~/annex$ git annex unused+    unused  (checking for unused data...) ok++But... When I run it from a subdirectory, it shows a lot:++    ~/annex/Software$ git annex unused+    unused  (checking for unused data...) +      Some annexed data is no longer pointed to by any files in the repository:+        NUMBER  KEY+        1       SHA1:########################################+    ...+        921     SHA1:########################################+      (To see where data was previously used, try: git log --stat -S'KEY')+      (To remove unwanted data: git-annex dropunused NUMBER)+      ok++Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files.++I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b++> I'm very sorry you lost data.+> +> But, git annex unused absolutely does not let the current directory+> influence what it does. It always scans the entire repo from the top.+> And I've tested it just now to make sure that in a subdirectory+> it does the same thing as at the top. +> +> There are only two ways this could happen that I can think of:+> +> 1. If "Software" were a separate git repository than "~/annex".+> 2. If gitignores or something made `git ls-files`+>    not list the files when ran in the subdir. This seems *possible*,+>    but I don't know how to construct such an ignore.+> +> --[[Joey]]
+ doc/bugs/git_rename_detection_on_file_move.mdwn view
@@ -0,0 +1,13 @@+It's unfortunate that git-annex sorta defeats git's rename detection.++When an annexed file is moved to a different directory (specifically, a+directory that is shallower or deeper than the old directory),+the symlink often has to change. And so git log cannot --follow back+through the rename history, since all it has to go on is that symlink,+which it effectively sees as a one line file containing the symlink target.++One way to fix this might be to do the `git annex fix` *after* the rename+is committed. This would mean that a commit would result in new staged+changes for another commit, which is perhaps startling behavior.++The other way to fix it is to stop using symlinks, see [[todo/smudge]].
+ doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment view
@@ -0,0 +1,26 @@+[[!comment format=mdwn+ username="http://christian.amsuess.com/chrysn"+ nickname="chrysn"+ subject="use mini-branches"+ date="2011-03-09T23:47:48Z"+ content="""+if you go for the two-commits version, small intermediate branches (or git-commit-tree) could be used to create a tree like this:+++    *   commit 106eef2+    |\  Merge: 436e46f 9395665+    | | +    | |     the main commit+    | |   +    | * commit 9395665+    |/  +    |       intermediate move+    |  +    * commit 436e46f+    | +    |     ...++while the first commit (436e46f) has a \"`/subdir/foo → ../.git-annex/where_foo_is`\", the intermediate (9395665) has \"`/subdir/deeper/foo → ../.git-annex/where_foo_is`\", and the inal commit (106eef2) has \"`/subdir/deeper/foo → ../../.git-annex/where_foo_is`\".++`--follow` uses the intermediate commit to find the history, but the intermediate commit would neither show up in `git log --first-parent` nor affect `git diff HEAD^..` & co. (there could still be confusion over `git show`, though).+"""]]
+ doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment view
@@ -0,0 +1,27 @@+[[!comment format=mdwn+ username="praet"+ ip="81.240.159.215"+ subject="Use variable symlinks, relative to the repo's root ?"+ date="2011-03-10T16:50:28Z"+ content="""+It all boils down to the fact that the path to a relative symlink's target is determined relative to the symlink itself.++Now, if we define the symlink's target relative to the git repo's root (eg. using the $GIT_DIR environment variable, which can be a relative or absolute path itself), this unfortunately results in an absolute symlink, which would -for obvious reasons- only be usable locally:++    user@host:~$ mkdir -p tmp/{.git/annex,somefolder}+    user@host:~$ export GIT_DIR=~/tmp+    user@host:~$ touch $GIT_DIR/.git/annex/realfile+    user@host:~$ ln -s $GIT_DIR/.git/annex/realfile $GIT_DIR/somefolder/file+    user@host:~$ ls -al $GIT_DIR/somefolder/+    total 12+    drwxr-x--- 2 user group 4096 2011-03-10 16:54 .+    drwxr-x--- 4 user group 4096 2011-03-10 16:53 ..+    lrwxrwxrwx 1 user group   33 2011-03-10 16:54 file -> /home/user/tmp/.git/annex/realfile+    user@host:~$++So, what we need is the ability to record the actual variable name (instead of it's value) in our symlinks.++It *is* possible, using [variable/variant symlinks](http://en.wikipedia.org/wiki/Symbolic_link#Variable_symbolic_links), yet I'm unsure as to whether or not this is available on Linux systems, and even if it is, it would introduce compatibility issues in multi-OS environments.++Thoughts on this?+"""]]
+ doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-16T03:03:19Z"+ content="""+Interesting, I had not heard of variable symlinks before. AFAIK linux does not have them.+"""]]
+ doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment view
@@ -0,0 +1,34 @@+[[!comment format=mdwn+ username="praet"+ ip="81.240.27.89"+ subject="Brainfart"+ date="2011-03-20T20:11:27Z"+ content="""+Haven't given these any serious thought (which will become apparent in a moment) but hoping they will give birth to some less retarded ideas:++---++### Bait'n'switch++- pre-commit: Replace all staged symlinks (when pointing to annexed files) with plaintext files containing the key of their respective annexed content, re-stage, and add their paths (relative to repo root) to .gitignore.+- post-commit: Replace the plaintext files with (git annex fix'ed) symlinks.++In doing so, the blobs to be committed can remain unaltered, irrespective of their related files' depth in the directory hierarchy.++To prevent git from reporting ALL annexed files as unstaged changes after running post-commit hook, their paths would need to be added to .gitignore.++This wouldn't cause any issues when adding files, very little when modifying files (would need some alterations to \"git annex unlock\"), BUT would make git totally oblivious to removals...++---++### Manifest-based (re)population+- Keep a manifest of all annexed files (key + relative path)+- DON'T track the symlinks (.gitignore)+- Populate/update the directory structure using a post-commit hook.++... thus circumventing the issue entirely, yet diffstats (et al.) would be rather uninformative.++---++***Wide open to suggestions, criticism, mocking laughter and finger-pointing :)***+"""]]
+ doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="praet"+ ip="81.242.56.203"+ subject="comment 5"+ date="2011-03-21T19:58:34Z"+ content="""+In the meantime, would it be acceptable to split the pre-commit hook+into two discrete parts?++This would allow to (if preferred) defer \"git annex fix\" until+post-commit while still keeping the safety net for unlocked files.+"""]]
+ doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn view
@@ -0,0 +1,24 @@+Current:++    % git annex status+    git-annex: unknown command++Better: ++    % git annex status+    git-annex: status: unknown command++Current:++    % git annex fsck+    [...]+    git-annex: 18 failed++Better:++    % git annex fsck+    [...]+    git-annex: fsck: 18 failed+++etc pp.
+ doc/bugs/ordering.mdwn view
@@ -0,0 +1,12 @@+One would expect "git annex get foo bar" to first retrieve foo, and then+bar. Actually though, it will operate on them in alphabetical order+(probably). This is annoying when you wanted to 1st list the most important+files to get. Maybe you'll run out of time before all can be gotten. The+workaround of course is to run "git annex get" twice.++This ordering comes from "git ls-files". git-annex passes it all the files+the user specified. This is a useful optimisation -- earlier it would+run "git ls-files" once per parameter, and so "git annex get *" could be+rather slow. But, it produces this ordering problem.++[[done]]
+ doc/bugs/problem_commit_normal_links.mdwn view
@@ -0,0 +1,59 @@+Dear All,++thank you for this wonderful tool!++I am having an issue when I try to commit a normal link++diokletian*194-> mkdir test++diokletian*195-> cd test++diokletian*196-> git init++Initialized empty Git repository in /home/henrus/test/.git/++diokletian*197-> git annex init new++init new [master (root-commit) 49f5f91] git-annex setup++ 1 files changed, 1 insertions(+), 0 deletions(-)++ create mode 100644 .gitattributes++[master 76496ff] git annex init++ 1 files changed, 1 insertions(+), 0 deletions(-)++ create mode 100644 .git-annex/uuid.log++ok++diokletian*198-> mkdir subdir++diokletian*199-> ln -s subdir link++diokletian*200-> git add link++diokletian*201-> git commit -m "ok"++[master f12f62d] ok++ 1 files changed, 1 insertions(+), 0 deletions(-)++ create mode 120000 link++diokletian*202-> ln -s subdir/ link2++diokletian*203-> git add link2++diokletian*204-> git commit -m "not ok"++git-annex: Prelude.head: empty list++The trailing slash seems to make a difference!++Best Regards,++Henrik++> Thanks for the bug report. This is fixed in 0.17. --[[Joey]]  [[!tag done]]
+ doc/bugs/problems_with_utf8_names.mdwn view
@@ -0,0 +1,104 @@+There are problems with displaying filenames in UTF8 encoding, as shown here:++    $ echo $LANG+    en_GB.UTF-8+    $ git init+    $ git annex init test+    [...]+    $ touch "Umlaut Ü.txt"+    $ git annex add Uml*+    add Umlaut Ã.txt ok+    (Recording state in git...)+    $ find -name U\* | hexdump -C+    00000000  2e 2f 55 6d 6c 61 75 74  20 c3 9c 2e 74 78 74 0a  |./Umlaut ...txt.|+    00000010+    $ git annex find | hexdump -C+    00000000  55 6d 6c 61 75 74 20 c3  83 c2 9c 2e 74 78 74 0a  |Umlaut .....txt.|+    00000010+    $++It looks like the common latin1-to-UTF8 encoding. Functionality other than otuput seems not to be affected.++> Yes, I believe that git-annex is reading filename data from git+> as a stream of char8s, and not decoding unicode in it into logical+> characters.+> Haskell then I guess, tries to unicode encode it when it's output to+> the console.+> This only seems to matter WRT its output to the console; the data+> does not get mangled internally and so it accesses the right files+> under the hood.+> +> I am too new to haskell to really have a handle on how to handle+> unicode and other encodings issues with it. In general, there are three+> valid approaches: --[[Joey]] +> +> 1. Convert all input data to unicode and be unicode clean end-to-end+>    internally. Problimatic here since filenames may not necessarily be+>    encoded in utf-8 (an archive could have historical filenames using+>    varying encodings), and you don't want which files are accessed to+>    depend on locale settings.+>    > I tried to do this by making parts of GitRepo call+>    > Codec.Binary.UTF8.String.decodeString when reading filenames from+>    > git. This seemed to break attempts to operate on the files,+>    > weirdly encoded strings were seen in syscalls in strace.+> 1. Keep input and internal data un-decoded, but decode it when+>    outputting a filename (assuming the filename is encoded using the+>    user's configured encoding), and allow haskell's output encoding to then+>    encode it according to the user's locale configuration.+>    > This is now [[implemented|done]]. I'm not very happy that I have to watch+>    > out for any place that a filename is output and call `filePathToString`+>    > on it, but there are really not too many such places in git-annex.+>    >+>    > Note that this only affects filenames apparently. +>    > (Names of files in the annex, and also some places where names+>    > of keys are displayed.) Utf-8 in the uuid.map file etc seems+>    > to be handled cleanly.+> 1. Avoid encodings entirely. Mostly what I'm doing now; probably+>    could find a way to disable encoding of console output. Then the raw+>    filename would be displayed, which should work ok. git-annex does+>    not really need to pull apart filenames; they are almost entirely+>    opaque blobs. I guess that the `--exclude` option is the exception+>    to that, but it is currently not unicode safe anyway. (Update: tried+>    `--exclude` again, seems it is unicode clean..)+>    One other possible+>    issue would be that this could cause problems if git-annex were+>    translated.+>    > On second thought, I switched to this. Any decoding of a filename+>    > is going to make someone unhappy; the previous approach broke+>    > non-utf8 filenames.++----++Simpler test case:++<pre>+import Codec.Binary.UTF8.String+import System.Environment++main = do+        args <- getArgs+        let file = decodeString $ head args+        putStrLn $ "file is: " ++ file+        putStr =<< readFile file+</pre>++If I pass this a filename like 'ü', it will fail, and notice+the bad encoding of the filename in the error message:++<pre>+$ echo hi > ü; runghc foo.hs ü+file is: ü+foo.hs: �: openFile: does not exist (No such file or directory)+</pre>++On the other hand, if I remove the decodeString, it prints the filename+wrong, while accessing it right:++<pre>+$ runghc foo.hs ü+file is: üa+hi+</pre>++The only way that seems to consistently work is to delay decoding the+filename to places where it's output. But then it's easy to miss some.
+ doc/bugs/scp_interrupt_to_background.mdwn view
@@ -0,0 +1,2 @@+When getting a file with scp, SIGINT is blocked, exposing the git+subcommand fork to background bug again. [[done]]
@@ -0,0 +1,54 @@+When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. -- RichiH++Optionally, editing the meta-data should change the times in all annexes.++> Thing is, git does not preserve file timestamps much at all. +> It's not uncommon for a `git checkout` to or `git update` to+> mess up timestamps. This is why things like metastore exist (and+> metastore should work ok with git annexed files too). Trying to +> make annexed file symlinks have better timestamp handling than regular+> files in git seems pointless. --[[Joey]]++> > Improving an area where git is (not yet?) good at still makes sense, imo. Photos and the like need absolute timestamps more than source code which is fine with relative timestamps (local builds & updates). Maintaining global timestamps for source code could even cause a lot of unwanted effects. As it is, this issue is the only, but a major, blocker for me before I can start adapting git-annex. As I have three different use cases for it, this is a shame. Unfortunately, I don't speak any Haskell so scratching my own itch isn't do-able (without major effort and not soon, at least). Is there a realistic chance that you will tackle this nonetheless or is this WONTFIX? -- RichiH++>>> Not quite WONTFIX. git-annex should at least, when adding new files,+>>> preserve their timestamp in the symlink it creates.+>>>+>>> Since it doesn't have anything to do with maintaining the symlinks+>>> during an update, or a clone, etc, maintaining the permissions of them +>>> is also out of scope, and it's best to just use metastore if you need+>>> it. Otherwise, git-annex would have to reimplement metastore, and is+>>> unlikely to do it better.++>>>> OK, thanks for the clarification. Would it be acceptable for you to put the timestamps into the metastore with vanilla git? If such an option existed, everyone would be able to benefit and not just me. -- RichiH++>>>>> I've now committed to git changes to make git-annex add make+>>>>> symlinks that reflect the original file's mtime. (It's not possible+>>>>> to set the ctime of a symlink; nor would you want to as messing with+>>>>> ctimes can break backup software ... and atime doesn't much matter.)+>>>>> +>>>>> So all you have to do is make the pre-commit hook call+>>>>> [metastore](http://david.hardeman.nu/software.php). The hook+>>>>> would look like this: ---[[Joey]]++	#!/bin/sh+	git annex pre-commit .+	metastore --save+	git add .metadata++>>>>>> Thanks a lot. Doing this in a new git-annex repo from the start should at least ensure local consistency and I assume I can simply add a post-pull hook to restore the mtimes on all all other repositories? -- RichiH++>>>>>>> This is even better:++    #!/bin/sh+    if ! type metastore >/dev/null; then echo "$0: metastore is not installed; exiting"; exit 1; fi+    git annex pre-commit .+    metastore --save+    git add .metadata++>>>>>>> -- RichiH++>>>>>>>> After getting to actually play with this from different machines with a bare git as central instance for several distributed repos, the metastore trick does not work. The .metadata is causing merge conflicts for every pull. I removed the "done" tag from this issue. -- RichiH++>>>>>>>>> softbox sounds _really_ nice. File systems need to preserve mtimes. Oviously, it would be nice if git-annex exposed this to the upper layer instead of relying on this FUSE implementation, or the next, or the other totally cool thing around the corner to implement it again and again. +>>>>>>>>> I talked to the author of metastore; he is aware that the format is merge-unfriendly but never needed merges for himself. He is aware that this is not ideal for something like git. He does not have the time to implement a text storage instead of binary and I lack the skills to do it. If metastore is used, all it would need to do is introduce a new version of the store (it's versioned, apparently) and save metadata in text, one file per line. xattr would need to be ASCII-armoured, the rest could be plain text. I still think storing this directly in git-annex would make the most sense. Introducing a metadata storage file per storage object in .git/annex and using the object file's name as index is impossible because several softlinks might point to one object so it would need to be done per-softlink :/ -- RichiH
+ doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn view
@@ -0,0 +1,50 @@+Make test fails when git doesn't know what identity to give to commits++<pre>++Testing 1:blackbox:0:git-annex init+Cases: 30  Tried: 7  Errors: 0  Failures: 0+*** Please tell me who you are.++Run++  git config --global user.email "you@example.com"+  git config --global user.name "Your Name"++to set your account's default identity.+Omit --global to set the identity only in this repository.++fatal: empty ident  <jtang@lenny.localdomain> not allowed+### Failure in: 1:blackbox:0:git-annex init+init failed+Testing 1:blackbox:1:git-annex add:0+Cases: 30  Tried: 8  Errors: 0  Failures: 1+*** Please tell me who you are.+</pre>++I guess most users testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.++Update: I pulled the master on my rhel5 test machine and moved my .gitconfig out of the way, the tests passes and continues but I still get a "warning message" from git. ++<pre>+Testing 1:blackbox:3:git-annex unannex:1:with content                         +Cases: 30  Tried: 12  Errors: 0  Failures: 0[master fce0cde] content removed from git annex+ Committer: Jimmy Tang <jtang@removed.removed.tcd.ie>+Your name and email address were configured automatically based+on your username and hostname. Please check that they are accurate.+You can suppress this message by setting them explicitly:++    git config --global user.name "Your Name"+    git config --global user.email you@example.com++After doing this, you may fix the identity used for this commit with:++    git commit --amend --reset-author++ 2 files changed, 1 insertions(+), 2 deletions(-)+ delete mode 120000 foo+</pre>++I guess it also depends a bit on how git figures out who it is is committing and how the machine in question is configured with hostnames and domain names.++> Fixed that. [[done]] --[[Joey]]
+ doc/bugs/tmp_file_handling.mdwn view
@@ -0,0 +1,13 @@+git-annex deletes all tmp files on shutdown, if everything succeeded.+This presents 2 problems:++1. If git-annex is rsyncing something and another one is run, it will+   delete the running instance's tmp files.+2. If a long-running rsync transfer is interrupted partway through, the+   tmp file was expensive to obtain, and one needs to avoid running+   git-annex to do anything else until that transfer can be resumed and+   finished.++--[[Joey]] ++[[done]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn view
@@ -0,0 +1,19 @@+It seems that commit bc5c54c987f548505a3877e8a0e460abe0b2a081 introduced some linux specific things...++<pre>+hsc2hs Touch.hsc+Touch.hsc: In function ‘main’:+Touch.hsc:46: error: ‘UTIME_OMIT’ undeclared (first use in this function)+Touch.hsc:46: error: (Each undeclared identifier is reported only once+Touch.hsc:46: error: for each function it appears in.)+Touch.hsc:48: error: ‘UTIME_NOW’ undeclared (first use in this function)+Touch.hsc:67: error: ‘AT_FDCWD’ undeclared (first use in this function)+Touch.hsc:68: error: ‘AT_SYMLINK_NOFOLLOW’ undeclared (first use in this function)+compiling Touch_hsc_make.c failed+command was: /usr/bin/gcc -c -m32 -I/Library/Frameworks/GHC.framework/Versions/612/usr/lib/ghc-6.12.3/include/ Touch_hsc_make.c -o Touch_hsc_make.o+make: *** [Touch.hs] Error 1+</pre>++I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex.++[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-16T16:07:26Z"+ content="""+Hmm.. is utimensat available at all?++I've committed an update that may convince at least some compilers to expose this newer POSIX stuff. I don't know if it will help, please let me know.+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-03-16T16:49:18Z"+ content="""+Just pulled the changes, it still fails to build. utimensat doesn't seem to exist on OSX 10.6.6.+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-16T17:46:40Z"+ content="""+Alright, I've added #idefs and the symlink timestamp mirroring feature will be unavailable on OSX until I get a version that works there.+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 4"+ date="2011-03-16T20:32:01Z"+ content="""+Just tried it out on my mac and it's working again. I guess this issue could be closed for now.+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 5"+ date="2011-03-20T18:12:59Z"+ content="""+I'm leaving this bug open because this feature, however minor is not available on OSX and BSD. ++I have added a partial implementation using lutimes(3), which should be available on the BSDs. However, it's ifdefed out due to a casting problem: The TimeSpec uses a CTime, while lutimes uses a CLong. These data types may be internally the same on some or all platforms, so if you want this feature you can try changing the \"ifdef 0\" in Touch.hsc to 1 and try it, see if \"git annex add\" mirrors file modification time in created symlinks, and let me know.+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment view
@@ -0,0 +1,42 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 6"+ date="2011-03-20T20:48:41Z"+ content="""+ok, pulling the latest master and building on OSX now does this...++<pre>+ghc -O2 -Wall -ignore-package monads-fd --make git-annex+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )++Touch.hsc:24:0:+    The type signature for `touchBoth' lacks an accompanying binding++Touch.hsc:27:26: Not in scope: `touchBoth'+make: *** [git-annex] Error 1+</pre>++changing the #if 0 to 1 gives this...++<pre>+ghc -O2 -Wall -ignore-package monads-fd --make git-annex+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )++Touch.hsc:95:43:+    Couldn't match expected type `CLong' against inferred type `CTime'+    In the second argument of `(\ hsc_ptr+                                    -> pokeByteOff hsc_ptr 0)', namely+        `(sec :: CLong)'+    In a stmt of a 'do' expression:+        (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong)+    In the expression:+        do { (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong);+             (\ hsc_ptr -> pokeByteOff hsc_ptr 4) ptr (0 :: CLong) }+make: *** [git-annex] Error 1+</pre>+++it seems that commit 6634b6a6b84a924f6f6059b5bea61f449d056eee has broken support for OSX.++"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 7"+ date="2011-03-20T22:06:25Z"+ content="""+Fixed that, and removed the impossible cast so it can be built with #if 1+"""]]
+ doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 8"+ date="2011-03-21T08:52:18Z"+ content="""+Just tried building both of the code paths, and they seem to build and somewhat function on OSX. I have yet to confirm the functionality is working correctly, but so far it's looking good. (I somewhat care less about the utimes/mtimes of my files since I care more about the content :) )+"""]]
+ doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn view
@@ -0,0 +1,14 @@+git's index broke and I was unable to restore it. While this is not git-annex' problem, it should still be possible to get my data in an un-annexed state.++    % git status+    fatal: index file smaller than expected+    % git annex unannex foo+    fatal: index file smaller than expected+    % git annex uninit+    fatal: index file smaller than expected+    uninit  +      pre-commit hook (/path/to/git-annex/.git/hooks/pre-commit) contents modified; not deleting. Edit it to remove call to git annex.+    ok+    %++Ttbomk, the softlinks and objects are enough to un-annex the files; side-stepping git's index if necessary.
+ doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-03T01:40:50Z"+ content="""+They rely on git-ls-files to get a list of files that are checked into git, in order to tell what to unannex.+"""]]
+ doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-03T08:55:18Z"+ content="""+Given that the softlinks contain all needed information (if the object exists, locally), an emergency way to get files \"out\" of git-annex would be nice. I am aware that one can script it, but a canonical way is always better, especially when things go south.+"""]]
+ doc/bugs/unannex_vs_unlock_hook_confusion.mdwn view
@@ -0,0 +1,15 @@+See [[forum/unannex_alternatives]] for problem description.++If an unannex is followed by a "git add; git commit", git-annex's hook thinks+that you have used git annex unlock on the file and are+now committing a changed version, and the right thing to do there is to add the+new content to the annex and update the symlink accordingly.++Can we tell the difference between an unannexed file that has yet to be committed+and has been re-added as a normal file, vs an unlocked file? --[[Joey||++> Hmm, not really. An unannexed file's content will have been dropped from+> the backend, but that's about the only difference. Perhaps unannex should+> just commit the removal of the file itself? --[[Joey]]++> [[done]], staged changes committed at end.
+ doc/bugs/unhappy_without_UTF8_locale.mdwn view
@@ -0,0 +1,41 @@+Try unsetting LANG and passing git-annex unicode filenames.++	joey@gnu:~/tmp/aa>git annex add ./Üa+	add add add add git-annex: <stdout>: commitAndReleaseBuffer: invalid+	argument (Invalid or incomplete multibyte or wide character)++> Interestingly, I can get the same crash in the de_DE.UTF-8 locale+> with certian input filenames, while in en_US.UTF-8, it's ok.+> The workaround below avoided the problem in de_DE.UTF-8. --[[Joey]]++> Put in the utf-8 forcing workaround for now. [[done]] --[[Joey]] ++## underlying haskell problem and workaround++The same problem can be seen with a simple haskell program:++	import System.Environment+	import Codec.Binary.UTF8.String+	main = do+	        args <- getArgs+	        putStrLn $ decodeString $ args !! 0++	joey@gnu:~/src/git-annex>LANG= runghc ~/foo.hs Ü+	foo.hs: <stdout>: hPutChar: invalid argument (Invalid or incomplete multibyte or wide character)++(The call to `decodeString` is necessary to make the input+unicode string be displayed properly in a utf8 locale, but+does not contribute to this problem.)++I guess that haskell is setting the IO encoding to latin1, which+is [documented](http://haskell.org/ghc/docs/latest/html/libraries/base/System-IO.html#v:latin1)+to error out on characters > 255. ++So this program doesn't have the problem -- but may output garbage+on non-utf-8 capable terminals:++	import System.IO+	main = do+ 		hSetEncoding stdout utf8+	        args <- getArgs+	        putStrLn $ decodeString $ args !! 0
+ doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn view
@@ -0,0 +1,16 @@+I upgraded another one of my git-annex clones.  The upgrade worked fine (i.e.+according to the manual) on two other clones before, but this time something is+different.++After 'git pull' and 'git annex upgrade', which took a long time and seemed to+have succeeded, there are no staged changes in git.  Instead there are lots of+untracked directories in .git-annex.  Aside from that, nothing seems to be+wrong.++At the time I had git-annex version 0.20110329 and I've been using the SHA1+backend since version 1.++> Yes, I agree with Jimmy, it's the same bug. So I'll be closing this one.+> Please keep us informed how the workaround committed to git-annex+> yesterday for the case insensativity issue works out. [[dup|done]]+> --[[Joey]]
+ doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-03T02:26:20Z"+ content="""+I'm not sure how this happened, as far as I can see, and based on my testing, `git annex upgrade` does stage the location log files. OTOH, I vaguely rememeber needing to stage some of them when I was doing my own upgrades, but that was a while ago, and I don't remember the details.++Your upgrade seems to have gone ok from the file lists you sent, so you can just: `git add .git-annex; git commit`+"""]]
+ doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="gernot"+ ip="213.168.117.192"+ subject="comment 2"+ date="2011-04-03T15:35:52Z"+ content="""+'git add .git-annex' didn't do anything.  That's when I noticed that this+repository is on a case-insensitive HFS+ file system.++So, if I get this right it's not a new bug, but similar to this situation:+[[git-annex_directory_hashing_problems_on_osx]]++Assuming that it was the file system's fault, I went ahead and upgraded yet+another clone.  That one (on an ext3 file system) had neither staged changes+nor left-over untracked files.  Everything seems to just have fallen right into+place.  Is that possible or still weird?++"""]]
+ doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 3"+ date="2011-04-03T16:05:39Z"+ content="""+Yes you seem to have come across the same bug that I had initially reported :P+"""]]
+ doc/bugs/weird_local_clone_confuses.mdwn view
@@ -0,0 +1,20 @@+See+<http://www.git.code-experiments.com/blog/2011/01/manage-large-files-with-git-annex-by-joey-hess.html>++If a local repo is cloned with "git clone orig/.git new", then git-annex in+new cannot see origin. ++the .git/config has "url=/.../orig/.git". Apparently git is ok with that+weird construction; probably it treats it as a bare git repo. But git-annex+just sees a directory w/o a .git subdir, and gives up.++---++Just tested, and the new support for bare repositories didn't solve this.+(Because config.bare is not set.)++I think this is not something git-annex should go out of its way to+support. [[done]]+--[[Joey]] ++Later.. Fixed this after all. --[[Joey]]
+ doc/cheatsheet.mdwn view
@@ -0,0 +1,15 @@+A suppliment to the [[walkthrough]].++[[!toc]]++[[!inline feeds=no show=0 template=walkthrough pagenames="""+	walkthrough/using_Amazon_S3+	walkthrough/using_bup+	walkthrough/using_the_web+	walkthrough/using_the_SHA1_backend+	walkthrough/migrating_data_to_a_new_backend+	walkthrough/untrusted_repositories+	walkthrough/what_to_do_when_you_lose_a_repository+	walkthrough/recover_data_from_lost+found+	walkthrough/Internet_Archive_via_S3+"""]]
+ doc/comments.mdwn view
@@ -0,0 +1,9 @@+[[!sidebar content="""+[[!inline pages="comment_pending(*)" feedfile=pendingmoderation+description="comments pending moderation" show=-1]]+Comments in the [[!commentmoderation desc="moderation queue"]]:+[[!pagecount pages="comment_pending(*)"]]+"""]]++Recent comments posted to this site:+[[!inline pages="comment(*)" template="comment"]]
+ doc/contact.mdwn view
@@ -0,0 +1,10 @@+Joey Hess <joey@kitenet.net> is the author of git-annex. If you need to+talk about something privatly, email me.++The [[forum]] is the best place to discuss git-annex.++The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home)+is a good mailing list for users who want to use git-annex in the context+of managing their large personal files.++For realtime chat, use the `#vcs-home` channel on irc.oftc.net.
+ doc/copies.mdwn view
@@ -0,0 +1,38 @@+The WORM and SHA1 key-value [[backends]] store data inside+your git repository's `.git` directory, not in some external data store.++It's important that data not get lost by an ill-considered `git annex drop`+command.  So, then using those backends, git-annex can be configured to try+to keep N copies of a file's content available across all repositories. +(Although [[untrusted_repositories|trust]] don't count toward this total.)++By default, N is 1; it is configured by annex.numcopies. This default+can be overridden on a per-file-type basis by the annex.numcopies+setting in `.gitattributes` files. The --numcopies switch allows+temporarily using a different value.++`git annex drop` attempts to check with other git remotes, to check that N+copies of the file exist. If enough repositories cannot be verified to have+it, it will retain the file content to avoid data loss. Note that+[[trusted_repositories|trust]] are not explicitly checked.++For example, consider three repositories: Server, Laptop, and USB. Both Server+and USB have a copy of a file, and N=1. If on Laptop, you `git annex get+$file`, this will transfer it from either Server or USB (depending on which+is available), and there are now 3 copies of the file.++Suppose you want to free up space on Laptop again, and you `git annex drop` the file+there. If USB is connected, or Server can be contacted, git-annex can check+that it still has a copy of the file, and the content is removed from+Laptop. But if USB is currently disconnected, and Server also cannot be+contacted, it can't verify that it is safe to drop the file, and will+refuse to do so.++With N=2, in order to drop the file content from Laptop, it would need access+to both USB and Server.++Note that different repositories can be configured with different values of+N. So just because Laptop has N=2, this does not prevent the number of+copies falling to 1, when USB and Server have N=1. To avoid this,+configure it in `.gitattributes`, which is shared between repositories+using git.
+ doc/design.mdwn view
@@ -0,0 +1,4 @@+git-annex's high-level design is mostly inherent in the data that it+stores in git, and alongside git. See [[internals]] for details.++See [[encryption]] for design of encryption elements.
+ doc/design/encryption.mdwn view
@@ -0,0 +1,117 @@+This was the design doc for [[/encryption]] and is preserved for+the curious. For an example of using git-annex with an encrypted S3 remote,+see [[walkthrough/using_Amazon_S3]].++[[!toc]]++## encryption backends++It makes sense to support multiple encryption backends. So, there+should be a way to tell what backend is responsible for a given filename+in an encrypted remote. (And since special remotes can also store files+unencrypted, differentiate from those as well.)++The rest of this page will describe a single encryption backend using GPG.+Probably only one will be needed, but who knows? Maybe that backend will+turn out badly designed, or some other encryptor needed. Designing+with more than one encryption backend in mind helps future-proofing.++## encryption key management++[[!template id=note text="""+The basis of this scheme was originally developed by Lars Wirzenius et al+[for Obnam](http://braawi.org/obnam/encryption/).+"""]]++Data is encrypted by gpg, using a symmetric cipher.+The cipher is itself checked into your git repository, encrypted using one or+more gpg public keys. This scheme allows new gpg private keys to be given+access to content that has already been stored in the remote.++Different encrypted remotes need to be able to each use different ciphers.+Allowing multiple ciphers to be used within a single remote would add a lot+of complexity, so is not planned to be supported.+Instead, if you want a new cipher, create a new S3 bucket, or whatever.+There does not seem to be much benefit to using the same cipher for+two different encrypted remotes.++So, the encrypted cipher could just be stored with the rest of a remote's+configuration in `remotes.log` (see [[internals]]). When `git+annex intiremote` makes a remote, it can generate a random symmetric+cipher, and encrypt it with the specified gpg key. To allow another gpg+public key access, update the encrypted cipher to be encrypted to both gpg+keys.++## filename enumeration++If the names of files are encrypted or securely hashed, or whatever is+chosen, this makes it harder for git-annex (let alone untrusted third parties!)+to get a list of the files that are stored on a given enrypted remote.+But, does git-annex really ever need to do such an enumeration?++Apparently not. `git annex unused --from remote` can now check for+unused data that is stored on a remote, and it does so based only on+location log data for the remote. This assumes that the location log is+kept accurately.++What about `git annex fsck --from remote`? Such a command should be able to,+for each file in the repository, contact the encrypted remote to check+if it has the file. This can be done without enumeration, although it will+mean running gpg once per file fscked, to get the encrypted filename.++So, the files stored in the remote should be encrypted. But, it needs+to be a repeatable encryption, so they cannot just be gpg encrypted,+that would yeild a new name each time. Instead, HMAC is used. Any hash+could be used with HMAC; currently SHA1 is used.++It was suggested that it might not be wise to use the same cipher for both+gpg and HMAC. Being paranoid, it's best not to tie the security of one+to the security of the other. So, the encrypted cipher described above is+actually split in two; half is used for HMAC, and half for gpg.++----++Does the HMAC cipher need to be gpg encrypted? Imagine if it were+stored in plainext in the git repository. Anyone who can access+the git repository already knows the actual filenames, and typically also+the content hashes of annexed content. Having access to the HMAC cipher+could perhaps be said to only let them verify that data they already+know.++While this seems a pretty persuasive argument, I'm not 100% convinced, and+anyway, most times that the HMAC cipher is needed, the gpg cipher is also+needed. Keeping the HMAC cipher encrypted does slow down two things:+dropping content from encrypted remotes, and checking if encrypted remotes+really have content. If it's later determined to be safe to not encrypt the+HMAC cipher, the current design allows changing that, even for existing+remotes.++## other use of the symmetric cipher++The symmetric cipher can be used to encrypt other content than the content+sent to the remote. In particular, it may make sense to encrypt whatever+access keys are used by the special remote with the cipher, and store that+in remotes.log. This way anyone whose gpg key has been given access to +the cipher can get access to whatever other credentials are needed to+use the special remote.++## risks++A risk of this scheme is that, once the symmetric cipher has been obtained, it+allows full access to all the encrypted content. This scheme does not allow+revoking a given gpg key access to the cipher, since anyone with such a key+could have already decrypted the cipher and stored a copy. ++If git-annex stores the decrypted symmetric cipher in memory, then there+is a risk that it could be intercepted from there by an attacker. Gpg+amelorates these type of risks by using locked memory. For git-annex, note+that an attacker with local machine access can tell at least all the+filenames and metadata of files stored in the encrypted remote anyway,+and can access whatever content is stored locally.++This design does not support obfuscating the size of files by chunking+them, as that would have added a lot of complexity, for dubious benefits.+If the untrusted party running the encrypted remote wants to know file sizes,+they could correlate chunks that are accessed together. Encrypting data+changes the original file size enough to avoid it being used as a direct+fingerprint at least.
+ doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-04-03T20:03:14Z"+ content="""+New encryption keys could be used for different directories/files/patterns/times/whatever. One could then encrypt this new key for the public keys of other people/machines and push them out along with the actual data. This would allow some level of access restriction or future revocation. git-annex would need to keep track of which files can be decrypted with which keys. I am undecided if that information needs to be encrypted or not.++Encrypted object files should be checksummed in encrypted form so that it's possible to verify integrity without knowing any keys. Same goes for encrypted keys, etc.++Chunking files in this context seems like needless overkill. This might make sense to store a DVD image on CDs or similar, at some point. But not for encryption, imo. Coming up with sane chunk sizes for all use cases is literally impossible and as you pointed out, correlation by the remote admin is trivial.+"""]]
+ doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-04-05T18:41:49Z"+ content="""+I see no use case for verifying encrypted object files w/o access to the encryption key. And possible use cases for not allowing anyone to verify your data.++If there are to be multiple encryption keys usable within a single encrypted remote, than they would need to be given some kind of name (a since symmetric key is used, there is no pubkey to provide a name), and the name encoded in the files stored in the remote. While certainly doable I'm not sold that adding a layer of indirection is worthwhile. It only seems it would be worthwhile if setting up a new encrypted remote was expensive to do. Perhaps that could be the case for some type of remote other than S3 buckets.+"""]]
+ doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 3"+ date="2011-04-05T23:24:17Z"+ content="""+Assuming you're storing your encrypted annex with me and I with you, our regular cron jobs to verify all data will catch corruption in each other's annexes.++Checksums of the encrypted objects could be optional, mitigating any potential attack scenarios.++It's not only about the cost of setting up new remotes. It would also be a way to keep data in one annex while making it accessible only in a subset of them. For example, I might need some private letters at work, but I don't want my work machine to be able to access them all.+"""]]
+ doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-04-07T19:59:30Z"+ content="""+@Richard the easy way to deal with that scenario is to set up a remote that work can access, and only put in it files work should be able to see. Needing to specify which key a file should be encrypted to when putting it in a remote that supported multiple keys would add another level of complexity which that avoids.++Of course, the right approach is probably to have a separate repository for work. If you don't trust it with seeing file contents, you probably also don't trust it with the contents of your git repository.+"""]]
+ doc/distributed_version_control.mdwn view
@@ -0,0 +1,13 @@+In git, there can be multiple clones of a repository, each clone can +be independently modified, and clones can push or pull changes to+one-another to get back in sync.++git-annex preserves that fundamental distributed nature of git, while+dropping the requirement that, once in sync, each clone contains all the data+that was committed to each other clone. Instead of storing the content+of a file in the repository, git-annex stores a pointer to the content.++Each git-annex repository is responsible for storing some of the content,+and can copy it to or from other repositories. [[Location_tracking]]+information is committed to git, to let repositories inform other+repositories what file contents they have available.
+ doc/download.mdwn view
@@ -0,0 +1,14 @@+The main git repository for git-annex is `git://git-annex.branchable.com/`++(You can push changes to this wiki from that anonymous git checkout.)++Other mirrors of the git repository:++* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]+* [at github](https://github.com/joeyh/git-annex)++To download a tarball of a particular release, use an url like+<http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/0.20110522>++Some operating systems include git-annex in easily prepackaged form and+others need some manual work. See [[install]] for details.
+ doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://peter-simons.myopenid.com/"+ ip="84.189.2.244"+ subject="Please provide stable tarballs or zipfiles"+ date="2011-03-22T13:06:58Z"+ content="""+I'm trying to package git annex for ArchLinux and NixOS. That task would be a *lot* easier, if there were proper release archives available for download. The Gitweb site offers to create snapshot tarballs on the fly, but those tarballs have a different SHA hash every time they're generated, so they cannot be used for the purposes of a distribution. A simple solution for this problem would be to enable snapshots in zip format (because zip files look the same every time they're generated).+"""]]
+ doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-03-22T14:01:37Z"+ content="""+maybe snag tarballs from <http://packages.debian.org/experimental/git-annex> ? +"""]]
+ doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-22T18:09:21Z"+ content="""+The tarballs produced by gitweb are actually stable. They are wrapped in a gz file with a varying timestamp however. It might be nice if gitweb passed --no-name to gzip to avoid that inconsistency.++git-annex also has a [pristine-tar](http://kitenet.net/~joey/code/pristine-tar/) branch in git that can be used to recreate the tarballs I upload to Debian.+"""]]
+ doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment view
@@ -0,0 +1,9 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"+ nickname="Fraser"+ subject="gitweb supplies --no-name as of 1.7.5.1"+ date="2011-05-19T08:19:02Z"+ content="""+git v1.7.5.1 fixes the gitweb gzip issue.  If the git instance is updated we+can have stable distributions (and I can finally write a FreeBSD port ^_^)+"""]]
+ doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 5"+ date="2011-05-19T16:10:35Z"+ content="""+Hmm, I've upgraded to that version, but I see nothing in its changelog, commit log, code, or runtime behavior to indicate that it's producing stable gzip output.+"""]]
+ doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"+ nickname="Fraser"+ subject="comment 6"+ date="2011-05-22T23:02:39Z"+ content="""+Whups, the fix landed in git's `maint' branch just after 1.7.5 but 1.7.5.1 was+tagged on a different branch.++Will look closer in future, and let you know when it's really released.+"""]]
+ doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"+ nickname="Fraser"+ subject="comment 7"+ date="2011-05-27T01:27:37Z"+ content="""+v1.7.5.3 has it.+"""]]
+ doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 8"+ date="2011-05-28T16:04:51Z"+ content="""+And that is now installed on kitenet.net and verified to work.+"""]]
+ doc/encryption.mdwn view
@@ -0,0 +1,35 @@+git-annex mostly does not use encryption. Anyone with access to a git+repository can see all the filenames in it, its history, and can access+any annexed file contents.++Encryption is needed when using [[special_remotes]] like Amazon S3, where+file content is sent to an untrusted party who does not have access to the+git repository.++Such an encrypted remote uses strong GPG encryption on the contents of files,+as well as HMAC hashing of the filenames. The size of the encrypted files,+and access patterns of the data, should be the only clues to what is+stored in such a remote.++You should decide whether to use encryption with a special remote before+any data is stored in it. So, `git annex initremote` requires you+to specify "encryption=none" when first setting up a remote in order+to disable encryption.++If you want to use encryption, run `git annex initremote` with+"encryption=USERID". The value will be passed to `gpg` to find encryption keys.+Typically, you will say "encryption=2512E3C7" to use a specific gpg key.+Or, you might say "encryption=joey@kitenet.net" to search for matching keys.++The [[encryption_design|design/encryption]] allows additional encryption keys+to be added on to a special remote later. Once a key is added, it is able+to access content that has already been stored in the special remote.+To add a new key, just run `git annex initremote` again, specifying the+new encryption key:++	git annex initremote myremote encryption=788A3F4C++Note that once a key has been given access to a remote, it's not+possible to revoke that access, short of deleting the remote. See+[[encryption_design|design/encryption]] for other security risks+associated with encryption.
+ doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="zooko"+ ip="75.220.153.232"+ subject="Tahoe-LAFS comes with encryption"+ date="2011-05-18T04:32:14Z"+ content="""+The Tahoe-LAFS special remote automatically encrypts and adds cryptography integrity checks/digital signatures. For that special remote you should not use the git-annex encryption scheme.++Tahoe-LAFS encryption generates a new independent key for each file. This means that you can share access to one of the files without thereby sharing access to all of them, and it means that individual files can be deduplicated among multiple users.+"""]]
+ doc/feeds.mdwn view
@@ -0,0 +1,3 @@+Aggregating git-annex mentions from elsewhere on the net..++* [[!aggregate expirecount=25 name="identica" feedurl="http://identi.ca/api/statusnet/tags/timeline/gitannex.rss" url="http://identi.ca/tag/gitannex"]]
+ doc/forum.mdwn view
@@ -0,0 +1,3 @@+This is a place to discuss using git-annex. If you need help, advice, or anything, post about it here.++[[!inline pages="forum/* and !*/Discussion" archive=yes rootpage=forum postformtext="Add a new thread titled:"]]
+ doc/forum/Behaviour_of_fsck.mdwn view
@@ -0,0 +1,13 @@+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or so blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.++Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.++    git annex fsck++should show only 'errors' and maybe a simple indicator showing the status (show a spinner or dots?) and when ++    git annex fsck PATH/FILE ++it should have the current behaviour? ++Right now the current fsck behaviour might get annoying for anyone who would want to run fsck with repos with lots of big files.
+ doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-24T17:45:08Z"+ content="""+I tend to agree that the default output of fsck is not quite right. I often use git annex fsck -q. A progress spinner display is a good idea.+"""]]
+ doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment view
@@ -0,0 +1,19 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-03-26T10:57:41Z"+ content="""+After some thought, perhaps the default fsck output should be at least machine readable and copy and pasteable i.e.++<pre>+$ git annex fsck+Files with errors++    file1+    file2++</pre>++so I can then copy the list of borked files and then just paste it into a for loop in my shell to recover the files. it's just an idea.+"""]]
+ doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment view
@@ -0,0 +1,19 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 3"+ date="2011-03-28T01:16:21Z"+ content="""+Another nice thing would be a summary of _what_ is wrong. I.e.++    % git fsck+    [...]+    git-annex: 100 total failed+      50 checksum failed+      50 not enough copies exit++And the same/similar for all other failure modes.+++-- RichiH+"""]]
+ doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-03-25T11:23:04Z"+ content="""+FWIW, I wanted to suggest exactly the same thing.+"""]]
+ doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn view
@@ -0,0 +1,1 @@+Moved to [[bugs|bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__]] --[[Joey]]
+ doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn view
@@ -0,0 +1,9 @@+Consider the following two use cases:++* I have a git-annex repo on a portable medium and carry it around between several machines. I use it on a non-important system with the most current git-annex installed, automagic upgrade happens. I am now forced to upgrade git-annex on all other machines. Bonus points if this happens in the background and I don't even notice it until it's too late.++* My system crashes and I use a rescue CD to access local data, including git-annex. The rescue CD includes a newer version of git-annex and once my system is restored, I am forced to upgrade git-annex locally.++My suggestion would be not to upgrade automatically, but to either ask the user if this is OK or to error out and request that they run git annex update by hand.++Optionally, this could be done via a local config variable which should default to error or ask, not upgrade.
+ doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-18T00:38:51Z"+ content="""+These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.++(Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)++Update: Upgrades from v1 to v2 will no longer be handled automatically+now.+"""]]
+ doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn view
@@ -0,0 +1,5 @@+The instructions for building git-annex on [[install/Debian]] stable don't seem to be valid anymore.++1. `dpkg-checkbuilddeps` is looking for the wrong packages, e.g. libghc-missingh-dev instead of libghc6-missingh-dev.++2. Not all dependencies are available in the Squeeze repositories anymore (at least not Crypto and hS3), if I am not mistaken.
+ doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-26T15:27:49Z"+ content="""+I have updated the instructions.+"""]]
+ doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment view
@@ -0,0 +1,21 @@+[[!comment format=mdwn+ username="gernot"+ ip="213.196.216.21"+ subject="comment 2"+ date="2011-04-26T18:56:44Z"+ content="""+Thanks for the update, Joey. I think you forgot to change libghc-missingh-dev to libghc6-missingh-dev for the copy & paste instructions though.++Also, after having checked that I have everything installed I'm still getting this error:++	...+	[15 of 77] Compiling Annex            ( Annex.hs, Annex.o )++	Annex.hs:19:35:+		Module `Control.Monad.State' does not export `state'+	make[1]: *** [git-annex] Error 1+	make[1]: Leaving directory `/home/gernot/dev/git-annex'+	dh_auto_build: make -j1 returned exit code 2+	make: *** [binary] Error 2++"""]]
+ doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-04-26T23:40:33Z"+ content="""+Both problems fixed.+"""]]
+ doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn view
@@ -0,0 +1,12 @@+This is a tip for users who wish to use remotes which are based on OSX systems and have used macports to install some of the required utilities for git-annex to work.++The default behaviour of OSX's sshd is to have a "highly restricted" restricted environment. The defaults that it allows is++    jtang@x00:~ $ ssh x00 echo \$PATH+    /usr/bin:/bin:/usr/sbin:/sbin++One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and then in your own `~/.ssh/environment` file you could add something like (the below is an example)++    PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin++If the above is not done, cloning from the OSX host will fail if git is not installed in /usr/bin (which it probably won't be).
+ doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn view
@@ -0,0 +1,17 @@+This isn't really a bug of git-annex, but a problem with haskell-platform/ghc6.12.x so this post might need to be moved to a better place (maybe tips).++OSX's haskell-platform doesn't have the dynamic libraries available, as far as I know it just isn't supported therefore git-annex will always be statically built on OSX, so wrappers like <http://tsocks.sourceforge.net/> or [[!google dsocks]] for preloading connect() calls won't work. ++<pre>+jtang@x00:~/annex $ tsocks git annex get .+dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib++error: git-annex died of signal 5+</pre>++The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like <http://bent.latency.net/bent/git/goto-san-connect-1.85/src/connect.html>.++<pre>+host remotemyhost+        ProxyCommand connect -S proxy.mydomain:1080 -R local %h %p+</pre>
+ doc/forum/Problems_with_large_numbers_of_files.mdwn view
@@ -0,0 +1,8 @@+I'm trying to use git-annex to archive scientific data. I'm often dealing with large numbers of files, sometimes 10k or more. When I try to git-annex add these files I get this error:+++    Stack space overflow: current size 8388608 bytes.+    Use `+RTS -Ksize' to increase it.+++This is with the latest version of git-annex and a current version of git on OS 10.6.7. After this error occurs, I am unable to un-annex the files and I'm forced to recover from a backup.
+ doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment view
@@ -0,0 +1,22 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 1"+ date="2011-04-05T07:27:46Z"+ content="""+Heh, cool, I was thinking throwing about 28million files at git-annex. Let me know how it goes, I suspect you have just run into a default limits OSX problem.++You probably just need to up some system limits (you will need to read the error messages that first appear) then do something like++<pre>+# this is really for the run time, you can set these settings in /etc/sysctl.conf+sudo sysctl -w kern.maxproc=2048+sudo sysctl -w kern.maxprocperuid=1024++# tell launchd about having higher limits+sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf+sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf+</pre>++There are other system limits which you can check by doing a \"ulimit -a\", once you make the above changes, you will need to reboot to make the changes take affect. I am unsure if the above will help as it is an example of what I did on 10.6.6 a few months ago to fix some forking issues. From the error you got you will probably need to increase the stacksize to something bigger or even make it unlimited if you feel lucky, the default stacksize on OSX is 8192, try making it say 10times that size first and see what happens.+"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment view
@@ -0,0 +1,25 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-04-05T17:46:03Z"+ content="""+This message comes from ghc's runtime memory manager. Apparently your ghc defaults to limiting the stack to 80 mb.+Mine seems to limit it slightly higher -- I have seen haskell programs successfully grow as large as 350 mb, although generally not intentionally. :)++Here's how to adjust the limit at runtime, obviously you'd want a larger number:++<pre>+# git-annex +RTS -K100 -RTS find+Stack space overflow: current size 100 bytes.+Use `+RTS -Ksize -RTS' to increase it.+</pre>++I've tried to avoid git-annex using quantities of memory that scale with the number of files in the repo, and I think in general successfully -- I run it on 32 mb and 128 mb machines, FWIW. There are some tricky cases, and haskell makes it easy to accidentally write code that uses much more memory than would be expected.++One well known case is `git annex unused`, which *has* to build a structure of every annexed file. I have been considering using a bloom filter or something to avoid that.++Another possible case is when running a command like `git annex add`, and passing it a lot of files/directories. Some code tries to preserve the order of your input after passing it through `git ls-files` (which destroys ordering), and to do so it needs to buffer both the input and the result in ram.++It's possible to build git-annex with memory profiling and generate some quite helpful profiling data. Edit the Makefile and add this to GHCFLAGS: `-prof -auto-all -caf-all -fforce-recomp` then when running git-annex, add the parameters: `+RTS -p -RTS` , and look for the git-annex.prof file.+"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-04-05T18:02:05Z"+ content="""+Oh, you'll need profiling builds of various haskell libraries to build with profiling support. If that's not easily accomplished, if you could show me the form of the command you're running, and also how git annex unannex fails, that would be helpful for investigating.+"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment view
@@ -0,0 +1,29 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM"+ nickname="Justin"+ subject="comment 4"+ date="2011-04-05T21:14:12Z"+ content="""+@joey++OK, I'll try increasing the stack size and see if that helps.++For reference, I was running:++git annex add .++on a directory containing about 100k files spread over many nested subdirectories. I actually have more than a dozen projects like this that I plan to keep in git annex, possibly in separate repositories if necessary. I could probably tar the data and then archive that, but I like the idea of being able to see the structure of my data even though the contents of the files are on a different machine.++After the crash, running:++git annex unannex++does nothing and returns instantly. What exactly is 'git annex add' doing? I know that it's moving files into the key-value store and adding symlinks, but I don't know what else it does.++--Justin++++If ++"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 5"+ date="2011-04-07T16:41:00Z"+ content="""+I think what is happening with \"git annex unannex\" is that \"git annex add\" crashes before it can \"git add\" the symlinks. unannex only looks at files that \"git ls-files\" shows, and so files that are not added to git are not seen. So, this can be recovered from by looking at git status and manually adding the symlinks to git, and then unannex.++That also suggests that \"git annex add .\" has done something before crashing. That's consistent with you passing it < 2 parameters; it's not just running out of memory trying to expand and preserve order of its parameters (like it might if you ran \"git annex add experiment-1/ experiment-2/\")++I'm pretty sure I know where the space leak is now. git-annex builds up a queue of git commands, so that it can run git a minimum number of times. Currently, this queue is only flushed at the end. I had been meaning to work on having it flush the queue periodically to avoid it growing without bounds, and I will prioritize doing that.++(The only other thing that \"git annex add\" does is record location log information.)+"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 6"+ date="2011-04-07T18:09:13Z"+ content="""+I've committed the queue flush improvements, so it will buffer up to 10240 git actions, and then flush the queue.++There may be other memory leaks at scale (besides the two I mentioned earlier), but this seems promising. I'm well into running `git annex add` on a half million files and it's using 18 mb ram and has flushed the queue several times. This run+will fail due to running out of inodes for the log files, not due to memory. :)+"""]]
+ doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 7"+ date="2011-04-08T21:55:36Z"+ content="""+http://xfs.org/index.php/XFS_FAQ#Q:_Performance:_mkfs.xfs_-n_size.3D64k_option+"""]]
+ doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn view
@@ -0,0 +1,3 @@+FAT32 does not support symlinks, so I wonder if there's going to be a problem with that.++Generally speaking, I am wondering about portability of git annex on windows and on android...
+ doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-07T19:13:14Z"+ content="""+See [[bugs/fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames.+"""]]
+ doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-03-19T15:37:22Z"+ content="""+Now it's fully supported, so long as you put a bare git repo on your key.+"""]]
+ doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn view
@@ -0,0 +1,15 @@+It would be extremely useful to have some additional ways to select files (for git annex copy/move/get and maybe others) based on the meta-information available to git-annex, rather than just by file or directory name.++An example of what I'd like to do is this:++    host1$ git annex copy --to usb-drive --missing-on host2++This would check location tracking information and copy each file from host1's annex which is not present on host2 onto the usb-drive annex -- i.e. it's what I want when I need to do a sneakernet synchronisation of host1 and host2 (for backup purposes, for example). Note that of course I could copy --to host2, assuming network connectivity, but that would take a long time.++There's probably other selectors that we can imagine; an obvious one could be --present-on <annex> -- useful for judiciously dropping only those files that you have easily available in a local annex (as you may want to keep files that are hard to make available even if --numcopies would nominally be satisfied).++Other similar ideas for file content selectors:++ * Files that have less than n, exactly n or more than n copies -- for when you need to satisfy your --numcopies policy over sneakernet.+ * Files that are present (or not present) on some trusted annex -- for making sure you have trusted copies of everything.+ * Boolean combinations of these filters -- "git annex drop --present-on lanserver1 --or --present-on lanserver2" or similar syntax, although obviously doing this in full generality may be quite fiddly.
+ doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn view
@@ -0,0 +1,7 @@+I found the command "git annex lock" very slow (much slower than the initial "git annex add" with SHA1), for a not so big directory, when run in a big repo.+It seems that each underlying git command is not fast, so I thought it would be better to run them once with all files as arguments.+I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?++Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!++Rafaël
+ doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment view
@@ -0,0 +1,16 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="fixed"+ date="2011-05-31T18:51:13Z"+ content="""+Running `git checkout` by hand is fine, of course.++Underlying problem is that git has some O(N) scalability of operations on the index with regards to the number of files in the repo. So a repo with a whole lot of files will have a big index, and any operation that changes the index, like the `git reset` this needs to do, has to read in the entire index, and write out a new, modified version. It seems that git could be much smarter about its index data structures here, but I confess I don't understand the index's data structures at all. I hope someone takes it on, as git's scalability to number of files in the repo is becoming a new pain point, now that scalability to large files is \"solved\". ;)++Still, it is possible to speed this up at git-annex's level. Rather than doing a `git reset` followed by a git checkout, it can just `git checkout HEAD -- file`, and since that's one command, it can then be fed into the queueing machinery in git-annex (that exists mostly to work around this git malfescence), and so only a single git command will need to be run to lock multiple files.++I've just implemented the above. In my music repo, this changed an lock of a CD's worth of files from taking ctrl-c long to 1.75 seconds. Enjoy!++(Hey, this even speeds up the one file case greatly, since `git reset -- file` is slooooow -- it seems to scan the *entire* repository tree. Yipes.)+"""]]
+ doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"+ nickname="Rafaël"+ subject="comment 2"+ date="2011-05-31T21:43:22Z"+ content="""+Nice!+So if I understand correctly, 'git reset -- file' was there to discard staged (but not commited) changes made to 'file', before checking out, so that it is equivalent to directly 'git checkout HEAD -- file' ?+I'm curious about the \"queueing machinery in git-annex\": does it end up calling the one git command with multiple files as arguments? does it correspond to the message \"(Recording state in git...)\" ?+Thanks!+++"""]]
+ doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-05-31T21:54:23Z"+ content="""+@Rafaël , you're correct on all counts.+"""]]
+ doc/forum/bainstorming:_git_annex_push___38___pull.mdwn view
@@ -0,0 +1,28 @@+Wouldn't it make sense to offer++    git annex pull++which would basically do++    git pull+    git annex get++and++    git annex push++which would do++    git annex commit .+    git annex put # (the proposed "send to default annex" command)+    git commit -a -m "$HOST $(date +%F-%H-%M-%S)" # or similar+    git push++Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote.++> Update:++This is useful:++    git config [--global] alias.annex-push '!git pull && git annex add . && git annex copy . --to $REMOTE --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push'+
+ doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment view
@@ -0,0 +1,11 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-05T18:05:00Z"+ content="""+Maybe, otoh, part of the point of git-annex is that the data may be too large to pull down all of it.++I find mr useful as a policy layer over top of git-annex, so \"mr update\" can pull down appropriate quantities of data from+appropriate locations.+"""]]
+ doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-05T20:52:52Z"+ content="""+No-so-subtle sarcasm taken and acknowledged :)++Arguably, git-annex should know about any local limits and not have them implemented via mr from the outside. I guess my concern boils down to having git-annex do the right thing all by itself with minimal user interaction. And while I really do appreciate the flexibility of chaining commands, I am a firm believer in exposing the common use cases as easily as possible.++And yes, I am fully aware that not all annexes are created equal. Point in case, I would never use git annex pull on my laptop, but I would git annex push extensively.+++"""]]
+ doc/forum/batch_check_on_remote_when_using_copy.mdwn view
@@ -0,0 +1,34 @@+When I copy my local repository with SHA* to a remote repo with SHA*, every single file is checked by itself which seems rather inefficient. When my remote is accessed via ssh, git-annex opens a new connections for every check. If you are not using a ssh key or key agent, this gets tedious...++For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.++Once all checks are done, one single transfer session should be started. Creating new sessions and waiting for TCP's slowstart to get going is a lot less than efficient.+++-- RichiH++> (Use of SHA is irrelevant here, copy does not checksum anything.)+> +> I think what you're seeing is+> that `git annex copy --to remote` is slow, going to the remote repository+> every time to see if it has the file, while `git annex copy --from remote`+> is fast, since it looks at what files are locally present.+> +> That is something I mean to improve. At least `git annex copy --fast --to remote`+> could easily do a fast copy of all files that are known to be missing from+> the remote repository. When local and remote git repos are not 100% in sync,+> relying on that data could miss some files that the remote doesn't have anymore,+> but local doesn't know it dropped. That's why it's a candidate for `--fast`.+> +> I've just implemented that.+> +> While I do hope to improve ssh usage so that it sshs once, and feeds+> `git-annex-shell` a series of commands to run, that is a much longer-term+> thing. --[[Joey]]++>> FYI, in a repo with 1228 files, all small, repos _completely in sync_.++    % git annex copy . --to foo # 1200 seconds+    % git annex copy . --to foo --fast # 20 seconds++>> RichiH
+ doc/forum/can_git-annex_replace_ddm__63__.mdwn view
@@ -0,0 +1,13 @@+Hi,+a few years ago I wrote a tool called 'ddm'.  The code is overengineered and the script is more complicated then it should be,+but I think it demonstrates some good use cases, and I wonder how well git-annex can fulfill the requirements for those use cases - maybe I should remove ddm and start hacking with git-annex instead.++To answer this question, you should read the section about the possible dataset types on http://dieter.plaetinck.be/ddm_a_distributed_data_manager.html, and the example at the bottom of that page. it demonstrates the idea behind the "selection" dataset to always try to keep a subset (the most appropriate, based on the output of some script) of files "checked out".+the introduction section on https://github.com/Dieterbe/ddm/raw/358f7cf92c0ba7b336dc97638351d4e324461afa/MANUAL should further clarify things, as well as give some more good use cases (as you can see it's a bit more about [semi-]automated workflows then purely tracking what's where)++So I'm not sure, maybe the way to go for me is to make git-annex my "housekeeping about which data is where" backend and make ddm into a set of policies and tools on top of git-annex.++Any input?++Thanks,+Dieter
+ doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-02-14T22:08:54Z"+ content="""+Yes, there is value in layering something over git-annex to use a policy to choose what goes where.++I use [mr](http://kitenet.net/~joey/code/mr/) to update and manage all my repositories, and since mr can be made to run arbitrary commands when doing eg, an update, I use its config file as such a policy layer. For example, my podcasts are pulled into my sound repository in a subdirectory; boxes that consume podcasts run \"git pull; git annex get podcasts --exclude=\"*/out/*\"; git annex drop podcasts/*/out\". I move podcasts to \"out\" directories once done with them (I have yet to teach mpd to do that for me..), and the next time I run \"mr update\" to update everything, it pulls down new ones and removes old ones.++I don't see any obstacle to doing what you want. May be that you'd need better querying facilities in git-annex (so the policy layer can know what is available where), or finer control (--exclude is a good enough hammer for me, but maybe not for you).+"""]]
+ doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment view
@@ -0,0 +1,19 @@+[[!comment format=mdwn+ username="http://dieter-be.myopenid.com/"+ nickname="dieter"+ subject="comment 2"+ date="2011-02-16T21:32:04Z"+ content="""+thanks Joey,++is it possible to run some git annex command that tells me, for a specific directory, which files are available in an other remote? (and which remote, and which filenames?)+I guess I could run that, do my own policy thingie, and run `git annex get` for the files I want.++For your podcast use case (and some of my use cases) don't you think git [annex] might actually be overkill?  For example your podcasts use case, what value does git annex give over a simple rsync/rm script?+such a script wouldn't even need a data store to store its state, unlike git. it seems simpler and cleaner to me.++for the mpd thing, check http://alip.github.com/mpdcron/ (bad project name, it's a plugin based \"event handler\")+you should be able to write a simple plugin for mpdcron that does what you want (or even interface with mpd yourself from perl/python/.. to use its idle mode to get events)++Dieter+"""]]
+ doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-16T03:01:17Z"+ content="""+Whups, the comment above got stuck in moderation queue for 27 days. I will try to check that more frequently.++In the meantime, I've implemented \"git annex whereis\" -- enjoy!++I find keeping my podcasts in the annex useful because it allows me to download individual episodes or poscasts easily when low bandwidth is available (ie, dialup), or over sneakernet. And generally keeps everything organised.+"""]]
+ doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn view
@@ -0,0 +1,9 @@+I'm not sure if this is my stupidity or if it's a bug, but++    git annex copy --force --to REMOTE . ++just zip's through really quickly and doesn't actually force a copy to a remote location. This is just following up on the [[git-annex directory hashing problems on osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a ++    git annex copy --force --from REMOTE .++to pull down files from a remote.
+ doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-03T16:49:01Z"+ content="""+How remote is REMOTE? If it's a directory on the same computer, then git-annex copy --to is actually quickly checking that each file is present on the remote, and when it is, skipping copying it again.++If the remote is ssh, git-annex copy talks to the remote to see if it has the file. This makes copy --to slow, as Rich [[complained_before|forum/batch_check_on_remote_when_using_copy]]. :)++So, copy --to does not trust location tracking information (unless --fast is specified), which means that it should be doing exactly what you want it to do in your situation -- transferring every file that is really not present in the destination repository already.++Neither does copy --from, by the way. It always checks if each file is present in the current repository's annex before trying to download it.+"""]]
+ doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-04-03T16:59:47Z"+ content="""+Remote as in \"another physical machine\". I assumed that++    git annex copy --force --to REMOTE .++would have not trusted the contents in the current directory (or the remote that is being copied to) and then just go off and re-download/upload all the files and overwrite what is already there. I expected the combination of *--force* and copy *--to* that it would not bother to check if the files are there or not and just copy it regardless of the outcome.+"""]]
+ doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 3"+ date="2011-04-03T17:12:35Z"+ content="""+On second thought maybe the current behaviour is better than what I am suggesting that the force command should do. I guess it's better to be safe than sorry.+"""]]
+ doc/forum/git-annex_communication_channels.mdwn view
@@ -0,0 +1,10 @@+Thought I'd ask how y'all are finding the current communication by this forum/website/git repo only.++Would there be a benefit to having an irc channel for git-annex?++Maybe a mailing list? (Any persuasive reason why it would be better than this forum?)++Are the existing RSS feeds on this site, for eg, new [[comments]] and posts to this forum, sufficient to keep up with+things?++--[[Joey]]
+ doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment view
@@ -0,0 +1,17 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 1"+ date="2011-03-28T15:48:08Z"+ content="""+No matter what you end up doing, I would appreciate a git-annex-announce@ list.++I really like the persistence of ikiwiki, but it's not ideal for quick communication. I would be fine with IRC and/or ML. The advantage of a ML over ikiwiki is that it doesn't seem to be as \"wasteful\" to mix normal chat with actual problem-solving.  But maybe that's merely my own perception.++Speaking of RSS: I thought I had added a wishlist item to ikiwiki about providing per-subsite RSS feeds. For example there is no (obvious) way to subscribe to changes in http://git-annex.branchable.com/forum/git-annex_communication_channels/ .++FWIW, I resorted to tagging my local clone of git-annex to keep track of what I've read, already.+++-- RichiH+"""]]
+ doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2011-03-28T18:35:50Z"+ content="""+I think the forums/website currently is sufficient, I do at times wish there was a mailing list or anonymous git push to the wiki as I find editing posts through the web browser is some times tedious (the lack of !fmt or alt-q bugs me at times ;) ). The main advantage of keeping stuff on the site/forum is that everything gets saved and passed on to anyone who checks out the git repo of the code base.+"""]]
+ doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 3"+ date="2011-03-28T20:47:23Z"+ content="""+Push access to the non-code bits of git-annex' ikiwiki would be very welcome indeed. Given the choice, I would rather edit everything in Vim than in a browser. -- RichiH+"""]]
+ doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"+ nickname="Yaroslav"+ subject="comment 4"+ date="2011-04-13T17:53:26Z"+ content="""+.1 cents: Having IRC would be really nice for seeking quick help.   E.g. like I was trying to do now, google lead me to this page.+"""]]
+ doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ"+ nickname="Christian"+ subject="comment 5"+ date="2011-04-14T11:24:59Z"+ content="""+I would also like an git-annex channel. Would be #git-annex@OFTC ok?+"""]]
+ doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 6"+ date="2011-04-15T19:32:08Z"+ content="""+We seem to be using #vcs-home @ OFTC for now. madduck is fine with it and joeyh pokes his head in there, as well. I just added a CIA bot to #vcs-home and this comment is a test if pushing works. -- RichiH+"""]]
+ doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="anonymous git push"+ date="2011-05-19T19:21:51Z"+ content="""+@Jimmy mentioned anonymous git push -- that is now enabled for this wiki. Enjoy!++I may try to spend more time on #vcs-home -- or I can be summoned there from my other lurking places on irc, I guess.+"""]]
+ doc/forum/git-annex_on_OSX.mdwn view
@@ -0,0 +1,1 @@+See [[install/OSX]].
+ doc/forum/hashing_objects_directories.mdwn view
@@ -0,0 +1,27 @@+I'm wondering how easy the addition of hashing to the directories of the objects would be.++Currently a tree directory structure becomes a flat two level tree under the .git/annex/objects directory ([[internals]]).  This, through the 555 mode on the directory prevents the accidental destruction of content, which is _good_.  However file and directory numbers soon add up in there and as such any file-systems with sub directory limitations will quickly realize the limit (certainly quicker than maybe expected).++Suggestion is therefore to change from ++ `.git/annex/objects/SHA1:123456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678`++to ++ `.git/annex/objects/SHA1:1/2/3456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678`++or anything in between to a paranoid++ `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678`++Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...`++`git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f`++> This is done now with a 2-level hash. It also hashes .git-annex/ log+> files which were the worse problem really. Scales to hundreds of millions+> of files with each dir having 1024 or fewer contents. Example:+>+> `me -> .git/annex/objects/71/9t/WORM-s3-m1300247299--me/WORM-s3-m1300247299--me`+>+> --[[Joey]]
+ doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-14T16:12:49Z"+ content="""+My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. ++I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.)++There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning..+"""]]
+ doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-03-15T13:52:16Z"+ content="""+Can't you just use an underscore instead of a colon?++Would it be feasible to split directories dynamically? I.e. start with SHA1_123456789abcdef0123456789abcdef012345678/SHA1_123456789abcdef0123456789abcdef012345678 and, at a certain cut-off point, switch to shorter directory names? This could even be done per subdirectory and based purely on a locally-configured number. Different annexes on different file systems or with different file subsets might even have different thresholds. This would ensure scale while not forcing you to segment from the start. Also, while segmenting with longer directory names means a flatter tree, segments longer than four characters might not make too much sense. Segmenting too often could lead to some directories becoming too populated, bringing us back to the dynamic segmentation.++All of the above would make merging annexes by hand a _lot_ harder, but I don't know if this is a valid use case. And if all else fails, one could merge everything with the unsegemented directory names and start again from there.++-- RichiH+"""]]
+ doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-03-16T03:13:39Z"+ content="""+It is unfortunatly not possible to do system-dependant hashing, so long as git-annex stores symlinks to the content in git.++It might be possible to start without hashing, and add hashing for new files after a cutoff point. It would add complexity.++I'm currently looking at a 2 character hash directory segment, based on an md5sum of the key, which splits it into 1024 buckets. git uses just 256 buckets for its object directory, but then its objects tend to get packed away. I sorta hope that one level is enough, but guess I could go to 2 levels (objects/ab/cd/key), which would provide 1048576 buckets, probably plenty, as if you are storing more than a million files, you are probably using a modern enough system to have a filesystem that doesn't need hashing.+"""]]
+ doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 4"+ date="2011-03-16T04:06:19Z"+ content="""+The .git-annex/ directory is what really needs hashing.++Consider that when git looks for changes in there, it has to scan every file in the directory. With hashing, it should be able to more quickly identify just the subdirectories that contained changed files, by the directory mtimes.++And the real kicker is that when committing there, git has to create a tree object containing every single file, even if only 1 file changed. That will be a lot of extra work; with hashed subdirs it will instead create just 2 or 3 small tree objects leading down to the changed file. (Probably these trees both pack down to similar size pack files, not sure.)+"""]]
+ doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 5"+ date="2011-03-16T15:47:17Z"+ content="""+If you can't segment the names retroactively, it's better to start with segmenting, imo.++As subdirectories are cheap, going with ab/cd/rest or even ab/cd/ef/rest by default wouldn't hurt.++Your point about git not needing to create as many tree objects is a kicker indeed. If I were you, I would default to segmentation.+"""]]
+ doc/forum/incompatible_versions__63__.mdwn view
@@ -0,0 +1,1 @@+Are versions 0.14 and 0.20110522 incompatible? I can't seem to copy files from a system running 0.14 to one running 20110522.
+ doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-06-08T00:40:54Z"+ content="""+They are not. See [[upgrades]]+"""]]
+ doc/forum/migrate_existing_git_repository_to_git-annex.mdwn view
@@ -0,0 +1,66 @@+I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history.++I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons:++* --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/).+* annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them++Any suggestions how to proceed?++EDIT 3/2/2010+I finally got it working for my purposes. Hardest part was preserving the branches while injecting the new `git annex setup` base commit.++#### Clone repository+    git clone original migrate+    cd migrate+    git checkout mybranch+    git checkout master+    git remote rm origin++#### Inject `git annex setup` base commit and repair branches+    git symbolic-ref HEAD refs/heads/newroot+    git rm --cached *+    git clean -f -d+    git annex init master+    echo \*.rpm annex.backend=SHA1 >> .gitattributes+    git commit -m "store rpms in git annex" .gitattributes+    git cherry-pick $(git rev-list --reverse master | head -1)+    git rebase --onto newroot newroot master+    git rebase --onto master mybranch~1 mybranch+    git branch -d newroot++#### Migrate repository+    mkdir .temp+    cp .git-annex/* .temp/+    MYWORKDIR=$(pwd) git filter-branch \+     --tag-name-filter cat \+     --tree-filter '+        mkdir -p .git-annex;+        cp ${MYWORKDIR}/.temp/* .git-annex/;+        for rpm in $(git ls-files | grep "\.rpm$"); do+            echo;+            git annex add $rpm;+            annexdest=$(readlink $rpm);+            if [ -e .git-annex/$(basename $annexdest).log ]; then+                echo "FOUND $(basename $annexdest).log";+            else+                echo "COPY $(basename $annexdest).log";+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/;+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log ${MYWORKDIR}/.temp/;+            fi;+            ln -sf ${annexdest#../../} $rpm;+        done;+        git reset HEAD .git-rewrite;+        :+        ' -- $(git branch | cut -c 3-)+    rm -rf .temp+    git reset --hard+++TODO:++* Find a way to repair branches automatically (detect branch points and run appropriate `git rebase` commands)++I'll be happy to try any suggestions to improve this migration script.++P.S. Is there a way to edit comments?
+ doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-02-25T05:16:48Z"+ content="""+I don't know how to approach this yet, but I support the idea -- it would be great if there was a tool that could punch files out of git history and put them in the annex. (Of course with typical git history rewriting caveats.)++Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? +"""]]
+ doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment view
@@ -0,0 +1,60 @@+[[!comment format=mdwn+ username="tyger"+ ip="80.66.20.180"+ subject="comment 2"+ date="2011-03-01T14:07:50Z"+ content="""+My current workflow looks like this (I'm still experimenting):++### Create backup clone for migration++    git clone original migrate+    cd migrate+    for branch in $(git branch -a | grep remotes/origin | grep -v HEAD); do git checkout --track $branch; done++### Inject git annex initialization at repository base++    git symbolic-ref HEAD refs/heads/newroot+    git rm --cached *.rpm+    git clean -f -d+    git annex init master+    git cherry-pick $(git rev-list --reverse master | head -1)+    git rebase --onto newroot newroot master+    git rebase master mybranch # how to automate this for all branches?+    git branch -d newroot++### Start migration with tree filter++    echo \*.rpm annex.backend=SHA1 > .git/info/attributes+    MYWORKDIR=$(pwd) git filter-branch --tree-filter ' \+        if [ ! -d .git-annex ]; then \+            mkdir .git-annex; \+            cp ${MYWORKDIR}/.git-annex/uuid.log .git-annex/; \+            cp ${MYWORKDIR}/.gitattributes .; \+        fi+        for rpm in $(git ls-files | grep \"\.rpm$\"); do \+            echo; \+            git annex add $rpm; \+            annexdest=$(readlink $rpm); \+            if [ -e .git-annex/$(basename $annexdest).log ]; then \+                echo \"FOUND $(basename $annexdest).log\"; \+            else \+                echo \"COPY $(basename $annexdest).log\"; \+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/; \+            fi; \+            ln -sf ${annexdest#../../} $rpm; \+        done; \+        git reset HEAD .git-rewrite; \+        : \+        ' -- $(git branch | cut -c 3-)+    rm -rf .temp+    git reset --hard+++There are still some drawbacks:++* git history shows that git annex log files are modified with each checkin+* branches have to be rebased manually before starting migration+++"""]]
+ doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="tyger"+ ip="80.66.20.180"+ subject="comment 3"+ date="2011-03-02T08:15:37Z"+ content="""+> Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be?++It should sufficient to honor GIT_DIR/GIT_WORK_TREE/GIT_INDEX_FILE environment variables. git filter-branch sets GIT_WORK_TREE to ., but this can be mitigated by starting the filter script with 'GIT_WORK_TREE=$(pwd $GIT_WORK_TREE)'. E.g. GIT_DIR=/home/tyger/repo/.git, GIT_WORK_TREE=/home/tyger/repo/.git-rewrite/t, then git annex should be able to compute the correct relative path or maybe use absolute pathes in symlinks.++Another problem I observed is that git annex add automatically commits the symlink; this behaviour doesn't work well with filter-tree. git annex commits the wrong path (.git-rewrite/t/LINK instead of LINK). Also filter-tree doesn't expect that the filter script commmits anything; new files in the temporary work tree will be committed by filter-tree on each iteration of the filter script (missing files will be removed).+"""]]
+ doc/forum/migration_to_git-annex_and_rsync.mdwn view
@@ -0,0 +1,33 @@+When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the <del>attached</del> migrate.py file below, and it saved me roughly a day of backuping.++A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created.++**WARNING**: This is a quickly hacked-together script. It worked for me, but is untested apart from that. It's just a dozen lines of code, so have a look at it and make sure you understand what it does, and what migrate.sh looks like. Take special care as this tampers with your backups, and if something goes wrong, well...++First, have an up-to-date backup; then, git annex init / add etc as described in the [[walkthrough]]. In the directory in which you use git-annex, run:++    $ python migrate.py > migrate.sh++Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway).++After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data.++**migrate.py**:++    #!/usr/bin/env python+    +    import os+    from pipes import quote+    +    print "#!/bin/sh"+    print "set -e"+    print ""+    +    for (dirpath, dirnames, filenames) in os.walk("."):+    	for f in filenames:+    		fn = os.path.join(dirpath, f)+    		if os.path.islink(fn):+    			link = os.path.normpath(os.path.join(dirpath, os.readlink(fn)))+    			assert link.startswith(".git/annex/objects/")+    			print "mkdir -p %s"%quote(os.path.dirname(link))+    			print "mv %s %s"%(quote(fn), quote(link))
+ doc/forum/new_microfeatures.mdwn view
@@ -0,0 +1,53 @@+I'm soliciting ideas for new small features that let git-annex do things that currently have to be done manually or whatever.++Here are a few I've been considering:++---++* --numcopies would be a useful command line switch.+  > Update: Added. Also allows for things like `git annex drop --numcopies=2` when in a repo that normally needs 3 copies, if you need+  > to urgently free up space.+* A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. ++Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)++> Update: done --[[Joey]] ++---++[[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`++  > Contrary to the "basic" solution, I would love to have a git annex distribute which is smart enough to simply distribute all data according to certain rules. My ideal, personal use case during the next holidays where I will have two external disks, several SD cards with 32 GB each and a local disk with 20 GB (yes....) would be:++    cd ~/photos.annex # this repository does not have any objects!+    git annex inject --bare /path/to/SD/card  # this adds softlinks, but does **not** add anything to the index. it would calculate checksums (if enabled) and have to add a temporary location list, though+    git annex distribute # this checks the config. it would see that my two external disks have a low cost whereas the two remotes have a higher cost.+     # check numcopies. it's 3+     # copy to external disk one (cost x)+     # copy to external disk two (cost x)+     # copy to remote one (cost x * 2)+     # remove file from temporary tracking list+    git annex fsck # everything ok. yay!++Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH++> I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]++---++Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.++---++If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH++---++Really micro:++    % grep annex-push .git/config+        annex-push = !git pull && git annex add . && git annex copy . --to origin --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push+    %++-- RichiH+--[[Joey]]
+ doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 1"+ date="2011-06-01T17:36:50Z"+ content="""+I've been longing for an automated way of removing references to a remote assuming I know the exact uuid that I want to remove. i.e. I have lost a portable HDD due to a destructive process, I now want to delete all references to copies of data that was on that disk. Unless this feature exists, I would love to see it implemented.+"""]]
+ doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 2"+ date="2011-06-01T20:24:33Z"+ content="""+@jimmy [[walkthrough/what_to_do_when_you_lose_a_repository]].. I have not seen a convincing argument that removing the location tracking data entirely serves any purpose+"""]]
+ doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment view
@@ -0,0 +1,15 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"+ nickname="Rafaël"+ subject="git annex unlock --readonly"+ date="2011-06-02T11:34:42Z"+ content="""+This was already asked [here](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=606577), but I have a use case where I need to unlock with the files being hardlinked instead of copied (my fs does not support CoW), even though 'git annex lock' is now much faster ;-) . The idea is that 1) I want the external world see my repo \"as if\" it wasn't annexed (because of its own limitation to deal with soft links), and 2) I know what I do, and am sure that files won't be written to but only read.++My case is: the repo contains a snapshot A1 of a certain remote directory. Later I want to rsync this dir into a new snapshot A2. Of course, I want to transfer only new or changed files, with the --copy-dest=A1 (or --compare-dest) rsync's options. Unfortunately, rsync won't recognize soft-links from git-annex, and will re-transfer everything.+++Maybe I'm overusing git-annex ;-) but still, I find it is a legitimate use case, and even though there are workarounds (I don't even remember what I had to do), it would be much more straightforward to have 'git annex unlock --readonly' (or '--readonly-unsafe'?), ... or have rsync take soft-links into account, but I did not see the author ask for microfeatures ideas :) (it was discussed, and only some convoluted workarounds were proposed). Thanks.+++"""]]
+ doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"+ nickname="Rafaël"+ subject="git annex unused"+ date="2011-06-02T11:55:58Z"+ content="""+Before dropping unsused items, sometimes I want to check the content of the files manually.+But currently, from e.g. a sha1 key, I don't know how to find the corresponding file, except with+'find .git/annex/objects -type f -name 'SHA1-s1678--70....', wich is too slow (I'm in the case where \"git log --stat -S'KEY'\"+won't work, either because it is too slow or it was never commited). By the way,+is it documented somewhere how to determine the 2 (nested) sub-directories in which a given+(by name) object is located?++So I would like 'git-annex unused' be able to give me the list of *paths* to the unused items.+Also, I would really appreciate a command like 'git annex unused --log NUMBER [NUMBER2...]' which would do for me the suggested command+\"git log --stat -S'KEY'\", where NUMBER is from the 'git annex unused' output.+Thanks.+"""]]
+ doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"+ nickname="Rafaël"+ subject="git annex unused"+ date="2011-06-02T19:51:49Z"+ content="""+ps: concerning the command 'find .git/annex/objects -type f -name 'SHA1-s1678--70....' from my previous comment, it is \"significantly\" faster to search for the containing directory which have the same name: 'find .git/annex/objects -maxdepth 2 -mindepth 2 -type d -name 'SHA1-s1678--70....'. I am just curious: what is the need to have each file object in its own directory, itself nested under two more sub-directories?+"""]]
+ doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn view
@@ -0,0 +1,3 @@+This works with bind-mount, I might try with softlinks as well.++Going through git's data on push/pull can take ages on a spindle disk even if the repo is rather small in size. This is especially true if you are used to ssd speeds, but ssd storage is expensive. Storing the annex objects on a cheap spindle disk and everything else on a ssd makes things a _lot_ faster.
+ doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-04-02T17:48:29Z"+ content="""+Either option should work fine, but git gc --aggressive will probably avoid most of git's seeking.+"""]]
+ doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment view
@@ -0,0 +1,20 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 2"+ date="2011-04-02T21:34:24Z"+ content="""+I'll give it a try as soon as I get rid of this:++    % git annex fsck+fatal: index file smaller than expected+fatal: index file smaller than expected+    % git status+fatal: index file smaller than expected+    %    ++And no, I am not sure where that is coming from all of a sudden... (it might have to do with a hard lockup of the whole system due to a faulty hdd I tested, but I didn't do anything to it for ages before that lock-up. So meh. Also, this is prolly off topic in here)+++Richard+"""]]
+ doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment view
@@ -0,0 +1,10 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 3"+ date="2011-04-03T01:48:57Z"+ content="""+For future reference, git can recover from a corrupted index file with `rm .git/index; git reset --mixed`.++Of course, you lose any staged changes that were in the old index file, and may need to re-stage some files.+"""]]
+ doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"+ nickname="Richard"+ subject="comment 4"+ date="2011-04-03T09:03:22Z"+ content="""+Thanks a lot. I tried various howtos around the net, but none of them worked; yours did. (I tried it in one of the copies of the broken repo which I keep around for obvious reasons).+"""]]
+ doc/forum/relying_on_git_for_numcopies.mdwn view
@@ -0,0 +1,47 @@+**&lt;out-of-date-warning&gt;**The main problems this is supposed to solve are addressed in a different way with [[todo/hidden files]] and the `--fast` option introduced in [[batch check on remote when using copy]], so while this is not technically obsolete, the main reasons for it are gone. --[[chrysn]]**&lt;/out-of-date-warning&gt;**++This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]]++Summary+=========++Currently, [[location tracking]] is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file until another repository takes over responsibility.++git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion.++Modified git-annex-drop behavior+==========================++The most important (if not only) git-annex command that is affected by this is `git annex drop`. Currently, for dropping a large number of files, every file is checked with another (or multiple, if so configured) host if it's safe to delete.++The new behavior would be to++* decrement the location tracking counter for all files to be dropped,+* commit that change,+* try to push it to at least as many repositories that the numcopies constraints are met,+* revert if that fails,+* otherwise really drop the files from the backend.++Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex itself, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.)++Implications for trust+==============++The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case:++> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home, so they can be deleted on demand.+>+> She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed.++Dereferencing could be implemented as `git annex drop --no-rm` (or `move --no-rm`), freeing space is similar to `dropunused`.++A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before.++Advantages / Disadvantages+=====================++The advantage of this proposal is that the round trips required for dropping something could be greatly reduced.++There should also be simplifications in the `git annex drop` command as it doesn't need to take care of locking any more (git should already do that between checking if HEAD is a parent of the pushed commit and replacing HEAD).++Besides being a major change in git-annex (with the requirement to track hosts' git-annex versions for migration, as the new trust system is incompatible with the old one), no disadvantages of that stragegy are known to the author (hoping for discussion below).
+ doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment view
@@ -0,0 +1,36 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-02-22T18:44:28Z"+ content="""+I see the following problems with this scheme:++- Disallows removal of files when disconnected. It's currently safe to force that, as long as+  git-annex tells you enough other repos are belived to have the file. Just as long as you+  only force on one machine (say your laptop). With your scheme, if you drop a file while +  disconnected, any other host could see that the counter is still at N, because your+  laptop had the file last time it was online, and can decide to drop the file, and lose the last +version.++- pushing a changed counter commit to other repos is tricky, because they're not bare, and +  the network topology to get the commit pulled into the other repo could vary.++- Merging counter files issues.  If the counter file doesn't automerge, two repos dropping the same file will conflict. But, if it does automerge, it breaks the counter conflict detection.++- Needing to revert commits is going to be annoying. An actual git revert+  could probably not reliably be done. It's need to construct a revert+  and commit it as a new commit. And then try to push that to remotes, and+  what if *that* push conflicts?++- I do like the pre-removal dropping somewhat as an alternative to+  trust checking. I think that can be done with current git-annex though,+  just remove the files from the location log, but keep them in-annex.+  Dropping a file only looks at repos that the location log says have a+  file; so other repos can have retained a copy of a file secretly like+  this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. ++- I don't see any reduced round trips. It still has to contact N other+  repos on drop. Now, rather than checking that they have a file, it needs+  to push a change to them.+"""]]
+ doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment view
@@ -0,0 +1,18 @@+[[!comment format=mdwn+ username="http://christian.amsuess.com/chrysn"+ nickname="chrysn"+ subject="comment 2"+ date="2011-02-23T16:43:59Z"+ content="""+i'll comment on each of the points separately, well aware that even a single little leftover issue can show that my plan is faulty:++* force removal: well, yes -- but the file that is currently force-removed on the laptop could just as well be the last of its kind itself. i see the problem, but am not sure if it's fatal (after all, if we rely on out-of-band knowledge when forcing something, we could just as well ask a little more)+* non-bare repos: pushing is tricky with non-bare repos now just as well; a post-commit hook could auto-accept counter changes. (but pushing causes problems with counters anyway, doesn't it?)+* merging: i'd have them auto-merge. git-annex will have to check the validity of the current state anyway, and a situation in which a counter-decrementing commit is not a fast-forward one would be reverted in the next step (or upon discovery, in case the next step never took place).+* reverting: my wording was bad as \"revert\" is already taken in git-lingo. the correct term for what i was thinking of is \"reset\". (as the commit could not be pushed, it would be rolled back completely).+    * we might have to resort to reverting, though, if the commit has already been pused to a first server of many.+* [[todo/hidden files]]: yes, this solves pre-removal dropping :-)+* round trips: it's not the number of servers, it's the number of files (up to 30k in my case). it seems to me that an individual request was made for every single file i wanted to drop (that would be N*M roundtrips for N affected servers and M files, and N roundtrips with git managed numcopies)++all together, it seems to be a bit more complicated than i imagined, although not completely impossible. a combination of [[todo/hidden files]] and maybe a simpler reduction of the number of requests might though achieve the important goals as well.+"""]]
+ doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://christian.amsuess.com/chrysn"+ nickname="chrysn"+ subject="relation to [[todo/branching]]"+ date="2011-02-23T21:48:14Z"+ content="""+the non-bare repository issue would go away if this was combined with the \"alternate\" approach to [[todo/branching]]. (with the \"fleshed out proposal\" of branching, this would not work at all for lack of shared commits.)+"""]]
+ doc/forum/rsync_over_ssh__63__.mdwn view
@@ -0,0 +1,2 @@+[Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure?+-- Michael K.
+ doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joey.kitenet.net/"+ nickname="joey"+ subject="comment 1"+ date="2011-03-06T15:59:37Z"+ content="""+Everything is done over ssh unless both repos are on the same system (or unless you NFS mount a repo)+"""]]
+ doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://m-f-k.myopenid.com/"+ ip="92.194.43.135"+ subject="comment 2"+ date="2011-03-06T16:33:19Z"+ content="""+Great! This was the only thing about git-annex which could have kept me from using it. --Michael+"""]]
+ doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn view
@@ -0,0 +1,1 @@+This is just a comment on git-annex building on haskell platform 2011.2.0.0 on archlinux. It just works.
+ doc/forum/sparse_git_checkouts_with_annex.mdwn view
@@ -0,0 +1,31 @@+I checked in my music collection into git annex (about 25000 files) and i'm really impressed by the performance of git annex (after i've done an git-repack). Now i'm also moving my movies into the same git-annex, but i have the following layout of my disk drives:++* small raid-1 for important stuff (music, documents), which is also backupped (aka: raid)+* big bulk data store (aka: media)++In the git-annex the following layout of files is used:++* documents/ <- on raid+* music/ <- on raid+* videos/ <- on media++Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (possible since version 1.7.0)++* raid: .git-annex/, documents/ and music+* media: .git-annex/, videos/++As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this?++Before:++     $ ls .git-annex+     00 01 02....++After:++     $ ls .git-annex+     documents/ music/ videos/+     $ ls .git-annex/documents+     00 01 02....++This would make it possible to checkout only the part of the log files which i'm interested in.
+ doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment view

file too large to diff

+ doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment view

file too large to diff

+ doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment view

file too large to diff

+ doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment view

file too large to diff

+ doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment view

file too large to diff

+ doc/forum/unannex_alternatives.mdwn view

file too large to diff

+ doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment view

file too large to diff

+ doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment view

file too large to diff

+ doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment view

file too large to diff

+ doc/forum/wishlist:_command_options_changes.mdwn view

file too large to diff

+ doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment view

file too large to diff

+ doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment view

file too large to diff

+ doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment view

file too large to diff

+ doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn view

file too large to diff

+ doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment view

file too large to diff

+ doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment view

file too large to diff

+ doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn view

file too large to diff

+ doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment view

file too large to diff

+ doc/forum/wishlist:_git-annex_replicate.mdwn view

file too large to diff

+ doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment view

file too large to diff

+ doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment view

file too large to diff

+ doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn view

file too large to diff

+ doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_status.mdwn view

file too large to diff

+ doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment view

file too large to diff

+ doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex.mdwn view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment view

file too large to diff

+ doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment view

file too large to diff

+ doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn view

file too large to diff

+ doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn view

file too large to diff

+ doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment view

file too large to diff

+ doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment view

file too large to diff

+ doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment view

file too large to diff

+ doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn view

file too large to diff

+ doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn view

file too large to diff

+ doc/forum/wishlist:alias_system.mdwn view

file too large to diff

+ doc/forum/working_without_git-annex_commits.mdwn view

file too large to diff

+ doc/future_proofing.mdwn view

file too large to diff

+ doc/git-annex-shell.mdwn view

file too large to diff

+ doc/git-annex.mdwn view

file too large to diff

+ doc/git-union-merge.mdwn view

file too large to diff

+ doc/index.mdwn view

file too large to diff

+ doc/install.mdwn view

file too large to diff

+ doc/install/Debian.mdwn view

file too large to diff

+ doc/install/Fedora.mdwn view

file too large to diff

+ doc/install/FreeBSD.mdwn view

file too large to diff

+ doc/install/OSX.mdwn view

file too large to diff

+ doc/install/Ubuntu.mdwn view

file too large to diff

+ doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment view

file too large to diff

+ doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment view

file too large to diff

+ doc/internals.mdwn view

file too large to diff

+ doc/location_tracking.mdwn view

file too large to diff

+ doc/logo.png view

file too large to diff

+ doc/logo_small.png view

file too large to diff

+ doc/news.mdwn view

file too large to diff

+ doc/news/LWN_article.mdwn view

file too large to diff

+ doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn view

file too large to diff

+ doc/news/version_0.20110522.mdwn view

file too large to diff

+ doc/news/version_0.20110601.mdwn view

file too large to diff

+ doc/news/version_0.20110610.mdwn view

file too large to diff

+ doc/news/version_3.20110624.mdwn view

file too large to diff

+ doc/news/version_3.20110702.mdwn view

file too large to diff

+ doc/not.mdwn view

file too large to diff

+ doc/repomap.png view

file too large to diff

+ doc/special_remotes.mdwn view

file too large to diff

+ doc/special_remotes/S3.mdwn view

file too large to diff

+ doc/special_remotes/bup.mdwn view

file too large to diff

+ doc/special_remotes/directory.mdwn view

file too large to diff

+ doc/special_remotes/hook.mdwn view

file too large to diff

+ doc/special_remotes/rsync.mdwn view

file too large to diff

+ doc/special_remotes/web.mdwn view

file too large to diff

+ doc/summary.mdwn view

file too large to diff

+ doc/templates/bare.tmpl view

file too large to diff

+ doc/templates/walkthrough.tmpl view

file too large to diff

+ doc/todo.mdwn view

file too large to diff

+ doc/todo/S3.mdwn view

file too large to diff

+ doc/todo/add_--exclude_option_to_git_annex_find.mdwn view

file too large to diff

+ doc/todo/add_a_git_backend.mdwn view

file too large to diff

+ doc/todo/auto_remotes.mdwn view

file too large to diff

+ doc/todo/auto_remotes/discussion.mdwn view

file too large to diff

+ doc/todo/backendSHA1.mdwn view

file too large to diff

+ doc/todo/branching.mdwn view

file too large to diff

+ doc/todo/cache_key_info.mdwn view

file too large to diff

+ doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment view

file too large to diff

+ doc/todo/checkout.mdwn view

file too large to diff

+ doc/todo/done.mdwn view

file too large to diff

+ doc/todo/file_copy_progress_bar.mdwn view

file too large to diff

+ doc/todo/fsck.mdwn view

file too large to diff

+ doc/todo/git-annex-shell.mdwn view

file too large to diff

+ doc/todo/git-annex_unused_eats_memory.mdwn view

file too large to diff

+ doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn view

file too large to diff

+ doc/todo/gitrm.mdwn view

file too large to diff

+ doc/todo/hidden_files.mdwn view

file too large to diff

+ doc/todo/immutable_annexed_files.mdwn view

file too large to diff

+ doc/todo/network_remotes.mdwn view

file too large to diff

+ doc/todo/object_dir_reorg_v2.mdwn view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment view

file too large to diff

+ doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment view

file too large to diff

+ doc/todo/parallel_possibilities.mdwn view

file too large to diff

+ doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment view

file too large to diff

+ doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment view

file too large to diff

+ doc/todo/pushpull.mdwn view

file too large to diff

+ doc/todo/rsync.mdwn view

file too large to diff

+ doc/todo/smudge.mdwn view

file too large to diff

+ doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment view

file too large to diff

+ doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment view

file too large to diff

+ doc/todo/speed_up_fsck.mdwn view

file too large to diff

+ doc/todo/support-non-utf8-locales.mdwn view

file too large to diff

+ doc/todo/support_S3_multipart_uploads.mdwn view

file too large to diff

file too large to diff

+ doc/todo/tahoe_lfs_for_reals.mdwn view

file too large to diff

+ doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment view

file too large to diff

+ doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment view

file too large to diff

+ doc/todo/union_mounting.mdwn view

file too large to diff

+ doc/todo/use_cp_reflink.mdwn view

file too large to diff

+ doc/todo/using_url_backend.mdwn view

file too large to diff

+ doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn view

file too large to diff

+ doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment view

file too large to diff

+ doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn view

file too large to diff

+ doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment view

file too large to diff

+ doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment view

file too large to diff

+ doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment view

file too large to diff

+ doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn view

file too large to diff

+ doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment view

file too large to diff

+ doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment view

file too large to diff

+ doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment view

file too large to diff

+ doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn view

file too large to diff

+ doc/todo/wishlist:_swift_backend.mdwn view

file too large to diff

+ doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment view

file too large to diff

+ doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment view

file too large to diff

+ doc/transferring_data.mdwn view

file too large to diff

+ doc/trust.mdwn view

file too large to diff

+ doc/upgrades.mdwn view

file too large to diff

+ doc/upgrades/SHA_size.mdwn view

file too large to diff

+ doc/use_case/Alice.mdwn view

file too large to diff

+ doc/use_case/Bob.mdwn view

file too large to diff

+ doc/users.mdwn view

file too large to diff

+ doc/users/chrysn.mdwn view

file too large to diff

+ doc/users/fmarier.mdwn view

file too large to diff

+ doc/users/joey.mdwn view

file too large to diff

+ doc/walkthrough.mdwn view

file too large to diff

+ doc/walkthrough/Internet_Archive_via_S3.mdwn view

file too large to diff

+ doc/walkthrough/adding_a_remote.mdwn view

file too large to diff

+ doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment view

file too large to diff

+ doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment view

file too large to diff

+ doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment view

file too large to diff

+ doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment view

file too large to diff

+ doc/walkthrough/adding_files.mdwn view

file too large to diff

+ doc/walkthrough/backups.mdwn view

file too large to diff

+ doc/walkthrough/creating_a_repository.mdwn view

file too large to diff

+ doc/walkthrough/fsck:_verifying_your_data.mdwn view

file too large to diff

+ doc/walkthrough/fsck:_when_things_go_wrong.mdwn view

file too large to diff

+ doc/walkthrough/getting_file_content.mdwn view

file too large to diff

+ doc/walkthrough/migrating_data_to_a_new_backend.mdwn view

file too large to diff

+ doc/walkthrough/modifying_annexed_files.mdwn view

file too large to diff

+ doc/walkthrough/more.mdwn view

file too large to diff

+ doc/walkthrough/moving_file_content_between_repositories.mdwn view

file too large to diff

+ doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment view

file too large to diff

+ doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment view

file too large to diff

+ doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment view

file too large to diff

+ doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment view

file too large to diff

+ doc/walkthrough/recover_data_from_lost+found.mdwn view

file too large to diff

+ doc/walkthrough/removing_files.mdwn view

file too large to diff

+ doc/walkthrough/removing_files:_When_things_go_wrong.mdwn view

file too large to diff

+ doc/walkthrough/renaming_files.mdwn view

file too large to diff

+ doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn view

file too large to diff

+ doc/walkthrough/untrusted_repositories.mdwn view

file too large to diff

+ doc/walkthrough/unused_data.mdwn view

file too large to diff

+ doc/walkthrough/using_Amazon_S3.mdwn view

file too large to diff

+ doc/walkthrough/using_bup.mdwn view

file too large to diff

+ doc/walkthrough/using_ssh_remotes.mdwn view

file too large to diff

+ doc/walkthrough/using_the_SHA1_backend.mdwn view

file too large to diff

+ doc/walkthrough/using_the_web.mdwn view

file too large to diff

+ doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn view

file too large to diff

+ git-annex-shell.hs view

file too large to diff

+ git-annex.cabal view

file too large to diff

+ git-annex.hs view

file too large to diff

+ git-union-merge.hs view

file too large to diff

+ mdwn2man view

file too large to diff

+ test.hs view

file too large to diff