packages feed

git-annex 5.20140221 → 5.20140227

raw patch · 103 files changed

+1443/−607 lines, 103 filesdep +http-clientdep +lifted-basedep +regex-compatdep −xml-conduit

Dependencies added: http-client, lifted-base, regex-compat

Dependencies removed: xml-conduit

Files

Annex/CatFile.hs view
@@ -87,8 +87,7 @@ 		| modeguaranteed = catObject ref 		| otherwise = L.take 8192 <$> catObject ref -{- Looks up the file mode corresponding to the Ref using the running- - cat-file.+{- Looks up the key corresponding to the Ref using the running cat-file.  -  - Currently this always has to look in HEAD, because cat-file --batch  - does not offer a way to specify that we want to look up a tree object
Annex/Content.hs view
@@ -221,7 +221,7 @@  -} prepGetViaTmpChecked :: Key -> Annex Bool -> Annex Bool prepGetViaTmpChecked key getkey = do-	tmp <- fromRepo $ gitAnnexTmpLocation key+	tmp <- fromRepo $ gitAnnexTmpObjectLocation key  	e <- liftIO $ doesFileExist tmp 	alreadythere <- if e@@ -250,7 +250,7 @@  prepTmp :: Key -> Annex FilePath prepTmp key = do-	tmp <- fromRepo $ gitAnnexTmpLocation key+	tmp <- fromRepo $ gitAnnexTmpObjectLocation key 	createAnnexDirectory (parentDir tmp) 	return tmp @@ -514,10 +514,8 @@ downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool downloadUrl urls file = go =<< annexWebDownloadCommand <$> Annex.getGitConfig   where-  	go Nothing = do-		opts <- map Param . annexWebOptions <$> Annex.getGitConfig-		headers <- getHttpHeaders-		anyM (\u -> Url.withUserAgent $ Url.download u headers opts file) urls+  	go Nothing = Url.withUrlOptions $ \uo ->+		anyM (\u -> Url.download u file uo) urls 	go (Just basecmd) = liftIO $ anyM (downloadcmd basecmd) urls 	downloadcmd basecmd url = 		boolSystem "sh" [Param "-c", Param $ gencmd url basecmd]
Annex/Init.hs view
@@ -120,7 +120,7 @@ #ifdef mingw32_HOST_OS 	return True #else-	tmp <- fromRepo gitAnnexTmpDir+	tmp <- fromRepo gitAnnexTmpMiscDir 	let f = tmp </> "gaprobe" 	createAnnexDirectory tmp 	liftIO $ writeFile f ""@@ -157,7 +157,7 @@ #ifdef mingw32_HOST_OS 	return False #else-	tmp <- fromRepo gitAnnexTmpDir+	tmp <- fromRepo gitAnnexTmpMiscDir 	let f = tmp </> "gaprobe" 	createAnnexDirectory tmp 	liftIO $ do
Annex/Journal.hs view
@@ -35,11 +35,11 @@  -} setJournalFile :: JournalLocked -> FilePath -> String -> Annex () setJournalFile _jl file content = do+	tmp <- fromRepo gitAnnexTmpMiscDir 	createAnnexDirectory =<< fromRepo gitAnnexJournalDir-	createAnnexDirectory =<< fromRepo gitAnnexTmpDir+	createAnnexDirectory tmp 	-- journal file is written atomically 	jfile <- fromRepo $ journalFile file-	tmp <- fromRepo gitAnnexTmpDir 	let tmpfile = tmp </> takeFileName jfile 	liftIO $ do 		writeBinaryFile tmpfile content
+ Annex/MetaData.hs view
@@ -0,0 +1,61 @@+{- git-annex metadata+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Annex.MetaData where++import Common.Annex+import qualified Annex+import Types.MetaData+import Logs.MetaData+import Annex.CatFile++import qualified Data.Set as S+import qualified Data.Map as M+import Data.Time.Calendar+import Data.Time.Clock+import Data.Time.Clock.POSIX++tagMetaField :: MetaField+tagMetaField = mkMetaFieldUnchecked "tag"++yearMetaField :: MetaField+yearMetaField = mkMetaFieldUnchecked "year"++monthMetaField :: MetaField+monthMetaField = mkMetaFieldUnchecked "month"++{- Adds metadata for a file that has just been ingested into the+ - annex, but has not yet been committed to git.+ -+ - When the file has been modified, the metadata is copied over+ - from the old key to the new key. Note that it looks at the old key as+ - committed to HEAD -- the new key may or may not have already been staged+ - in th annex.+ -+ - Also, can generate new metadata, if configured to do so.+ -}+genMetaData :: Key -> FilePath -> FileStatus -> Annex ()+genMetaData key file status = do+	maybe noop (flip copyMetaData key) =<< catKeyFileHEAD file+	whenM (annexGenMetaData <$> Annex.getGitConfig) $ do+		metadata <- getCurrentMetaData key+		let metadata' = genMetaData' status metadata+		unless (metadata' == emptyMetaData) $+			addMetaData key metadata'++{- Generates metadata from the FileStatus.+ - Does not overwrite any existing metadata values. -}+genMetaData' :: FileStatus -> MetaData -> MetaData+genMetaData' status old = MetaData $ M.fromList $ filter isnew+	[ (yearMetaField, S.singleton $ toMetaValue $ show y)+	, (monthMetaField, S.singleton $ toMetaValue $ show m)+	]+  where+	isnew (f, _) = S.null (currentMetaDataValues f old)+	(y, m, _d) = toGregorian $ utctDay $ +		posixSecondsToUTCTime $ realToFrac $+			modificationTime status
Annex/ReplaceFile.hs view
@@ -24,7 +24,7 @@  -} replaceFile :: FilePath -> (FilePath -> Annex ()) -> Annex () replaceFile file a = do-	tmpdir <- fromRepo gitAnnexTmpDir+	tmpdir <- fromRepo gitAnnexTmpMiscDir 	void $ createAnnexDirectory tmpdir 	bracketIO (setup tmpdir) nukeFile $ \tmpfile -> do 		a tmpfile@@ -36,4 +36,4 @@ 		return tmpfile 	fallback tmpfile _ = do 		createDirectoryIfMissing True $ parentDir file-		rename tmpfile file+		moveFile tmpfile file
Annex/Url.hs view
@@ -1,13 +1,15 @@-{- Url downloading, with git-annex user agent.+{- Url downloading, with git-annex user agent and configured http+ - headers and wget/curl options.  -- - Copyright 2013 Joey Hess <joey@kitenet.net>+ - Copyright 2013-2014 Joey Hess <joey@kitenet.net>  -  - Licensed under the GNU GPL version 3 or higher.  -}  module Annex.Url ( 	module U,-	withUserAgent,+	withUrlOptions,+	getUrlOptions, 	getUserAgent, ) where @@ -23,5 +25,18 @@ getUserAgent = Annex.getState $  	Just . fromMaybe defaultUserAgent . Annex.useragent -withUserAgent :: (Maybe U.UserAgent -> IO a) -> Annex a-withUserAgent a = liftIO . a =<< getUserAgent+getUrlOptions :: Annex U.UrlOptions+getUrlOptions = U.UrlOptions+	<$> getUserAgent+	<*> headers+	<*> options+  where+	headers = do+		v <- annexHttpHeadersCommand <$> Annex.getGitConfig+		case v of+			Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd])+			Nothing -> annexHttpHeaders <$> Annex.getGitConfig+	options = map Param . annexWebOptions <$> Annex.getGitConfig++withUrlOptions :: (U.UrlOptions -> IO a) -> Annex a+withUrlOptions a = liftIO . a =<< getUrlOptions
Annex/View.hs view
@@ -5,11 +5,10 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE CPP #-}- module Annex.View where  import Common.Annex+import Annex.View.ViewedFile import Types.View import Types.MetaData import qualified Git@@ -28,22 +27,16 @@ import Annex.CatFile import Logs.MetaData import Logs.View+import Utility.Glob import Utility.FileMode import Types.Command import Config import CmdLine.Action  import qualified Data.Set as S-import System.Path.WildMatch+import qualified Data.Map as M import "mtl" Control.Monad.Writer -#ifdef WITH_TDFA-import Text.Regex.TDFA-import Text.Regex.TDFA.String-#else-import Text.Regex-#endif- {- Each visible ViewFilter in a view results in another level of  - subdirectory nesting. When a file matches multiple ways, it will appear  - in multiple subdirectories. This means there is a bit of an exponential@@ -127,42 +120,13 @@ combineViewFilter (FilterValues _) newglob@(FilterGlob _) = 	(newglob, Widening) combineViewFilter (FilterGlob oldglob) new@(FilterValues s)-	| all (matchGlob (compileGlob oldglob) . fromMetaValue) (S.toList s) = (new, Narrowing)+	| all (matchGlob (compileGlob oldglob CaseInsensative) . fromMetaValue) (S.toList s) = (new, Narrowing) 	| otherwise = (new, Widening) combineViewFilter (FilterGlob old) newglob@(FilterGlob new) 	| old == new = (newglob, Unchanged)-	| matchGlob (compileGlob old) new = (newglob, Narrowing)+	| matchGlob (compileGlob old CaseInsensative) new = (newglob, Narrowing) 	| otherwise = (newglob, Widening) -{- Converts a filepath used in a reference branch to the- - filename that will be used in the view.- -- - No two filepaths from the same branch should yeild the same result,- - so all directory structure needs to be included in the output file- - in some way. However, the branch's directory structure is not relevant- - in the view.- -- - So, from dir/subdir/file.foo, generate file_{dir;subdir}.foo- -- - (To avoid collisions with a filename that already contains {foo},- - that is doubled to {{foo}}.)- -}-fileViewFromReference :: MkFileView-fileViewFromReference f = concat-	[ double base-	, if null dirs then "" else "_{" ++ double (intercalate ";" dirs) ++ "}"-	, double $ concat extensions-	]-  where-	(path, basefile) = splitFileName f-	dirs = filter (/= ".") $ map dropTrailingPathSeparator (splitPath path)-	(base, extensions) = splitShortExtensions basefile--	double = replace "{" "{{" . replace "}" "}}"--fileViewReuse :: MkFileView-fileViewReuse = takeFileName- {- Generates views for a file from a branch, based on its metadata  - and the filename used in the branch.  -@@ -176,10 +140,10 @@  - evaluate this function with the view parameter and reuse  - the result. The globs in the view will then be compiled and memoized.  -}-fileViews :: View -> MkFileView -> FilePath -> MetaData -> [FileView]-fileViews view = +viewedFiles :: View -> MkViewedFile -> FilePath -> MetaData -> [ViewedFile]+viewedFiles view =  	let matchers = map viewComponentMatcher (viewComponents view)-	in \mkfileview file metadata ->+	in \mkviewedfile file metadata -> 		let matches = map (\m -> m metadata) matchers 		in if any isNothing matches 			then []@@ -187,8 +151,8 @@ 				let paths = pathProduct $ 					map (map toViewPath) (visible matches) 				in if null paths-					then [mkfileview file]-					else map (</> mkfileview file) paths+					then [mkviewedfile file]+					else map (</> mkviewedfile file) paths   where 	visible = map (fromJust . snd) . 		filter (viewVisible . fst) .@@ -205,31 +169,9 @@ 	matcher = case viewFilter viewcomponent of 		FilterValues s -> \values -> S.intersection s values 		FilterGlob glob ->-			let regex = compileGlob glob+			let cglob = compileGlob glob CaseInsensative 			in \values -> -				S.filter (matchGlob regex . fromMetaValue) values--compileGlob :: String -> Regex-compileGlob glob = -#ifdef WITH_TDFA-	case compile (defaultCompOpt {caseSensitive = False}) defaultExecOpt regex of-		Right r -> r-		Left _ -> error $ "failed to compile regex: " ++ regex-#else-	mkRegexWithOpts regex False True-#endif-  where-	regex = '^':wildToRegex glob--matchGlob :: Regex -> String -> Bool-matchGlob regex val = -#ifdef WITH_TDFA-	case execute regex val of-		Right (Just _) -> True-		_ -> False-#else-	isJust $ matchRegex regex val-#endif+				S.filter (matchGlob cglob . fromMetaValue) values  toViewPath :: MetaValue -> FilePath toViewPath = concatMap escapeslash . fromMetaValue@@ -268,23 +210,28 @@   where 	combinel xs ys = [combine x y | x <- xs, y <- ys] -{- Extracts the metadata from a fileview, based on the view that was used- - to construct it. -}-fromView :: View -> FileView -> MetaData-fromView view f = foldr (uncurry updateMetaData) newMetaData (zip fields values)+{- Extracts the metadata from a ViewedFile, based on the view that was used+ - to construct it.+ -+ - Derived metadata is excluded.+ -}+fromView :: View -> ViewedFile -> MetaData+fromView view f = MetaData $+	M.fromList (zip fields values) `M.difference` derived   where 	visible = filter viewVisible (viewComponents view) 	fields = map viewField visible-	paths = splitDirectories $ dropFileName f-	values = map fromViewPath paths+	paths = splitDirectories (dropFileName f)+	values = map (S.singleton . fromViewPath) paths+	MetaData derived = getViewedFileMetaData f  {- Constructing a view that will match arbitrary metadata, and applying- - it to a file yields a set of FileViews which all contain the same+ - it to a file yields a set of ViewedFile which all contain the same  - MetaFields that were present in the input metadata  - (excluding fields that are not visible). -} prop_view_roundtrips :: FilePath -> MetaData -> Bool -> Bool prop_view_roundtrips f metadata visible = null f || viewTooLarge view ||-	all hasfields (fileViews view fileViewFromReference f metadata)+	all hasfields (viewedFiles view viewedFileFromReference f metadata)   where 	view = View (Git.Ref "master") $ 		map (\(mf, mv) -> ViewComponent mf (FilterValues $ S.filter (not . null . fromMetaValue) mv) visible)@@ -292,11 +239,32 @@ 	visiblefields = sort (map viewField $ filter viewVisible (viewComponents view)) 	hasfields fv = sort (map fst (fromMetaData (fromView view fv))) == visiblefields +{- A directory foo/bar/baz/ is turned into metadata fields+ - /=foo, foo/=bar, foo/bar/=baz.+ -+ - Note that this may generate MetaFields that legalField rejects.+ - This is necessary to have a 1:1 mapping between directory names and+ - fields. So this MetaData cannot safely be serialized. -}+getDirMetaData :: FilePath -> MetaData+getDirMetaData d = MetaData $ M.fromList $ zip fields values+  where+	dirs = splitDirectories d+	fields = map (mkMetaFieldUnchecked . addTrailingPathSeparator . joinPath)+		(inits dirs)+	values = map (S.singleton . toMetaValue . fromMaybe "" . headMaybe)+		(tails dirs)++getWorkTreeMetaData :: FilePath -> MetaData+getWorkTreeMetaData = getDirMetaData . dropFileName++getViewedFileMetaData :: FilePath -> MetaData+getViewedFileMetaData = getDirMetaData . dirFromViewedFile . takeFileName+ {- Applies a view to the currently checked out branch, generating a new  - branch for the view.  -} applyView :: View -> Annex Git.Branch-applyView view = applyView' fileViewFromReference view+applyView view = applyView' viewedFileFromReference getWorkTreeMetaData view  {- Generates a new branch for a View, which must be a more narrow  - version of the View originally used to generate the currently@@ -304,18 +272,18 @@  - in view, not any others.  -} narrowView :: View -> Annex Git.Branch-narrowView = applyView' fileViewReuse+narrowView = applyView' viewedFileReuse getViewedFileMetaData  {- Go through each file in the currently checked out branch.  - If the file is not annexed, skip it, unless it's a dotfile in the top.- - Look up the metadata of annexed files, and generate any FileViews,+ - Look up the metadata of annexed files, and generate any ViewedFiles,  - and stage them.  -  - Currently only works in indirect mode. Must be run from top of  - repository.  -}-applyView' :: MkFileView -> View -> Annex Git.Branch-applyView' mkfileview view = do+applyView' :: MkViewedFile -> (FilePath -> MetaData) -> View -> Annex Git.Branch+applyView' mkviewedfile getfilemetadata view = do 	top <- fromRepo Git.repoPath 	(l, clean) <- inRepo $ Git.LsFiles.inRepo [top] 	liftIO . nukeFile =<< fromRepo gitAnnexViewIndex@@ -329,10 +297,11 @@ 			void $ stopUpdateIndex uh 			void clean   where-	genfileviews = fileViews view mkfileview -- enables memoization+	genviewedfiles = viewedFiles view mkviewedfile -- enables memoization 	go uh hasher f (Just (k, _)) = do 		metadata <- getCurrentMetaData k-		forM_ (genfileviews f metadata) $ \fv -> do+		let metadata' = getfilemetadata f `unionMetaData` metadata+		forM_ (genviewedfiles f metadata') $ \fv -> do 			stagesymlink uh hasher fv =<< inRepo (gitAnnexLink fv k) 	go uh hasher f Nothing 		| "." `isPrefixOf` f = do@@ -381,7 +350,7 @@  - Note that removes must be handled before adds. This is so  - that moving a file from x/foo/ to x/bar/ adds back the metadata for x.  -}-withViewChanges :: (FileView -> Key -> CommandStart) -> (FileView -> Key -> CommandStart) -> Annex ()+withViewChanges :: (ViewedFile -> Key -> CommandStart) -> (ViewedFile -> Key -> CommandStart) -> Annex () withViewChanges addmeta removemeta = do 	makeabs <- flip fromTopFilePath <$> gitRepo 	(diffs, cleanup) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef
+ Annex/View/ViewedFile.hs view
@@ -0,0 +1,75 @@+{- filenames (not paths) used in views+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++module Annex.View.ViewedFile (+	ViewedFile,+	MkViewedFile,+	viewedFileFromReference,+	viewedFileReuse,+	dirFromViewedFile,+	prop_viewedFile_roundtrips,+) where++import Common.Annex++type FileName = String+type ViewedFile = FileName++type MkViewedFile = FilePath -> ViewedFile++{- Converts a filepath used in a reference branch to the+ - filename that will be used in the view.+ -+ - No two filepaths from the same branch should yeild the same result,+ - so all directory structure needs to be included in the output filename+ - in some way.+ -+ - So, from dir/subdir/file.foo, generate file_%dir%subdir%.foo+ -}+viewedFileFromReference :: MkViewedFile+viewedFileFromReference f = concat+	[ escape base+	, if null dirs then "" else "_%" ++ intercalate "%" (map escape dirs) ++ "%"+	, escape $ concat extensions+	]+  where+	(path, basefile) = splitFileName f+	dirs = filter (/= ".") $ map dropTrailingPathSeparator (splitPath path)+	(base, extensions) = splitShortExtensions basefile++	{- To avoid collisions with filenames or directories that contain+	 - '%', and to allow the original directories to be extracted+	 - from the ViewedFile, '%' is escaped to '\%' (and '\' to '\\').+	 -}+	escape :: String -> String+	escape = replace "%" "\\%" . replace "\\" "\\\\"++{- For use when operating already within a view, so whatever filepath+ - is present in the work tree is already a ViewedFile. -}+viewedFileReuse :: MkViewedFile+viewedFileReuse = takeFileName++{- Extracts from a ViewedFile the directory where the file is located on+ - in the reference branch. -}+dirFromViewedFile :: ViewedFile -> FilePath+dirFromViewedFile = joinPath . drop 1 . sep [] ""+  where+  	sep l _ [] = reverse l+	sep l curr (c:cs)+		| c == '%' = sep (reverse curr:l) "" cs+		| c == '\\' = case cs of+			(c':cs') -> sep l (c':curr) cs'+			[] -> sep l curr cs+		| otherwise = sep l (c:curr) cs++prop_viewedFile_roundtrips :: FilePath -> Bool+prop_viewedFile_roundtrips f+	-- Relative filenames wanted, not directories.+	| any (isPathSeparator) (end f ++ beginning f) = True+	| otherwise = dir == dirFromViewedFile (viewedFileFromReference f)+  where+	dir = joinPath $ beginning $ splitDirectories f
Assistant.hs view
@@ -49,11 +49,13 @@ import Assistant.Types.UrlRenderer #endif import qualified Utility.Daemon-import Utility.LogFile import Utility.ThreadScheduler import Utility.HumanTime-import Annex.Perms import qualified Build.SysConfig as SysConfig+#ifndef mingw32_HOST_OS+import Utility.LogFile+import Annex.Perms+#endif  import System.Log.Logger import Network.Socket (HostName)@@ -70,8 +72,8 @@ startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = do 	Annex.changeState $ \s -> s { Annex.daemon = True } 	pidfile <- fromRepo gitAnnexPidFile-	logfile <- fromRepo gitAnnexLogFile #ifndef mingw32_HOST_OS+	logfile <- fromRepo gitAnnexLogFile 	createAnnexDirectory (parentDir logfile) 	logfd <- liftIO $ openLog logfile 	if foreground@@ -93,11 +95,12 @@ 			start (Utility.Daemon.daemonize logfd (Just pidfile) False) Nothing #else 	-- Windows is always foreground, and has no log file.-	liftIO $ Utility.Daemon.lockPidFile pidfile-	start id $ do-		case startbrowser of-			Nothing -> Nothing-			Just a -> Just $ a Nothing Nothing+	when (foreground || not foreground) $ do+		liftIO $ Utility.Daemon.lockPidFile pidfile+		start id $ do+			case startbrowser of+				Nothing -> Nothing+				Just a -> Just $ a Nothing Nothing #endif   where   	desc
Assistant/Restart.hs view
@@ -30,6 +30,7 @@ #else import Utility.WinProcess #endif+import Data.Default  {- Before the assistant can be restarted, have to remove our   - gitAnnexUrlFile and our gitAnnexPidFile. Pausing the watcher is also@@ -81,7 +82,7 @@ 				( return url 				, delayed $ waiturl urlfile 				)-	listening url = catchBoolIO $ fst <$> exists url [] Nothing+	listening url = catchBoolIO $ fst <$> exists url def 	delayed a = do 		threadDelay 100000 -- 1/10th of a second 		a
Assistant/Threads/Committer.hs view
@@ -448,7 +448,7 @@ 			let segments = segmentXargs $ map keyFilename keysources 			concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) 		, do-			tmpdir <- fromRepo gitAnnexTmpDir+			tmpdir <- fromRepo gitAnnexTmpMiscDir 			liftIO $ Lsof.queryDir tmpdir 		)
Assistant/Threads/NetWatcher.hs view
@@ -24,7 +24,9 @@ import Data.Word (Word32) import Assistant.NetMessager #else+#ifdef linux_HOST_OS #warning Building without dbus support; will poll for network connection changes+#endif #endif  netWatcherThread :: NamedThread
Assistant/Threads/SanityChecker.hs view
@@ -27,7 +27,6 @@ import qualified Git.Config import Utility.ThreadScheduler import qualified Assistant.Threads.Watcher as Watcher-import Utility.LogFile import Utility.Batch import Utility.NotificationBroadcaster import Config@@ -43,6 +42,9 @@ #ifdef WITH_WEBAPP import Assistant.WebApp.Types #endif+#ifndef mingw32_HOST_OS+import Utility.LogFile+#endif  import Data.Time.Clock.POSIX import qualified Data.Text as T@@ -214,10 +216,10 @@ 			checkLogSize $ n + 1   where 	filesize f = fromIntegral . fileSize <$> liftIO (getFileStatus f)-#endif -oneMegabyte :: Int-oneMegabyte = 1000000+	oneMegabyte :: Int+	oneMegabyte = 1000000+#endif  oneHour :: Int oneHour = 60 * 60
Assistant/Threads/TransferPoller.hs view
@@ -35,7 +35,7 @@ 		{- Downloads are polled by checking the size of the 		 - temp file being used for the transfer. -} 		| transferDirection t == Download = do-			let f = gitAnnexTmpLocation (transferKey t) g+			let f = gitAnnexTmpObjectLocation (transferKey t) g 			sz <- liftIO $ catchMaybeIO $ 				fromIntegral . fileSize <$> getFileStatus f 			newsize t info sz
Assistant/Threads/Upgrader.hs view
@@ -89,10 +89,10 @@  getDistributionInfo :: Assistant (Maybe GitAnnexDistribution) getDistributionInfo = do-	ua <- liftAnnex Url.getUserAgent+	uo <- liftAnnex Url.getUrlOptions 	liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do 		hClose h-		ifM (Url.downloadQuiet distributionInfoUrl [] [] tmpfile ua)+		ifM (Url.downloadQuiet distributionInfoUrl tmpfile uo) 			( readish <$> readFileStrict tmpfile 			, return Nothing 			)
Assistant/Threads/Watcher.hs view
@@ -23,7 +23,6 @@ import Assistant.Alert import Utility.DirWatcher import Utility.DirWatcher.Types-import qualified Utility.Lsof as Lsof import qualified Annex import qualified Annex.Queue import qualified Git@@ -40,6 +39,9 @@ import Git.Types import Config import Utility.ThreadScheduler+#ifndef mingw32_HOST_OS+import qualified Utility.Lsof as Lsof+#endif  import Data.Bits.Utils import Data.Typeable
Assistant/WebApp/Configurators/IA.hs view
@@ -190,8 +190,8 @@  getRepoInfo :: RemoteConfig -> Widget getRepoInfo c = do-	ua <- liftAnnex Url.getUserAgent-	exists <- liftIO $ catchDefaultIO False $ fst <$> Url.exists url [] ua+	uo <- liftAnnex Url.getUrlOptions+	exists <- liftIO $ catchDefaultIO False $ fst <$> Url.exists url uo 	[whamlet| <a href="#{url}">   Internet Archive item
Assistant/WebApp/Configurators/Local.hs view
@@ -114,9 +114,9 @@  - browsed to a directory with git-annex and run it from there. -} defaultRepositoryPath :: Bool -> IO FilePath defaultRepositoryPath firstrun = do-	cwd <- liftIO getCurrentDirectory-	home <- myHomeDir #ifndef mingw32_HOST_OS+	home <- myHomeDir+	cwd <- liftIO getCurrentDirectory 	if home == cwd && firstrun 		then inhome 		else ifM (legit cwd <&&> canWrite cwd)@@ -127,7 +127,7 @@ 	-- On Windows, always default to ~/Desktop/annex or ~/annex, 	-- no cwd handling because the user might be able to write 	-- to the entire drive.-	inhome+	if firstrun then inhome else inhome #endif   where 	inhome = do@@ -136,9 +136,11 @@ 			( relHome $ desktop </> gitAnnexAssistantDefaultDir 			, return $ "~" </> gitAnnexAssistantDefaultDir 			)+#ifndef mingw32_HOST_OS 	-- Avoid using eg, standalone build's git-annex.linux/ directory 	-- when run from there. 	legit d = not <$> doesFileExist (d </> "git-annex")+#endif  newRepositoryForm :: FilePath -> Hamlet.Html -> MkMForm RepositoryPath newRepositoryForm defpath msg = do
Assistant/WebApp/Configurators/Pairing.hs view
@@ -12,7 +12,6 @@  import Assistant.Pairing import Assistant.WebApp.Common-import Assistant.WebApp.Configurators import Assistant.Types.Buddies import Annex.UUID #ifdef WITH_PAIRING@@ -32,6 +31,7 @@ import Assistant.Types.NetMessager import Assistant.NetMessager import Assistant.WebApp.RepoList+import Assistant.WebApp.Configurators import Assistant.WebApp.Configurators.XMPP #endif import Utility.UserInfo
Assistant/XMPP/Git.hs view
@@ -187,7 +187,7 @@ 		v <- liftIO $ getEnv "GIT_ANNEX_TMP_DIR" 		case v of 			Nothing -> do-				tmp <- liftAnnex $ fromRepo gitAnnexTmpDir+				tmp <- liftAnnex $ fromRepo gitAnnexTmpMiscDir 				return $ tmp </> "xmppgit" 			Just d -> return $ d </> "xmppgit"
Build/DistributionUpdate.hs view
@@ -21,6 +21,11 @@  makeinfos :: Annex () makeinfos = do+	void $ inRepo $ runBool +		[ Param "commit"+		, Param "-m"+		, Param $ "publishing git-annex " ++ version+		] 	basedir <- liftIO getRepoDir 	version <- liftIO getChangelogVersion 	now <- liftIO getCurrentTime@@ -44,7 +49,7 @@ 	void $ inRepo $ runBool  		[ Param "commit" 		, Param "-m"-		, Param $ "publishing git-annex " ++ version+		, Param $ "updated info files for git-annex " ++ version 		] 	void $ inRepo $ runBool 		[ Param "annex"
CHANGELOG view
@@ -1,3 +1,38 @@+git-annex (5.20140227) unstable; urgency=medium++  * metadata: Field names limited to alphanumerics and a few whitelisted+    punctuation characters to avoid issues with views, etc.+  * metadata: Field names are now case insensative.+  * When constructing views, metadata is available about the location of the+    file in the view's reference branch. Allows incorporating parts of the+    directory hierarchy in a view.+    For example `git annex view tag=* podcasts/=*` makes a view in the form+    tag/showname.+  * --metadata field=value can now use globs to match, and matches+    case insensatively, the same as git annex view field=value does.+  * annex.genmetadata can be set to make git-annex automatically set+    metadata (year and month) when adding files.+  * Make annex.web-options be used in several places that call curl.+  * Fix handling of rsync remote urls containing a username,+    including rsync.net.+  * Preserve metadata when staging a new version of an annexed file.+  * metadata: Support --json+  * webapp: Fix creation of box.com and Amazon S3 and Glacier+    repositories, broken in 5.20140221.+  * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid+    locking files, which is not needed by git-annex's use of webdav, and+    does not work on Box.com.+  * webdav: Fix path separator bug when used on Windows.+  * repair: Optimise unpacking of pack files, and avoid repeated error+    messages about corrupt pack files.+  * Add build dep on regex-compat to fix build on mipsel, which lacks+    regex-tdfa.+  * Disable test suite on sparc, which is missing optparse-applicative.+  * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp+    for only partially transferred objects.++ -- Joey Hess <joeyh@debian.org>  Thu, 27 Feb 2014 11:34:19 -0400+ git-annex (5.20140221) unstable; urgency=medium    * metadata: New command that can attach metadata to files.
Command/Add.hs view
@@ -19,6 +19,7 @@ import Annex.Content.Direct import Annex.Perms import Annex.Link+import Annex.MetaData import qualified Annex import qualified Annex.Queue #ifdef WITH_CLIBS@@ -95,7 +96,7 @@ lockDown file = ifM crippledFileSystem 	( liftIO $ catchMaybeIO nohardlink 	, do-		tmp <- fromRepo gitAnnexTmpDir+		tmp <- fromRepo gitAnnexTmpMiscDir 		createAnnexDirectory tmp 		eitherToMaybe <$> tryAnnexIO (go tmp) 	)@@ -145,26 +146,32 @@ ingest (Just source) = do 	backend <- chooseBackend $ keyFilename source 	k <- genKey source backend-	cache <- liftIO $ genInodeCache $ contentLocation source-	case (cache, inodeCache source) of-		(_, Nothing) -> go k cache-		(Just newc, Just c) | compareStrong c newc -> go k cache+	ms <- liftIO $ catchMaybeIO $ getFileStatus $ contentLocation source+	let mcache = toInodeCache =<< ms+	case (mcache, inodeCache source) of+		(_, Nothing) -> go k mcache ms+		(Just newc, Just c) | compareStrong c newc -> go k mcache ms 		_ -> failure "changed while it was being added"   where-	go k cache = ifM isDirect ( godirect k cache , goindirect k cache )+	go k mcache ms = ifM isDirect+		( godirect k mcache ms+		, goindirect k mcache ms+		) -	goindirect (Just (key, _)) mcache = do+	goindirect (Just (key, _)) mcache ms = do 		catchAnnex (moveAnnex key $ contentLocation source) 			(undo (keyFilename source) key)+		maybe noop (genMetaData key (keyFilename source)) ms 		liftIO $ nukeFile $ keyFilename source 		return $ (Just key, mcache)-	goindirect Nothing _ = failure "failed to generate a key"+	goindirect _ _ _ = failure "failed to generate a key" -	godirect (Just (key, _)) (Just cache) = do+	godirect (Just (key, _)) (Just cache) ms = do 		addInodeCache key cache+		maybe noop (genMetaData key (keyFilename source)) ms 		finishIngestDirect key source 		return $ (Just key, Just cache)-	godirect _ _ = failure "failed to generate a key"+	godirect _ _ _ = failure "failed to generate a key"  	failure msg = do 		warning $ keyFilename source ++ " " ++ msg@@ -211,15 +218,15 @@ 	l <- inRepo $ gitAnnexLink file key 	replaceFile file $ makeAnnexLink l -#ifdef WITH_CLIBS-#ifndef __ANDROID__ 	-- touch symlink to have same time as the original file, 	-- as provided in the InodeCache 	case mcache of+#if defined(WITH_CLIBS) && ! defined(__ANDROID__) 		Just c -> liftIO $ touch file (TimeSpec $ inodeCacheToMtime c) False-		Nothing -> noop-#endif+#else+		Just _ -> noop #endif+		Nothing -> noop  	return l
Command/AddUrl.hs view
@@ -114,7 +114,7 @@ 			 - it later. -} 			sizedkey <- addSizeUrlKey videourl key 			prepGetViaTmpChecked sizedkey $ do-				tmp <- fromRepo $ gitAnnexTmpLocation key+				tmp <- fromRepo $ gitAnnexTmpObjectLocation key 				showOutput 				ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do 					liftIO $ createDirectoryIfMissing True (parentDir tmp)@@ -134,8 +134,7 @@ 			setUrlPresent key url 			next $ return True 		| otherwise = do-			headers <- getHttpHeaders-			(exists, samesize) <- Url.withUserAgent $ Url.check url headers $ keySize key+			(exists, samesize) <- Url.withUrlOptions $ Url.check url (keySize key) 			if exists && samesize 				then do 					setUrlPresent key url@@ -163,7 +162,7 @@ 	 - downloads, as the dummy key for a given url is stable. -} 	dummykey <- addSizeUrlKey url =<< Backend.URL.fromUrl url Nothing 	prepGetViaTmpChecked dummykey $ do-		tmp <- fromRepo $ gitAnnexTmpLocation dummykey+		tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey 		showOutput 		ifM (runtransfer dummykey tmp) 			( do@@ -192,8 +191,7 @@  -} addSizeUrlKey :: URLString -> Key -> Annex Key addSizeUrlKey url key = do-	headers <- getHttpHeaders-	size <- snd <$> Url.withUserAgent (Url.exists url headers)+	size <- snd <$> Url.withUrlOptions (Url.exists url) 	return $ key { keySize = size }  cleanup :: URLString -> FilePath -> Key -> Maybe FilePath -> Annex Bool@@ -212,10 +210,9 @@  nodownload :: Bool -> URLString -> FilePath -> Annex Bool nodownload relaxed url file = do-	headers <- getHttpHeaders 	(exists, size) <- if relaxed 		then pure (True, Nothing)-		else Url.withUserAgent $ Url.exists url headers+		else Url.withUrlOptions (Url.exists url) 	if exists 		then do 			key <- Backend.URL.fromUrl url size
Command/DropUnused.hs view
@@ -27,7 +27,7 @@ 	withUnusedMaps (start numcopies) ps  start :: NumCopies -> UnusedMaps -> Int -> CommandStart-start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpLocation)+start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpObjectLocation)  perform :: NumCopies -> Key -> CommandPerform perform numcopies key = maybe droplocal dropremote =<< Remote.byNameWithUUID =<< from
Command/Fsck.hs view
@@ -147,7 +147,7 @@ 		] 	withtmp a = do 		pid <- liftIO getPID-		t <- fromRepo gitAnnexTmpDir+		t <- fromRepo gitAnnexTmpObjectDir 		createAnnexDirectory t 		let tmp = t </> "fsck" ++ show pid ++ "." ++ keyFile key 		let cleanup = liftIO $ catchIO (removeFile tmp) (const noop)@@ -466,7 +466,8 @@  - To guard against time stamp damange (for example, if an annex directory  - is copied without -a), the fsckstate file contains a time that should  - be identical to its modification time.- - (This is not possible to do on Windows.)+ - (This is not possible to do on Windows, and so the timestamp in+ - the file will only be equal or greater than the modification time.)  -} recordStartTime :: Annex () recordStartTime = do@@ -477,10 +478,10 @@ 		withFile f WriteMode $ \h -> do #ifndef mingw32_HOST_OS 			t <- modificationTime <$> getFileStatus f-			hPutStr h $ showTime $ realToFrac t #else-			noop+			t <- getPOSIXTime #endif+			hPutStr h $ showTime $ realToFrac t   where 	showTime :: POSIXTime -> String 	showTime = show@@ -494,15 +495,18 @@ 	f <- fromRepo gitAnnexFsckState 	liftIO $ catchDefaultIO Nothing $ do 		timestamp <- modificationTime <$> getFileStatus f-#ifndef mingw32_HOST_OS-		t <- readishTime <$> readFile f-		return $ if Just (realToFrac timestamp) == t+		let fromstatus = Just (realToFrac timestamp)+		fromfile <- readishTime <$> readFile f+		return $ if matchingtimestamp fromfile fromstatus 			then Just timestamp 			else Nothing-#else-		return $ Just timestamp-#endif   where 	readishTime :: String -> Maybe POSIXTime 	readishTime s = utcTimeToPOSIXSeconds <$> 		parseTime defaultTimeLocale "%s%Qs" s+	matchingtimestamp fromfile fromstatus =+#ifndef mingw32_HOST_OS+		fromfile == fromstatus+#else+		fromfile >= fromstatus+#endif
Command/ImportFeed.hs view
@@ -121,10 +121,10 @@ downloadFeed :: URLString -> Annex (Maybe Feed) downloadFeed url = do 	showOutput-	ua <- Url.getUserAgent+	uo <- Url.getUrlOptions 	liftIO $ withTmpFile "feed" $ \f h -> do 		fileEncoding h-		ifM (Url.download url [] [] f ua)+		ifM (Url.download url f uo) 			( parseFeedString <$> hGetContentsStrict h 			, return Nothing 			)
Command/Info.hs view
@@ -201,7 +201,7 @@ 	showSizeKeys <$> cachedReferencedData  tmp_size :: Stat-tmp_size = staleSize "temporary directory size" gitAnnexTmpDir+tmp_size = staleSize "temporary object directory size" gitAnnexTmpObjectDir  bad_data_size :: Stat bad_data_size = staleSize "bad keys size" gitAnnexBadDir
Command/MetaData.hs view
@@ -10,6 +10,7 @@ import Common.Annex import qualified Annex import Command+import Annex.MetaData import Logs.MetaData import Types.MetaData @@ -17,7 +18,7 @@ import Data.Time.Clock.POSIX  def :: [Command]-def = [withOptions [setOption, tagOption, untagOption] $+def = [withOptions [setOption, tagOption, untagOption, jsonOption] $ 	command "metadata" paramPaths seek 	SectionMetaData "sets metadata of a file"] @@ -55,14 +56,16 @@ perform _ [] k = next $ cleanup k perform now ms k = do 	oldm <- getCurrentMetaData k-	let m = foldl' unionMetaData newMetaData $ map (modMeta oldm) ms+	let m = foldl' unionMetaData emptyMetaData $ map (modMeta oldm) ms 	addMetaData' k m now 	next $ cleanup k 	 cleanup :: Key -> CommandCleanup cleanup k = do-	m <- getCurrentMetaData k-	showLongNote $ unlines $ concatMap showmeta $ fromMetaData $ currentMetaData m+	l <- map unwrapmeta . fromMetaData <$> getCurrentMetaData k+	maybeShowJSON l+	showLongNote $ unlines $ concatMap showmeta l 	return True   where-	showmeta (f, vs) = map (\v -> fromMetaField f ++ "=" ++ fromMetaValue v) $ S.toList vs+	unwrapmeta (f, v) = (fromMetaField f, map fromMetaValue (S.toList v))+	showmeta (f, vs) = map ((f ++ "=") ++) vs
Command/PreCommit.hs view
@@ -14,6 +14,7 @@ import qualified Command.Fix import Annex.Direct import Annex.View+import Annex.View.ViewedFile import Logs.View import Logs.MetaData import Types.View@@ -52,12 +53,12 @@ startDirect :: [String] -> CommandStart startDirect _ = next $ next $ preCommitDirect -addViewMetaData :: View -> FileView -> Key -> CommandStart+addViewMetaData :: View -> ViewedFile -> Key -> CommandStart addViewMetaData v f k = do 	showStart "metadata" f 	next $ next $ changeMetaData k $ fromView v f -removeViewMetaData :: View -> FileView -> Key -> CommandStart+removeViewMetaData :: View -> ViewedFile -> Key -> CommandStart removeViewMetaData v f k = do 	showStart "metadata" f 	next $ next $ changeMetaData k $ unsetMetaData $ fromView v f
Command/Unlock.hs view
@@ -36,7 +36,7 @@ 	unlessM (checkDiskSpace Nothing key 0) $ error "cannot unlock"  	src <- calcRepo $ gitAnnexLocation key-	tmpdest <- fromRepo $ gitAnnexTmpLocation key+	tmpdest <- fromRepo $ gitAnnexTmpObjectLocation key 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest) 	showAction "copying" 	ifM (liftIO $ copyFileExternal src tmpdest)
Command/Unused.hs view
@@ -63,7 +63,7 @@ checkUnused = chain 0 	[ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast 	, check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir False-	, check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpDir True+	, check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpObjectDir True 	]   where 	findunused True = do
Command/VAdd.hs view
@@ -10,11 +10,11 @@ import Common.Annex import Command import Annex.View-import Command.View (paramView, parseViewParam, checkoutViewBranch)+import Command.View (parseViewParam, checkoutViewBranch)  def :: [Command]-def = [notBareRepo $ notDirect $-	command "vadd" paramView seek SectionMetaData "add subdirs to current view"]+def = [notBareRepo $ notDirect $ command "vadd" (paramRepeating "FIELD=GLOB")+	seek SectionMetaData "add subdirs to current view"]  seek :: CommandSeek seek = withWords start
Command/View.hs view
@@ -14,6 +14,7 @@ import qualified Git.Ref import qualified Git.Branch import Types.MetaData+import Annex.MetaData import Types.View import Annex.View import Logs.View@@ -43,12 +44,19 @@ 	next $ checkoutViewBranch view applyView  paramView :: String-paramView = paramPair (paramRepeating "FIELD=VALUE") (paramRepeating "TAG")+paramView = paramPair (paramRepeating "TAG") (paramRepeating "FIELD=VALUE") +{- Parse field=value+ -+ - Note that the field may not be a legal metadata field name,+ - but it's let through anyway.+ - This is useful when matching on directory names with spaces,+ - which are not legal MetaFields.+ -} parseViewParam :: String -> (MetaField, String) parseViewParam s = case separate (== '=') s of 	(tag, []) -> (tagMetaField, tag)-	(field, wanted) -> either error (\f -> (f, wanted)) (mkMetaField field)+	(field, wanted) -> (mkMetaFieldUnchecked field, wanted)  mkView :: [String] -> Annex View mkView params = do
Config.hs view
@@ -79,11 +79,3 @@ setCrippledFileSystem b = do 	setConfig (annexConfig "crippledfilesystem") (Git.Config.boolConfig b) 	Annex.changeGitConfig $ \c -> c { annexCrippledFileSystem = b }--{- Gets the http headers to use. -}-getHttpHeaders :: Annex [String]-getHttpHeaders = do-	v <- annexHttpHeadersCommand <$> Annex.getGitConfig-	case v of-		Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd])-		Nothing -> annexHttpHeaders <$> Annex.getGitConfig
Git/Command.hs view
@@ -15,9 +15,6 @@ import Git import Git.Types import qualified Utility.CoProcess as CoProcess-#ifdef mingw32_HOST_OS-import Git.FilePath-#endif import Utility.Batch  {- Constructs a git command line operating on the specified repo. -}
Git/Queue.hs view
@@ -17,16 +17,17 @@ 	flush, ) where -import qualified Data.Map as M-import System.IO-import System.Process- import Utility.SafeCommand import Common import Git import Git.Command import qualified Git.UpdateIndex +import qualified Data.Map as M+#ifndef mingw32_HOST_OS+import System.Process+#endif+ {- Queable actions that can be performed in a git repository.  -} data Action@@ -147,8 +148,9 @@ runAction repo (UpdateIndexAction streamers) = 	-- list is stored in reverse order 	Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers-runAction repo action@(CommandAction {}) = +runAction repo action@(CommandAction {}) = do #ifndef mingw32_HOST_OS+	let p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } 	withHandle StdinHandle createProcessSuccess p $ \h -> do 		fileEncoding h 		hPutStr h $ intercalate "\0" $ toCommand $ getFiles action@@ -162,6 +164,5 @@ 			void $ boolSystem "git" (gitparams ++ [f]) #endif   where-	p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } 	gitparams = gitCommandLine 		(Param (getSubcommand action):getParams action) repo
Git/Repair.hs view
@@ -75,24 +75,35 @@ 			return True 		else return False +{- Explodes all pack files, and deletes them.+ -+ - First moves all pack files to a temp dir, before unpacking them each in+ - turn.+ -+ - This is because unpack-objects will not unpack a pack file if it's in the+ - git repo.+ -+ - Also, this prevents unpack-objects from possibly looking at corrupt+ - pack files to see if they contain an object, while unpacking a+ - non-corrupt pack file.+ -} explodePacks :: Repo -> IO Bool-explodePacks r = do-	packs <- listPackFiles r-	if null packs-		then return False-		else do-			putStrLn "Unpacking all pack files."-			mapM_ go packs-			return True+explodePacks r = go =<< listPackFiles r   where-	go packfile = withTmpFileIn (localGitDir r) "pack" $ \tmp _ -> do-		moveFile packfile tmp-		nukeFile $ packIdxFile packfile-		allowRead tmp-		-- May fail, if pack file is corrupt.-		void $ tryIO $-			pipeWrite [Param "unpack-objects", Param "-r"] r $ \h ->+	go [] = return False+	go packs = withTmpDir "packs" $ \tmpdir -> do+		putStrLn "Unpacking all pack files."+		forM_ packs $ \packfile -> do+			moveFile packfile (tmpdir </> takeFileName packfile)+			nukeFile $ packIdxFile packfile+		forM_ packs $ \packfile -> do+			let tmp = tmpdir </> takeFileName packfile+			allowRead tmp+			-- May fail, if pack file is corrupt.+			void $ tryIO $+				pipeWrite [Param "unpack-objects", Param "-r"] r $ \h -> 				L.hPut h =<< L.readFile tmp+		return True  {- Try to retrieve a set of missing objects, from the remotes of a  - repository. Returns any that could not be retreived.
Limit.hs view
@@ -5,8 +5,6 @@  - Licensed under the GNU GPL version 3 or higher.  -} -{-# LANGUAGE CPP #-}- module Limit where  import Common.Annex@@ -29,19 +27,14 @@ import Logs.Unused import Logs.Location import Git.Types (RefDate(..))+import Utility.Glob import Utility.HumanTime import Utility.DataUnits  import Data.Time.Clock.POSIX import qualified Data.Set as S import qualified Data.Map as M-import System.Path.WildMatch -#ifdef WITH_TDFA-import Text.Regex.TDFA-import Text.Regex.TDFA.String-#endif- {- Checks if there are user-specified limits. -} limited :: Annex Bool limited = (not . Utility.Matcher.isEmpty) <$> getMatcher'@@ -82,33 +75,21 @@ addInclude = addLimit . limitInclude  limitInclude :: MkLimit-limitInclude glob = Right $ const $ return . matchglob glob+limitInclude glob = Right $ const $ return . matchGlobFile glob  {- Add a limit to skip files that match the glob. -} addExclude :: String -> Annex () addExclude = addLimit . limitExclude  limitExclude :: MkLimit-limitExclude glob = Right $ const $ return . not . matchglob glob+limitExclude glob = Right $ const $ return . not . matchGlobFile glob -{- Could just use wildCheckCase, but this way the regex is only compiled- - once. Also, we use regex-TDFA when available, because it's less buggy- - in its support of non-unicode characters. -}-matchglob :: String -> MatchInfo -> Bool-matchglob glob (MatchingFile fi) =-#ifdef WITH_TDFA-	case cregex of-		Right r -> case execute r (matchFile fi) of-			Right (Just _) -> True-			_ -> False-		Left _ -> error $ "failed to compile regex: " ++ regex-  where-	cregex = compile defaultCompOpt defaultExecOpt regex-	regex = '^':wildToRegex glob-#else-	wildCheckCase glob (matchFile fi)-#endif-matchglob _ (MatchingKey _) = False+matchGlobFile :: String -> (MatchInfo -> Bool)+matchGlobFile glob = go+	where+		cglob = compileGlob glob CaseSensative -- memoized+		go (MatchingKey _) = False+		go (MatchingFile fi) = matchGlob cglob (matchFile fi)  {- Adds a limit to skip files not believed to be present  - in a specfied repository. Optionally on a prior date. -}@@ -270,9 +251,13 @@ limitMetaData :: MkLimit limitMetaData s = case parseMetaData s of 	Left e -> Left e-	Right (f, v) -> Right $ const $ checkKey (check f v)+	Right (f, v) ->+		let cglob = compileGlob (fromMetaValue v) CaseInsensative+		in Right $ const $ checkKey (check f cglob)   where-  	check f v k = S.member v . metaDataValues f <$> getCurrentMetaData k+  	check f cglob k = not . S.null +		. S.filter (matchGlob cglob . fromMetaValue) +		. metaDataValues f <$> getCurrentMetaData k  addTimeLimit :: String -> Annex () addTimeLimit s = do
Locations.hs view
@@ -23,8 +23,9 @@ 	annexLocation, 	gitAnnexDir, 	gitAnnexObjectDir,-	gitAnnexTmpDir,-	gitAnnexTmpLocation,+	gitAnnexTmpMiscDir,+	gitAnnexTmpObjectDir,+	gitAnnexTmpObjectLocation, 	gitAnnexBadDir, 	gitAnnexBadLocation, 	gitAnnexUnusedLog,@@ -180,13 +181,17 @@ gitAnnexObjectDir :: Git.Repo -> FilePath gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r </> objectDir -{- .git/annex/tmp/ is used for temp files -}-gitAnnexTmpDir :: Git.Repo -> FilePath-gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"+{- .git/annex/misctmp/ is used for random temp files -}+gitAnnexTmpMiscDir :: Git.Repo -> FilePath+gitAnnexTmpMiscDir r = addTrailingPathSeparator $ gitAnnexDir r </> "misctmp" +{- .git/annex/tmp/ is used for temp files for key's contents -}+gitAnnexTmpObjectDir :: Git.Repo -> FilePath+gitAnnexTmpObjectDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"+ {- The temp file to use for a given key's content. -}-gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath-gitAnnexTmpLocation key r = gitAnnexTmpDir r </> keyFile key+gitAnnexTmpObjectLocation :: Key -> Git.Repo -> FilePath+gitAnnexTmpObjectLocation key r = gitAnnexTmpObjectDir r </> keyFile key  {- .git/annex/bad/ is used for bad files found during fsck -} gitAnnexBadDir :: Git.Repo -> FilePath
Logs/MetaData.hs view
@@ -28,10 +28,10 @@  module Logs.MetaData ( 	getCurrentMetaData,-	getMetaData, 	addMetaData, 	addMetaData', 	currentMetaData,+	copyMetaData, ) where  import Common.Annex@@ -55,7 +55,7 @@ getCurrentMetaData :: Key -> Annex MetaData getCurrentMetaData = currentMetaData . collect <$$> getMetaData   where-	collect = foldl' unionMetaData newMetaData . map value . S.toAscList+	collect = foldl' unionMetaData emptyMetaData . map value . S.toAscList  {- Adds in some metadata, which can override existing values, or unset  - them, but otherwise leaves any existing metadata as-is. -}@@ -129,9 +129,26 @@  	go c _ [] = c 	go c newer (l:ls)-		| unique == newMetaData = go c newer ls+		| unique == emptyMetaData = go c newer ls 		| otherwise = go (l { value = unique } : c) 			(unionMetaData unique newer) ls 	  where 		older = value l 		unique = older `differenceMetaData` newer++{- Copies the metadata from the old key to the new key.+ -+ - The exact content of the metadata file is copied, so that the timestamps+ - remain the same, and because this is more space-efficient in the git+ - repository.+ - + - Any metadata already attached to the new key is not preserved.+ -}+copyMetaData :: Key -> Key -> Annex ()+copyMetaData oldkey newkey+	| oldkey == newkey = noop+	| otherwise = do+		l <- getMetaData oldkey+		unless (S.null l) $+			Annex.Branch.change (metaDataLogFile newkey) $+				const $ showLog l
Logs/Transfer.hs view
@@ -186,7 +186,7 @@ 		| transferDirection t == Upload = 			liftIO $ readMVar metervar 		| otherwise = do-			f <- fromRepo $ gitAnnexTmpLocation (transferKey t)+			f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t) 			liftIO $ catchDefaultIO 0 $ 				fromIntegral . fileSize <$> getFileStatus f
Makefile view
@@ -253,7 +253,7 @@ distributionupdate: 	git pull 	cabal configure-	ghc --make Build/DistributionUpdate+	ghc --make Build/DistributionUpdate -XPackageImports 	./Build/DistributionUpdate  .PHONY: git-annex git-union-merge git-recover-repository tags build-stamp
Remote/Git.hs view
@@ -158,9 +158,7 @@ 				| haveconfig r' -> return r' 				| otherwise -> configlist_failed 			Left _ -> configlist_failed-	| Git.repoIsHttp r = do-		headers <- getHttpHeaders-		store $ geturlconfig headers+	| Git.repoIsHttp r = store geturlconfig 	| Git.GCrypt.isEncrypted r = handlegcrypt =<< getConfigMaybe (remoteConfig r "uuid") 	| Git.repoIsUrl r = return r 	| otherwise = store $ safely $ onLocal r $ do @@ -185,11 +183,11 @@ 				return $ Right r' 			Left l -> return $ Left l -	geturlconfig headers = do-		ua <- Url.getUserAgent+	geturlconfig = do+		uo <- Url.getUrlOptions 		v <- liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do 			hClose h-			ifM (Url.downloadQuiet (Git.repoLocation r ++ "/config") headers [] tmpfile ua)+			ifM (Url.downloadQuiet (Git.repoLocation r ++ "/config") tmpfile uo) 				( pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] 				, return $ Left undefined 				)@@ -255,14 +253,14 @@  -} inAnnex :: Remote -> Key -> Annex (Either String Bool) inAnnex rmt key-	| Git.repoIsHttp r = checkhttp =<< getHttpHeaders+	| Git.repoIsHttp r = checkhttp 	| Git.repoIsUrl r = checkremote 	| otherwise = checklocal   where   	r = repo rmt-	checkhttp headers = do+	checkhttp = do 		showChecking r-		ifM (anyM (\u -> Url.withUserAgent $ Url.checkBoth u headers (keySize key)) (keyUrls rmt key))+		ifM (Url.withUrlOptions $ \uo -> anyM (\u -> Url.checkBoth u (keySize key) uo) (keyUrls rmt key)) 			( return $ Right True 			, return $ Left "not found" 			)@@ -287,7 +285,7 @@ #ifndef mingw32_HOST_OS 	locs' = locs #else-	locs' = map (replace "\\" "/") (annexLocations key)+	locs' = map (replace "\\" "/") locs #endif  dropKey :: Remote -> Key -> Annex Bool
Remote/Glacier.hs view
@@ -73,16 +73,16 @@ glacierSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) glacierSetup mu mcreds c = do 	u <- maybe (liftIO genUUID) return mu-	glacierSetup' (isJust mu) u mcreds c-glacierSetup' :: Bool -> UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID)-glacierSetup' enabling u mcreds c = do+	c' <- setRemoteCredPair c (AWS.creds u) mcreds+	glacierSetup' (isJust mu) u c'+glacierSetup' :: Bool -> UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+glacierSetup' enabling u c = do 	c' <- encryptionSetup c 	let fullconfig = c' `M.union` defaults 	unless enabling $ 		genVault fullconfig u 	gitConfigSpecialRemote u fullconfig "glacier" "true"-	c'' <- setRemoteCredPair fullconfig (AWS.creds u) mcreds-	return (c'', u)+	return (c', u)   where 	remotename = fromJust (M.lookup "name" c) 	defvault = remotename ++ "-" ++ fromUUID u
Remote/Rsync.hs view
@@ -112,26 +112,26 @@ 		| otherwise = True  rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex ([CommandParam], RsyncUrl)-rsyncTransport gc rawurl-	| rsyncUrlIsShell rawurl =-		(\rsh -> return (rsyncShell rsh, resturl)) =<<+rsyncTransport gc url+	| rsyncUrlIsShell url =+		(\rsh -> return (rsyncShell rsh, url)) =<< 		case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of 			"ssh":sshopts -> do 				let (port, sshopts') = sshReadPort sshopts-				    host = takeWhile (/=':') resturl+				    userhost = takeWhile (/=':') url 				-- Connection caching 				(Param "ssh":) <$> sshCachingOptions-					(host, port)+					(userhost, port) 					(map Param $ loginopt ++ sshopts') 			"rsh":rshopts -> return $ map Param $ "rsh" : 				loginopt ++ rshopts 			rsh -> error $ "Unknown Rsync transport: " 				++ unwords rsh-	| otherwise = return ([], rawurl)+	| otherwise = return ([], url)   where-	(login,resturl) = case separate (=='@') rawurl of-		(h, "") -> (Nothing, h)-		(l, h)  -> (Just l, h)+	login = case separate (=='@') url of+		(_h, "") -> Nothing+		(l, _)  -> Just l 	loginopt = maybe [] (\l -> ["-l",l]) login 	fromNull as xs = if null xs then as else xs @@ -247,7 +247,7 @@ withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a withRsyncScratchDir a = do 	p <- liftIO getPID-	t <- fromRepo gitAnnexTmpDir+	t <- fromRepo gitAnnexTmpObjectDir 	createAnnexDirectory t 	let tmp = t </> "rsynctmp" </> show p 	nuke tmp
Remote/S3.hs view
@@ -76,9 +76,10 @@ s3Setup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) s3Setup mu mcreds c = do 	u <- maybe (liftIO genUUID) return mu-	s3Setup' u mcreds c-s3Setup' :: UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID)-s3Setup' u mcreds c = if isIA c then archiveorg else defaulthost+	c' <- setRemoteCredPair c (AWS.creds u) mcreds+	s3Setup' u c'+s3Setup' :: UUID -> RemoteConfig -> Annex (RemoteConfig, UUID)+s3Setup' u c = if isIA c then archiveorg else defaulthost   where 	remotename = fromJust (M.lookup "name" c) 	defbucket = remotename ++ "-" ++ fromUUID u@@ -92,8 +93,7 @@ 		 	use fullconfig = do 		gitConfigSpecialRemote u fullconfig "s3" "true"-		c' <- setRemoteCredPair fullconfig (AWS.creds u) mcreds-		return (c', u)+		return (fullconfig, u)  	defaulthost = do 		c' <- encryptionSetup c
Remote/Web.hs view
@@ -14,7 +14,6 @@ import qualified Git import qualified Git.Construct import Annex.Content-import Config import Config.Cost import Logs.Web import Types.Key@@ -117,9 +116,8 @@ 			return $ Left "quvi support needed for this url" #endif 		DefaultDownloader -> do-			headers <- getHttpHeaders-			Url.withUserAgent $ catchMsgIO .-				Url.checkBoth u' headers (keySize key)+			Url.withUrlOptions $ catchMsgIO .+				Url.checkBoth u' (keySize key)   where   	firsthit [] miss _ = return miss 	firsthit (u:rest) _ a = do
Remote/WebDAV.hs view
@@ -14,10 +14,15 @@ import qualified Data.ByteString.UTF8 as B8 import qualified Data.ByteString.Lazy.UTF8 as L8 import qualified Data.ByteString.Lazy as L-import Network.URI (normalizePathSegments) import qualified Control.Exception as E+import qualified Control.Exception.Lifted as EL+#if MIN_VERSION_DAV(0,6,0)+import Network.HTTP.Client (HttpException(..))+#else import Network.HTTP.Conduit (HttpException(..))+#endif import Network.HTTP.Types+import System.Log.Logger (debugM) import System.IO.Error  import Common.Annex@@ -33,8 +38,8 @@ import Utility.Metered import Annex.Content import Annex.UUID+import Remote.WebDAV.DavUrl -type DavUrl = String type DavUser = B8.ByteString type DavPass = B8.ByteString @@ -82,10 +87,10 @@ 	let url = fromMaybe (error "Specify url=") $ 		M.lookup "url" c 	c' <- encryptionSetup c-	creds <- getCreds c' u+	creds <- maybe (getCreds c' u) (return . Just) mcreds 	testDav url creds 	gitConfigSpecialRemote u c' "webdav" "true"-	c'' <- setRemoteCredPair c' (davCreds u) mcreds+	c'' <- setRemoteCredPair c' (davCreds u) creds 	return (c'', u)  store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool@@ -105,7 +110,7 @@  storeHelper :: Remote -> Key -> DavUrl -> DavUser -> DavPass -> L.ByteString -> IO Bool storeHelper r k baseurl user pass b = catchBoolIO $ do-	davMkdir tmpurl user pass+	mkdirRecursiveDAV tmpurl user pass 	storeChunks k tmpurl keyurl chunksize storer recorder finalizer   where 	tmpurl = tmpLocation baseurl k@@ -114,11 +119,10 @@ 	storer urls = storeChunked chunksize urls storehttp b 	recorder url s = storehttp url (L8.fromString s) 	finalizer srcurl desturl = do-		void $ catchMaybeHttp (deleteContent desturl user pass)-		davMkdir (urlParent desturl) user pass-		moveContent srcurl (B8.fromString desturl) user pass-	storehttp url v = putContent url user pass-		(contentType, v)+		void $ tryNonAsync (deleteDAV desturl user pass)+		mkdirRecursiveDAV (urlParent desturl) user pass+		moveDAV srcurl desturl user pass+	storehttp url = putDAV url user pass  retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False@@ -128,7 +132,7 @@ 	davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $ 		withStoredFiles r k baseurl user pass onerr $ \urls -> do 			meteredWriteFileChunks meterupdate d urls $ \url -> do-				mb <- davGetUrlContent url user pass+				mb <- getDAV url user pass 				case mb of 					Nothing -> throwIO "download failed" 					Just b -> return b@@ -148,7 +152,7 @@  	feeder _ _ [] _ = noop 	feeder user pass (url:urls) h = do-		mb <- davGetUrlContent url user pass+		mb <- getDAV url user pass 		case mb of 			Nothing -> throwIO "download failed" 			Just b -> do@@ -160,7 +164,7 @@ 	-- Delete the key's whole directory, including any chunked 	-- files, etc, in a single action. 	let url = davLocation baseurl k-	isJust <$> catchMaybeHttp (deleteContent url user pass)+	isJust . eitherToMaybe <$> tryNonAsync (deleteDAV url user pass)  checkPresent :: Remote -> Key -> Annex (Either String Bool) checkPresent r k = davAction r noconn go@@ -173,7 +177,7 @@ 	  where 		check [] = return $ Right True 		check (url:urls) = do-			v <- davUrlExists url user pass+			v <- existsDAV url user pass 			if v == Right True 				then check urls 				else return v@@ -182,7 +186,7 @@ 		 - or if there's a problem accessing it, 		 - or perhaps this was an intermittent error. -} 		onerr url = do-			v <- davUrlExists url user pass+			v <- existsDAV url user pass 			return $ if v == Right True 				then Left $ "failed to read " ++ url 				else v@@ -199,11 +203,11 @@ withStoredFiles r k baseurl user pass onerr a 	| isJust $ chunkSize $ config r = do 		let chunkcount = keyurl ++ chunkCount-		v <- davGetUrlContent chunkcount user pass+		v <- getDAV chunkcount user pass 		case v of 			Just s -> a $ listChunks keyurl $ L8.toString s 			Nothing -> do-				chunks <- probeChunks keyurl $ \u -> (== Right True) <$> davUrlExists u user pass+				chunks <- probeChunks keyurl $ \u -> (== Right True) <$> existsDAV u user pass 				if null chunks 					then onerr chunkcount 					else a chunks@@ -231,46 +235,12 @@ toDavPass :: String -> DavPass toDavPass = B8.fromString -{- The directory where files(s) for a key are stored. -}-davLocation :: DavUrl -> Key -> DavUrl-davLocation baseurl k = addTrailingPathSeparator $-	davUrl baseurl $ hashDirLower k </> keyFile k--{- Where we store temporary data for a key as it's being uploaded. -}-tmpLocation :: DavUrl -> Key -> DavUrl-tmpLocation baseurl k = addTrailingPathSeparator $-	davUrl baseurl $ "tmp" </> keyFile k--davUrl :: DavUrl -> FilePath -> DavUrl-davUrl baseurl file = baseurl </> file--davUrlExists :: DavUrl -> DavUser -> DavPass -> IO (Either String Bool)-davUrlExists url user pass = decode <$> catchHttp get-  where-	decode (Right _) = Right True-#if ! MIN_VERSION_http_conduit(1,9,0)-	decode (Left (Left (StatusCodeException status _)))-#else-	decode (Left (Left (StatusCodeException status _ _)))-#endif-		| statusCode status == statusCode notFound404 = Right False-	decode (Left e) = Left $ showEitherException e-#if ! MIN_VERSION_DAV(0,4,0)-	get = getProps url user pass-#else-	get = getProps url user pass Nothing-#endif--davGetUrlContent :: DavUrl -> DavUser -> DavPass -> IO  (Maybe L.ByteString)-davGetUrlContent url user pass = fmap (snd . snd) <$>-	catchMaybeHttp (getPropsAndContent url user pass)- {- Creates a directory in WebDAV, if not already present; also creating  - any missing parent directories. -}-davMkdir :: DavUrl -> DavUser -> DavPass -> IO ()-davMkdir url user pass = go url+mkdirRecursiveDAV :: DavUrl -> DavUser -> DavPass -> IO ()+mkdirRecursiveDAV url user pass = go url   where-	make u = makeCollection u user pass+	make u = mkdirDAV u user pass  	go u = do 		r <- E.try (make u) :: IO (Either E.SomeException Bool)@@ -287,64 +257,25 @@ 			 - to use this directory will fail. -} 			Left _ -> return () -{- Catches HTTP and IO exceptions. -}-catchMaybeHttp :: IO a -> IO (Maybe a)-catchMaybeHttp a = (Just <$> a) `E.catches`-	[ E.Handler $ \(_e :: HttpException) -> return Nothing-	, E.Handler $ \(_e :: E.IOException) -> return Nothing-	]--{- Catches HTTP and IO exceptions -}-catchHttp :: IO a -> IO (Either EitherException a)-catchHttp a = (Right <$> a) `E.catches`-	[ E.Handler $ \(e :: HttpException) -> return $ Left $ Left e-	, E.Handler $ \(e :: E.IOException) -> return $ Left $ Right e-	]--type EitherException = Either HttpException E.IOException--showEitherException :: EitherException -> String-#if ! MIN_VERSION_http_conduit(1,9,0)-showEitherException (Left (StatusCodeException status _)) =-#else-showEitherException (Left (StatusCodeException status _ _)) =-#endif-	show $ statusMessage status-showEitherException (Left httpexception) = show httpexception-showEitherException (Right ioexception) = show ioexception--throwIO :: String -> IO a-throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing--urlParent :: DavUrl -> DavUrl-urlParent url = dropTrailingPathSeparator $-	normalizePathSegments (dropTrailingPathSeparator url ++ "/..")-  where- {- Test if a WebDAV store is usable, by writing to a test file, and then  - deleting the file. Exits with an IO error if not. -} testDav :: String -> Maybe CredPair -> Annex () testDav baseurl (Just (u, p)) = do 	showSideAction "testing WebDAV server"-	test "make directory" $ davMkdir baseurl user pass-	test "write file" $ putContent testurl user pass-		(contentType, L.empty)-	test "delete file" $ deleteContent testurl user pass+	test "make directory" $ mkdirRecursiveDAV baseurl user pass+	test "write file" $ putDAV testurl user pass L.empty+	test "delete file" $ deleteDAV testurl user pass   where 	test desc a = liftIO $-		either (\e -> throwIO $ "WebDAV failed to " ++ desc ++ ": " ++ showEitherException e)+		either (\e -> throwIO $ "WebDAV failed to " ++ desc ++ ": " ++ show e) 			(const noop)-			=<< catchHttp a+			=<< tryNonAsync a  	user = toDavUser u 	pass = toDavPass p 	testurl = davUrl baseurl "git-annex-test" testDav _ Nothing = error "Need to configure webdav username and password." -{- Content-Type to use for files uploaded to WebDAV. -}-contentType :: Maybe B8.ByteString-contentType = Just $ B8.fromString "application/octet-stream"- getCreds :: RemoteConfig -> UUID -> Annex (Maybe CredPair) getCreds c u = getRemoteCredPairFor "webdav" c (davCreds u) @@ -354,3 +285,114 @@ 	, credPairEnvironment = ("WEBDAV_USERNAME", "WEBDAV_PASSWORD") 	, credPairRemoteKey = Just "davcreds" 	}++{- Content-Type to use for files uploaded to WebDAV. -}+contentType :: Maybe B8.ByteString+contentType = Just $ B8.fromString "application/octet-stream"++throwIO :: String -> IO a+throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing++debugDAV :: DavUrl -> String -> IO ()+debugDAV msg url = debugM "DAV" $ msg ++ " " ++ url++{---------------------------------------------------------------------+ - Low-level DAV operations, using the new DAV monad when available.+ ---------------------------------------------------------------------}++putDAV :: DavUrl -> DavUser -> DavPass -> L.ByteString -> IO ()+putDAV url user pass b = do+	debugDAV "PUT" url+#if MIN_VERSION_DAV(0,6,0)+	goDAV url user pass $ putContentM (contentType, b)+#else+	putContent url user pass (contentType, b)+#endif++getDAV :: DavUrl -> DavUser -> DavPass -> IO (Maybe L.ByteString)+getDAV url user pass = do+	debugDAV "GET" url+	eitherToMaybe <$> tryNonAsync go+  where+#if MIN_VERSION_DAV(0,6,0)+	go = goDAV url user pass $ snd <$> getContentM+#else+	go = snd . snd <$> getPropsAndContent url user pass+#endif++deleteDAV :: DavUrl -> DavUser -> DavPass -> IO ()+deleteDAV url user pass = do+	debugDAV "DELETE" url+#if MIN_VERSION_DAV(0,6,0)+	goDAV url user pass delContentM+#else+	deleteContent url user pass+#endif++moveDAV :: DavUrl -> DavUrl -> DavUser -> DavPass -> IO ()+moveDAV url newurl user pass = do+	debugDAV ("MOVE to " ++ newurl ++ " from ") url+#if MIN_VERSION_DAV(0,6,0)+	goDAV url user pass $ moveContentM newurl'+#else+	moveContent url newurl' user pass+#endif+  where+	newurl' = B8.fromString newurl++mkdirDAV :: DavUrl -> DavUser -> DavPass -> IO Bool+mkdirDAV url user pass = do+	debugDAV "MKDIR" url+#if MIN_VERSION_DAV(0,6,0)+	goDAV url user pass mkCol+#else+	makeCollection url user pass+#endif++existsDAV :: DavUrl -> DavUser -> DavPass -> IO (Either String Bool)+existsDAV url user pass = do+	debugDAV "EXISTS" url+	either (Left . show) id <$> tryNonAsync check+  where+	ispresent = return . Right+#if MIN_VERSION_DAV(0,6,0)+	check = goDAV url user pass $ do+		setDepth Nothing+		EL.catchJust+			(matchStatusCodeException notFound404)+			(getPropsM >> ispresent True)+			(const $ ispresent False)+#else+	check = E.catchJust+		(matchStatusCodeException notFound404)+#if ! MIN_VERSION_DAV(0,4,0)+		(getProps url user pass >> ispresent True)+#else+		(getProps url user pass Nothing >> ispresent True)+#endif+		(const $ ispresent False)+#endif++matchStatusCodeException :: Status -> HttpException -> Maybe ()+#if MIN_VERSION_DAV(0,6,0)+matchStatusCodeException want (StatusCodeException s _ _)+#else+matchStatusCodeException want (StatusCodeException s _)+#endif+	| s == want = Just ()+	| otherwise = Nothing+matchStatusCodeException _ _ = Nothing++#if MIN_VERSION_DAV(0,6,0)+goDAV :: DavUrl -> DavUser -> DavPass -> DAVT IO a -> IO a+goDAV url user pass a = choke $ evalDAVT url $ do+	setCreds user pass+	a+  where+	choke :: IO (Either String a) -> IO a+	choke f = do+		x <- f+		case x of+			Left e -> error e+			Right r -> return r+#endif
+ Remote/WebDAV/DavUrl.hs view
@@ -0,0 +1,44 @@+{- WebDAV urls.+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++{-# LANGUAGE CPP #-}++module Remote.WebDAV.DavUrl where++import Types+import Locations++import Network.URI (normalizePathSegments)+import System.FilePath.Posix+#ifdef mingw32_HOST_OS+import Data.String.Utils+#endif++type DavUrl = String++{- The directory where files(s) for a key are stored. -}+davLocation :: DavUrl -> Key -> DavUrl+davLocation baseurl k = addTrailingPathSeparator $+	davUrl baseurl $ hashdir </> keyFile k+  where+#ifndef mingw32_HOST_OS+	hashdir = hashDirLower k+#else+	hashdir = replace "\\" "/" (hashDirLower k)+#endif++{- Where we store temporary data for a key as it's being uploaded. -}+tmpLocation :: DavUrl -> Key -> DavUrl+tmpLocation baseurl k = addTrailingPathSeparator $+	davUrl baseurl $ "tmp" </> keyFile k++davUrl :: DavUrl -> FilePath -> DavUrl+davUrl baseurl file = baseurl </> file++urlParent :: DavUrl -> DavUrl+urlParent url = dropTrailingPathSeparator $+	normalizePathSegments (dropTrailingPathSeparator url ++ "/..")
Test.hs view
@@ -55,6 +55,7 @@ import qualified Annex.Init import qualified Annex.CatFile import qualified Annex.View+import qualified Annex.View.ViewedFile import qualified Logs.View import qualified Utility.Path import qualified Utility.FileMode@@ -151,6 +152,7 @@ 	, testProperty "prop_metadata_serialize" Types.MetaData.prop_metadata_serialize 	, testProperty "prop_branchView_legal" Logs.View.prop_branchView_legal 	, testProperty "prop_view_roundtrips" Annex.View.prop_view_roundtrips+	, testProperty "prop_viewedFile_rountrips" Annex.View.ViewedFile.prop_viewedFile_roundtrips 	]  {- These tests set up the test environment, but also test some basic parts
Types/GitConfig.hs view
@@ -49,6 +49,7 @@ 	, annexAutoUpgrade :: AutoUpgrade 	, annexExpireUnused :: Maybe (Maybe Duration) 	, annexSecureEraseCommand :: Maybe String+	, annexGenMetaData :: Bool 	, coreSymlinks :: Bool 	, gcryptId :: Maybe String 	}@@ -81,6 +82,7 @@ 	, annexExpireUnused = maybe Nothing Just . parseDuration 		<$> getmaybe (annex "expireunused") 	, annexSecureEraseCommand = getmaybe (annex "secure-erase-command")+	, annexGenMetaData = getbool (annex "genmetadata") False 	, coreSymlinks = getbool "core.symlinks" True 	, gcryptId = getmaybe "core.gcrypt-id" 	}
Types/MetaData.hs view
@@ -17,7 +17,7 @@ 	MetaSerializable, 	toMetaField, 	mkMetaField,-	tagMetaField,+	mkMetaFieldUnchecked, 	fromMetaField, 	toMetaValue, 	mkMetaValue,@@ -25,7 +25,7 @@ 	unsetMetaData, 	fromMetaValue, 	fromMetaData,-	newMetaData,+	emptyMetaData, 	updateMetaData, 	unionMetaData, 	differenceMetaData,@@ -48,6 +48,7 @@ import qualified Data.Set as S import qualified Data.Map as M import Data.Char+import qualified Data.CaseInsensitive as CI  newtype MetaData = MetaData (M.Map MetaField (S.Set MetaValue)) 	deriving (Show, Eq, Ord)@@ -57,7 +58,8 @@ newtype CurrentlySet = CurrentlySet Bool 	deriving (Read, Show, Eq, Ord, Arbitrary) -newtype MetaField = MetaField String+{- Fields are case insensitive. -}+newtype MetaField = MetaField (CI.CI String) 	deriving (Read, Show, Eq, Ord)  data MetaValue = MetaValue CurrentlySet String@@ -81,7 +83,7 @@ 	serialize (MetaData m) = unwords $ concatMap go $ M.toList m 	  where 		go (f, vs) = serialize f : map serialize (S.toList vs)-	deserialize = Just . getfield newMetaData . words+	deserialize = Just . getfield emptyMetaData . words 	  where 		getfield m [] = m 		getfield m (w:ws) = maybe m (getvalues m ws) (deserialize w)@@ -91,8 +93,8 @@ 			Nothing -> getfield m l  instance MetaSerializable MetaField where-	serialize (MetaField f) = f-	deserialize = Just . MetaField+	serialize (MetaField f) = CI.original f+	deserialize = Just . mkMetaFieldUnchecked  {- Base64 problimatic values. -} instance MetaSerializable MetaValue where@@ -116,19 +118,39 @@ 	deserialize "-" = Just (CurrentlySet False) 	deserialize _ = Nothing -{- Fields cannot be empty, contain whitespace, or start with "+-" as- - that would break the serialization. -}+mkMetaField :: String -> Either String MetaField+mkMetaField f = maybe (Left $ badField f) Right (toMetaField f)++badField :: String -> String+badField f = "Illegal metadata field name, \"" ++ f ++ "\""++{- Does not check that the field name is valid. Use with caution. -}+mkMetaFieldUnchecked :: String -> MetaField+mkMetaFieldUnchecked = MetaField . CI.mk+ toMetaField :: String -> Maybe MetaField toMetaField f-	| legalField f = Just $ MetaField f+	| legalField f = Just $ MetaField $ CI.mk f 	| otherwise = Nothing +{- Fields cannot be empty, contain whitespace, or start with "+-" as+ - that would break the serialization.+ -+ - Additionally, fields should not contain any form of path separator, as+ - that would break views.+ -+ - So, require they have an alphanumeric first letter, with the remainder+ - being either alphanumeric or a small set of shitelisted common punctuation.+ -} legalField :: String -> Bool-legalField f-	| null f = False-	| any isSpace f = False-	| any (`isPrefixOf` f) ["+", "-"] = False-	| otherwise = True+legalField [] = False+legalField (c1:cs)+	| not (isAlphaNum c1) = False+	| otherwise = all legalchars cs+  where+	legalchars c+		| isAlphaNum c = True+		| otherwise = c `elem` "_-."  toMetaValue :: String -> MetaValue toMetaValue = MetaValue (CurrentlySet True)@@ -144,7 +166,7 @@ unsetMetaData (MetaData m) = MetaData $ M.map (S.map unsetMetaValue) m  fromMetaField :: MetaField -> String-fromMetaField (MetaField f) = f+fromMetaField (MetaField f) = CI.original f  fromMetaValue :: MetaValue -> String fromMetaValue (MetaValue _ f) = f@@ -152,8 +174,8 @@ fromMetaData :: MetaData -> [(MetaField, S.Set MetaValue)] fromMetaData (MetaData m) = M.toList m -newMetaData :: MetaData-newMetaData = MetaData M.empty+emptyMetaData :: MetaData+emptyMetaData = MetaData M.empty  {- Can be used to set a value, or to unset it, depending on whether  - the MetaValue has CurrentlySet or not. -}@@ -202,10 +224,10 @@  - Note that the new MetaData does not include all the   - values set in the input metadata. It only contains changed values. -} modMeta :: MetaData -> ModMeta -> MetaData-modMeta _ (AddMeta f v) = updateMetaData f v newMetaData-modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) newMetaData+modMeta _ (AddMeta f v) = updateMetaData f v emptyMetaData+modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) emptyMetaData modMeta m (SetMeta f v) = updateMetaData f v $-	foldr (updateMetaData f) newMetaData $+	foldr (updateMetaData f) emptyMetaData $ 		map unsetMetaValue $ S.toList $ currentMetaDataValues f m  {- Parses field=value, field+=value, field-=value -}@@ -227,15 +249,6 @@   where 	(f, v) = separate (== '=') p -mkMetaField :: String -> Either String MetaField-mkMetaField f = maybe (Left $ badField f) Right (toMetaField f)--badField :: String -> String-badField f = "Illegal metadata field name, \"" ++ f ++ "\""--tagMetaField :: MetaField-tagMetaField = MetaField "tag"- {- Avoid putting too many fields in the map; extremely large maps make  - the seriaization test slow due to the sheer amount of data.  - It's unlikely that more than 100 fields of metadata will be used. -}@@ -248,13 +261,13 @@ 	arbitrary = MetaValue <$> arbitrary <*> arbitrary  instance Arbitrary MetaField where-	arbitrary = MetaField <$> arbitrary `suchThat` legalField +	arbitrary = MetaField . CI.mk <$> arbitrary `suchThat` legalField   prop_metadata_sane :: MetaData -> MetaField -> MetaValue -> Bool prop_metadata_sane m f v = and 	[ S.member v $ metaDataValues f m' 	, not (isSet v) || S.member v (currentMetaDataValues f m')-	, differenceMetaData m' newMetaData == m'+	, differenceMetaData m' emptyMetaData == m' 	]   where 	m' = updateMetaData f v m
Types/View.hs view
@@ -35,10 +35,6 @@ instance Arbitrary ViewComponent where 	arbitrary = ViewComponent <$> arbitrary <*> arbitrary <*> arbitrary -{- Only files with metadata matching the view are displayed. -}-type FileView = FilePath-type MkFileView = FilePath -> FileView- data ViewFilter 	= FilterValues (S.Set MetaValue) 	| FilterGlob String
− Utility/.Base64.hs.swp

binary file changed (12288 → absent bytes)

+ Utility/Glob.hs view
@@ -0,0 +1,57 @@+{- file globbing+ -+ - This uses TDFA when available, with a fallback to regex-compat.+ - TDFA is less buggy in its support for non-unicode characters.+ -+ - Copyright 2014 Joey Hess <joey@kitenet.net>+ -+ - Licensed under the GNU GPL version 3 or higher.+ -}++{-# LANGUAGE CPP #-}++module Utility.Glob (+	Glob,+	GlobCase(..),+	compileGlob,+	matchGlob+) where++import System.Path.WildMatch++#ifdef WITH_TDFA+import Text.Regex.TDFA+import Text.Regex.TDFA.String+#else+import Text.Regex+#endif++newtype Glob = Glob Regex++data GlobCase = CaseSensative | CaseInsensative++{- Compiles a glob to a regex, that can be repeatedly used. -}+compileGlob :: String -> GlobCase -> Glob+compileGlob glob globcase = Glob $+#ifdef WITH_TDFA+	case compile (defaultCompOpt {caseSensitive = casesentitive}) defaultExecOpt regex of+		Right r -> r+		Left _ -> error $ "failed to compile regex: " ++ regex+#else+	mkRegexWithOpts regex casesentitive True+#endif+  where+	regex = '^':wildToRegex glob+	casesentitive = case globcase of+		CaseSensative -> True+		CaseInsensative -> False++matchGlob :: Glob -> String -> Bool+matchGlob (Glob regex) val = +#ifdef WITH_TDFA+	case execute regex val of+		Right (Just _) -> True+		_ -> False+#else+	isJust $ matchRegex regex val+#endif
Utility/LogFile.hs view
@@ -11,7 +11,9 @@  import Common +#ifndef mingw32_HOST_OS import System.Posix.Types+#endif  #ifndef mingw32_HOST_OS openLog :: FilePath -> IO Fd
Utility/Path.hs view
@@ -18,7 +18,6 @@ import Control.Applicative  #ifdef mingw32_HOST_OS-import Data.Char import qualified System.FilePath.Posix as Posix #else import System.Posix.Files
Utility/Url.hs view
@@ -10,6 +10,7 @@ module Utility.Url ( 	URLString, 	UserAgent,+	UrlOptions(..), 	check, 	checkBoth, 	exists,@@ -23,6 +24,7 @@ import qualified Network.Browser as Browser import Network.HTTP import Data.Either+import Data.Default  import qualified Build.SysConfig @@ -32,14 +34,24 @@  type UserAgent = String +data UrlOptions = UrlOptions+	{ userAgent :: Maybe UserAgent+	, reqHeaders :: Headers+	, reqParams :: [CommandParam]+	}++instance Default UrlOptions+  where+	def = UrlOptions Nothing [] []+ {- Checks that an url exists and could be successfully downloaded,  - also checking that its size, if available, matches a specified size. -}-checkBoth :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO Bool-checkBoth url headers expected_size ua = do-	v <- check url headers expected_size ua+checkBoth :: URLString -> Maybe Integer -> UrlOptions -> IO Bool+checkBoth url expected_size uo = do+	v <- check url expected_size uo 	return (fst v && snd v)-check :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO (Bool, Bool)-check url headers expected_size = handle <$$> exists url headers+check :: URLString -> Maybe Integer -> UrlOptions -> IO (Bool, Bool)+check url expected_size = handle <$$> exists url   where 	handle (False, _) = (False, False) 	handle (True, Nothing) = (True, True)@@ -55,8 +67,8 @@  - Uses curl otherwise, when available, since curl handles https better  - than does Haskell's Network.Browser.  -}-exists :: URLString -> Headers -> Maybe UserAgent -> IO (Bool, Maybe Integer)-exists url headers ua = case parseURIRelaxed url of+exists :: URLString -> UrlOptions -> IO (Bool, Maybe Integer)+exists url uo = case parseURIRelaxed url of 	Just u 		| uriScheme u == "file:" -> do 			s <- catchMaybeIO $ getFileStatus (unEscapeString $ uriPath u)@@ -70,7 +82,7 @@ 					Just ('2':_:_) -> return (True, extractsize output) 					_ -> dne 			else do-				r <- request u headers HEAD ua+				r <- request u HEAD uo 				case rspCode r of 					(2,_,_) -> return (True, size r) 					_ -> return (False, Nothing)@@ -78,12 +90,12 @@   where 	dne = return (False, Nothing) -	curlparams = addUserAgent ua $+	curlparams = addUserAgent uo $ 		[ Param "-s" 		, Param "--head" 		, Param "-L", Param url 		, Param "-w", Param "%{http_code}"-		] ++ concatMap (\h -> [Param "-H", Param h]) headers+		] ++ concatMap (\h -> [Param "-H", Param h]) (reqHeaders uo) ++ (reqParams uo)  	extractsize s = case lastMaybe $ filter ("Content-Length:" `isPrefixOf`) (lines s) of 		Just l -> case lastMaybe $ words l of@@ -94,9 +106,10 @@ 	size = liftM Prelude.read . lookupHeader HdrContentLength . rspHeaders  -- works for both wget and curl commands-addUserAgent :: Maybe UserAgent -> [CommandParam] -> [CommandParam]-addUserAgent Nothing ps = ps-addUserAgent (Just ua) ps = ps ++ [Param "--user-agent", Param ua] +addUserAgent :: UrlOptions -> [CommandParam] -> [CommandParam]+addUserAgent uo ps = case userAgent uo of+	Nothing -> ps+	Just ua -> ps ++ [Param "--user-agent", Param ua]   {- Used to download large files, such as the contents of keys.  -@@ -105,15 +118,15 @@  - would not be appropriate to test at configure time and build support  - for only one in.  -}-download :: URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool+download :: URLString -> FilePath -> UrlOptions -> IO Bool download = download' False  {- No output, even on error. -}-downloadQuiet :: URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool+downloadQuiet :: URLString -> FilePath -> UrlOptions -> IO Bool downloadQuiet = download' True -download' :: Bool -> URLString -> Headers -> [CommandParam] -> FilePath -> Maybe UserAgent -> IO Bool-download' quiet url headers options file ua = +download' :: Bool -> URLString -> FilePath -> UrlOptions -> IO Bool+download' quiet url file uo =  	case parseURIRelaxed url of 		Just u 			| uriScheme u == "file:" -> do@@ -124,7 +137,7 @@ 			| otherwise -> ifM (inPath "wget") (wget , curl) 		_ -> return False   where-	headerparams = map (\h -> Param $ "--header=" ++ h) headers+	headerparams = map (\h -> Param $ "--header=" ++ h) (reqHeaders uo) 	wget = go "wget" $ headerparams ++ quietopt "-q" ++ wgetparams 	{- Regular wget needs --clobber to continue downloading an existing 	 - file. On Android, busybox wget is used, which does not@@ -142,7 +155,7 @@ 	curl = go "curl" $ headerparams ++ quietopt "-s" ++ 		[Params "-f -L -C - -# -o"] 	go cmd opts = boolSystem cmd $-		addUserAgent ua $ options++opts++[File file, File url]+		addUserAgent uo $ reqParams uo++opts++[File file, File url] 	quietopt s 		| quiet = [Param s] 		| otherwise = []@@ -157,14 +170,14 @@  - Unfortunately, does not handle https, so should only be used  - when curl is not available.  -}-request :: URI -> Headers -> RequestMethod -> Maybe UserAgent -> IO (Response String)-request url headers requesttype ua = go 5 url+request :: URI -> RequestMethod -> UrlOptions -> IO (Response String)+request url requesttype uo = go 5 url   where 	go :: Int -> URI -> IO (Response String) 	go 0 _ = error "Too many redirects " 	go n u = do 		rsp <- Browser.browse $ do-			maybe noop Browser.setUserAgent ua+			maybe noop Browser.setUserAgent (userAgent uo) 			Browser.setErrHandler ignore 			Browser.setOutHandler ignore 			Browser.setAllowRedirects False@@ -174,7 +187,7 @@ 			(3,0,x) | x /= 5 -> redir (n - 1) u rsp 			_ -> return rsp 	addheaders req = setHeaders req (rqHeaders req ++ userheaders)-	userheaders = rights $ map parseHeader headers+	userheaders = rights $ map parseHeader (reqHeaders uo) 	ignore = const noop 	redir n u rsp = case retrieveHeaders HdrLocation rsp of 		[] -> return rsp
Utility/WebApp.hs view
@@ -23,7 +23,6 @@ import System.Log.Logger import qualified Data.CaseInsensitive as CI import Network.Socket-import Control.Exception import "crypto-api" Crypto.Random import qualified Web.ClientSession as CS import qualified Data.ByteString.Lazy as L@@ -38,6 +37,10 @@ import Control.Concurrent #ifdef __ANDROID__ import Data.Endian+#endif+#if defined(__ANDROID__) || defined (mingw32_HOST_OS)+#else+import Control.Exception (bracketOnError) #endif  localhost :: HostName
Utility/WinProcess.hs view
@@ -11,9 +11,5 @@  import Utility.PID -import System.Win32.Process-import Foreign.C-import Control.Exception- foreign import ccall unsafe "terminatepid" 	terminatePID :: PID -> IO ()
debian/changelog view
@@ -1,3 +1,38 @@+git-annex (5.20140227) unstable; urgency=medium++  * metadata: Field names limited to alphanumerics and a few whitelisted+    punctuation characters to avoid issues with views, etc.+  * metadata: Field names are now case insensative.+  * When constructing views, metadata is available about the location of the+    file in the view's reference branch. Allows incorporating parts of the+    directory hierarchy in a view.+    For example `git annex view tag=* podcasts/=*` makes a view in the form+    tag/showname.+  * --metadata field=value can now use globs to match, and matches+    case insensatively, the same as git annex view field=value does.+  * annex.genmetadata can be set to make git-annex automatically set+    metadata (year and month) when adding files.+  * Make annex.web-options be used in several places that call curl.+  * Fix handling of rsync remote urls containing a username,+    including rsync.net.+  * Preserve metadata when staging a new version of an annexed file.+  * metadata: Support --json+  * webapp: Fix creation of box.com and Amazon S3 and Glacier+    repositories, broken in 5.20140221.+  * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid+    locking files, which is not needed by git-annex's use of webdav, and+    does not work on Box.com.+  * webdav: Fix path separator bug when used on Windows.+  * repair: Optimise unpacking of pack files, and avoid repeated error+    messages about corrupt pack files.+  * Add build dep on regex-compat to fix build on mipsel, which lacks+    regex-tdfa.+  * Disable test suite on sparc, which is missing optparse-applicative.+  * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp+    for only partially transferred objects.++ -- Joey Hess <joeyh@debian.org>  Thu, 27 Feb 2014 11:34:19 -0400+ git-annex (5.20140221) unstable; urgency=medium    * metadata: New command that can attach metadata to files.
debian/control view
@@ -6,6 +6,7 @@ 	ghc (>= 7.4), 	libghc-mtl-dev (>= 2.1.1), 	libghc-missingh-dev,+	libghc-data-default-dev, 	libghc-hslogger-dev, 	libghc-pcre-light-dev, 	libghc-sha-dev,@@ -51,11 +52,12 @@ 	libghc-http-dev, 	libghc-feed-dev, 	libghc-regex-tdfa-dev [!mipsel !s390],+	libghc-regex-compat-dev [mipsel s390], 	libghc-tasty-dev (>= 0.7) [!mipsel !sparc], 	libghc-tasty-hunit-dev [!mipsel !sparc], 	libghc-tasty-quickcheck-dev [!mipsel !sparc], 	libghc-tasty-rerun-dev [!mipsel !sparc],-	libghc-optparse-applicative-dev,+	libghc-optparse-applicative-dev [!sparc], 	lsof [!kfreebsd-i386 !kfreebsd-amd64], 	ikiwiki, 	perlmagick,
+ doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment view
@@ -0,0 +1,13 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawk2BFMjJW081uX4aJdhStmSFPBUGzL92ZU"+ nickname="Frédéric"+ subject="where do I put the SSH keys on Android"+ date="2014-02-26T20:26:38Z"+ content="""+@mebus :+You can put your SSH keys here :++/sdcard/git-annex.home/.ssh/id_rsa++/sdcard/git-annex.home/.ssh/id_rsa.pub+"""]]
+ doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn view
@@ -0,0 +1,19 @@+### Please describe the problem.++The assistant regulary ends up trying to perform repair (I don't know why, it happens fairly often, once a week or so). When it does so, it ends up creating a huge (2.4G) .git/objects directory, and a git prune-packed process uses so much I/O the machine really slows down.++### What steps will reproduce the problem?++I don't have any reliable way to reproduce it. The repository ends up being attempted to be repaired around once a week. This week the repair (and the slowdown) also happened on a second computer.++### What version of git-annex are you using? On what operating system?++git-annex version: 5.20140221-gbdfc8e1 (using the standalone 64bit builds)++This is on an up-to-date Arch Linux. It also happened on Fedora 20.++### Please provide any additional information below.++The daemon.log is fairly long, but not particulary interesting: [[https://ssl.zerodogg.org/~zerodogg/private/tmp/daemon.log-2014-02-25.1]]++The «resource vanished (Broken pipe)» at the end is the result of me killing the prune-packed in order to be able to use the machine again.
+ doc/bugs/Can_not_Drop_Unused_Files_With_Spaces.mdwn view
@@ -0,0 +1,22 @@+I have a repository at rsync.net, even though following files are shown as unused I can not drop them.++Running unused,++    git annex unused --from cloud                                            +    unused cloud (checking for unused data...) (checking annex/direct/master...) +      Some annexed data on cloud is not used by any files:+        NUMBER  KEY+        1       SHA256E-s4189547--43aef42540e7f50fc454ab3a2ce4aa28a13b57cccff725359cea0470eb88704b. Bir.mp3+        2       SHA256E-s853765--c0964d3af493d78b7b8393a2aefdd8c290390a03c8cb5cccdcac4647c0fc52a0. 1.jpg+        3       SHA256E-s8706267--e34988b70048a512ad0f431a2a91fa7dd553f96c2bd6caca0bcef928bdfafb93. 3.mp3+      (To see where data was previously used, try: git log --stat -S'KEY')+      +      To remove unwanted data: git-annex dropunused --from cloud NUMBER++show these then running,++    git annex dropunused 1-3 --force++reports ok for each drop operation but rerunning git annex unused --from cloud still shows these three files as unused. I am using git-annex on mac os x (current dmg) on a direct repo. I have similar problems dropping files on the current repo even though I drop unused they still show up as unused.++> [[fixed|done]] --[[Joey]]
doc/bugs/Creating_a_box.com_repository_fails.mdwn view
@@ -35,3 +35,7 @@ > Seems that [DAV-0.6 is badly broken](http://bugs.debian.org/737902). > I have adjusted the cabal file to refuse to build with that broken > version.+> +>> Update: Had to work around additional breakage in DAV-0.6. It's+>> fully tested and working now, although not yet uploaded to Debian+>> unstable. [[done]] --[[Joey]]
+ doc/bugs/enableremote_broken_with_direct_mode__63__.mdwn view
@@ -0,0 +1,41 @@+### Please describe the problem.+I have 2 regular annex locations plus one on glacier. I recently cloned one location, converted it to direct mode, then tried to enableremote glacier.++When I ran the webapp everything seemed to be ok, it showed file transfers to glacier which appeared to be completing very quickly. When I looked in the log file however, I saw the files weren't actually transferring and I was getting lots of errors:++  Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use glacier++In order to get glacier working I had to manually copy the creds file to the new repo. The transfer errors should also made obvious in the webapp.++### What steps will reproduce the problem?+[[!format sh """+git clone server:/location+git annex init "laptop"+git remote add "server" server:/location+git annex direct++Then I ran:+git annex enableremote glacier+Which prints:+(merging origin/git-annex origin/synced/git-annex into git-annex...)+(Recording state in git...)+enableremote glacier (gpg) ok+(Recording state in git...)+"""]]+But in fact it hasn't actually copied the creds to use it.+++### What version of git-annex are you using? On what operating system?+5.20140221, debian.++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]++> Should be fixed in 5.20140227. [[done]] --[[Joey]]
+ doc/bugs/pages_of_packfile_errors.mdwn view
@@ -0,0 +1,30 @@+### Please describe the problem.++A repair that runs for ages. In the log file, pages and pages and pages of:++error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index+warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed+error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index+warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed+error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index+warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed+error: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack does not match index+warning: packfile /Volumes/BandZbackup2/annex/.git/objects/pack/pack-f0ae2f5cc83f11eab406518b9f06a344acf9c93c.pack cannot be accessed++### What steps will reproduce the problem?++Running git-annex, plugging in my external drive++### What version of git-annex are you using? On what operating system?++Auto-updated latest, I thought, but the about page says: Version: 5.20131230-g9a495e6 ++### Please provide any additional information below.++[[!format sh """+# If you can, paste a complete transcript of the problem occurring here.+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log+++# End of transcript or log.+"""]]
doc/bugs/rsync_transport:_username_not_respected.mdwn view
@@ -36,3 +36,8 @@  # End of transcript or log. """]]++> Argh! How did that break? I know it used to work.+> I have fixed it, unfortunately the fix was too late for today's release,+> but it will be available in autobuilds shortly.+> [[fixed|done]] --[[Joey]]
doc/design/metadata.mdwn view
@@ -29,7 +29,7 @@   relevant metadata from the files.     TODO: It's not clear that   removing a file should nuke all the metadata used to filter it into the-  branch (especially if it's derived metadata like the year).+  branch   Currently, only metadata used for visible subdirs is added and removed   this way.   Also, this is not usable in direct mode because deleting the@@ -56,21 +56,9 @@  Also auto add metadata when adding files to view branches. See below. -## derived metadata--This is probably not stored anywhere. It's computed on demand by a pure-function from the other metadata. -(Should be a general mechanism for this. (It probably generalizes to-sql queries if we want to go that far.))--### data metadata--TODO From the ctime, some additional -metadata is derived, at least year=yyyy and probably also month, etc.--### directory hierarchy metadata+## directory hierarchy metadata -TODO From the original filename used in the master branch, when+From the original filename used in the master branch, when constructing a view, generate fields. For example foo/bar/baz.mp3 would get /=foo, foo/=bar, foo/bar/=baz, and .=mp3. @@ -82,11 +70,10 @@ without locking the view into using it.   Complication: When refining a view, it only looks at the filenames in-the view, so it would need to map from+the view, so it has to map from those filenames to derive the same metadata, unless there is persistent storage. Luckily, the filenames used in the views currently include the-subdirs (although not quite in a parseable format, would need some small-changes).+subdirs.  # other uses for metadata @@ -185,21 +172,15 @@ * Git has a complex set of rules for what is legal in a ref name.   View branch names will need to filter out any illegal stuff. **done** +* Metadata should be copied to the new key when adding a modified version+  of a file. **done**+ * Filesystems that are not case sensative (including case preserving OSX)   will cause problems if view branches try to use different cases for -  2 directories representing the value of some metadata. But, users-  probably want at least case-preserving metadata values. +  2 directories representing a metadata field.   -  Solution might be to compare metadata case-insensitively, and-  pick one representation consistently, so if, for example an author-  field uses mixed case, it will be used in the view branch.--  Alternatively, it could escape `A` to `_A` when such a filesystem-  is detected and avoid collisions that way (double `_` to escape it).-  This latter option is ugly, but so are non-posix filesystems.. and it-  also solves any similar issues with case-colliding filenames.--  TODO: Check current state of this.+  Solution might be to compare fields names case-insensitively, and+  pick one representation consistently. **done**  * Assistant needs to know about views, so it can update metadata when   files are moved around inside them. TODO@@ -207,3 +188,7 @@ * What happens if git annex add or the assistant add a new file while on a   view? If the file is not also added to the master branch, it will be lost   when exiting the view. TODO++* The filename mangling can result in a filename in a view+  that is too long for its containing filesystem. Should detect and do+  something reasonable to avoid. TODO
+ doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment view
@@ -0,0 +1,22 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawm3vKzS4eOWYpKMoYXqMIjNsIg_nYF-loU"+ nickname="Konubinix"+ subject="Already existing metadata implementation "+ date="2014-02-22T21:45:25Z"+ content="""+Hi,++I love the idea behing storing metadata.++I suggest to exchange ideas (and maybe code) with projects already implementing metadata systems.++I have tried several implementations and particularly noticed tmsu (http://tmsu.org/). This tool stores tags into a sqlite database and uses also a SHA-256 fingerprint of the file to be aware of file moves. It provides a fuse view of the tags with the ability to change tags by moving files (like in the git annex metadata view).++Paul Ruane is particularly responsive on the mailing list and he already supports git annexed files (with SHAE-256 fingerprint) (see the end of the thread https://groups.google.com/forum/#!topic/tmsu/A5EGpnCcJ2w).++Even if you cannot reuse the project, they are interresting ideas that might be worth looking at like the implications of tags: a file tagged \"film\" being automatically tagged \"video\".++Tagsistant (http://www.tagsistant.net/) may also be a good source of inspirations. I just don't like the fact that it uses a backstore of tagged files.++Thanks for reading.+"""]]
+ doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment view
@@ -0,0 +1,14 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 2"+ date="2014-02-25T09:51:17Z"+ content="""+Some additional ideas for metadata...++Instead of having a simplistic scheme like 'field=value' it might be advantageous to consider a scheme like 'attribute=XXX, value=YYY, unit=ZZZ' that way you could do intesting things with the metadata like adding counters to things, and allow for doing interesting queries like give me all 'things' tagged with a unit of \"audio_file\", this assumes one had trawled through an entire annex and then tagged all files based on type with the unix file tool or something like that.++The above idea is already in use in irods and its a really nice and powerful way to let users add meta-data and to build up more interesting use cases and tools.++btw, I plan on taking a look at seeing if I can map some of the meta that we have in work into this new git-annex feature to see how well/bad it works. Either way this feature looks cool! +1!!!+"""]]
+ doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment view
@@ -0,0 +1,12 @@+[[!comment format=mdwn+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"+ nickname="Jimmy"+ subject="comment 3"+ date="2014-02-25T09:57:09Z"+ content="""+actually in your mp3 example you could have ....++ATTRIBUTE=sample_rate, VALUE=22100, UNIT=Hertz++another example use case is to always be consistent with the AVU order then you could stick in ntriples from RDF to do other cool things by looking up various linked data sources -- see http://www.w3.org/2001/sw/RDFCore/ntriples/ and http://www.freebase.com/, actually this would be quite cool if git-annex examined the mp3's id3 tag, the created an ntriple styled entry can be automatically parsed with the web-based annex gui and automatically pull in additional meta-data from the likes of freebase. I guess the list of ideas can just only get bigger with this potential metadata capability.+"""]]
doc/design/roadmap.mdwn view
@@ -6,10 +6,10 @@  * Month 1 [[!traillink assistant/encrypted_git_remotes]] * Month 2 [[!traillink assistant/disaster_recovery]]-* Month 3 user-driven features and polishing [[todo/direct_mode_guard]] [[assistant/upgrading]]-* Month 4 [[Windows_webapp|assistant/Windows]], Linux arm, [[todo/support_for_writing_external_special_remotes]]+* Month 3 user-driven features and polishing [[!traillink todo/direct_mode_guard]] [[!traillink assistant/upgrading]]+* Month 4 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, [[!traillink todo/support_for_writing_external_special_remotes]] * Month 5 user-driven features and polishing-* **Month 6 get Windows out of beta, [[metadata and views|design/metadata]]**+* **Month 6 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]]** * Month 7 user-driven features and polishing * Month 8 [[!traillink assistant/telehash]] * Month 9 [[!traillink assistant/gpgkeys]] [[!traillink assistant/sshpassword]]
doc/devblog/day_108__new_use_for_location_tracking.mdwn view
@@ -4,7 +4,7 @@ point in the past. For example, `git annex get --in=here@{yesterday}` will get any files that have been dropped over the past day. -While git-annex's location tracking info is stored in git and so versioned,+While git-annex's location tracking info is stored in git and thus versioned, very little of it makes use of past versions of the location tracking info (only `git annex log`). I'm happy to have finally found a use for it!
+ doc/devblog/day_120__more_metadata.mdwn view
@@ -0,0 +1,17 @@+When generating a view, there's now a way to reuse part of the directory+hierarchy of the parent branch. For example, `git annex view tag=* podcasts/=*`+makes a view where the first level is the tags, and the second level is+whatever `podcasts/*` directories the files were in.++Also, year and month metadata can be automatically recorded when+adding files to the annex. I made this only be done when annex.genmetadata+is turned on, to avoid polluting repositories that don't want to use metadata.++It would be nice if there was a way to add a hook script that's run+when files are added, to collect their metadata. I am not sure yet if+I am going to add that to git-annex though. It's already possible to do via+the regular git `post-commit` hook. Just make it look at the commit to see+what files were added, and then run `git annex metadata` to set their+metadata appropriately. It would be good to at least have an example of+such a script to eg, extract EXIF or ID3 metadata. Perhaps someone can+contribute one?
+ doc/devblog/day_121__special_remote_maintenance.mdwn view
@@ -0,0 +1,23 @@+Turns out that in the last release I broke making box.com, Amazon S3 and+Glacier remotes from the webapp. Fixed that.++Also, dealt with changes in the haskell DAV library that broke support for+box.com, and worked around an exception handling bug in the library.++I think I should try to enhance the test suite so it can run live tests+on special remotes, which would at least have caught the some of these+recent problems...++----++Since metadata is tied to a particular key, editing an annexed file,+which causes the key to change, made the metadata seem to get lost.++I've now fixed this; it copies the metadata from the old version to the new+one. (Taking care to copy the log file identically, so git can reuse its+blob.) ++That meant that `git annex add` has to check every file it adds to see if+there's an old version. Happily, that check is fairly fast; I benchmarked my+laptop running 2500 such checks a second. So it's not going to slow things+down appreciably.
+ doc/devblog/day_122_more_windows_porting.mdwn view
@@ -0,0 +1,4 @@+More Windows porting. Made the build completely `-Wall` safe on Windows.+Fixed some DOS path separator bugs that were preventing WebDav from+working. Have now tested both box.com and Amazon S3 to be completely+working in the webapp on Windows.
+ doc/devblog/day_123__stuck.mdwn view
@@ -0,0 +1,13 @@+Not a lot accomplished today. Some release prep, followed up to a few bug+reports.++Split git-annex's .git/annex/tmp into two directories. .git/annex/tmp will+now be used only for partially transferred objects, while+.git/annex/misctmp will be used for everything else. In particular this+allows symlinking .git/annex/tmp to a ram disk, if you want to do that.+(It's not possible for .git/annex/misctmp to be on a different filesystem+from the rest of the repository for various reasons.)++Beat on Windows XMPP for several more painful hours. Got all the haskell+bindings installed, except for gnuidn. And patched network-client-xmpp to+build without gnuidn. Have not managed to get it to link.
− doc/forum/Can_not_Drop_Unused_Files_With_Spaces.mdwn 
@@ -1,20 +0,0 @@-I have a repository at rsync.net, even though following files are shown as unused I can not drop them.--Running unused,--    git annex unused --from cloud                                            -    unused cloud (checking for unused data...) (checking annex/direct/master...) -      Some annexed data on cloud is not used by any files:-        NUMBER  KEY-        1       SHA256E-s4189547--43aef42540e7f50fc454ab3a2ce4aa28a13b57cccff725359cea0470eb88704b. Bir.mp3-        2       SHA256E-s853765--c0964d3af493d78b7b8393a2aefdd8c290390a03c8cb5cccdcac4647c0fc52a0. 1.jpg-        3       SHA256E-s8706267--e34988b70048a512ad0f431a2a91fa7dd553f96c2bd6caca0bcef928bdfafb93. 3.mp3-      (To see where data was previously used, try: git log --stat -S'KEY')-      -      To remove unwanted data: git-annex dropunused --from cloud NUMBER--show these then running,--    git annex dropunused 1-3 --force--reports ok for each drop operation but rerunning git annex unused --from cloud still shows these three files as unused. I am using git-annex on mac os x (current dmg) on a direct repo. I have similar problems dropping files on the current repo even though I drop unused they still show up as unused.
+ doc/forum/Can_not_delete_Repository.mdwn view
@@ -0,0 +1,3 @@+I have one repository that I deleted a while back. When I mark it as dead in command line it disappears from git annex info however when I run webapp it pops back webapp shows it as syncing disabled. When I try to delete it from the webapp it does not delete. I tried shutting down the daemon mark it as dead again then run git annex forget --drop-dead --force but running it makes the repo active again instead of deleting it.++Repo in question was a S3 repo. I tried deleting it using both its name and uuid.
+ doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn view
@@ -0,0 +1,1 @@+Is it possible to convert a regular git annex repo (git clone then git annex init in the folder), to an rsync remote. I have an annex with alot of remotes which makes the sync operation take a really long time. I would like to convert some of those remotes to rsync. This particular repo has a TB of data so I would like to avoid dropping content from the remote than re download everything.
+ doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn view
@@ -0,0 +1,5 @@+Is there any way to find all files that do not have a certain field assigned in metadata. E.g. I want to find all files that do not have an author field set and++    git-annex find --not --metadata "author=*"++doesn't give any results.
+ doc/forum/Too_big_to_fsck.mdwn view
@@ -0,0 +1,20 @@+Hi,++My Webapp isn't working: +    +    $ git-annex webapp error: refs/gcrypt/gitception+ does not point to a valid object! +    error: refs/remotes/Beta/git-annex does not point to a valid object! +    error: refs/remotes/Beta/master does not point to a valid object! +    fatal: unable to read tree 656e7db5be172f01c0b6994d01f1a08d1273af12++So I tried to repair it: ++    $ git-annex repair Running git fsck ... +    Stack space overflow: current size 8388608 bytes. Use `+RTS -Ksize -RTS' to increase it.++So I tried to follow your advice here and increase the stack: ++    $ git-annex +RTS -K35000000 -RTS fsck +    git-annex: Most RTS options are disabled. Link with -rtsopts to enable them.++I wasn't sure what to do next, so any help would be appreciated.
+ doc/forum/git_annex_with_local_apache_webdav_server.mdwn view
@@ -0,0 +1,23 @@+Hi, ++trying to make git annex work locally with an apache webdav server.++I have the webdav server working without issue on computers. When we try to init the repository there we get the following error:++WEBDAV_USERNAME=user WEBDAV_PASSWORD=xxxxxx git annex initremote webdavtest type=webdav url=http://webdavserver/webdavsgare/annextest4 encryption=none +initremote webdavtest (testing WebDAV server...)++git-annex: WebDAV failed to delete file: "Locked": user error+failed+git-annex: initremote: 1 failed+++Does anyone have any thoughts? I can post config of webdav if it helps, though the error I receive in the error_log of apache is as follows:++This resource is locked and an "If:" header was not supplied to allow access to the resource.  [423, #0]++but I can manage the files through command line, web interface and mounted drive with no issue.++thank you in advance.++Damien
doc/git-annex.mdwn view
@@ -715,21 +715,30 @@          	git annex metadata annexscreencast.ogv -t video -t screencast -s author+=Alice -* `view [field=value ...] [tag ...]`+* `view [tag ...] [field=value ...] [location/=value]`    Uses metadata to build a view branch of the files in the current branch,   and checks out the view branch. Only files in the current branch whose   metadata matches all the specified field values and tags will be   shown in the view.++  Once within a view, you can make additional directories, and+  copy or move files into them. When you commit, the metadata will+  be updated to correspond to your changes.      Multiple values for a metadata field can be specified, either by using   a glob (`field="*"`) or by listing each wanted value. The resulting view   will put files in subdirectories according to the value of their fields. -  Once within a view, you can make additional directories, and-  copy or move files into them. When you commit, the metadata will-  be updated to correspond to your changes.+  There are fields corresponding to the path to the file. So a file+  "foo/bar/baz/file" has fields "/=foo", "foo/=bar", and "foo/bar/=baz".+  These location fields can be used the same as other metadata to construct+  the view. +  For example, `/=podcasts` will only include files from the podcasts+  directory in the view, while `podcasts/=*` will preserve the+  subdirectories of the podcasts directory in the view.+ * `vpop [N]`    Switches from the currently active view back to the previous view.@@ -737,12 +746,12 @@    The optional number tells how many views to pop. -* `vfilter [field=value ...] [tag ...]`+* `vfilter [tag ...] [field=value ...] [location/=value]`    Filters the current view to only the files that have the-  specified values and tags.+  specified field values, tags, and locations. -* `vadd [field=glob ...]`+* `vadd [field=glob ...] [location/=glob]`    Changes the current view, adding an additional level of directories   to categorize the files.@@ -942,7 +951,7 @@   Rather than the normal output, generate JSON. This is intended to be   parsed by programs that use git-annex. Each line of output is a JSON   object. Note that JSON output is only usable with some git-annex commands,-  like info, find, and whereis.+  like info, find, whereis, and metadata.  * `--debug` @@ -1133,10 +1142,11 @@   The size can be specified with any commonly used units, for example,   "0.5 gb" or "100 KiloBytes" -* `--metadata field=value`+* `--metadata field=glob` -  Matches only files that have a metadata field attached with the specified-  value.+  Matches only files that have a metadata field attached with a value that+  matches the glob. The values of metadata fields are matched case+  insensitively.  * `--want-get` @@ -1269,6 +1279,12 @@    Note that setting numcopies to 0 is very unsafe. +* `annex.genmetadata`++  Set this to `true` to make git-annex automatically generate some metadata+  when adding files to the repository. In particular, it stores+  year and month metadata, from the file's modification date.+ * `annex.queuesize`    git-annex builds a queue of git commands, in order to combine similar@@ -1509,8 +1525,7 @@  * `annex.web-options` -  Options to use when using wget or curl to download a file from the web.-  (wget is always used in preference to curl if available.)+  Options to pass when running wget or curl.   For example, to force ipv4 only, set it to "-4"  * `annex.quvi-options`
doc/install/fromscratch.mdwn view
@@ -5,6 +5,7 @@   * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer)   * [mtl](http://hackage.haskell.org.package/mtl) (2.1.1 or newer)   * [MissingH](http://github.com/jgoerzen/missingh/wiki)+  * [data-default](http://hackage.haskell.org/package/data-default)   * [utf8-string](http://hackage.haskell.org/package/utf8-string)   * [SHA](http://hackage.haskell.org/package/SHA)   * [cryptohash](http://hackage.haskell.org/package/cryptohash) (optional but recommended)@@ -25,6 +26,7 @@   * [extensible-exceptions](http://hackage.haskell.org/package/extensible-exceptions)   * [feed](http://hackage.haskell.org/package/feed)   * [async](http://hackage.haskell.org/package/async)+  * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)   * [stm](http://hackage.haskell.org/package/stm)     (version 2.3 or newer) * Optional haskell stuff, used by the [[assistant]] and its webapp@@ -35,7 +37,6 @@   * [yesod-static](http://hackage.haskell.org/package/yesod-static)   * [yesod-default](http://hackage.haskell.org/package/yesod-default)   * [data-default](http://hackage.haskell.org/package/data-default)-  * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)   * [http-types](http://hackage.haskell.org/package/http-types)   * [wai](http://hackage.haskell.org/package/wai)   * [wai-logger](http://hackage.haskell.org/package/wai-logger)
doc/internals.mdwn view
@@ -27,6 +27,42 @@ changed, and `.map` files contain a list of file(s) in the work directory that contain the key. +# `.git/annex/tmp/`++This directory contains partially transferred objects.++# `.git/annex/misctmp/`++This is a temp directory for miscellaneous other temp files.++While .git/annex/objects and .git/annex/tmp can be put on different+filesystems if desired, .git/annex/misctmp +has to be on the same filesystem as the work tree and git repository.++# `.git/annex/bad/`++git-annex fsck puts any bad objects it finds in here.++# `.git/annex/transfers/`++Contains information files for uploads and downloads that are in progress,+as well as any that have failed. Used especially by the assistant.+It is safe to delete these files.++# `.git/annex/ssh/`++ssh connection caching files are written in here.++# `.git/annex/index`++This is a git index file which git-annex uses for commits to the git-annex+branch.++# `.git/annex/journal/`++git-annex uses this to journal changes to the git-annex branch,+before committing a set of changes.+ ## The git-annex branch  This branch is managed by git-annex, with the contents listed below.
+ doc/metadata.mdwn view
@@ -0,0 +1,41 @@+git-annex allows you to store arbitrary metadata about files stored in the+git-annex repository. The metadata is stored in the `git-annex` branch, and+so is automatically kept in sync with the rest of git-annex's state, such+as [[location_tracking]] information.++Some of the things you can do with metadata include:++* Using `git annex metadata file` to show all +  the metadata associated with a file.+* [[tips/metadata_driven_views]]+* Limiting the files git-annex commands act on to those with+  or without particular metadata.+  For example `git annex find --metadata tag=foo --or --metadata tag=bar`+* Using it in [[preferred_content]] expressions. +  For example "tag=important or not author=me"++Each file (actually the underlying key) can have any number of metadata+fields, which each can have any number of values. For example, to tag+files, the `tag` field is typically used, with values set to each tag that+applies to the file.++The field names are limited to alphanumerics (and `[_-.]`), and are case+insensitive. The metadata values can contain absolutely anything you+like -- but you're recommended to keep it simple and reasonably short.++Here are some recommended metadata fields to use:++* `tag` - With each tag being a different value.+* `year`, `month` - When this particular version of the file came into+  being.+  +To make git-annex automatically set the year and month when adding files,+run `git config annex.genmetadata true`++git-annex's metadata can be updated in a distributed fashion. For example,+two users, each with their own clone of a repository, can set and unset+metadata at the same time, even for the same field of the same file. +When they push their changes, `git annex merge` will combine their+metadata changes in a consistent and (probably) intuitive way.++See [[the metadata design page|design/metadata]] for more details.
− doc/news/version_5.20140127.mdwn 
@@ -1,41 +0,0 @@-git-annex 5.20140127 released with [[!toggle text="these changes"]]-[[!toggleable text="""-   * sync --content: New option that makes the content of annexed files be-     transferred. Similar to the assistant, this honors any configured-     preferred content expressions.-   * Remove --json option from commands not supporting it.-   * status: Support --json.-   * list: Fix specifying of files to list.-   * Allow --all to be mixed with matching options like --copies and --in-     (but not --include and --exclude).-   * numcopies: New command, sets global numcopies value that is seen by all-     clones of a repository.-   * The annex.numcopies git config setting is deprecated. Once the numcopies-     command is used to set the global number of copies, any annex.numcopies-     git configs will be ignored.-   * assistant: Make the prefs page set the global numcopies.-   * Add lackingcopies, approxlackingcopies, and unused to-     preferred content expressions.-   * Client, transfer, incremental backup, and archive repositories-     now want to get content that does not yet have enough copies.-   * Client, transfer, and source repositories now do not want to retain-     unused file contents.-   * assistant: Checks daily for unused file contents, and when possible-     moves them to a repository (such as a backup repository) that-     wants to retain them.-   * assistant: annex.expireunused can be configured to cause unused-     file contents to be deleted after some period of time.-   * webapp: Nudge user to see if they want to expire old unused file-     contents when a lot of them seem to be piling up in the repository.-   * repair: Check git version at run time.-   * assistant: Run the periodic git gc in batch mode.-   * added annex.secure-erase-command config option.-   * Optimise non-bare http remotes; no longer does a 404 to the wrong-     url every time before trying the right url. Needs annex-bare to be-     set to false, which is done when initially probing the uuid of a-     http remote.-   * webapp: After upgrading a git repository to git-annex, fix-     bug that made it temporarily not be synced with.-   * whereis: Support --all.-   * All commands that support --all also support a --key option,-     which limits them to acting on a single key."""]]
+ doc/news/version_5.20140227.mdwn view
@@ -0,0 +1,32 @@+git-annex 5.20140227 released with [[!toggle text="these changes"]]+[[!toggleable text="""+   * metadata: Field names limited to alphanumerics and a few whitelisted+     punctuation characters to avoid issues with views, etc.+   * metadata: Field names are now case insensative.+   * When constructing views, metadata is available about the location of the+     file in the view's reference branch. Allows incorporating parts of the+     directory hierarchy in a view.+     For example `git annex view tag=* podcasts/=*` makes a view in the form+     tag/showname.+   * --metadata field=value can now use globs to match, and matches+     case insensatively, the same as git annex view field=value does.+   * annex.genmetadata can be set to make git-annex automatically set+     metadata (year and month) when adding files.+   * Make annex.web-options be used in several places that call curl.+   * Fix handling of rsync remote urls containing a username,+     including rsync.net.+   * Preserve metadata when staging a new version of an annexed file.+   * metadata: Support --json+   * webapp: Fix creation of box.com and Amazon S3 and Glacier+     repositories, broken in 5.20140221.+   * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid+     locking files, which is not needed by git-annex's use of webdav, and+     does not work on Box.com.+   * webdav: Fix path separator bug when used on Windows.+   * repair: Optimise unpacking of pack files, and avoid repeated error+     messages about corrupt pack files.+   * Add build dep on regex-compat to fix build on mipsel, which lacks+     regex-tdfa.+   * Disable test suite on sparc, which is missing optparse-applicative.+   * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp+     for only partially transferred objects."""]]
doc/tips/metadata_driven_views.mdwn view
@@ -1,5 +1,5 @@ git-annex now has support for storing -[[arbitrary metadata|design/metadata]] about annexed files. For example, this can be+[[arbitrary metadata|metadata]] about annexed files. For example, this can be used to tag files, to record the author of a file, etc. The metadata is synced around between repositories with the other information git-annex keeps track of.@@ -14,6 +14,12 @@ Let's get started by setting some tags on files. No views yet, just some metadata: +[[!template id=note text="""+To avoid needing to manually tag files with the year (and month),+run `annex.genmetadata true`, and git-annex will do it for you+when adding files.+"""]]+ 	# git annex metadata --tag todo work/2014/* 	# git annex metadata --untag todo work/2014/done/* 	# git annex metadata --tag urgent work/2014/presentation_for_tomorrow.odt@@ -24,8 +30,8 @@ 	# git annex metadata --tag done videos/old 	# git annex metadata --tag new videos/lotsofcats.ogv 	# git annex metadata --tag sound podcasts-	# git annex metadata --tag done podcasts/old-	# git annex metadata --tag new podcasts/recent+	# git annex metadata --tag done podcasts/*/old+	# git annex metadata --tag new podcasts/*/recent  So, you had a bunch of different kinds of files sorted into a directory structure. But that didn't really reflect how you approach the files.@@ -39,6 +45,12 @@ 	Switched to branch 'views/_' 	ok +[[!template id=note text="""+Notice that a single file may appear in multiple directories+depending on its tags. For example, `lotsofcats.ogv` is in+both `new/` and `video/`.+"""]]+ This searched for all files with any tag, and created a new git branch that sorts the files according to their tags. @@ -51,10 +63,6 @@ 	video 	sound -Notice that a single file may appear in multiple directories-depending on its tags. For example, `lotsofcats.ogv` is in-both `new/` and `video/`.- Ah, but you're at work now, and don't want to be distracted by cat videos. Time to filter the view: @@ -81,10 +89,12 @@ originally started from. You can also use `git checkout` to switch between views and other branches. -Beyond simple tags, you can add whatever kinds of metadata you like, and-use that metadata in more elaborate views. For example, let's add a year-field.+## fields +Beyond simple tags and directories, you can add whatever kinds of metadata+you like, and use that metadata in more elaborate views. For example, let's+add a year field.+ 	# git checkout master 	# git annex metadata --set year=2014 work/2014 	# git annex metadata --set year=2013 work/2013@@ -117,5 +127,26 @@ 	done 	  |-- 2014 	  `-- 2013++## location fields++Let's switch to a view containing only new podcasts. And since the+podcasts are organized into one subdirectory per show, let's+include those subdirectories in the view.++	# git checkout master+	# git annex view tag=new podcasts/=*+	# tree -d+	This_Developers_Life+	Escape_Pod+	GitMinutes+	The_Haskell_Cast+	StarShipSofa++That's an example of using part of the directory layout of the original+branch to inform the view. Every file gets fields automatically set up+corresponding to the directory it's in. So a file"foo/bar/baz/file" has+fields "/=foo", "foo/=bar", and "foo/bar/=baz". These location fields+can be used the same as other metadata to construct the view.  This has probably only scratched the surface of what you can do with views.
+ doc/todo/Views_Demo.mdwn view
@@ -0,0 +1,13 @@+Joey,++I've been thinking about leveraging git-annex for a workgroup document repository and I have just watched your views demo. The timing of the demo is great because I need to deploy a document repository with per-document metadata and your views concept seems like a great mechanism for associating metadata to documents and for displaying that metadata.++While I don't expect to use your views concept for my workgroup repostory, a later iteration might do.++The metadata in my use case begins with all the weird metadata seen on a book's copyright page. In addition, per-document provenance, like how one found the document and (if we're lucky) a URL where the latest version of the document may be found.  Metadata values may be simple strings or may be markdown text.++So, are you considering a metadata syntax that can support complex metadata? One example is multiple authors. Another issue is complex metadata values, like key=abstract and value="markdown text...".++FWIW,++Bob
+ doc/todo/Views_Demo/comment_1_d7c83a0e9a83e4a05aa74a34a7e1cf19._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joeyh.name/"+ ip="209.250.56.172"+ subject="comment 1"+ date="2014-02-24T18:17:04Z"+ content="""+All that should work fine. All metadata fields are multivalued, and the value can be any arbitrary data.+"""]]
+ doc/todo/ctrl_c_handling.mdwn view
@@ -0,0 +1,5 @@+Sometimes I start off a large file transfer to a new remote (a la "git-annex copy . --to glacier").++I believe all of the special remotes transfer the files one at a time, which is good, and provides a sensible place to interrupt a copy/move operation.++Wish: When I press ctrl+c in the terminal, git-annex will catch that and finish it's current transfer and then exit cleanly (ie: no odd backtraces in the special remote code). For the case where the file currently being transfered also needs to be killed (ie: it's a big .iso) then subsequent ctrl+c's can do that.
+ doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joeyh.name/"+ ip="209.250.56.172"+ subject="comment 1"+ date="2014-02-21T21:36:14Z"+ content="""+This really depends on the remote, some can resume where they were interrupted, such as rsync, and some cannot, such as glacier (and, er, encrypted rsync).+"""]]
+ doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://grossmeier.net/"+ nickname="greg"+ subject="very remote specific"+ date="2014-02-21T22:11:16Z"+ content="""+Yeah, this is very remote specific and probably means adding the functionality there as well (eg: in the glacier.py code, not only in git-annex haskell). Maybe I should file bugs there accordingly :)+"""]]
+ doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment view
@@ -0,0 +1,8 @@+[[!comment format=mdwn+ username="http://joeyh.name/"+ ip="209.250.56.172"+ subject="comment 3"+ date="2014-02-21T22:34:14Z"+ content="""+Hmm, I forget if it's possible for git-annex to mask SIGINT when it runs glacier or rsync, so that the child process does not receive it, but the parent git-annex does.+"""]]
doc/todo/windows_support.mdwn view
@@ -8,21 +8,7 @@   or perhaps easier,   <http://hackage.haskell.org/package/Win32-services-wrapper> -* XMPP library not yet built.-  -  This should work to install the deps, using libs from cygwin--	cabal install libxml-sax --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\libxml2-	cabal install gnuidn --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\-	cabal install gnutls --extra-lib-dirs=C:\\cygwin\\lib --extra-include-dirs=C:\\cygwin\\usr\\include\\--  While the 1st line works, the rest fail oddly. Looks like lack of-  quoting when cabal runs c2hs and gcc, as "Haskell Platform" is-  taken as 2 filenames. Needs investigation why this happens here-  and not other times..--  Also needs gsasl, which is not in cygwin. -  See <http://josefsson.org/gsasl4win/README.html>+* XMPP library not yet built. (See below.)  * View debug log is empty in windows -- all logs go to console.   This messes up a few parts of UI that direct user to the debug log.@@ -30,6 +16,8 @@   (and possibly gpg) are not prompted there anymore.  * Local pairing seems to fail, after acking on Linux box, it stalls.+  (Also, of course, the Windows box is unlikely to have a ssh server,+  so only pairing with a !Windows box will work.)  * gcrypt is not ported to windows (and as a shell script, may need   to be rewritten)@@ -43,9 +31,6 @@  ## minor problems -* Does not work with Cygwin's build of git (that git does not consistently-  support use of DOS style paths, which git-annex uses on Windows). -  Must use Msysgit. * rsync special remotes with a rsyncurl of a local directory are known   buggy. (git-annex tells rsync C:foo and it thinks it means a remote host   named C...)@@ -58,7 +43,21 @@  ## stuff needing testing -* test S3 and box.com setup in webapp now that they should work.. * test that adding a repo on a removable drive works; that git is synced to   it and files can be transferred to it and back * Does stopping in progress transfers work in the webapp?++## trying to build XMPP++1. gnutls-$LATEST.zip from <http://josefsson.org/gnutls4win/>+2. pkg-config from+   <http://sourceforge.net/projects/pkgconfiglite/files/latest/download?source=files>+3. libxml2 from mingw:+   <http://sourceforge.net/projects/mingw/files/MSYS/Extension/libxml2/libxml2-2.7.6-1/>+   both the -dll and the -dev+3. gsasl from <ftp://alpha.gnu.org/gnu/gsasl/>+3. gnuidn from <ftp://ftp.gnu.org/gnu/libidn/libidn-1.28-win32.zip>+3. Extract all the above into the Haskell platform's mingw directory. Note+   that pkg-config needs to be moved out of a named subdirectory.+4. Run in DOS prompt (not cygwin!): cabal install gnutls libxml-sax gsasl gnuidn network-protocol-xmpp+
git-annex.1 view
@@ -660,31 +660,40 @@ .IP  git annex metadata annexscreencast.ogv \-t video \-t screencast \-s author+=Alice .IP-.IP "\fBview [field=value ...] [tag ...]\fP"+.IP "\fBview [tag ...] [field=value ...] [location/=value]\fP" Uses metadata to build a view branch of the files in the current branch, and checks out the view branch. Only files in the current branch whose metadata matches all the specified field values and tags will be shown in the view. .IP+Once within a view, you can make additional directories, and+copy or move files into them. When you commit, the metadata will+be updated to correspond to your changes.+.IP Multiple values for a metadata field can be specified, either by using a glob (\fBfield="*"\fP) or by listing each wanted value. The resulting view will put files in subdirectories according to the value of their fields. .IP-Once within a view, you can make additional directories, and-copy or move files into them. When you commit, the metadata will-be updated to correspond to your changes.+There are fields corresponding to the path to the file. So a file+"foo/bar/baz/file" has fields "/=foo", "foo/=bar", and "foo/bar/=baz".+These location fields can be used the same as other metadata to construct+the view. .IP+For example, \fB/=podcasts\fP will only include files from the podcasts+directory in the view, while \fBpodcasts/=*\fP will preserve the+subdirectories of the podcasts directory in the view.+.IP .IP "\fBvpop [N]\fP" Switches from the currently active view back to the previous view. Or, from the first view back to original branch. .IP The optional number tells how many views to pop. .IP-.IP "\fBvfilter [field=value ...] [tag ...]\fP"+.IP "\fBvfilter [tag ...] [field=value ...] [location/=value]\fP" Filters the current view to only the files that have the-specified values and tags.+specified field values, tags, and locations. .IP-.IP "\fBvadd [field=glob ...]\fP"+.IP "\fBvadd [field=glob ...] [location/=glob]\fP" Changes the current view, adding an additional level of directories to categorize the files. .IP@@ -859,7 +868,7 @@ Rather than the normal output, generate JSON. This is intended to be parsed by programs that use git\-annex. Each line of output is a JSON object. Note that JSON output is only usable with some git\-annex commands,-like info, find, and whereis.+like info, find, whereis, and metadata. .IP .IP "\fB\-\-debug\fP" Show debug messages.@@ -1025,9 +1034,10 @@ The size can be specified with any commonly used units, for example, "0.5 gb" or "100 KiloBytes" .IP-.IP "\fB\-\-metadata field=value\fP"-Matches only files that have a metadata field attached with the specified-value.+.IP "\fB\-\-metadata field=glob\fP"+Matches only files that have a metadata field attached with a value that+matches the glob. The values of metadata fields are matched case+insensitively. .IP .IP "\fB\-\-want\-get\fP" Matches files that the preferred content settings for the repository@@ -1145,6 +1155,11 @@ .IP Note that setting numcopies to 0 is very unsafe. .IP+.IP "\fBannex.genmetadata\fP"+Set this to \fBtrue\fP to make git\-annex automatically generate some metadata+when adding files to the repository. In particular, it stores+year and month metadata, from the file's modification date.+.IP .IP "\fBannex.queuesize\fP" git\-annex builds a queue of git commands, in order to combine similar commands for speed. By default the size of the queue is limited to@@ -1351,8 +1366,7 @@ as described above. .IP .IP "\fBannex.web\-options\fP"-Options to use when using wget or curl to download a file from the web.-(wget is always used in preference to curl if available.)+Options to pass when running wget or curl. For example, to force ipv4 only, set it to "\-4" .IP .IP "\fBannex.quvi\-options\fP"
git-annex.cabal view
@@ -1,5 +1,5 @@ Name: git-annex-Version: 5.20140221+Version: 5.20140227 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess <joey@kitenet.net>@@ -93,7 +93,8 @@    extensible-exceptions, dataenc, SHA, process, json,    base (>= 4.5 && < 4.9), monad-control, MonadCatchIO-transformers,    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,-   SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3)+   SafeSemaphore, uuid, random, dlist, unix-compat, async, stm (>= 2.3),+   data-default, case-insensitive   CC-Options: -Wall   GHC-Options: -Wall   Extensions: PackageImports@@ -122,6 +123,8 @@   if flag(TDFA)     Build-Depends: regex-tdfa     CPP-Options: -DWITH_TDFA+  else+    Build-Depends: regex-compat      if flag(CryptoHash)     Build-Depends: cryptohash (>= 0.10.0)@@ -133,7 +136,7 @@    if flag(WebDAV)     Build-Depends: DAV ((>= 0.3 && < 0.6) || > 0.6),-     http-conduit, xml-conduit, http-types+     http-client, http-conduit, http-types, lifted-base     CPP-Options: -DWITH_WEBDAV    if flag(Assistant) && ! os(solaris)@@ -173,7 +176,7 @@   if flag(Webapp)     Build-Depends:      yesod, yesod-default, yesod-static, yesod-form, yesod-core,-     case-insensitive, http-types, transformers, wai, wai-logger, warp,+     http-types, transformers, wai, wai-logger, warp,      blaze-builder, crypto-api, hamlet, clientsession,      template-haskell, data-default, aeson, network-conduit     CPP-Options: -DWITH_WEBAPP