git-annex 7.20191017 → 7.20191024

raw patch · 16 files changed

+197/−49 lines, 16 files

Files

• Annex/AdjustedBranch.hs +16/−6
• CHANGELOG +22/−0
• Command/Drop.hs +1/−3
• Command/EnableTor.hs +6/−6
• Command/Smudge.hs +32/−14
• Database/Handle.hs +1/−1
• Database/Keys.hs +5/−1
• Database/Keys/SQL.hs +32/−1
• Git/Tree.hs +5/−3
• NEWS +9/−0
• Types/GitConfig.hs +2/−0
• Utility/InodeCache.hs +18/−1
• Utility/Su.hs +17/−8
• doc/git-annex-smudge.mdwn +7/−0
• doc/git-annex.mdwn +23/−4
• git-annex.cabal +1/−1

@@ -136,9 +136,6 @@ fromAdjustedBranch :: Branch -> OrigBranch fromAdjustedBranch b = maybe b snd (adjustedToOriginal b) -originalBranch :: Annex (Maybe OrigBranch)-originalBranch = fmap fromAdjustedBranch <$> inRepo Git.Branch.current- {- Enter an adjusted version of current branch (or, if already in an  - adjusted version of a branch, changes the adjustment of the original  - branch).@@ -227,15 +224,28 @@ adjustToCrippledFileSystem = do 	warning "Entering an adjusted branch where files are unlocked as this filesystem does not support locked files." 	checkVersionSupported-	whenM (isNothing <$> originalBranch) $+	whenM (isNothing <$> inRepo Git.Branch.current) $ 		void $ inRepo $ Git.Branch.commitCommand Git.Branch.AutomaticCommit 			[ Param "--quiet" 			, Param "--allow-empty" 			, Param "-m" 			, Param "commit before entering adjusted unlocked branch" 			]-	unlessM (enterAdjustedBranch (LinkAdjustment UnlockAdjustment)) $-		warning "Failed to enter adjusted branch!"+	inRepo Git.Branch.current >>= \case+		Just currbranch -> case getAdjustment currbranch of+			Just curradj | curradj == adj -> return ()+			_ -> do+				let adjbranch = originalToAdjusted currbranch adj+				ifM (inRepo (Git.Ref.exists $ adjBranch adjbranch))+					( unlessM (checkoutAdjustedBranch adjbranch []) $+						failedenter+					, unlessM (enterAdjustedBranch adj) $+						failedenter+					)+		Nothing -> failedenter+  where+	adj = LinkAdjustment UnlockAdjustment+	failedenter = warning "Failed to enter adjusted branch!"  setBasisBranch :: BasisBranch -> Ref -> Annex () setBasisBranch (BasisBranch basis) new = 

@@ -1,3 +1,25 @@+git-annex (7.20191024) upstream; urgency=medium++  * Changed git add/git commit -a default behavior back to what it was+    before v7; they add file contents to git, not to the annex. +    (However, if a file was annexed before, they will still add it to+    the annex, to avoid footgun.)+  * Configuring annex.largefiles overrides that; once git-annex has+    been told which files are large git add/git commit -a will annex them.+  * Added annex.gitaddtoannex configuration. Setting it to false prevents+    git add from adding files to the annex even when annex.largefiles+    is configured. (Unless the file was annexed before.)+  * smudge: Made git add smarter about renamed annexed files. It can tell+    when an annexed file was renamed, and will add it to the annex, +    and not to git, unless annex.largefiles tells it to do otherwise.+  * init: Fix a failure when used in a submodule on a crippled filesystem.+  * sync: Fix crash when there are submodules and an adjusted branch is+    checked out.+  * enable-tor: Deal with pkexec changing to root's home directory+    when running a command.++ -- Joey Hess <id@joeyh.name>  Fri, 25 Oct 2019 13:04:59 -0400+ git-annex (7.20191017) upstream; urgency=medium    * initremote: Added --sameas option, allows for two special remotes that

@@ -120,10 +120,8 @@  performRemote :: Key -> AssociatedFile -> NumCopies -> Remote -> CommandPerform performRemote key afile numcopies remote = do-	-- Filter the remote it's being dropped from out of the lists of+	-- Filter the uuid it's being dropped from out of the lists of 	-- places assumed to have the key, and places to check.-	-- When the local repo has the key, that's one additional copy,-	-- as long as the local repo is not untrusted. 	(tocheck, verified) <- verifiableCopies key [uuid] 	doDrop uuid Nothing key afile numcopies [uuid] verified tocheck 		( \proof -> do 

@@ -46,15 +46,12 @@ #else start _os = do #endif-	uuid <- getUUID-	when (uuid == NoUUID) $-		giveup "This can only be run in a git-annex repository." #ifndef mingw32_HOST_OS 	curruserid <- liftIO getEffectiveUserID 	if curruserid == 0 		then case readish =<< headMaybe os of 			Nothing -> giveup "Need user-id parameter."-			Just userid -> go uuid userid+			Just userid -> go userid 		else starting "enable-tor" (ActionItemOther Nothing) $ do 			gitannex <- liftIO readProgramFile 			let ps = [Param (cmdname cmd), Param (show curruserid)]@@ -67,10 +64,13 @@ 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps 				) #else-	go uuid 0+	go 0 #endif   where-	go uuid userid = do+	go userid = do+		uuid <- getUUID+		when (uuid == NoUUID) $+			giveup "This can only be run in a git-annex repository." 		(onionaddr, onionport) <- liftIO $ 			addHiddenService torAppName userid (fromUUID uuid) 		storeP2PAddress $ TorAnnex onionaddr onionport

@@ -1,6 +1,6 @@ {- git-annex command  -- - Copyright 2015-2018 Joey Hess <id@joeyh.name>+ - Copyright 2015-2019 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU AGPL version 3 or higher.  -}@@ -19,9 +19,11 @@ import qualified Git.BuildVersion import Git.FilePath import qualified Git-import qualified Git.Ref+import qualified Annex import Backend import Utility.Metered+import Annex.InodeSentinal+import Utility.InodeCache  import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L@@ -143,21 +145,37 @@ 		filepath <- liftIO $ absPath file 		return $ not $ dirContains repopath filepath --- New files are annexed as configured by annex.largefiles, with a default--- of annexing them.--- --- If annex.largefiles is not configured for a file, and a file with its--- name is already in the index, preserve its annexed/not annexed state.--- This prevents accidental conversions when annex.largefiles is being--- set/unset on the fly rather than being set in gitattributes or .git/config.+-- If annex.largefiles is configured, matching files are added to the+-- annex. But annex.gitaddtoannex can be set to false to disable that.+--+-- When annex.largefiles is not configured, files are normally not+-- added to the annex, so will be added to git. But some heuristics+-- are used to avoid bad behavior:+--+-- If the index already contains the file, preserve its annexed/not annexed+-- state. This prevents accidental conversions.+--+-- Otherwise, when the file's inode is the same as one that was used for+-- annexed content before, annex it. This handles cases such as renaming an+-- unlocked annexed file followed by git add, which the user naturally+-- expects to behave the same as git mv. shouldAnnex :: FilePath -> Maybe Key -> Annex Bool-shouldAnnex file moldkey = do-	matcher <- largeFilesMatcher-	checkFileMatcher' matcher file whenempty+shouldAnnex file moldkey = ifM (annexGitAddToAnnex <$> Annex.getGitConfig)+	( checkmatcher checkheuristics+	, checkheuristics+	)   where-	whenempty = case moldkey of+	checkmatcher d = do+		matcher <- largeFilesMatcher+		checkFileMatcher' matcher file d+	+	checkheuristics = case moldkey of 		Just _ -> return True-		Nothing -> isNothing <$> catObjectMetaData (Git.Ref.fileRef file)+		Nothing -> checkknowninode++	checkknowninode = withTSDelta (liftIO . genInodeCache file) >>= \case+		Nothing -> pure False+		Just ic -> Database.Keys.isInodeKnown ic =<< sentinalStatus  emitPointer :: Key -> IO () emitPointer = S.putStr . formatPointer

@@ -99,7 +99,7 @@ 	putMVar jobs $ QueryJob $ 		liftIO . putMVar res =<< tryNonAsync a 	(either throwIO return =<< takeMVar res)-		`catchNonAsync` (const $ error "sqlite query crashed")+		`catchNonAsync` (\e -> error $ "sqlite query crashed: " ++ show e)  {- Writes a change to the database.  -

@@ -1,6 +1,6 @@ {- Sqlite database of information about Keys  -- - Copyright 2015-2018 Joey Hess <id@joeyh.name>+ - Copyright 2015-2019 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU AGPL version 3 or higher.  -}@@ -19,6 +19,7 @@ 	addInodeCaches, 	getInodeCaches, 	removeInodeCaches,+	isInodeKnown, 	runWriter, ) where @@ -186,6 +187,9 @@  removeInodeCaches :: Key -> Annex () removeInodeCaches = runWriterIO . SQL.removeInodeCaches . toIKey++isInodeKnown :: InodeCache -> SentinalStatus -> Annex Bool+isInodeKnown i s = or <$> runReaderIO ((:[]) <$$> SQL.isInodeKnown i s)  {- Looks at staged changes to find when unlocked files are copied/moved,  - and updates associated files in the keys database.

@@ -1,6 +1,6 @@ {- Sqlite database of information about Keys  -- - Copyright 2015-2016 Joey Hess <id@joeyh.name>+ - Copyright 2015-2019 Joey Hess <id@joeyh.name>  -  - Licensed under the GNU AGPL version 3 or higher.  -}@@ -23,6 +23,9 @@ import Database.Persist.TH import Data.Time.Clock import Control.Monad+import Data.Maybe+import qualified Data.Text as T+import qualified Data.Conduit.List as CL  share [mkPersist sqlSettings, mkMigrate "migrateKeysDb"] [persistLowerCase| Associated@@ -116,3 +119,31 @@ removeInodeCaches :: IKey -> WriteHandle -> IO () removeInodeCaches ik = queueDb $ 	deleteWhere [ContentKey ==. ik]++{- Check if the inode is known to be used for an annexed file.+ -+ - This is currently slow due to the lack of indexes.+ -}+isInodeKnown :: InodeCache -> SentinalStatus -> ReadHandle -> IO Bool+isInodeKnown i s = readDb query+  where+	query +		| sentinalInodesChanged s =+			withRawQuery likesql [] $ isJust <$> CL.head+		| otherwise =+			isJust <$> selectFirst [ContentCache ==. si] []+	+	si = toSInodeCache i+			+	likesql = T.concat+		[ "SELECT key FROM content WHERE "+		, T.intercalate " OR " $ map mklike (likeInodeCacheWeak i)+		, " LIMIT 1"+		]++	mklike p = T.concat+		[ "cache LIKE "+		, "'I \"" -- SInodeCache serializes as I "..."+		, T.pack p+		, "\"'"+		]

@@ -100,7 +100,7 @@ mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive   where 	send h = do-		forM_ l $ \i -> hPutStr h $ case i of+		forM_ l $ \i ->	hPutStr h $ case i of 			TreeBlob f fm s -> mkTreeOutput fm BlobObject s f 			RecordedSubTree f s _ -> mkTreeOutput treeMode TreeObject s f 			NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree"@@ -127,7 +127,9 @@ 	deriving (Show, Eq)  treeItemToTreeContent :: TreeItem -> TreeContent-treeItemToTreeContent (TreeItem f m s) = TreeBlob f m s+treeItemToTreeContent (TreeItem f m s) = case toTreeItemType m of+	Just TreeSubmodule -> TreeCommit f m s+	_ -> TreeBlob f m s  treeItemToLsTreeItem :: TreeItem -> LsTree.TreeItem treeItemToLsTreeItem (TreeItem f mode sha) = LsTree.TreeItem@@ -235,7 +237,7 @@ 				let !modified' = modified || slmodified || wasmodified 				go h modified' (subtree : c) depth intree is' 			Just CommitObject -> do-				let ti = TreeCommit (LsTree.file i) (LsTree.mode i)  (LsTree.sha i)+				let ti = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) 				go h wasmodified (ti:c) depth intree is 			_ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") 		| otherwise = return (c, wasmodified, i:is)

@@ -1,3 +1,12 @@+git-annex (7.20191024) UNRELEASED; urgency=medium++  When annex.largefiles is not configured, `git add` and `git commit -a`+  add files to git, not to the annex. If you have gotten used to `git add`+  adding all files to the annex, you can get that behavior back by running:+  git config annex.largefiles anything++ -- Joey Hess <id@joeyh.name>  Thu, 24 Oct 2019 13:46:52 -0400+ git-annex (7.20190912) upstream; urgency=medium    This version of git-annex uses repository version 7 for all repositories.

@@ -78,6 +78,7 @@ 	, annexAriaTorrentOptions :: [String] 	, annexCrippledFileSystem :: Bool 	, annexLargeFiles :: Maybe String+	, annexGitAddToAnnex :: Bool 	, annexAddSmallFiles :: Bool 	, annexFsckNudge :: Bool 	, annexAutoUpgrade :: AutoUpgrade@@ -147,6 +148,7 @@ 	, annexAriaTorrentOptions = getwords (annex "aria-torrent-options") 	, annexCrippledFileSystem = getbool (annex "crippledfilesystem") False 	, annexLargeFiles = getmaybe (annex "largefiles")+	, annexGitAddToAnnex = getbool (annex "gitaddtoannex") True 	, annexAddSmallFiles = getbool (annex "addsmallfiles") True 	, annexFsckNudge = getbool (annex "fscknudge") True 	, annexAutoUpgrade = toAutoUpgrade $ getmaybe (annex "autoupgrade")

@@ -23,6 +23,7 @@ 	showInodeCache, 	genInodeCache, 	toInodeCache,+	likeInodeCacheWeak,  	InodeCacheKey, 	inodeCacheToKey,@@ -77,7 +78,7 @@  - The weak mtime comparison treats any mtimes that are within 2 seconds  - of one-another as the same. This is because FAT has only a 2 second  - resolution. When a FAT filesystem is used on Linux, higher resolution- - timestamps maybe be are cached and used by Linux, but they are lost+ - timestamps maybe are cached and used by Linux, but they are lost  - on unmount, so after a remount, the timestamp can appear to have changed.  -} compareWeak :: InodeCache -> InodeCache -> Bool@@ -148,6 +149,22 @@ 		, show size 		, show mtime 		]++-- Generates patterns that can be used in a SQL LIKE query to match+-- serialized inode caches that are weakly the same as the provided+-- InodeCache.+--+-- Like compareWeak, the size has to match, while the mtime can differ+-- by anything less than 2 seconds.+likeInodeCacheWeak :: InodeCache -> [String]+likeInodeCacheWeak (InodeCache (InodeCachePrim _ size mtime)) =+	lowresl ++ highresl+  where+	lowresl = map mkpat [t, t+1, t-1]+	highresl = map (++ " %") lowresl+	t = lowResTime mtime+	mkpat t' = "% " ++ ssz ++ " " ++ show t'+ 	ssz = show size  readInodeCache :: String -> Maybe InodeCache readInodeCache s = case words s of

@@ -61,24 +61,33 @@ -- decide based on the system's configuration whether sudo should be used. mkSuCommand :: String -> [CommandParam] -> IO (Maybe SuCommand) #ifndef mingw32_HOST_OS-mkSuCommand cmd ps = firstM (\(SuCommand _ p _) -> inPath p) =<< selectcmds+mkSuCommand cmd ps = do+	pwd <- getCurrentDirectory+	firstM (\(SuCommand _ p _) -> inPath p) =<< selectcmds pwd   where-	selectcmds = ifM (inx <||> (not <$> atconsole))-		( return (graphicalcmds ++ consolecmds)-		, return consolecmds+	selectcmds pwd = ifM (inx <||> (not <$> atconsole))+		( return (graphicalcmds pwd ++ consolecmds pwd)+		, return (consolecmds pwd) 		) 	 	inx = isJust <$> getEnv "DISPLAY" 	atconsole = queryTerminal stdInput  	-- These will only work when the user is logged into a desktop.-	graphicalcmds =+	graphicalcmds pwd = 		[ SuCommand (MayPromptPassword SomePassword) "gksu" 			[Param shellcmd] 		, SuCommand (MayPromptPassword SomePassword) "kdesu" 			[Param "-c", Param shellcmd]-		, SuCommand (MayPromptPassword SomePassword) "pkexec"-			([Param cmd] ++ ps)+		-- pkexec does not run the command in the current+		-- working directory, but in root's HOME.+		, SuCommand (MayPromptPassword SomePassword) "pkexec" $+			[Param "sh", Param "-c", Param $ unwords+				[ "cd", shellEscape pwd+				, "&&"+				, shellcmd+				]+			] 		-- Available in Debian's menu package; knows about lots of 		-- ways to gain root. 		, SuCommand (MayPromptPassword SomePassword) "su-to-root"@@ -89,7 +98,7 @@ 		] 	 	-- These will only work when run in a console.-	consolecmds = +	consolecmds _pwd =  		[ SuCommand (WillPromptPassword RootPassword) "su" 			[Param "-c", Param shellcmd] 		, SuCommand (MayPromptPassword UserPassword) "sudo"

@@ -17,6 +17,13 @@ When adding a file with `git add`, the annex.largefiles config is consulted to decide if a given file should be added to git as-is, or if its content are large enough to need to use git-annex.+The annex.gitaddtoannex setting overrides that; setting it to false+prevents `git add` from adding files to the annex.++However, if git-annex can tell that a file was annexed before,+it will still be added to the annex even when those configs would normally+prevent it. Two examples of this are adding a modified version of an+annexed file, and moving an annexed file to a new filename and adding that.  The git configuration to use this command as a filter driver is as follows. This is normally set up for you by git-annex init, so you should

@@ -891,12 +891,30 @@ * `annex.largefiles`    Used to configure which files are large enough to be added to the annex.-  Default: All files.+  It is an expression that matches the large files, eg+  "*.mp3 or largerthan(500kb)"    Overrides any annex.largefiles attributes in `.gitattributes` files. -  See <https://git-annex.branchable.com/tips/largefiles> for details.+  This configures the behavior of both git-annex and git when adding+  files to the repository. By default, `git-annex add` adds all files+  to the annex, and `git add` adds files to git (unless they were added+  to the annex previously). When annex.largefiles is configured, both+  `git annex add` and `git add` will add matching large files to the+  annex, and the other files to git. +  Other git-annex commands also honor annex.largefiles, including+  `git annex import`, `git annex addurl`, `git annex importfeed`+  and the assistant.++  See <https://git-annex.branchable.com/tips/largefiles> for syntax+  documentation and more.++* `annex.gitaddtoannex`++  Setting this to false will prevent `git add` from honoring the +  annex.largefiles configuration.+ * `annex.addsmallfiles`    Controls whether small files (not matching annex.largefiles)@@ -1664,9 +1682,10 @@ 	* annex.backend=WORM 	*.ogg annex.backend=SHA256E -There is a annex.largefiles attribute; which is used to configure which+There is a annex.largefiles attribute, which is used to configure which files are large enough to be added to the annex.-See <https://git-annex.branchable.com/tips/largefiles> for details.+See the documentation above of the annex.largefiles git config+and <https://git-annex.branchable.com/tips/largefiles> for details.  The numcopies setting can also be configured on a per-file-type basis via the `annex.numcopies` attribute in `.gitattributes` files. This overrides

@@ -1,5 +1,5 @@ Name: git-annex-Version: 7.20191017+Version: 7.20191024 Cabal-Version: >= 1.8 License: AGPL-3 Maintainer: Joey Hess <id@joeyh.name>