diff --git a/Annex/AdjustedBranch.hs b/Annex/AdjustedBranch.hs --- a/Annex/AdjustedBranch.hs +++ b/Annex/AdjustedBranch.hs @@ -136,9 +136,6 @@ fromAdjustedBranch :: Branch -> OrigBranch fromAdjustedBranch b = maybe b snd (adjustedToOriginal b) -originalBranch :: Annex (Maybe OrigBranch) -originalBranch = fmap fromAdjustedBranch <$> inRepo Git.Branch.current - {- Enter an adjusted version of current branch (or, if already in an - adjusted version of a branch, changes the adjustment of the original - branch). @@ -227,15 +224,28 @@ adjustToCrippledFileSystem = do warning "Entering an adjusted branch where files are unlocked as this filesystem does not support locked files." checkVersionSupported - whenM (isNothing <$> originalBranch) $ + whenM (isNothing <$> inRepo Git.Branch.current) $ void $ inRepo $ Git.Branch.commitCommand Git.Branch.AutomaticCommit [ Param "--quiet" , Param "--allow-empty" , Param "-m" , Param "commit before entering adjusted unlocked branch" ] - unlessM (enterAdjustedBranch (LinkAdjustment UnlockAdjustment)) $ - warning "Failed to enter adjusted branch!" + inRepo Git.Branch.current >>= \case + Just currbranch -> case getAdjustment currbranch of + Just curradj | curradj == adj -> return () + _ -> do + let adjbranch = originalToAdjusted currbranch adj + ifM (inRepo (Git.Ref.exists $ adjBranch adjbranch)) + ( unlessM (checkoutAdjustedBranch adjbranch []) $ + failedenter + , unlessM (enterAdjustedBranch adj) $ + failedenter + ) + Nothing -> failedenter + where + adj = LinkAdjustment UnlockAdjustment + failedenter = warning "Failed to enter adjusted branch!" setBasisBranch :: BasisBranch -> Ref -> Annex () setBasisBranch (BasisBranch basis) new = diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,25 @@ +git-annex (7.20191024) upstream; urgency=medium + + * Changed git add/git commit -a default behavior back to what it was + before v7; they add file contents to git, not to the annex. + (However, if a file was annexed before, they will still add it to + the annex, to avoid footgun.) + * Configuring annex.largefiles overrides that; once git-annex has + been told which files are large git add/git commit -a will annex them. + * Added annex.gitaddtoannex configuration. Setting it to false prevents + git add from adding files to the annex even when annex.largefiles + is configured. (Unless the file was annexed before.) + * smudge: Made git add smarter about renamed annexed files. It can tell + when an annexed file was renamed, and will add it to the annex, + and not to git, unless annex.largefiles tells it to do otherwise. + * init: Fix a failure when used in a submodule on a crippled filesystem. + * sync: Fix crash when there are submodules and an adjusted branch is + checked out. + * enable-tor: Deal with pkexec changing to root's home directory + when running a command. + + -- Joey Hess Fri, 25 Oct 2019 13:04:59 -0400 + git-annex (7.20191017) upstream; urgency=medium * initremote: Added --sameas option, allows for two special remotes that diff --git a/Command/Drop.hs b/Command/Drop.hs --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -120,10 +120,8 @@ performRemote :: Key -> AssociatedFile -> NumCopies -> Remote -> CommandPerform performRemote key afile numcopies remote = do - -- Filter the remote it's being dropped from out of the lists of + -- Filter the uuid it's being dropped from out of the lists of -- places assumed to have the key, and places to check. - -- When the local repo has the key, that's one additional copy, - -- as long as the local repo is not untrusted. (tocheck, verified) <- verifiableCopies key [uuid] doDrop uuid Nothing key afile numcopies [uuid] verified tocheck ( \proof -> do diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs --- a/Command/EnableTor.hs +++ b/Command/EnableTor.hs @@ -46,15 +46,12 @@ #else start _os = do #endif - uuid <- getUUID - when (uuid == NoUUID) $ - giveup "This can only be run in a git-annex repository." #ifndef mingw32_HOST_OS curruserid <- liftIO getEffectiveUserID if curruserid == 0 then case readish =<< headMaybe os of Nothing -> giveup "Need user-id parameter." - Just userid -> go uuid userid + Just userid -> go userid else starting "enable-tor" (ActionItemOther Nothing) $ do gitannex <- liftIO readProgramFile let ps = [Param (cmdname cmd), Param (show curruserid)] @@ -67,10 +64,13 @@ [ "Failed to run as root:" , gitannex ] ++ toCommand ps ) #else - go uuid 0 + go 0 #endif where - go uuid userid = do + go userid = do + uuid <- getUUID + when (uuid == NoUUID) $ + giveup "This can only be run in a git-annex repository." (onionaddr, onionport) <- liftIO $ addHiddenService torAppName userid (fromUUID uuid) storeP2PAddress $ TorAnnex onionaddr onionport diff --git a/Command/Smudge.hs b/Command/Smudge.hs --- a/Command/Smudge.hs +++ b/Command/Smudge.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2015-2018 Joey Hess + - Copyright 2015-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -19,9 +19,11 @@ import qualified Git.BuildVersion import Git.FilePath import qualified Git -import qualified Git.Ref +import qualified Annex import Backend import Utility.Metered +import Annex.InodeSentinal +import Utility.InodeCache import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L @@ -143,21 +145,37 @@ filepath <- liftIO $ absPath file return $ not $ dirContains repopath filepath --- New files are annexed as configured by annex.largefiles, with a default --- of annexing them. --- --- If annex.largefiles is not configured for a file, and a file with its --- name is already in the index, preserve its annexed/not annexed state. --- This prevents accidental conversions when annex.largefiles is being --- set/unset on the fly rather than being set in gitattributes or .git/config. +-- If annex.largefiles is configured, matching files are added to the +-- annex. But annex.gitaddtoannex can be set to false to disable that. +-- +-- When annex.largefiles is not configured, files are normally not +-- added to the annex, so will be added to git. But some heuristics +-- are used to avoid bad behavior: +-- +-- If the index already contains the file, preserve its annexed/not annexed +-- state. This prevents accidental conversions. +-- +-- Otherwise, when the file's inode is the same as one that was used for +-- annexed content before, annex it. This handles cases such as renaming an +-- unlocked annexed file followed by git add, which the user naturally +-- expects to behave the same as git mv. shouldAnnex :: FilePath -> Maybe Key -> Annex Bool -shouldAnnex file moldkey = do - matcher <- largeFilesMatcher - checkFileMatcher' matcher file whenempty +shouldAnnex file moldkey = ifM (annexGitAddToAnnex <$> Annex.getGitConfig) + ( checkmatcher checkheuristics + , checkheuristics + ) where - whenempty = case moldkey of + checkmatcher d = do + matcher <- largeFilesMatcher + checkFileMatcher' matcher file d + + checkheuristics = case moldkey of Just _ -> return True - Nothing -> isNothing <$> catObjectMetaData (Git.Ref.fileRef file) + Nothing -> checkknowninode + + checkknowninode = withTSDelta (liftIO . genInodeCache file) >>= \case + Nothing -> pure False + Just ic -> Database.Keys.isInodeKnown ic =<< sentinalStatus emitPointer :: Key -> IO () emitPointer = S.putStr . formatPointer diff --git a/Database/Handle.hs b/Database/Handle.hs --- a/Database/Handle.hs +++ b/Database/Handle.hs @@ -99,7 +99,7 @@ putMVar jobs $ QueryJob $ liftIO . putMVar res =<< tryNonAsync a (either throwIO return =<< takeMVar res) - `catchNonAsync` (const $ error "sqlite query crashed") + `catchNonAsync` (\e -> error $ "sqlite query crashed: " ++ show e) {- Writes a change to the database. - diff --git a/Database/Keys.hs b/Database/Keys.hs --- a/Database/Keys.hs +++ b/Database/Keys.hs @@ -1,6 +1,6 @@ {- Sqlite database of information about Keys - - - Copyright 2015-2018 Joey Hess + - Copyright 2015-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -19,6 +19,7 @@ addInodeCaches, getInodeCaches, removeInodeCaches, + isInodeKnown, runWriter, ) where @@ -186,6 +187,9 @@ removeInodeCaches :: Key -> Annex () removeInodeCaches = runWriterIO . SQL.removeInodeCaches . toIKey + +isInodeKnown :: InodeCache -> SentinalStatus -> Annex Bool +isInodeKnown i s = or <$> runReaderIO ((:[]) <$$> SQL.isInodeKnown i s) {- Looks at staged changes to find when unlocked files are copied/moved, - and updates associated files in the keys database. diff --git a/Database/Keys/SQL.hs b/Database/Keys/SQL.hs --- a/Database/Keys/SQL.hs +++ b/Database/Keys/SQL.hs @@ -1,6 +1,6 @@ {- Sqlite database of information about Keys - - - Copyright 2015-2016 Joey Hess + - Copyright 2015-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -23,6 +23,9 @@ import Database.Persist.TH import Data.Time.Clock import Control.Monad +import Data.Maybe +import qualified Data.Text as T +import qualified Data.Conduit.List as CL share [mkPersist sqlSettings, mkMigrate "migrateKeysDb"] [persistLowerCase| Associated @@ -116,3 +119,31 @@ removeInodeCaches :: IKey -> WriteHandle -> IO () removeInodeCaches ik = queueDb $ deleteWhere [ContentKey ==. ik] + +{- Check if the inode is known to be used for an annexed file. + - + - This is currently slow due to the lack of indexes. + -} +isInodeKnown :: InodeCache -> SentinalStatus -> ReadHandle -> IO Bool +isInodeKnown i s = readDb query + where + query + | sentinalInodesChanged s = + withRawQuery likesql [] $ isJust <$> CL.head + | otherwise = + isJust <$> selectFirst [ContentCache ==. si] [] + + si = toSInodeCache i + + likesql = T.concat + [ "SELECT key FROM content WHERE " + , T.intercalate " OR " $ map mklike (likeInodeCacheWeak i) + , " LIMIT 1" + ] + + mklike p = T.concat + [ "cache LIKE " + , "'I \"" -- SInodeCache serializes as I "..." + , T.pack p + , "\"'" + ] diff --git a/Git/Tree.hs b/Git/Tree.hs --- a/Git/Tree.hs +++ b/Git/Tree.hs @@ -100,7 +100,7 @@ mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive where send h = do - forM_ l $ \i -> hPutStr h $ case i of + forM_ l $ \i -> hPutStr h $ case i of TreeBlob f fm s -> mkTreeOutput fm BlobObject s f RecordedSubTree f s _ -> mkTreeOutput treeMode TreeObject s f NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree" @@ -127,7 +127,9 @@ deriving (Show, Eq) treeItemToTreeContent :: TreeItem -> TreeContent -treeItemToTreeContent (TreeItem f m s) = TreeBlob f m s +treeItemToTreeContent (TreeItem f m s) = case toTreeItemType m of + Just TreeSubmodule -> TreeCommit f m s + _ -> TreeBlob f m s treeItemToLsTreeItem :: TreeItem -> LsTree.TreeItem treeItemToLsTreeItem (TreeItem f mode sha) = LsTree.TreeItem @@ -235,7 +237,7 @@ let !modified' = modified || slmodified || wasmodified go h modified' (subtree : c) depth intree is' Just CommitObject -> do - let ti = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + let ti = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) go h wasmodified (ti:c) depth intree is _ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") | otherwise = return (c, wasmodified, i:is) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -1,3 +1,12 @@ +git-annex (7.20191024) UNRELEASED; urgency=medium + + When annex.largefiles is not configured, `git add` and `git commit -a` + add files to git, not to the annex. If you have gotten used to `git add` + adding all files to the annex, you can get that behavior back by running: + git config annex.largefiles anything + + -- Joey Hess Thu, 24 Oct 2019 13:46:52 -0400 + git-annex (7.20190912) upstream; urgency=medium This version of git-annex uses repository version 7 for all repositories. diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -78,6 +78,7 @@ , annexAriaTorrentOptions :: [String] , annexCrippledFileSystem :: Bool , annexLargeFiles :: Maybe String + , annexGitAddToAnnex :: Bool , annexAddSmallFiles :: Bool , annexFsckNudge :: Bool , annexAutoUpgrade :: AutoUpgrade @@ -147,6 +148,7 @@ , annexAriaTorrentOptions = getwords (annex "aria-torrent-options") , annexCrippledFileSystem = getbool (annex "crippledfilesystem") False , annexLargeFiles = getmaybe (annex "largefiles") + , annexGitAddToAnnex = getbool (annex "gitaddtoannex") True , annexAddSmallFiles = getbool (annex "addsmallfiles") True , annexFsckNudge = getbool (annex "fscknudge") True , annexAutoUpgrade = toAutoUpgrade $ getmaybe (annex "autoupgrade") diff --git a/Utility/InodeCache.hs b/Utility/InodeCache.hs --- a/Utility/InodeCache.hs +++ b/Utility/InodeCache.hs @@ -23,6 +23,7 @@ showInodeCache, genInodeCache, toInodeCache, + likeInodeCacheWeak, InodeCacheKey, inodeCacheToKey, @@ -77,7 +78,7 @@ - The weak mtime comparison treats any mtimes that are within 2 seconds - of one-another as the same. This is because FAT has only a 2 second - resolution. When a FAT filesystem is used on Linux, higher resolution - - timestamps maybe be are cached and used by Linux, but they are lost + - timestamps maybe are cached and used by Linux, but they are lost - on unmount, so after a remount, the timestamp can appear to have changed. -} compareWeak :: InodeCache -> InodeCache -> Bool @@ -148,6 +149,22 @@ , show size , show mtime ] + +-- Generates patterns that can be used in a SQL LIKE query to match +-- serialized inode caches that are weakly the same as the provided +-- InodeCache. +-- +-- Like compareWeak, the size has to match, while the mtime can differ +-- by anything less than 2 seconds. +likeInodeCacheWeak :: InodeCache -> [String] +likeInodeCacheWeak (InodeCache (InodeCachePrim _ size mtime)) = + lowresl ++ highresl + where + lowresl = map mkpat [t, t+1, t-1] + highresl = map (++ " %") lowresl + t = lowResTime mtime + mkpat t' = "% " ++ ssz ++ " " ++ show t' + ssz = show size readInodeCache :: String -> Maybe InodeCache readInodeCache s = case words s of diff --git a/Utility/Su.hs b/Utility/Su.hs --- a/Utility/Su.hs +++ b/Utility/Su.hs @@ -61,24 +61,33 @@ -- decide based on the system's configuration whether sudo should be used. mkSuCommand :: String -> [CommandParam] -> IO (Maybe SuCommand) #ifndef mingw32_HOST_OS -mkSuCommand cmd ps = firstM (\(SuCommand _ p _) -> inPath p) =<< selectcmds +mkSuCommand cmd ps = do + pwd <- getCurrentDirectory + firstM (\(SuCommand _ p _) -> inPath p) =<< selectcmds pwd where - selectcmds = ifM (inx <||> (not <$> atconsole)) - ( return (graphicalcmds ++ consolecmds) - , return consolecmds + selectcmds pwd = ifM (inx <||> (not <$> atconsole)) + ( return (graphicalcmds pwd ++ consolecmds pwd) + , return (consolecmds pwd) ) inx = isJust <$> getEnv "DISPLAY" atconsole = queryTerminal stdInput -- These will only work when the user is logged into a desktop. - graphicalcmds = + graphicalcmds pwd = [ SuCommand (MayPromptPassword SomePassword) "gksu" [Param shellcmd] , SuCommand (MayPromptPassword SomePassword) "kdesu" [Param "-c", Param shellcmd] - , SuCommand (MayPromptPassword SomePassword) "pkexec" - ([Param cmd] ++ ps) + -- pkexec does not run the command in the current + -- working directory, but in root's HOME. + , SuCommand (MayPromptPassword SomePassword) "pkexec" $ + [Param "sh", Param "-c", Param $ unwords + [ "cd", shellEscape pwd + , "&&" + , shellcmd + ] + ] -- Available in Debian's menu package; knows about lots of -- ways to gain root. , SuCommand (MayPromptPassword SomePassword) "su-to-root" @@ -89,7 +98,7 @@ ] -- These will only work when run in a console. - consolecmds = + consolecmds _pwd = [ SuCommand (WillPromptPassword RootPassword) "su" [Param "-c", Param shellcmd] , SuCommand (MayPromptPassword UserPassword) "sudo" diff --git a/doc/git-annex-smudge.mdwn b/doc/git-annex-smudge.mdwn --- a/doc/git-annex-smudge.mdwn +++ b/doc/git-annex-smudge.mdwn @@ -17,6 +17,13 @@ When adding a file with `git add`, the annex.largefiles config is consulted to decide if a given file should be added to git as-is, or if its content are large enough to need to use git-annex. +The annex.gitaddtoannex setting overrides that; setting it to false +prevents `git add` from adding files to the annex. + +However, if git-annex can tell that a file was annexed before, +it will still be added to the annex even when those configs would normally +prevent it. Two examples of this are adding a modified version of an +annexed file, and moving an annexed file to a new filename and adding that. The git configuration to use this command as a filter driver is as follows. This is normally set up for you by git-annex init, so you should diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -891,12 +891,30 @@ * `annex.largefiles` Used to configure which files are large enough to be added to the annex. - Default: All files. + It is an expression that matches the large files, eg + "*.mp3 or largerthan(500kb)" Overrides any annex.largefiles attributes in `.gitattributes` files. - See for details. + This configures the behavior of both git-annex and git when adding + files to the repository. By default, `git-annex add` adds all files + to the annex, and `git add` adds files to git (unless they were added + to the annex previously). When annex.largefiles is configured, both + `git annex add` and `git add` will add matching large files to the + annex, and the other files to git. + Other git-annex commands also honor annex.largefiles, including + `git annex import`, `git annex addurl`, `git annex importfeed` + and the assistant. + + See for syntax + documentation and more. + +* `annex.gitaddtoannex` + + Setting this to false will prevent `git add` from honoring the + annex.largefiles configuration. + * `annex.addsmallfiles` Controls whether small files (not matching annex.largefiles) @@ -1664,9 +1682,10 @@ * annex.backend=WORM *.ogg annex.backend=SHA256E -There is a annex.largefiles attribute; which is used to configure which +There is a annex.largefiles attribute, which is used to configure which files are large enough to be added to the annex. -See for details. +See the documentation above of the annex.largefiles git config +and for details. The numcopies setting can also be configured on a per-file-type basis via the `annex.numcopies` attribute in `.gitattributes` files. This overrides diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 7.20191017 +Version: 7.20191024 Cabal-Version: >= 1.8 License: AGPL-3 Maintainer: Joey Hess