diff --git a/Annex/AdjustedBranch.hs b/Annex/AdjustedBranch.hs --- a/Annex/AdjustedBranch.hs +++ b/Annex/AdjustedBranch.hs @@ -318,8 +318,8 @@ {- Update the currently checked out adjusted branch, merging the provided - branch into it. Note that the provided branch should be a non-adjusted - branch. -} -updateAdjustedBranch :: Branch -> (OrigBranch, Adjustment) -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Annex Bool -updateAdjustedBranch tomerge (origbranch, adj) mergeconfig commitmode = catchBoolIO $ +updateAdjustedBranch :: Branch -> (OrigBranch, Adjustment) -> [Git.Merge.MergeConfig] -> Annex Bool -> Git.Branch.CommitMode -> Annex Bool +updateAdjustedBranch tomerge (origbranch, adj) mergeconfig canresolvemerge commitmode = catchBoolIO $ join $ preventCommits go where adjbranch@(AdjBranch currbranch) = originalToAdjusted origbranch adj @@ -417,7 +417,7 @@ -- this commit will be a fast-forward. adjmergecommitff <- commitAdjustedTree' adjtree (BasisBranch mergecommit) [currbranch] showAction "Merging into adjusted branch" - ifM (autoMergeFrom adjmergecommitff (Just currbranch) mergeconfig commitmode) + ifM (autoMergeFrom adjmergecommitff (Just currbranch) mergeconfig canresolvemerge commitmode) ( reparent adjtree adjmergecommit =<< getcurrentcommit , return False ) diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs --- a/Annex/AutoMerge.hs +++ b/Annex/AutoMerge.hs @@ -43,18 +43,18 @@ - Callers should use Git.Branch.changed first, to make sure that - there are changes from the current branch to the branch being merged in. -} -autoMergeFrom :: Git.Ref -> Maybe Git.Ref -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Annex Bool -autoMergeFrom branch currbranch mergeconfig commitmode = do +autoMergeFrom :: Git.Ref -> Maybe Git.Ref -> [Git.Merge.MergeConfig] -> Annex Bool -> Git.Branch.CommitMode -> Annex Bool +autoMergeFrom branch currbranch mergeconfig canresolvemerge commitmode = do showOutput case currbranch of Nothing -> go Nothing Just b -> go =<< inRepo (Git.Ref.sha b) where go old = ifM isDirect - ( mergeDirect currbranch old branch (resolveMerge old branch False) mergeconfig commitmode + ( mergeDirect currbranch old branch resolvemerge mergeconfig commitmode , do r <- inRepo (Git.Merge.merge branch mergeconfig commitmode) - <||> (resolveMerge old branch False <&&> commitResolvedMerge commitmode) + <||> (resolvemerge <&&> commitResolvedMerge commitmode) -- Merging can cause new associated files to appear -- and the smudge filter will add them to the database. -- To ensure that this process sees those changes, @@ -62,6 +62,11 @@ Database.Keys.closeDb return r ) + where + resolvemerge = ifM canresolvemerge + ( resolveMerge old branch False + , return False + ) {- Resolves a conflicted merge. It's important that any conflicts be - resolved in a way that itself avoids later merge conflicts, since diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs --- a/Annex/Ssh.hs +++ b/Annex/Ssh.hs @@ -193,8 +193,10 @@ c <- Annex.getState Annex.concurrency case c of - Concurrent {} -> makeconnection socketlock - NonConcurrent -> return () + Concurrent {} + | annexUUID (remoteGitConfig gc) /= NoUUID -> + makeconnection socketlock + _ -> return () lockFileCached socketlock where @@ -207,11 +209,15 @@ -- When we can start the connection in batch mode, -- ssh won't prompt to the console. (_, connected) <- liftIO $ processTranscript "ssh" - (["-o", "BatchMode=true"] ++ toCommand startps) + (["-o", "BatchMode=true"] + ++ toCommand startps) Nothing - unless connected $ - prompt $ void $ liftIO $ + unless connected $ do + ok <- prompt $ liftIO $ boolSystem "ssh" startps + unless ok $ + warning $ "Unable to run git-annex-shell on remote " ++ + Git.repoDescribe (gitConfigRepo (remoteGitConfig gc)) -- Parameters to get ssh connected to the remote host, -- by asking it to run a no-op command. diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs --- a/Annex/Transfer.hs +++ b/Annex/Transfer.hs @@ -118,7 +118,9 @@ void $ liftIO $ tryIO $ writeTransferInfoFile info tfile return (Just lockhandle, False) - , return (Nothing, True) + , do + liftIO $ dropLock lockhandle + return (Nothing, True) ) #else prep tfile _mode info = catchPermissionDenied (const prepfailed) $ do diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs --- a/Assistant/Sync.hs +++ b/Assistant/Sync.hs @@ -211,7 +211,8 @@ else return Nothing haddiverged <- liftAnnex Annex.Branch.forceUpdate forM_ normalremotes $ \r -> - liftAnnex $ Command.Sync.mergeRemote r currentbranch Command.Sync.mergeConfig + liftAnnex $ Command.Sync.mergeRemote r + currentbranch Command.Sync.mergeConfig def return (catMaybes failed, haddiverged) where wantpull gc = remoteAnnexPull gc diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -1,6 +1,6 @@ {- git-annex assistant git merge thread - - - Copyright 2012 Joey Hess + - Copyright 2012-2017 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -15,6 +15,7 @@ import qualified Annex.Branch import qualified Git import qualified Git.Branch +import qualified Git.Ref import qualified Command.Sync {- This thread watches for changes to .git/refs/, and handles incoming @@ -63,14 +64,14 @@ diverged <- liftAnnex Annex.Branch.forceUpdate when diverged $ queueDeferredDownloads "retrying deferred download" Later - | "/synced/" `isInfixOf` file = - mergecurrent =<< liftAnnex (join Command.Sync.getCurrBranch) - | otherwise = noop + | otherwise = mergecurrent where changedbranch = fileToBranch file - mergecurrent currbranch@(Just b, _) - | equivBranches changedbranch b = + mergecurrent = + mergecurrent' =<< liftAnnex (join Command.Sync.getCurrBranch) + mergecurrent' currbranch@(Just b, _) + | changedbranch `isRelatedTo` b = whenM (liftAnnex $ inRepo $ Git.Branch.changed b changedbranch) $ do debug [ "merging", Git.fromRef changedbranch @@ -78,14 +79,22 @@ ] void $ liftAnnex $ Command.Sync.merge currbranch Command.Sync.mergeConfig + def Git.Branch.AutomaticCommit changedbranch - mergecurrent _ = noop + mergecurrent' _ = noop -equivBranches :: Git.Ref -> Git.Ref -> Bool -equivBranches x y = base x == base y +{- Is the first branch a synced branch or remote tracking branch related + - to the second branch, which should be merged into it? -} +isRelatedTo :: Git.Ref -> Git.Ref -> Bool +isRelatedTo x y + | basex /= takeDirectory basex ++ "/" ++ basey = False + | "/synced/" `isInfixOf` Git.fromRef x = True + | "refs/remotes/" `isPrefixOf` Git.fromRef x = True + | otherwise = False where - base = takeFileName . Git.fromRef + basex = Git.fromRef $ Git.Ref.base x + basey = Git.fromRef $ Git.Ref.base y isAnnexBranch :: FilePath -> Bool isAnnexBranch f = n `isSuffixOf` f diff --git a/Build/OSXMkLibs.hs b/Build/OSXMkLibs.hs --- a/Build/OSXMkLibs.hs +++ b/Build/OSXMkLibs.hs @@ -25,6 +25,7 @@ import Utility.Exception import Utility.Env import Utility.Misc +import Utility.Split import qualified Data.Map as M import qualified Data.Set as S diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,38 @@ +git-annex (6.20170520) unstable; urgency=medium + + * move --to=here moves from all reachable remotes to the local repository. + * initremote, enableremote: Support gpg subkeys suffixed with an + exclamation mark, which forces gpg to use a specific subkey. + * Improve progress display when watching file size, in cases where + a transfer does not resume. + * Fix transfer log file locking problem when running concurrent + transfers. + * Avoid concurrent git-config setting problem when running concurrent + threads. + * metadata: When setting metadata of a file that did not exist, + no error message was displayed, unlike getting metadata and most other + git-annex commands. Fixed this oversight. + * Added annex.resolvemerge configuration, which can be set to false to + disable the usual automatic merge conflict resolution done by git-annex + sync and the assistant. + * sync: Added --no-resolvemerge option. + * Avoid error about git-annex-shell not being found when + syncing with -J with a git remote where git-annex-shell is not + installed. + * Fix bug that prevented transfer locks from working when + run on SMB or other filesystem that does not support fcntl locks + and hard links. + * assistant: Merge changes from refs/remotes/foo/master into master. + Previously, only sync branches were merged. This makes regular git push + into a repository watched by the assistant auto-merge. + * Makefile: Install completions for the fish and zsh shells + when git-annex is built with optparse-applicative-0.14. + * assistant: Don't trust OSX FSEvents's eventFlagItemModified to be called + when the last writer of a file closes it; apparently that sometimes + does not happen, which prevented files from being quickly added. + + -- Joey Hess Mon, 12 Jun 2017 13:37:16 -0400 + git-annex (6.20170519) unstable; urgency=medium * Ssh password prompting improved when using -J for concurrency. diff --git a/CmdLine/Action.hs b/CmdLine/Action.hs --- a/CmdLine/Action.hs +++ b/CmdLine/Action.hs @@ -16,6 +16,7 @@ import Types.Concurrency import Messages.Concurrent import Types.Messages +import Remote.List import Control.Concurrent.Async import Control.Exception (throwIO) @@ -57,6 +58,12 @@ ws <- Annex.getState Annex.workers (st, ws') <- if null ws then do + -- Generate the remote list now, to avoid + -- each thread generating it, which would + -- be more expensive and could cause + -- threads to contend over eg, calls to + -- setConfig. + _ <- remoteList st <- dupState return (st, replicate (n-1) (Left st)) else do diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -1,6 +1,6 @@ {- git-annex command-line option parsing - - - Copyright 2010-2015 Joey Hess + - Copyright 2010-2017 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -97,6 +97,7 @@ cmdParams :: CmdParamsDesc -> Parser CmdParams cmdParams paramdesc = many $ argument str ( metavar paramdesc + <> action "file" ) parseAutoOption :: Parser Bool @@ -105,10 +106,10 @@ <> help "automatic mode" ) -parseRemoteOption :: Parser RemoteName -> Parser (DeferredParse Remote) -parseRemoteOption p = DeferredParse +parseRemoteOption :: RemoteName -> DeferredParse Remote +parseRemoteOption = DeferredParse . (fromJust <$$> Remote.byNameWithUUID) - . Just <$> p + . Just data FromToOptions = FromRemote (DeferredParse Remote) @@ -120,18 +121,18 @@ parseFromToOptions :: Parser FromToOptions parseFromToOptions = - (FromRemote <$> parseFromOption) - <|> (ToRemote <$> parseToOption) + (FromRemote . parseRemoteOption <$> parseFromOption) + <|> (ToRemote . parseRemoteOption <$> parseToOption) -parseFromOption :: Parser (DeferredParse Remote) -parseFromOption = parseRemoteOption $ strOption +parseFromOption :: Parser RemoteName +parseFromOption = strOption ( long "from" <> short 'f' <> metavar paramRemote <> help "source remote" <> completeRemotes ) -parseToOption :: Parser (DeferredParse Remote) -parseToOption = parseRemoteOption $ strOption +parseToOption :: Parser RemoteName +parseToOption = strOption ( long "to" <> short 't' <> metavar paramRemote <> help "destination remote" <> completeRemotes diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -41,7 +41,9 @@ ( withFilesInGit a params , if null params then giveup needforce - else seekActions $ prepFiltered a (getfiles [] params) + else do + checkFileOrDirectoryExists params + seekActions $ prepFiltered a (getfiles [] params) ) where getfiles c [] = return (reverse c) @@ -243,12 +245,15 @@ seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [FilePath] -> Annex [FilePath] seekHelper a params = do - forM_ params $ \p -> - unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ do - toplevelWarning False (p ++ " not found") - Annex.incError + checkFileOrDirectoryExists params inRepo $ \g -> concat . concat <$> forM (segmentXargsOrdered params) (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) + +checkFileOrDirectoryExists :: [FilePath] -> Annex () +checkFileOrDirectoryExists ps = forM_ ps $ \p -> + unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ do + toplevelWarning False (p ++ " not found") + Annex.incError notSymlink :: FilePath -> IO Bool notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f diff --git a/Command/Copy.hs b/Command/Copy.hs --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -52,7 +52,10 @@ | autoMode o = want <||> numCopiesCheck file key (<) | otherwise = return True want = case Command.Move.fromToOptions (moveOptions o) of - ToRemote dest -> (Remote.uuid <$> getParsed dest) >>= - wantSend False (Just key) (AssociatedFile (Just file)) - FromRemote _ -> - wantGet False (Just key) (AssociatedFile (Just file)) + Right (ToRemote dest) -> + (Remote.uuid <$> getParsed dest) >>= checkwantsend + Right (FromRemote _) -> checkwantget + Left Command.Move.ToHere -> checkwantget + + checkwantsend = wantSend False (Just key) (AssociatedFile (Just file)) + checkwantget = wantGet False (Just key) (AssociatedFile (Just file)) diff --git a/Command/Drop.hs b/Command/Drop.hs --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -45,7 +45,7 @@ <*> parseBatchOption parseDropFromOption :: Parser (DeferredParse Remote) -parseDropFromOption = parseRemoteOption $ strOption +parseDropFromOption = parseRemoteOption <$> strOption ( long "from" <> short 'f' <> metavar paramRemote <> help "drop content from a remote" <> completeRemotes diff --git a/Command/Fsck.hs b/Command/Fsck.hs --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -62,7 +62,7 @@ optParser :: CmdParamsDesc -> Parser FsckOptions optParser desc = FsckOptions <$> cmdParams desc - <*> optional (parseRemoteOption $ strOption + <*> optional (parseRemoteOption <$> strOption ( long "from" <> short 'f' <> metavar paramRemote <> help "check remote" <> completeRemotes diff --git a/Command/Get.hs b/Command/Get.hs --- a/Command/Get.hs +++ b/Command/Get.hs @@ -32,7 +32,7 @@ optParser :: CmdParamsDesc -> Parser GetOptions optParser desc = GetOptions <$> cmdParams desc - <*> optional parseFromOption + <*> optional (parseRemoteOption <$> parseFromOption) <*> parseAutoOption <*> optional (parseIncompleteOption <|> parseKeyOptions <|> parseFailedTransfersOption) <*> parseBatchOption diff --git a/Command/Merge.hs b/Command/Merge.hs --- a/Command/Merge.hs +++ b/Command/Merge.hs @@ -33,4 +33,4 @@ mergeSynced :: CommandStart mergeSynced = do prepMerge - mergeLocal mergeConfig =<< join getCurrBranch + mergeLocal mergeConfig def =<< join getCurrBranch diff --git a/Command/Move.hs b/Command/Move.hs --- a/Command/Move.hs +++ b/Command/Move.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010-2015 Joey Hess + - Copyright 2010-2017 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -27,20 +27,29 @@ data MoveOptions = MoveOptions { moveFiles :: CmdParams - , fromToOptions :: FromToOptions + , fromToOptions :: Either ToHere FromToOptions , keyOptions :: Maybe KeyOptions } +data ToHere = ToHere + optParser :: CmdParamsDesc -> Parser MoveOptions optParser desc = MoveOptions <$> cmdParams desc - <*> parseFromToOptions + <*> (parsefrom <|> parseto) <*> optional (parseKeyOptions <|> parseFailedTransfersOption) + where + parsefrom = Right . FromRemote . parseRemoteOption <$> parseFromOption + parseto = herespecialcase <$> parseToOption + where + herespecialcase "here" = Left ToHere + herespecialcase "." = Left ToHere + herespecialcase n = Right $ ToRemote $ parseRemoteOption n instance DeferredParseClass MoveOptions where finishParse v = MoveOptions <$> pure (moveFiles v) - <*> finishParse (fromToOptions v) + <*> either (pure . Left) (Right <$$> finishParse) (fromToOptions v) <*> pure (keyOptions v) seek :: MoveOptions -> CommandSeek @@ -61,10 +70,15 @@ start' :: MoveOptions -> Bool -> AssociatedFile -> Key -> ActionItem -> CommandStart start' o move afile key ai = case fromToOptions o of - FromRemote src -> checkFailedTransferDirection ai Download $ - fromStart move afile key ai =<< getParsed src - ToRemote dest -> checkFailedTransferDirection ai Upload $ - toStart move afile key ai =<< getParsed dest + Right (FromRemote src) -> + checkFailedTransferDirection ai Download $ + fromStart move afile key ai =<< getParsed src + Right (ToRemote dest) -> + checkFailedTransferDirection ai Upload $ + toStart move afile key ai =<< getParsed dest + Left ToHere -> + checkFailedTransferDirection ai Download $ + toHereStart move afile key ai showMoveAction :: Bool -> Key -> ActionItem -> Annex () showMoveAction move = showStart' (if move then "move" else "copy") @@ -171,14 +185,15 @@ return $ u /= Remote.uuid src && elem src remotes fromPerform :: Remote -> Bool -> Key -> AssociatedFile -> CommandPerform -fromPerform src move key afile = ifM (inAnnex key) - ( dispatch move True - , dispatch move =<< go - ) +fromPerform src move key afile = do + showAction $ "from " ++ Remote.name src + ifM (inAnnex key) + ( dispatch move True + , dispatch move =<< go + ) where go = notifyTransfer Download afile $ - download (Remote.uuid src) key afile forwardRetry $ \p -> do - showAction $ "from " ++ Remote.name src + download (Remote.uuid src) key afile forwardRetry $ \p -> getViaTmp (RemoteVerify src) key $ \t -> Remote.retrieveKeyFile src key afile t p dispatch _ False = stop -- failed @@ -198,3 +213,20 @@ ok <- Remote.removeKey src key next $ Command.Drop.cleanupRemote key src ok faileddropremote = giveup "Unable to drop from remote." + +{- Moves (or copies) the content of an annexed file from reachable remotes + - to the current repository. + - + - When moving, the content is removed from all the reachable remotes. -} +toHereStart :: Bool -> AssociatedFile -> Key -> ActionItem -> CommandStart +toHereStart move afile key ai + | move = go + | otherwise = stopUnless (not <$> inAnnex key) go + where + go = do + rs <- Remote.keyPossibilities key + forM_ rs $ \r -> + includeCommandAction $ do + showMoveAction move key ai + next $ fromPerform r move key afile + stop diff --git a/Command/PostReceive.hs b/Command/PostReceive.hs --- a/Command/PostReceive.hs +++ b/Command/PostReceive.hs @@ -48,4 +48,4 @@ updateInsteadEmulation :: CommandStart updateInsteadEmulation = do prepMerge - mergeLocal mergeConfig =<< join getCurrBranch + mergeLocal mergeConfig def =<< join getCurrBranch diff --git a/Command/Sync.hs b/Command/Sync.hs --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -76,8 +76,14 @@ , noContentOption :: Bool , contentOfOption :: [FilePath] , keyOptions :: Maybe KeyOptions + , resolveMergeOverride :: ResolveMergeOverride } +newtype ResolveMergeOverride = ResolveMergeOverride Bool + +instance Default ResolveMergeOverride where + def = ResolveMergeOverride False + optParser :: CmdParamsDesc -> Parser SyncOptions optParser desc = SyncOptions <$> (many $ argument str @@ -117,6 +123,9 @@ <> metavar paramPath )) <*> optional parseAllOption + <*> (ResolveMergeOverride <$> invertableSwitch "resolvemerge" True + ( help "do not automatically resolve merge conflicts" + )) -- Since prepMerge changes the working directory, FilePath options -- have to be adjusted. @@ -132,6 +141,7 @@ <*> pure (noContentOption v) <*> liftIO (mapM absPath (contentOfOption v)) <*> pure (keyOptions v) + <*> pure (resolveMergeOverride v) seek :: SyncOptions -> CommandSeek seek o = allowConcurrentOutput $ do @@ -150,7 +160,7 @@ -- These actions cannot be run concurrently. mapM_ includeCommandAction $ concat [ [ commit o ] - , [ withbranch (mergeLocal mergeConfig) ] + , [ withbranch (mergeLocal mergeConfig (resolveMergeOverride o)) ] , map (withbranch . pullRemote o mergeConfig) gitremotes , [ mergeAnnex ] ] @@ -219,11 +229,14 @@ , Git.Merge.MergeUnrelatedHistories ] -merge :: CurrBranch -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Git.Branch -> Annex Bool -merge (Just b, Just adj) mergeconfig commitmode tomerge = - updateAdjustedBranch tomerge (b, adj) mergeconfig commitmode -merge (b, _) mergeconfig commitmode tomerge = - autoMergeFrom tomerge b mergeconfig commitmode +merge :: CurrBranch -> [Git.Merge.MergeConfig] -> ResolveMergeOverride -> Git.Branch.CommitMode -> Git.Branch -> Annex Bool +merge currbranch mergeconfig resolvemergeoverride commitmode tomerge = case currbranch of + (Just b, Just adj) -> updateAdjustedBranch tomerge (b, adj) mergeconfig canresolvemerge commitmode + (b, _) -> autoMergeFrom tomerge b mergeconfig canresolvemerge commitmode + where + canresolvemerge = case resolvemergeoverride of + ResolveMergeOverride True -> getGitConfigVal annexResolveMerge + ResolveMergeOverride False -> return False syncBranch :: Git.Branch -> Git.Branch syncBranch = Git.Ref.underBase "refs/heads/synced" . fromDirectBranch . fromAdjustedBranch @@ -296,15 +309,15 @@ void $ inRepo $ Git.Branch.commit commitmode False commitmessage branch parents return True -mergeLocal :: [Git.Merge.MergeConfig] -> CurrBranch -> CommandStart -mergeLocal mergeconfig currbranch@(Just _, _) = +mergeLocal :: [Git.Merge.MergeConfig] -> ResolveMergeOverride -> CurrBranch -> CommandStart +mergeLocal mergeconfig resolvemergeoverride currbranch@(Just _, _) = go =<< needMerge currbranch where go Nothing = stop go (Just syncbranch) = do showStart "merge" $ Git.Ref.describe syncbranch - next $ next $ merge currbranch mergeconfig Git.Branch.ManualCommit syncbranch -mergeLocal _ (Nothing, madj) = do + next $ next $ merge currbranch mergeconfig resolvemergeoverride Git.Branch.ManualCommit syncbranch +mergeLocal _ _ (Nothing, madj) = do b <- inRepo Git.Branch.currentUnsafe ifM (isJust <$> needMerge (b, madj)) ( do @@ -365,7 +378,7 @@ next $ do showOutput stopUnless fetch $ - next $ mergeRemote remote branch mergeconfig + next $ mergeRemote remote branch mergeconfig (resolveMergeOverride o) where fetch = inRepoWithSshOptionsTo (Remote.repo remote) (Remote.gitconfig remote) $ Git.Command.runBool @@ -377,8 +390,8 @@ - were committed (or pushed changes, if this is a bare remote), - while the synced/master may have changes that some - other remote synced to this remote. So, merge them both. -} -mergeRemote :: Remote -> CurrBranch -> [Git.Merge.MergeConfig] -> CommandCleanup -mergeRemote remote currbranch mergeconfig = ifM isBareRepo +mergeRemote :: Remote -> CurrBranch -> [Git.Merge.MergeConfig] -> ResolveMergeOverride -> CommandCleanup +mergeRemote remote currbranch mergeconfig resolvemergeoverride = ifM isBareRepo ( return True , case currbranch of (Nothing, _) -> do @@ -390,7 +403,7 @@ ) where mergelisted getlist = and <$> - (mapM (merge currbranch mergeconfig Git.Branch.ManualCommit . remoteBranch remote) =<< getlist) + (mapM (merge currbranch mergeconfig resolvemergeoverride Git.Branch.ManualCommit . remoteBranch remote) =<< getlist) tomerge = filterM (changed remote) branchlist Nothing = [] branchlist (Just branch) = [branch, syncBranch branch] diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -58,6 +58,7 @@ , annexHttpHeaders :: [String] , annexHttpHeadersCommand :: Maybe String , annexAutoCommit :: Configurable Bool + , annexResolveMerge :: Configurable Bool , annexSyncContent :: Configurable Bool , annexDebug :: Bool , annexWebOptions :: [String] @@ -115,6 +116,8 @@ , annexHttpHeadersCommand = getmaybe (annex "http-headers-command") , annexAutoCommit = configurable True $ getmaybebool (annex "autocommit") + , annexResolveMerge = configurable True $ + getmaybebool (annex "resolvemerge") , annexSyncContent = configurable False $ getmaybebool (annex "synccontent") , annexDebug = getbool (annex "debug") False diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -64,18 +64,18 @@ {- With inotify, file closing is tracked to some extent, so an add event - will always be received for a file once its writer closes it, and - (typically) not before. This may mean multiple add events for the same file. - - - - fsevents behaves similarly, although different event types are used for - - creating and modification of the file. - - OTOH, with kqueue, add events will often be received while a file is - still being written to, and then no add event will be received once the - - writer closes it. -} + - writer closes it. + - + - fsevents sometimes behaves similarly, but has sometimes been + - seen to behave like kqueue. -} closingTracked :: Bool -#if (WITH_INOTIFY || WITH_FSEVENTS || WITH_WIN32NOTIFY) +#if (WITH_INOTIFY || WITH_WIN32NOTIFY) closingTracked = True #else -#if WITH_KQUEUE +#if (WITH_KQUEUE || WITH_FSEVENTS) closingTracked = False #else closingTracked = error "closingTracked not defined" diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -22,6 +22,7 @@ import Control.Concurrent import Control.Monad.IO.Class import qualified Data.Map as M +import Data.Char type KeyId = String @@ -157,12 +158,20 @@ - a key id, or a name; See the section 'HOW TO SPECIFY A USER ID' of - GnuPG's manpage.) -} findPubKeys :: GpgCmd -> String -> IO KeyIds -findPubKeys cmd for = KeyIds . parse . lines <$> readStrict cmd params +findPubKeys cmd for + -- pass forced subkey through as-is rather than + -- looking up the master key. + | isForcedSubKey for = return $ KeyIds [for] + | otherwise = KeyIds . parse . lines <$> readStrict cmd params where params = [Param "--with-colons", Param "--list-public-keys", Param for] parse = mapMaybe (keyIdField . splitc ':') keyIdField ("pub":_:_:_:f:_) = Just f keyIdField _ = Nothing + +{- "subkey!" tells gpg to force use of a specific subkey -} +isForcedSubKey :: String -> Bool +isForcedSubKey s = "!" `isSuffixOf` s && all isHexDigit (drop 1 s) type UserId = String diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs --- a/Utility/LockFile/PidLock.hs +++ b/Utility/LockFile/PidLock.hs @@ -122,18 +122,22 @@ setFileMode tmp (combineModes readModes) hPutStr h . show =<< mkPidLock hClose h - st <- getFileStatus tmp - let failedlock = do + let failedlock st = do dropLock $ LockHandle tmp st sidelock + nukeFile tmp return Nothing - let tooklock = return $ Just $ LockHandle lockfile' st sidelock + let tooklock st = return $ Just $ LockHandle lockfile' st sidelock ifM (linkToLock sidelock tmp lockfile') ( do nukeFile tmp - tooklock + -- May not have made a hard link, so stat + -- the lockfile + lckst <- getFileStatus lockfile' + tooklock lckst , do v <- readPidLock lockfile' hn <- getHostName + tmpst <- getFileStatus tmp case v of Just pl | isJust sidelock && hn == lockingHost pl -> do -- Since we have the sidelock, @@ -142,8 +146,8 @@ -- we know that the pidlock is -- stale, and can take it over. rename tmp lockfile' - tooklock - _ -> failedlock + tooklock tmpst + _ -> failedlock tmpst ) -- Linux's open(2) man page recommends linking a pid lock into place, diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs --- a/Utility/LockPool/STM.hs +++ b/Utility/LockPool/STM.hs @@ -49,9 +49,9 @@ -- A shared global variable for the lockPool. Avoids callers needing to -- maintain state for this implementation detail. +{-# NOINLINE lockPool #-} lockPool :: LockPool lockPool = unsafePerformIO (newTMVarIO M.empty) -{-# NOINLINE lockPool #-} -- Updates the LockPool, blocking as necessary if another thread is holding -- a conflicting lock. @@ -62,23 +62,23 @@ -- Keeping the whole Map in a TMVar accomplishes this, at the expense of -- sometimes retrying after unrelated changes in the map. waitTakeLock :: LockPool -> LockFile -> LockMode -> STM LockHandle -waitTakeLock pool file mode = do - m <- takeTMVar pool - v <- case M.lookup file m of - Just (LockStatus mode' n closelockfile) - | mode == LockShared && mode' == LockShared -> - return $ LockStatus mode (succ n) closelockfile - | n > 0 -> retry -- wait for lock - _ -> return $ LockStatus mode 1 noop - putTMVar pool (M.insert file v m) - newTMVar (pool, file) +waitTakeLock pool file mode = maybe retry return =<< tryTakeLock pool file mode -- Avoids blocking if another thread is holding a conflicting lock. tryTakeLock :: LockPool -> LockFile -> LockMode -> STM (Maybe LockHandle) -tryTakeLock pool file mode = - (Just <$> waitTakeLock pool file mode) - `orElse` - return Nothing +tryTakeLock pool file mode = do + m <- takeTMVar pool + let success v = do + putTMVar pool (M.insert file v m) + Just <$> newTMVar (pool, file) + case M.lookup file m of + Just (LockStatus mode' n closelockfile) + | mode == LockShared && mode' == LockShared -> + success $ LockStatus mode (succ n) closelockfile + | n > 0 -> do + putTMVar pool m + return Nothing + _ -> success $ LockStatus mode 1 noop -- Call after waitTakeLock or tryTakeLock, to register a CloseLockFile -- action to run when releasing the lock. diff --git a/Utility/Metered.hs b/Utility/Metered.hs --- a/Utility/Metered.hs +++ b/Utility/Metered.hs @@ -171,22 +171,27 @@ k = 1024 chunkOverhead = 2 * sizeOf (1 :: Int) -- GHC specific -{- Runs an action, watching a file as it grows and updating the meter. -} +{- Runs an action, watching a file as it grows and updating the meter. + - + - The file may already exist, and the action could throw the original file + - away and start over. To avoid reporting the original file size followed + - by a smaller size in that case, wait until the file starts growing + - before updating the meter for the first time. + -} watchFileSize :: (MonadIO m, MonadMask m) => FilePath -> MeterUpdate -> m a -> m a watchFileSize f p a = bracket - (liftIO $ forkIO $ watcher zeroBytesProcessed) + (liftIO $ forkIO $ watcher =<< getsz) (liftIO . void . tryIO . killThread) (const a) where watcher oldsz = do - v <- catchMaybeIO $ toBytesProcessed <$> getFileSize f - newsz <- case v of - Just sz | sz /= oldsz -> do - p sz - return sz - _ -> return oldsz threadDelay 500000 -- 0.5 seconds - watcher newsz + sz <- getsz + when (sz > oldsz) $ + p sz + watcher sz + getsz = catchDefaultIO zeroBytesProcessed $ + toBytesProcessed <$> getFileSize f data OutputHandler = OutputHandler { quietMode :: Bool diff --git a/doc/git-annex-adjust.mdwn b/doc/git-annex-adjust.mdwn --- a/doc/git-annex-adjust.mdwn +++ b/doc/git-annex-adjust.mdwn @@ -4,7 +4,7 @@ # SYNOPSIS -`git annex adjust --unlock|--fix` +git annex adjust `--unlock|--fix` # DESCRIPTION diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -14,14 +14,14 @@ * `--from=remote` - Use this option to copy the content of files from the specified + Copy the content of files from the specified remote to the local repository. Any files that are not available on the remote will be silently skipped. * `--to=remote` - Use this option to copy the content of files from the local repository + Copy the content of files from the local repository to the specified remote. * `--jobs=N` `-JN` @@ -37,12 +37,15 @@ * `--fast` - Avoid contacting the remote to check if it has every file when copying - --to it. + When copying content to a remote, avoid a round trip to check if the remote + already has content. This can be faster, but might skip copying content + to the remote in some cases. * `--force` - Force checking the remote for every file when copying --from it. + When copying content from a remote, ignore location tracking information + and always check if the remote has content. Can be useful if the location + tracking information is out of date. * `--all` diff --git a/doc/git-annex-group.mdwn b/doc/git-annex-group.mdwn --- a/doc/git-annex-group.mdwn +++ b/doc/git-annex-group.mdwn @@ -4,7 +4,7 @@ # SYNOPSIS -git annex group `repository groupname` +git annex group `repository [groupname]` # DESCRIPTION diff --git a/doc/git-annex-merge.mdwn b/doc/git-annex-merge.mdwn --- a/doc/git-annex-merge.mdwn +++ b/doc/git-annex-merge.mdwn @@ -12,6 +12,9 @@ that is done by the sync command, but without pushing or pulling any data. +When annex.resolvemerge is set to false, merge conflict resolution +will not be done. + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn --- a/doc/git-annex-move.mdwn +++ b/doc/git-annex-move.mdwn @@ -4,7 +4,7 @@ # SYNOPSIS -git annex move `[path ...] [--from=remote|--to=remote]` +git annex move `[path ...] [--from=remote|--to=remote|--to=here]` # DESCRIPTION @@ -14,14 +14,17 @@ * `--from=remote` - Use this option to move the content of files from the specified - remote to the local repository. + Move the content of files from the specified remote to the local repository. * `--to=remote` - Use this option to move the content of files from the local repository - to the specified remote. + Move the content of files from the local repository to the specified remote. +* `--to=here` + + Move the content of files from all reachable remotes to the local + repository. + * `--jobs=N` `-JN` Enables parallel transfers with up to the specified number of jobs @@ -49,6 +52,18 @@ * `--key=keyname` Use this option to move a specified key. + +* `--fast` + + When moving content to a remote, avoid a round trip to check if the remote + already has content. This can be faster, but might skip moving content + to the remote in some cases. + +* `--force` + + When moving content from a remote, ignore location tracking information + and always check if the remote has content. Can be useful if the location + tracking information is out of date. * file matching options diff --git a/doc/git-annex-resolvemerge.mdwn b/doc/git-annex-resolvemerge.mdwn --- a/doc/git-annex-resolvemerge.mdwn +++ b/doc/git-annex-resolvemerge.mdwn @@ -12,9 +12,28 @@ file to the tree, using variants of their filename. This is done automatically when using `git annex sync` or `git annex merge`. -Note that only merge conflicts that involve an annexed file are resolved. -Merge conflicts between two files that are not annexed will not be -automatically resolved. +Note that only merge conflicts that involve one or more annexed files +are resolved. Merge conflicts between two files that are not annexed +will not be automatically resolved. + +# EXAMPLE + +Suppose Alice commits a change to annexed file `foo`, and Bob commits +a different change to the same file `foo`. + +Merging between them will then fail, and git will present the +merge conflict as a file `foo` pointing to one version of the +git-annex symlink, with `git status` indicating that `foo` has an +unresolved conflict. + +Running `git annex resolvemerge` in this situation will resolve the merge +conflict, by replacing the file `foo` with files named like +`foo.variant-c696` and `foo.variant-f16a`. One of the files has the content +that Alice committed, and the other has the content that Bob committed. + +The user can then examine the two variants of the file, and either merge +the two changes into a single file, or rename one of them back to `foo` +and delete the other. # SEE ALSO diff --git a/doc/git-annex-sync.mdwn b/doc/git-annex-sync.mdwn --- a/doc/git-annex-sync.mdwn +++ b/doc/git-annex-sync.mdwn @@ -21,13 +21,9 @@ The content of annexed objects is not synced by default, but the --content option (see below) can make that also be synchronized. -Merge conflicts are automatically handled by sync. When two conflicting -versions of a file have been committed, both will be added to the tree, -under different filenames. For example, file "foo" would be replaced -with "foo.somekey" and "foo.otherkey". - -Note that syncing with a remote will not update the remote's working -tree with changes made to the local repository. However, those changes +Note that syncing with a remote will not normally update the remote's working +tree with changes made to the local repository. (Unless it's configured +with receive.denyCurrentBranch=updateInstead.) However, those changes are pushed to the remote, so they can be merged into its working tree by running "git annex sync" on the remote. @@ -112,6 +108,18 @@ parallel. Pulls are not done in parallel because that tends to be less efficient. When --content is synced, the files are processed in parallel as well. + +* `--resolvemerge`, `--no-resolvemerge` + + By default, merge conflicts are automatically handled by sync. When two + conflicting versions of a file have been committed, both will be added + to the tree, under different filenames. For example, file "foo" + would be replaced with "foo.variant-A" and "foo.variant-B". (See + [[git-annex-resolvemerge]](1) for details.) + + Use `--no-resolvemerge` to disable this automatic merge conflict + resolution. It can also be disabled by setting annex.resolvemerge + to false. # SEE ALSO diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1040,6 +1040,15 @@ To configure the behavior in all clones of the repository, this can be set in [[git-annex-config]]. +* `annex.resolvemerge` + + Set to false to prevent merge conflicts being automatically resolved + by the git-annex assitant, git-annex sync, git-annex merge, + and the git-annex post-receive hook. + + To configure the behavior in all clones of the repository, + this can be set in [[git-annex-config]]. + * `annex.synccontent` Set to true to make git-annex sync default to syncing content. diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 6.20170519 +Version: 6.20170520 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess