diff --git a/Annex/Init.hs b/Annex/Init.hs --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -40,6 +40,7 @@ import Utility.UserInfo import Utility.FileMode import Annex.Perms +import System.Posix.User #endif genDescription :: Maybe String -> Annex String @@ -138,9 +139,13 @@ createSymbolicLink f f2 nukeFile f2 preventWrite f - -- Should be unable to write to the file, but some crippled + -- Should be unable to write to the file, unless + -- running as root, but some crippled -- filesystems ignore write bit removals. - not <$> catchBoolIO (writeFile f "2" >> return True) + ifM ((== 0) <$> getRealUserID) + ( return True + , not <$> catchBoolIO (writeFile f "2" >> return True) + ) #endif checkCrippledFileSystem :: Annex () diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs --- a/Assistant/Threads/WebApp.hs +++ b/Assistant/Threads/WebApp.hs @@ -49,7 +49,6 @@ import Data.Text (pack, unpack) import qualified Network.Wai.Handler.WarpTLS as TLS import Network.Wai.Middleware.RequestLogger -import System.Log.Logger mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") @@ -138,9 +137,3 @@ #else return Nothing #endif - -{- Checks if debugging is actually enabled. -} -debugEnabled :: IO Bool -debugEnabled = do - l <- getRootLogger - return $ getLevel l <= Just DEBUG diff --git a/Assistant/WebApp/Configurators/Preferences.hs b/Assistant/WebApp/Configurators/Preferences.hs --- a/Assistant/WebApp/Configurators/Preferences.hs +++ b/Assistant/WebApp/Configurators/Preferences.hs @@ -30,7 +30,7 @@ , numCopies :: Int , autoStart :: Bool , autoUpgrade :: AutoUpgrade - , debugEnabled :: Bool + , enableDebug :: Bool } prefsAForm :: PrefsForm -> MkAForm PrefsForm @@ -44,7 +44,7 @@ <*> areq (selectFieldList autoUpgradeChoices) (bfs autoUpgradeLabel) (Just $ autoUpgrade d) <*> areq (checkBoxField `withNote` debugnote) - "Enable debug logging" (Just $ debugEnabled d) + "Enable debug logging" (Just $ enableDebug d) where diskreservenote = [whamlet|
Avoid downloading files from other repositories when there is too little free disk space.|] numcopiesnote = [whamlet|
Only drop a file after verifying that other repositories contain this many copies.|] @@ -98,8 +98,8 @@ liftIO $ if autoStart p then addAutoStartFile here else removeAutoStartFile here - setConfig (annexConfig "debug") (boolConfig $ debugEnabled p) - liftIO $ if debugEnabled p + setConfig (annexConfig "debug") (boolConfig $ enableDebug p) + liftIO $ if enableDebug p then enableDebugOutput else disableDebugOutput diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs --- a/Assistant/WebApp/Types.hs +++ b/Assistant/WebApp/Types.hs @@ -36,8 +36,6 @@ staticRoutes :: Static staticRoutes = $(embed "static") -mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") - data WebApp = WebApp { assistantData :: AssistantData , authToken :: AuthToken @@ -48,6 +46,8 @@ , noAnnex :: Bool , listenHost ::Maybe HostName } + +mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") instance Yesod WebApp where {- Require an auth token be set when accessing any (non-static) route -} diff --git a/Build/collect-ghc-options.sh b/Build/collect-ghc-options.sh new file mode 100644 --- /dev/null +++ b/Build/collect-ghc-options.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Generate --ghc-options to pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc +# and on to ld, cc, and cpp. +for w in $LDFLAGS; do + printf -- "-optl%s\n" "$w" +done +for w in $CFLAGS; do + printf -- "-optc%s\n" "$w" +done +for w in $CPPFLAGS; do + printf -- "-optc-Wp,%s\n" "$w" +done diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,32 @@ +git-annex (5.20150824) unstable; urgency=medium + + * Sped up downloads of files from ssh remotes, reducing the + non-data-transfer overhead 6x. + * sync: Support --jobs + * sync --content: Avoid unnecessary second pull from remotes when + no file transfers are made. + * External special remotes can now be built that can be used in readonly + mode, where git-annex downloads content from the remote using regular + http. + * Added WHEREIS to external special remote protocol. + * importfeed --relaxed: Avoid hitting the urls of items in the feed. + * Fix reversion in init when ran as root, introduced in version 5.20150731. + * Reorder declaration to fix build with yesod-core > 1.4.13. + Thanks, Michael Alan Dorman. + * Fix building without quvi and without database. + Thanks, Ben Boeckel. + * Avoid building the assistant on the hurd, since an inotify equivalent + is not yet implemented in git-annex for the hurd. + * --debug log messages are now timestamped with fractional seconds. + * --debug is passed along to git-annex-shell when git-annex is in debug mode. + * Makefile: Pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc and on to + ld, cc, and cpp. + * As a result of the Makefile changes, the Debian package is built + with various hardening options. Although their benefit to a largely + haskell program is unknown. + + -- Joey Hess Mon, 24 Aug 2015 14:11:05 -0700 + git-annex (5.20150812) unstable; urgency=medium * Added support for SHA3 hashed keys (in 8 varieties), when git-annex is diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -182,7 +182,7 @@ regulardownload url = do pathmax <- liftIO $ fileNameLengthLimit "." urlinfo <- if relaxedOption o - then pure $ Url.UrlInfo True Nothing Nothing + then pure Url.assumeUrlExists else Url.withUrlOptions (Url.getUrlInfo urlstring) file <- adjustFile o <$> case fileOption o of Just f -> pure f diff --git a/Command/Fsck.hs b/Command/Fsck.hs --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -519,10 +519,10 @@ data Incremental = NonIncremental + | ScheduleIncremental Duration UUID Incremental #ifdef WITH_DATABASE | StartIncremental FsckDb.FsckHandle | ContIncremental FsckDb.FsckHandle - | ScheduleIncremental Duration UUID Incremental #endif prepIncremental :: UUID -> Maybe IncrementalOpt -> Annex Incremental diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -148,7 +148,7 @@ ) Nothing -> return Nothing #else - mkquvi = return Nothing + mkquvi _ _ = return Nothing #endif {- Feeds change, so a feed download cannot be resumed. -} @@ -172,7 +172,9 @@ r <- Remote.claimingUrl url if Remote.uuid r == webUUID || rawOption opts then do - urlinfo <- Url.withUrlOptions (Url.getUrlInfo url) + urlinfo <- if relaxedOption opts + then pure Url.assumeUrlExists + else Url.withUrlOptions (Url.getUrlInfo url) maybeToList <$> addUrlFile (relaxedOption opts) url urlinfo f else do res <- tryNonAsync $ maybe diff --git a/Command/SendKey.hs b/Command/SendKey.hs --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -16,6 +16,8 @@ import qualified CmdLine.GitAnnexShell.Fields as Fields import Utility.Metered +import System.Log.Logger + cmd :: Command cmd = noCommit $ command "sendkey" SectionPlumbing @@ -44,10 +46,12 @@ fieldTransfer :: Direction -> Key -> (MeterUpdate -> Annex Bool) -> CommandStart fieldTransfer direction key a = do + liftIO $ debugM "fieldTransfer" "transfer start" afile <- Fields.getField Fields.associatedFile ok <- maybe (a $ const noop) (\u -> runner (Transfer direction (toUUID u) key) afile noRetry noObserver a) =<< Fields.getField Fields.remoteUUID + liftIO $ debugM "fieldTransfer" "transfer done" liftIO $ exitBool ok where {- Allow the key to be sent to the remote even if there seems to be diff --git a/Command/Sync.hs b/Command/Sync.hs --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -47,14 +47,16 @@ import Annex.Ssh import Annex.BloomFilter import Utility.Bloom +import Utility.OptParse import Control.Concurrent.MVar import qualified Data.Map as M cmd :: Command -cmd = command "sync" SectionCommon - "synchronize local repository with remotes" - (paramRepeating paramRemote) (seek <$$> optParser) +cmd = withGlobalOptions [jobsOption] $ + command "sync" SectionCommon + "synchronize local repository with remotes" + (paramRepeating paramRemote) (seek <$$> optParser) data SyncOptions = SyncOptions { syncWith :: CmdParams @@ -66,9 +68,8 @@ optParser :: CmdParamsDesc -> Parser SyncOptions optParser desc = SyncOptions <$> cmdParams desc - <*> switch - ( long "content" - <> help "also transfer file contents" + <*> invertableSwitch "content" False + ( help "also transfer file contents" ) <*> optional (strOption ( long "message" <> short 'm' <> metavar "MSG" @@ -102,7 +103,8 @@ -- Syncing involves many actions, any of which can independently -- fail, without preventing the others from running. - seekActions $ return $ concat + -- These actions cannot be run concurrently. + mapM_ includeCommandAction $ concat [ [ commit o ] , [ withbranch mergeLocal ] , map (withbranch . pullRemote) gitremotes @@ -115,14 +117,14 @@ -- branch on the remotes in the meantime, so pull -- and merge again to avoid our push overwriting -- those changes. - seekActions $ return $ concat + mapM_ includeCommandAction $ concat [ map (withbranch . pullRemote) gitremotes , [ commitAnnex, mergeAnnex ] ] - seekActions $ return $ concat - [ [ withbranch pushLocal ] - , map (withbranch . pushRemote) gitremotes - ] + + void $ includeCommandAction $ withbranch pushLocal + -- Pushes to remotes can run concurrently. + mapM_ (commandAction . withbranch . pushRemote) gitremotes {- Merging may delete the current directory, so go to the top - of the repo. This also means that sync always acts on all files in the @@ -380,7 +382,9 @@ - This ensures that preferred content expressions that match on - filenames work, even when in --all mode. - - - If any file movements were generated, returns true. + - Returns true if any file transfers were made. + - + - When concurrency is enabled, files are processed concurrently. -} seekSyncContent :: SyncOptions -> [Remote] -> Annex Bool seekSyncContent o rs = do @@ -392,15 +396,17 @@ (seekkeys mvar bloom) (const noop) [] + finishCommandActions liftIO $ not <$> isEmptyMVar mvar where seekworktree mvar l bloomfeeder = seekHelper LsFiles.inRepo l >>= mapM_ (\f -> ifAnnexed f (go (Right bloomfeeder) mvar (Just f)) noop) seekkeys mvar bloom getkeys = mapM_ (go (Left bloom) mvar Nothing) =<< getkeys - go ebloom mvar af k = do - void $ liftIO $ tryPutMVar mvar () - syncFile ebloom rs af k + go ebloom mvar af k = commandAction $ do + whenM (syncFile ebloom rs af k) $ + void $ liftIO $ tryPutMVar mvar () + return Nothing {- If it's preferred content, and we don't have it, get it from one of the - listed remotes (preferring the cheaper earlier ones). @@ -412,8 +418,10 @@ - - Drop it from each remote that has it, where it's not preferred content - (honoring numcopies). + - + - Returns True if any file transfers were made. -} -syncFile :: Either (Maybe (Bloom Key)) (Key -> Annex ()) -> [Remote] -> AssociatedFile -> Key -> Annex () +syncFile :: Either (Maybe (Bloom Key)) (Key -> Annex ()) -> [Remote] -> AssociatedFile -> Key -> Annex Bool syncFile ebloom rs af k = do locs <- loggedLocations k let (have, lack) = partition (\r -> Remote.uuid r `elem` locs) rs @@ -443,6 +451,8 @@ -- the sync failed. handleDropsFrom locs' rs "unwanted" True k af Nothing callCommandAction + + return (got || not (null putrs)) where wantget have = allM id [ pure (not $ null have) diff --git a/Command/Whereis.hs b/Command/Whereis.hs --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -14,6 +14,7 @@ import Remote import Logs.Trust import Logs.Web +import Remote.Web (getWebUrls) cmd :: Command cmd = noCommit $ withGlobalOptions (jsonOption : annexedMatchingOptions) $ @@ -60,6 +61,11 @@ unless (null safelocations) $ showLongNote pp pp' <- prettyPrintUUIDs "untrusted" untrustedlocations unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp' + + -- Since other remotes than the web remote can set urls + -- where a key can be downloaded, get and show all such urls + -- as a special case. + showRemote "web" =<< getWebUrls key forM_ (mapMaybe (`M.lookup` remotemap) locations) $ performRemote key if null safelocations then stop else next $ return True @@ -73,8 +79,7 @@ ls <- (++) <$> askremote <*> claimedurls - unless (null ls) $ showLongNote $ unlines $ - map (\l -> name remote ++ ": " ++ l) ls + showRemote (name remote) ls where askremote = maybe (pure []) (flip id key) (whereisKey remote) claimedurls = do @@ -83,3 +88,9 @@ . map getDownloader <$> getUrls key filterM (\u -> (==) <$> pure remote <*> claimingUrl u) us + +showRemote :: String -> [String] -> Annex () +showRemote n ls + | null ls = return () + | otherwise = showLongNote $ unlines $ + map (\l -> n ++ ": " ++ l) ls diff --git a/Config.hs b/Config.hs --- a/Config.hs +++ b/Config.hs @@ -5,6 +5,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + module Config where import Common.Annex @@ -14,6 +16,7 @@ import qualified Annex import Config.Cost import Types.Availability +import Git.Types type UnqualifiedConfigKey = String data ConfigKey = ConfigKey String @@ -41,10 +44,19 @@ unsetConfig :: ConfigKey -> Annex () unsetConfig (ConfigKey key) = void $ inRepo $ Git.Config.unset key +class RemoteNameable r where + getRemoteName :: r -> RemoteName + +instance RemoteNameable Git.Repo where + getRemoteName r = fromMaybe "" (Git.remoteName r) + +instance RemoteNameable RemoteName where + getRemoteName = id + {- A per-remote config setting in git config. -} -remoteConfig :: Git.Repo -> UnqualifiedConfigKey -> ConfigKey +remoteConfig :: RemoteNameable r => r -> UnqualifiedConfigKey -> ConfigKey remoteConfig r key = ConfigKey $ - "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key + "remote." ++ getRemoteName r ++ ".annex-" ++ key {- A global annex setting in git config. -} annexConfig :: UnqualifiedConfigKey -> ConfigKey diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi - $(CABAL) configure + $(CABAL) configure --ghc-options="$(shell Build/collect-ghc-options.sh)" git-annex: Build/SysConfig.hs $(CABAL) build @@ -81,7 +81,7 @@ --plugin=comments --set comments_pagespec="*" \ --exclude='news/.*' --exclude='design/assistant/blog/*' \ --exclude='bugs/*' --exclude='todo/*' --exclude='forum/*' \ - --exclude='users/*' --exclude='devblog/*' + --exclude='users/*' --exclude='devblog/*' --exclude='thanks' clean: rm -rf tmp dist git-annex $(mans) configure *.tix .hpc \ diff --git a/Messages.hs b/Messages.hs --- a/Messages.hs +++ b/Messages.hs @@ -32,6 +32,7 @@ setupConsole, enableDebugOutput, disableDebugOutput, + debugEnabled, commandProgressDisabled, ) where @@ -173,7 +174,7 @@ setupConsole = do s <- setFormatter <$> streamHandler stderr DEBUG - <*> pure (simpleLogFormatter "[$time] $msg") + <*> pure preciseLogFormatter updateGlobalLogger rootLoggerName (setLevel NOTICE . setHandlers [s]) {- This avoids ghc's output layer crashing on - invalid encoded characters in @@ -181,11 +182,21 @@ fileEncoding stdout fileEncoding stderr +{- Log formatter with precision into fractions of a second. -} +preciseLogFormatter :: LogFormatter a +preciseLogFormatter = tfLogFormatter "%F %X%Q" "[$time] $msg" + enableDebugOutput :: IO () enableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel DEBUG disableDebugOutput :: IO () disableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel NOTICE + +{- Checks if debugging is enabled. -} +debugEnabled :: IO Bool +debugEnabled = do + l <- getRootLogger + return $ getLevel l <= Just DEBUG {- Should commands that normally output progress messages have that - output disabled? -} diff --git a/Remote/External.hs b/Remote/External.hs --- a/Remote/External.hs +++ b/Remote/External.hs @@ -1,6 +1,6 @@ {- External special remote interface. - - - Copyright 2013 Joey Hess + - Copyright 2013-2015 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -13,9 +13,13 @@ import Types.Remote import Types.CleanupActions import Types.UrlContents +import Types.Key import qualified Git import Config +import Git.Config (isTrue, boolConfig) import Remote.Helper.Special +import Remote.Helper.ReadOnly +import Remote.Helper.Messages import Utility.Metered import Messages.Progress import Logs.Transfer @@ -23,6 +27,8 @@ import Logs.RemoteState import Logs.Web import Config.Cost +import Annex.Content +import Annex.Url import Annex.UUID import Creds @@ -40,17 +46,34 @@ } gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) -gen r u c gc = do - external <- newExternal externaltype u c - Annex.addCleanup (RemoteCleanup u) $ stopExternal external - cst <- getCost external r gc - avail <- getAvailability external r gc - return $ Just $ specialRemote c - (simplyPrepare $ store external) - (simplyPrepare $ retrieve external) - (simplyPrepare $ remove external) - (simplyPrepare $ checkKey external) - Remote +gen r u c gc + -- readonly mode only downloads urls; does not use external program + | remoteAnnexReadOnly gc = do + cst <- remoteCost gc expensiveRemoteCost + mk cst GloballyAvailable + readonlyStorer + retrieveUrl + readonlyRemoveKey + (checkKeyUrl r) + Nothing + Nothing + Nothing + | otherwise = do + external <- newExternal externaltype u c + Annex.addCleanup (RemoteCleanup u) $ stopExternal external + cst <- getCost external r gc + avail <- getAvailability external r gc + mk cst avail + (store external) + (retrieve external) + (remove external) + (checkKey external) + (Just (whereis external)) + (Just (claimurl external)) + (Just (checkurl external)) + where + mk cst avail tostore toretrieve toremove tocheckkey towhereis toclaimurl tocheckurl = do + let rmt = Remote { uuid = u , cost = cst , name = Git.repoDescribe r @@ -60,7 +83,7 @@ , removeKey = removeKeyDummy , checkPresent = checkPresentDummy , checkPresentCheap = False - , whereisKey = Nothing + , whereisKey = towhereis , remoteFsck = Nothing , repairRepo = Nothing , config = c @@ -73,10 +96,15 @@ , mkUnavailable = gen r u c $ gc { remoteAnnexExternalType = Just "!dne!" } , getInfo = return [("externaltype", externaltype)] - , claimUrl = Just (claimurl external) - , checkUrl = Just (checkurl external) + , claimUrl = toclaimurl + , checkUrl = tocheckurl } - where + return $ Just $ specialRemote c + (simplyPrepare tostore) + (simplyPrepare toretrieve) + (simplyPrepare toremove) + (simplyPrepare tocheckkey) + rmt externaltype = fromMaybe (error "missing externaltype") (remoteAnnexExternalType gc) externalSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID) @@ -86,12 +114,17 @@ M.lookup "externaltype" c (c', _encsetup) <- encryptionSetup c - external <- newExternal externaltype u c' - handleRequest external INITREMOTE Nothing $ \resp -> case resp of - INITREMOTE_SUCCESS -> Just noop - INITREMOTE_FAILURE errmsg -> Just $ error errmsg - _ -> Nothing - c'' <- liftIO $ atomically $ readTMVar $ externalConfig external + c'' <- case M.lookup "readonly" c of + Just v | isTrue v == Just True -> do + setConfig (remoteConfig (fromJust (M.lookup "name" c)) "readonly") (boolConfig True) + return c' + _ -> do + external <- newExternal externaltype u c' + handleRequest external INITREMOTE Nothing $ \resp -> case resp of + INITREMOTE_SUCCESS -> Just noop + INITREMOTE_FAILURE errmsg -> Just $ error errmsg + _ -> Nothing + liftIO $ atomically $ readTMVar $ externalConfig external gitConfigSpecialRemote u c'' "externaltype" externaltype return (c'', u) @@ -144,6 +177,13 @@ | k' == k -> Just $ return $ Left errmsg _ -> Nothing +whereis :: External -> Key -> Annex [String] +whereis external k = handleRequest external (WHEREIS k) Nothing $ \resp -> case resp of + WHEREIS_SUCCESS s -> Just $ return [s] + WHEREIS_FAILURE -> Just $ return [] + UNSUPPORTED_REQUEST -> Just $ return [] + _ -> Nothing + safely :: Annex Bool -> Annex Bool safely a = go =<< tryNonAsync a where @@ -460,3 +500,20 @@ _ -> Nothing where mkmulti (u, s, f) = (u, s, mkSafeFilePath f) + +retrieveUrl :: Retriever +retrieveUrl = fileRetriever $ \f k _p -> do + us <- getWebUrls k + unlessM (downloadUrl us f) $ + error "failed to download content" + +checkKeyUrl :: Git.Repo -> CheckPresent +checkKeyUrl r k = do + showChecking r + us <- getWebUrls k + anyM (\u -> withUrlOptions $ checkBoth u (keySize k)) us + +getWebUrls :: Key -> Annex [URLString] +getWebUrls key = filter supported <$> getUrls key + where + supported u = snd (getDownloader u) == WebDownloader diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs --- a/Remote/External/Types.hs +++ b/Remote/External/Types.hs @@ -97,6 +97,7 @@ | TRANSFER Direction Key FilePath | CHECKPRESENT Key | REMOVE Key + | WHEREIS Key deriving (Show) -- Does PREPARE need to have been sent before this request? @@ -120,6 +121,7 @@ ] formatMessage (CHECKPRESENT key) = [ "CHECKPRESENT", Proto.serialize key ] formatMessage (REMOVE key) = [ "REMOVE", Proto.serialize key ] + formatMessage (WHEREIS key) = [ "WHEREIS", Proto.serialize key ] -- Responses the external remote can make to requests. data Response @@ -141,6 +143,8 @@ | CHECKURL_CONTENTS Size FilePath | CHECKURL_MULTI [(URLString, Size, FilePath)] | CHECKURL_FAILURE ErrorMsg + | WHEREIS_SUCCESS String + | WHEREIS_FAILURE | UNSUPPORTED_REQUEST deriving (Show) @@ -163,6 +167,8 @@ parseCommand "CHECKURL-CONTENTS" = Proto.parse2 CHECKURL_CONTENTS parseCommand "CHECKURL-MULTI" = Proto.parse1 CHECKURL_MULTI parseCommand "CHECKURL-FAILURE" = Proto.parse1 CHECKURL_FAILURE + parseCommand "WHEREIS-SUCCESS" = Just . WHEREIS_SUCCESS + parseCommand "WHEREIS-FAILURE" = Proto.parse0 WHEREIS_FAILURE parseCommand "UNSUPPORTED-REQUEST" = Proto.parse0 UNSUPPORTED_REQUEST parseCommand _ = Proto.parseFail diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -200,7 +200,6 @@ tryGitConfigRead autoinit r | haveconfig r = return r -- already read | Git.repoIsSsh r = store $ do - liftIO $ print autoinit v <- Ssh.onRemote r (pipedconfig, return (Left $ error "configlist failed")) "configlist" [] configlistfields case v of Right r' @@ -446,9 +445,14 @@ let feeder = \n -> do meterupdate n writeSV v (fromBytesProcessed n) - let cleanup = do + + -- It can easily take 0.3 seconds to clean up after + -- the transferinfo, and all that's involved is shutting + -- down the process and associated thread cleanly. So, + -- do it in the background. + let cleanup = forkIO $ do void $ tryIO $ killThread tid - tryNonAsync $ + void $ tryNonAsync $ maybe noop (void . waitForProcess) =<< tryTakeMVar pidv bracketIO noop (const cleanup) (const $ a feeder) diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -18,6 +18,7 @@ import Config import Config.Cost import Remote.Helper.Special +import Remote.Helper.Messages import qualified Remote.Helper.AWS as AWS import Creds import Utility.Metered @@ -176,7 +177,7 @@ checkKey :: Remote -> CheckPresent checkKey r k = do - showAction $ "checking " ++ name r + showChecking r go =<< glacierEnv (config r) (uuid r) where go Nothing = error "cannot check glacier" diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs --- a/Remote/Helper/Chunked.hs +++ b/Remote/Helper/Chunked.hs @@ -8,6 +8,7 @@ module Remote.Helper.Chunked ( ChunkSize, ChunkConfig(..), + noChunks, describeChunkConfig, getChunkConfig, storeChunks, diff --git a/Remote/Helper/Messages.hs b/Remote/Helper/Messages.hs --- a/Remote/Helper/Messages.hs +++ b/Remote/Helper/Messages.hs @@ -5,15 +5,14 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + module Remote.Helper.Messages where import Common.Annex import qualified Git import qualified Types.Remote as Remote -showChecking :: Git.Repo -> Annex () -showChecking r = showAction $ "checking " ++ Git.repoDescribe r - class Checkable a where descCheckable :: a -> String @@ -22,6 +21,12 @@ instance Checkable (Remote.RemoteA a) where descCheckable = Remote.name + +instance Checkable String where + descCheckable = id + +showChecking :: Checkable a => a -> Annex () +showChecking v = showAction $ "checking " ++ descCheckable v cantCheck :: Checkable a => a -> e cantCheck v = error $ "unable to check " ++ descCheckable v diff --git a/Remote/Helper/ReadOnly.hs b/Remote/Helper/ReadOnly.hs --- a/Remote/Helper/ReadOnly.hs +++ b/Remote/Helper/ReadOnly.hs @@ -1,14 +1,21 @@ {- Adds readonly support to remotes. - - - Copyright 2013 Joey Hess + - Copyright 2013, 2015 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Helper.ReadOnly (adjustReadOnly) where +module Remote.Helper.ReadOnly + ( adjustReadOnly + , readonlyStoreKey + , readonlyStorer + , readonlyRemoveKey + ) where import Common.Annex import Types.Remote +import Types.StoreRetrieve +import Utility.Metered {- Adds support for read-only remotes, by replacing the - methods that write to a remote with dummies that fail. @@ -18,12 +25,22 @@ adjustReadOnly :: Remote -> Remote adjustReadOnly r | remoteAnnexReadOnly (gitconfig r) = r - { storeKey = \_ _ _ -> failbool - , removeKey = \_ -> failbool + { storeKey = readonlyStoreKey + , removeKey = readonlyRemoveKey , repairRepo = Nothing } | otherwise = r - where - failbool = do - warning "this remote is readonly" - return False + +readonlyStoreKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +readonlyStoreKey _ _ _ = readonlyFail + +readonlyRemoveKey :: Key -> Annex Bool +readonlyRemoveKey _ = readonlyFail + +readonlyStorer :: Storer +readonlyStorer _ _ _ = readonlyFail + +readonlyFail :: Annex Bool +readonlyFail = do + warning "this remote is readonly" + return False diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -36,6 +36,7 @@ import Types.StoreRetrieve import Types.Remote import Crypto +import Config import Config.Cost import Utility.Metered import Remote.Helper.Chunked as X @@ -44,7 +45,6 @@ import Annex.Content import Messages.Progress import qualified Git -import qualified Git.Command import qualified Git.Construct import qualified Data.ByteString.Lazy as L @@ -66,13 +66,10 @@ {- Sets up configuration for a special remote in .git/config. -} gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex () gitConfigSpecialRemote u c k v = do - set ("annex-"++k) v - set ("annex-uuid") (fromUUID u) + setConfig (remoteConfig remotename k) v + setConfig (remoteConfig remotename "uuid") (fromUUID u) where - set a b = inRepo $ Git.Command.run - [Param "config", Param (configsetting a), Param b] remotename = fromJust (M.lookup "name" c) - configsetting s = "remote." ++ remotename ++ "." ++ s -- Use when nothing needs to be done to prepare a helper. simplyPrepare :: helper -> Preparer helper @@ -165,18 +162,21 @@ (\_ -> return False) , removeKey = \k -> cip >>= removeKeyGen k , checkPresent = \k -> cip >>= checkPresentGen k - , cost = maybe - (cost baser) - (const $ cost baser + encryptedRemoteCostAdj) - (extractCipher c) + , cost = if isencrypted + then cost baser + encryptedRemoteCostAdj + else cost baser , getInfo = do l <- getInfo baser return $ l ++ [ ("encryption", describeEncryption c) , ("chunking", describeChunkConfig (chunkConfig cfg)) ] + , whereisKey = if noChunks (chunkConfig cfg) && not isencrypted + then whereisKey baser + else Nothing } cip = cipherKey c + isencrypted = isJust (extractCipher c) gpgopts = getGpgEncParams encr safely a = catchNonAsync a (\e -> warning (show e) >> return False) diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -38,22 +38,30 @@ - repository. -} git_annex_shell :: Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam])) git_annex_shell r command params fields - | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts ++ fieldopts) + | not $ Git.repoIsUrl r = do + shellopts <- getshellopts + return $ Just (shellcmd, shellopts ++ fieldopts) | Git.repoIsSsh r = do gc <- Annex.getRemoteGitConfig r u <- getRepoUUID r - sshparams <- toRepo r gc [Param $ sshcmd u gc] + shellopts <- getshellopts + let sshcmd = unwords $ + fromMaybe shellcmd (remoteAnnexShell gc) + : map shellEscape (toCommand shellopts) ++ + uuidcheck u ++ + map shellEscape (toCommand fieldopts) + sshparams <- toRepo r gc [Param sshcmd] return $ Just ("ssh", sshparams) | otherwise = return Nothing where dir = Git.repoPath r shellcmd = "git-annex-shell" - shellopts = Param command : File dir : params - sshcmd u gc = unwords $ - fromMaybe shellcmd (remoteAnnexShell gc) - : map shellEscape (toCommand shellopts) ++ - uuidcheck u ++ - map shellEscape (toCommand fieldopts) + getshellopts = do + debug <- liftIO debugEnabled + let params' = if debug + then Param "--debug" : params + else params + return (Param command : File dir : params') uuidcheck NoUUID = [] uuidcheck (UUID u) = ["--uuid", u] fieldopts diff --git a/Remote/Hook.hs b/Remote/Hook.hs --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -16,6 +16,7 @@ import Config.Cost import Annex.UUID import Remote.Helper.Special +import Remote.Helper.Messages import Utility.Env import Messages.Progress @@ -138,7 +139,7 @@ checkKey :: Git.Repo -> HookName -> CheckPresent checkKey r h k = do - showAction $ "checking " ++ Git.repoDescribe r + showChecking r v <- lookupHook h action liftIO $ check v where diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -27,6 +27,7 @@ import Annex.UUID import Annex.Ssh import Remote.Helper.Special +import Remote.Helper.Messages import Remote.Rsync.RsyncUrl import Crypto import Utility.Rsync @@ -222,7 +223,7 @@ checkKey :: Git.Repo -> RsyncOpts -> CheckPresent checkKey r o k = do - showAction $ "checking " ++ Git.repoDescribe r + showChecking r -- note: Does not currently differentiate between rsync failing -- to connect, and the file not being present. untilTrue (rsyncUrls o k) $ \u -> diff --git a/Remote/S3.hs b/Remote/S3.hs --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -39,6 +39,7 @@ import Config.Cost import Remote.Helper.Special import Remote.Helper.Http +import Remote.Helper.Messages import qualified Remote.Helper.AWS as AWS import Creds import Annex.UUID @@ -269,7 +270,7 @@ checkKey :: Remote -> S3Info -> Maybe S3Handle -> CheckPresent checkKey r info (Just h) k = do - showAction $ "checking " ++ name r + showChecking r #if MIN_VERSION_aws(0,10,0) rsp <- go return (isJust $ S3.horMetadata rsp) @@ -300,7 +301,7 @@ warnMissingCredPairFor "S3" (AWS.creds $ uuid r) error "No S3 credentials configured" Just geturl -> do - showAction $ "checking " ++ name r + showChecking r withUrlOptions $ checkBoth (geturl k) (keySize k) {- Generate the bucket if it does not already exist, including creating the diff --git a/Remote/Web.hs b/Remote/Web.hs --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -7,10 +7,11 @@ {-# LANGUAGE CPP #-} -module Remote.Web (remote) where +module Remote.Web (remote, getWebUrls) where import Common.Annex import Types.Remote +import Remote.Helper.Messages import qualified Git import qualified Git.Construct import Annex.Content @@ -53,7 +54,7 @@ , removeKey = dropKey , checkPresent = checkKey , checkPresentCheap = False - , whereisKey = Just getWebUrls + , whereisKey = Nothing , remoteFsck = Nothing , repairRepo = Nothing , config = c @@ -112,7 +113,7 @@ checkKey' :: Key -> [URLString] -> Annex (Either String Bool) checkKey' key us = firsthit us (Right False) $ \u -> do let (u', downloader) = getDownloader u - showAction $ "checking " ++ u' + showChecking u' case downloader of QuviDownloader -> #ifdef WITH_QUVI diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -25,6 +25,7 @@ import Config import Config.Cost import Remote.Helper.Special +import Remote.Helper.Messages import Remote.Helper.Http import qualified Remote.Helper.Chunked.Legacy as Legacy import Creds @@ -147,7 +148,7 @@ checkKey :: Remote -> ChunkConfig -> Maybe DavHandle -> CheckPresent checkKey r _ Nothing _ = error $ name r ++ " not configured" checkKey r chunkconfig (Just dav) k = do - showAction $ "checking " ++ name r + showChecking r case chunkconfig of LegacyChunks _ -> checkKeyLegacyChunked dav k _ -> do diff --git a/Utility/Hash.hs b/Utility/Hash.hs --- a/Utility/Hash.hs +++ b/Utility/Hash.hs @@ -31,9 +31,11 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.ByteString as S -import Crypto.Hash #ifdef WITH_CRYPTONITE -import Crypto.MAC.HMAC +import "cryptonite" Crypto.MAC.HMAC +import "cryptonite" Crypto.Hash +#else +import "cryptohash" Crypto.Hash #endif sha1 :: L.ByteString -> Digest SHA1 diff --git a/Utility/OptParse.hs b/Utility/OptParse.hs new file mode 100644 --- /dev/null +++ b/Utility/OptParse.hs @@ -0,0 +1,45 @@ +{- optparse-applicative additions + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.OptParse where + +import Options.Applicative +import Data.Monoid + +-- | A switch that can be enabled using --foo and disabled using --no-foo. +-- +-- The option modifier is applied to only the option that is *not* enabled +-- by default. For example: +-- +-- > invertableSwitch "recursive" True (help "do not recurse into directories") +-- +-- This example makes --recursive enabled by default, so +-- the help is shown only for --no-recursive. +invertableSwitch + :: String -- ^ long option + -> Bool -- ^ is switch enabled by default? + -> Mod FlagFields Bool -- ^ option modifier + -> Parser Bool +invertableSwitch longopt defv optmod = invertableSwitch' longopt defv + (if defv then mempty else optmod) + (if defv then optmod else mempty) + +-- | Allows providing option modifiers for both --foo and --no-foo. +invertableSwitch' + :: String -- ^ long option (eg "foo") + -> Bool -- ^ is switch enabled by default? + -> Mod FlagFields Bool -- ^ option modifier for --foo + -> Mod FlagFields Bool -- ^ option modifier for --no-foo + -> Parser Bool +invertableSwitch' longopt defv enmod dismod = collapse <$> many + ( flag' True (enmod <> long longopt) + <|> flag' False (dismod <> long nolongopt) + ) + where + nolongopt = "no-" ++ longopt + collapse [] = defv + collapse l = last l diff --git a/Utility/Process.hs b/Utility/Process.hs --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -31,6 +31,7 @@ withQuietOutput, feedWithQuietOutput, createProcess, + waitForProcess, startInteractiveProcess, stdinHandle, stdoutHandle, @@ -42,7 +43,7 @@ import qualified System.Process import qualified System.Process as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess) -import System.Process hiding (createProcess, readProcess) +import System.Process hiding (createProcess, readProcess, waitForProcess) import System.Exit import System.IO import System.Log.Logger @@ -345,22 +346,6 @@ processHandle :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> ProcessHandle processHandle (_, _, _, pid) = pid --- | Debugging trace for a CreateProcess. -debugProcess :: CreateProcess -> IO () -debugProcess p = do - debugM "Utility.Process" $ unwords - [ action ++ ":" - , showCmd p - ] - where - action - | piped (std_in p) && piped (std_out p) = "chat" - | piped (std_in p) = "feed" - | piped (std_out p) = "read" - | otherwise = "call" - piped Inherit = False - piped _ = True - -- | Shows the command that a CreateProcess will run. showCmd :: CreateProcess -> String showCmd = go . cmdspec @@ -385,9 +370,31 @@ (Just from, Just to, _, pid) <- createProcess p return (pid, to, from) --- | Wrapper around 'System.Process.createProcess' from System.Process, --- that does debug logging. +-- | Wrapper around 'System.Process.createProcess' that does debug logging. createProcess :: CreateProcess -> IO (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) createProcess p = do debugProcess p System.Process.createProcess p + +-- | Debugging trace for a CreateProcess. +debugProcess :: CreateProcess -> IO () +debugProcess p = do + debugM "Utility.Process" $ unwords + [ action ++ ":" + , showCmd p + ] + where + action + | piped (std_in p) && piped (std_out p) = "chat" + | piped (std_in p) = "feed" + | piped (std_out p) = "read" + | otherwise = "call" + piped Inherit = False + piped _ = True + +-- | Wrapper around 'System.Process.waitForProcess' that does debug logging. +waitForProcess :: ProcessHandle -> IO ExitCode +waitForProcess h = do + r <- System.Process.waitForProcess h + debugM "Utility.Process" ("process done " ++ show r) + return r diff --git a/Utility/Url.hs b/Utility/Url.hs --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -20,6 +20,7 @@ exists, UrlInfo(..), getUrlInfo, + assumeUrlExists, download, downloadQuiet, parseURIRelaxed @@ -103,6 +104,9 @@ , urlSize :: Maybe Integer , urlSuggestedFile :: Maybe FilePath } + +assumeUrlExists :: UrlInfo +assumeUrlExists = UrlInfo True Nothing Nothing {- Checks that an url exists and could be successfully downloaded, - also returning its size and suggested filename if available. -} diff --git a/debian/.changelog.swp b/debian/.changelog.swp deleted file mode 100644 Binary files a/debian/.changelog.swp and /dev/null differ diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,32 @@ +git-annex (5.20150824) unstable; urgency=medium + + * Sped up downloads of files from ssh remotes, reducing the + non-data-transfer overhead 6x. + * sync: Support --jobs + * sync --content: Avoid unnecessary second pull from remotes when + no file transfers are made. + * External special remotes can now be built that can be used in readonly + mode, where git-annex downloads content from the remote using regular + http. + * Added WHEREIS to external special remote protocol. + * importfeed --relaxed: Avoid hitting the urls of items in the feed. + * Fix reversion in init when ran as root, introduced in version 5.20150731. + * Reorder declaration to fix build with yesod-core > 1.4.13. + Thanks, Michael Alan Dorman. + * Fix building without quvi and without database. + Thanks, Ben Boeckel. + * Avoid building the assistant on the hurd, since an inotify equivalent + is not yet implemented in git-annex for the hurd. + * --debug log messages are now timestamped with fractional seconds. + * --debug is passed along to git-annex-shell when git-annex is in debug mode. + * Makefile: Pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc and on to + ld, cc, and cpp. + * As a result of the Makefile changes, the Debian package is built + with various hardening options. Although their benefit to a largely + haskell program is unknown. + + -- Joey Hess Mon, 24 Aug 2015 14:11:05 -0700 + git-annex (5.20150812) unstable; urgency=medium * Added support for SHA3 hashed keys (in 8 varieties), when git-annex is diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -32,17 +32,17 @@ libghc-stm-dev (>= 2.3), libghc-dbus-dev (>= 0.10.7) [linux-any], libghc-fdo-notify-dev (>= 0.3) [linux-any], - libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-shakespeare-dev (>= 2.0.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-clientsession-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-warp-dev (>= 3.0.0.5) [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-warp-tls-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-wai-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], - libghc-wai-extra-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], + libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-shakespeare-dev (>= 2.0.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-clientsession-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-warp-dev (>= 3.0.0.5) [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-warp-tls-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-wai-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], + libghc-wai-extra-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x], libghc-dav-dev (>= 1.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], libghc-persistent-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], libghc-persistent-template-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386], diff --git a/debian/git-annex.lintian-overrides b/debian/git-annex.lintian-overrides new file mode 100644 --- /dev/null +++ b/debian/git-annex.lintian-overrides @@ -0,0 +1,1 @@ +binary-or-shlib-defines-rpath diff --git a/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment b/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment new file mode 100644 --- /dev/null +++ b/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://woid.cryptobitch.de/foobar" + subject="@niklaas" + date="2015-08-17T13:42:26Z" + content=""" +Joey Hess is using xmonad. +"""]] diff --git a/doc/backends.mdwn b/doc/backends.mdwn --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -15,7 +15,7 @@ * `SHA512`, `SHA512E` -- Best SHA-2 hash, for the very paranoid. * `SHA384`, `SHA384E`, `SHA224`, `SHA224E` -- SHA-2 hashes for people who like unusual sizes. -* `SHA3_512`, `SHA_512E`, `SHA3_384`, `SHA3_384E`, `SHA3_256`, `SHA3_256E`, `SHA3_224`, `SHA3_224E` +* `SHA3_512`, `SHA3_512E`, `SHA3_384`, `SHA3_384E`, `SHA3_256`, `SHA3_256E`, `SHA3_224`, `SHA3_224E` -- SHA-3 hashes, for bleeding edge fun. * `SKEIN512`, `SKEIN512E`, `SKEIN256`, `SKEIN256E` -- [Skein hash](http://en.wikipedia.org/wiki/Skein_hash), diff --git a/doc/bugs/Build_failures_with_7.10.mdwn b/doc/bugs/Build_failures_with_7.10.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Build_failures_with_7.10.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. + +git-annex fails to build in the NixOS builder + +### What steps will reproduce the problem? + +cabal build + +### What version of git-annex are you using? On what operating system? + +5.20150812 + +### Please provide any additional information below. + +[The build log](http://hydra.cryp.to/build/1099681/nixlog/1/raw) has a full transcript of the build from one of the build machines, though I have reproduced this locally. + +The pertinent bit of the log (which is *not* at the end, though I think that's just a capture-both-stderr-and-stdin thing, since it was at the end when I repro'd it: + +[[!format sh """ +Assistant/WebApp/Types.hs:39:1: ‘WebApp’ is not in scope at a reify +"""]] + +If I'm reading things correctly (you can do some diffing of the inputs from [the page for the build](http://hydra.cryp.to/build/1099681)), I'm going to guess that it was the upgrade to yesod-1.4.14. + +I'll continue to look into it, but it would seem to touch on TH and Yesod stuff with which I am largely unfamiliar. + +> Fixed with mdorman's patch, [[done]] (will push a release in a day or 2) --[[Joey]] diff --git a/doc/bugs/OOM_while_configuring_git-annex.mdwn b/doc/bugs/OOM_while_configuring_git-annex.mdwn --- a/doc/bugs/OOM_while_configuring_git-annex.mdwn +++ b/doc/bugs/OOM_while_configuring_git-annex.mdwn @@ -44,3 +44,5 @@ checking sha224... sha224sum checking sha384...Setup: out of memory (requested 1048576 bytes) """]] + +> forwarded to cabal && [[done]] per my comment --[[Joey]] diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn b/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +git annex cannot get me my file contents from my remotes + +### What steps will reproduce the problem? + +``` +git clone myserver:/path/to/repo +cd repo +git annex get +``` + +### What version of git-annex are you using? On what operating system? +Using git-annex-5.20150327 on gentoo linux + +### Please provide any additional information below. + +The error messages I get egenrally look like: + +``` +get 2att/photos-glasgow/male/patrik.jpg (from clusterhost...) +git-annex: ../chymera/data/.git/annex/transfer/upload/b8415264-4b9a-40ca-b450-7e57507cdc06/lck.SHA256E-s814245--9dc6f1287ba683cae030e04ba7f94a73e566ce392c2d032f171094ddc342fa60.jpg: openFd: does not exist (No such file or directory) +git-annex-shell: sendkey: 1 failed +protocol version mismatch -- is your shell clean? +(see the rsync man page for an explanation) +rsync error: protocol incompatibility (code 2) at compat.c(176) [Receiver=3.1.1] + + rsync failed -- run git annex again to resume file transfer + + Unable to access these remotes: clusterhost + + Try making some of these repositories available: + 809074b6-e079-4ea1-b2f8-2d7840deda7d -- zenbookhost + a1ed6786-8b93-4a14-b00d-877b741e34da -- [clusterhost] +failed +``` diff --git a/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn b/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn --- a/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn +++ b/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn @@ -26,3 +26,5 @@ # End of transcript or log. """]] + +[[!tag moreinfo]] diff --git a/doc/bugs/hPutChar_error_message_with_UTF-8_chars_above_7F_in_filenames.mdwn b/doc/bugs/hPutChar_error_message_with_UTF-8_chars_above_7F_in_filenames.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/hPutChar_error_message_with_UTF-8_chars_above_7F_in_filenames.mdwn @@ -0,0 +1,78 @@ +### Please describe the problem. + +When using `--incremental` together with `git annex fsck`, the error +message "hPutChar: invalid argument (invalid character)" appears in the +"Only X of Y trustworthy copies exist" message when the filename +contains an UTF-8 character above U+007F. The only locale in which this +doesn't happen is "C.UTF-8". + +### What steps will reproduce the problem? + +- Create and add a file with an UTF-8 character in the file name above U+007F to git-annex +- Set `numcopies` high enough so `git annex fsck` will produce a warning about missing copies +- Execute `git annex fsck --incremental` + +I've created two test scripts on + that demonstrate +this error: + +- `git clone https://gist.github.com/ebf4d055f5500b257ed8.git` +- `cd ebf4d055f5500b257ed8` +- `./runme` + +You can specify a locale to `runme` as `$1` to experiment with different +locales. + +There's also a `test-all-locales` script that executes `./runme` with +all defined locales on the computer. Both scripts return 1 if the error +message appears, if it's gone, 0 is returned. + +### What version of git-annex are you using? On what operating system? + +Newest git-annex amd64 (5.20150812) from `downloads.kitenet.net`. + +### Please provide any additional information below. + +The `runme` script contains more information about this issue. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +Here are two excerpts of the test output using the "C" and +"C.UTF-8" locale: + +$ ./runme C +[snip] +================== git annex --incremental fsck ================== +fsck U00D8_Ø.txt (checksum...) + + Only 1 of 2 trustworthy copies exist of U00D8_ +git-annex: : hPutChar: invalid argument (invalid character) +failed +fsck ascii_only.txt (checksum...) + + Only 1 of 2 trustworthy copies exist of ascii_only.txt + Back it up with git-annex copy. +failed +(recording state in git...) +git-annex: fsck: 2 failed + +$ ./runme C.UTF-8 +[snip] +================== git annex --incremental fsck ================== +fsck U00D8_Ø.txt (checksum...) + + Only 1 of 2 trustworthy copies exist of U00D8_Ø.txt + Back it up with git-annex copy. +failed +fsck ascii_only.txt (checksum...) + + Only 1 of 2 trustworthy copies exist of ascii_only.txt + Back it up with git-annex copy. +failed +(recording state in git...) +git-annex: fsck: 2 failed + +# End of transcript or log. +"""]] diff --git a/doc/bugs/importfeed_--relaxed_fails_with_HEAD-rejecting_servers.mdwn b/doc/bugs/importfeed_--relaxed_fails_with_HEAD-rejecting_servers.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/importfeed_--relaxed_fails_with_HEAD-rejecting_servers.mdwn @@ -0,0 +1,38 @@ +### Please describe the problem. + +Calling "git annex importfeed --relaxed" on a url of a BBC podcast +fails just like a "git annex addurl --fast" (and not relaxed) on the +url of a show: + + addurl open.live.bbc.co.uk_mediaselector_5_redir_version_2.0_mediaset_audio_nondrm_download_proto_http_vpid_p02y8kfc.mp3 + unable to access url: http://open.live.bbc.co.uk/mediaselector/5/redir/version/2.0/mediaset/audio-nondrm-download/proto/http/vpid/p02y8kfc.mp3 + failed + git-annex: addurl: 1 failed + +I suppose that is because HEAD on that same URL returns 403 Forbidden +(addurl without fast works just fine). + +"git annex addurl --relaxed" works on the given url. + + +### What steps will reproduce the problem? + +I ran into this bug trying to importfeed various BBC podcasts. For instance: + + $ git annex importfeed --relaxed 'http://www.bbc.co.uk/programmes/p02pc9x6/episodes/downloads.rss' + (checking known urls...) + importfeed http://www.bbc.co.uk/programmes/p02pc9x6/episodes/downloads.rss + /tmp/feed3947 100%[=======================================================>] 8,39K --.-KB/s en 0,007s + addurl Comedy_of_the_Week/In_and_Out_of_the_Kitchen__Episode_1__The_Supplement.mp3 + unable to access url: http://open.live.bbc.co.uk/mediaselector/5/redir/version/2.0/mediaset/audio-nondrm-download/proto/http/vpid/p02yy1hn.mp3 + failed + + warning: problem downloading item + ok + + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20150731-1 on a quite up-to-date debian unstable. + +> Thanks for a nice test case. [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn @@ -0,0 +1,49 @@ +### Please describe the problem. + +It seems that the `synced/git-annex` branch (which I am a little confused about in the first place), doesn't get synced automatically by the assistant. + +I was expecting the assistant to regularly do the equivalent of `git annex sync`. However, after a client pushed changes to the `git-annex` branch, I had to manually do a `git annex sync` for the changes of the `synced/git-annex` branch to be merged down into the local `git-annex`. + +### What steps will reproduce the problem? + +I have 3 machines in this setup (to simplify: there are more, but those are sufficient). Let's call them foo, bar and quux. foo and quuex are connected to bar through password-less SSH connexions. + +foo commits a file to git-annex. the assistant syncs that to bar in the `synced/git-annex` branch. + +quux syncs with bar, and seems to ignore the `synced/git-annex` branch. + +git-annex sync on quux syncs the `synced/git-annex` branch into the local `git-annex`, working around the issue. + +### What version of git-annex are you using? On what operating system? + +foo is 5.20150610+gitg608172f-1~ndall+1 on Debian 7.8 (wheezy). + +bar and quux are 5.20150409+git126-ga29f683-1~ndall+1 and 5.20150610+gitg608172f-1~ndall+1 (respectively) on Ubuntu 12.04 (precise) . + +### Please provide any additional information below. + +I guess a more general question is how and how often do those branches get merged by the assistant... it's still unclear to me how this works. + +[[!format sh """ +$ sudo -u www-data -H git annex sync +(merging origin/synced/git-annex into git-annex...) +(recording state in git...) +commit ok +pull origin +Auto packing the repository in background for optimum performance. +See "git help gc" for manual housekeeping. + +Already up-to-date. +ok +push origin +Counting objects: 373637, done. +# End of transcript or log. +"""]] + +This was started at 20:23 UTC. Note that the sync had run previously under the assistant: + +
+[2015-07-22 20:09:06 UTC] RemoteControl: Syncing with origin
+
+ +Available, as usual, for further debugging. :) -- [[anarcat]] diff --git a/doc/bugs/sync__47__git-annex_branch_not_syncing_in_the_assistant.mdwn b/doc/bugs/sync__47__git-annex_branch_not_syncing_in_the_assistant.mdwn deleted file mode 100644 --- a/doc/bugs/sync__47__git-annex_branch_not_syncing_in_the_assistant.mdwn +++ /dev/null @@ -1,49 +0,0 @@ -### Please describe the problem. - -It seems that the `synced/git-annex` branch (which I am a little confused about in the first place), doesn't get synced automatically by the assistant. - -I was expecting the assistant to regularly do the equivalent of `git annex sync`. However, after a client pushed changes to the `git-annex` branch, I had to manually do a `git annex sync` for the changes of the `synced/git-annex` branch to be merged down into the local `git-annex`. - -### What steps will reproduce the problem? - -I have 3 machines in this setup (to simplify: there are more, but those are sufficient). Let's call them foo, bar and quux. foo and quuex are connected to bar through password-less SSH connexions. - -foo commits a file to git-annex. the assistant syncs that to bar in the `synced/git-annex` branch. - -quux syncs with bar, and seems to ignore the `synced/git-annex` branch. - -git-annex sync on quux syncs the `synced/git-annex` branch into the local `git-annex`, working around the issue. - -### What version of git-annex are you using? On what operating system? - -foo is 5.20150610+gitg608172f-1~ndall+1 on Debian 7.8 (wheezy). - -bar and quux are 5.20150409+git126-ga29f683-1~ndall+1 and 5.20150610+gitg608172f-1~ndall+1 (respectively) on Ubuntu 12.04 (precise) . - -### Please provide any additional information below. - -I guess a more general question is how and how often do those branches get merged by the assistant... it's still unclear to me how this works. - -[[!format sh """ -$ sudo -u www-data -H git annex sync -(merging origin/synced/git-annex into git-annex...) -(recording state in git...) -commit ok -pull origin -Auto packing the repository in background for optimum performance. -See "git help gc" for manual housekeeping. - -Already up-to-date. -ok -push origin -Counting objects: 373637, done. -# End of transcript or log. -"""]] - -This was started at 20:23 UTC. Note that the sync had run previously under the assistant: - -
-[2015-07-22 20:09:06 UTC] RemoteControl: Syncing with origin
-
- -Available, as usual, for further debugging. :) -- [[anarcat]] diff --git a/doc/design/external_special_remote_protocol.mdwn b/doc/design/external_special_remote_protocol.mdwn --- a/doc/design/external_special_remote_protocol.mdwn +++ b/doc/design/external_special_remote_protocol.mdwn @@ -133,6 +133,15 @@ Asks the remote to check if the url's content can currently be downloaded (without downloading it). The remote replies with one of `CHECKURL-FAILURE`, `CHECKURL-CONTENTS`, or `CHECKURL-MULTI`. +* `WHEREIS Key` + Asks the remote to provide any information it can about ways to access + the content of a key stored in it, such as eg, public urls. + This will be displayed to the user by eg, `git annex whereis`. The remote + replies with `WHEREIS-SUCCESS` or `WHEREIS-FAILURE`. + Note that users expect `git annex whereis` to run fast, without eg, + network access. + This is not needed when `SETURIPRESENT` is used, since such uris are + automatically displayed by `git annex whereis`. More optional requests may be added, without changing the protocol version, so if an unknown request is seen, reply with `UNSUPPORTED-REQUEST`. @@ -193,6 +202,12 @@ can contain spaces. * `CHECKURL-FAILURE` Indicates that the requested url could not be accessed. +* `WHEREIS-SUCCESS String` + Indicates a location of a key. Typically an url, the string can + be anything that it makes sense to display to the user about content + stored in the special remote. +* `WHEREIS-FAILURE` + Indicates that no location is known for a key. * `UNSUPPORTED-REQUEST` Indicates that the special remote does not know how to handle a request. @@ -276,16 +291,18 @@ * `SETURLPRESENT Key Url` Records an URL where the Key can be downloaded from. Note that this does not make git-annex think that the url is present on - the web special remote. + the web special remote. + Keep in mind that this stores the url in the git-annex branch. This can + result in bloat to the branch if the url is large and/or does not delta + pack well with other information (such as the names of keys) already + stored in the branch. * `SETURLMISSING Key Url` Records that the key can no longer be downloaded from the specified URL. * `SETURIPRESENT Key Uri` - Records a special URI where the Key can be downloaded from. + Records an URI where the Key can be downloaded from. For example, "ipfs:ADDRESS" is used for the ipfs special remote; - its CLAIMURL handler checks for such URIS and claims them. Setting - it present as an URI makes `git annex whereis` display the URI - as belonging to the special remote. + its CLAIMURL handler checks for such URIS and claims them. * `SETURIMISSING Key Uri` Records that the key can no longer be downloaded from the specified URI. @@ -325,6 +342,26 @@ quite a long time, especially when the git-annex assistant is using it. The assistant will detect when the system connects to a network, and will start a new process the next time it needs to use a remote. + +## readonly mode + +Some storage services allow downloading the content of a file using a +regular http connection, with no authentication. An external special remote +for such a storage service can support a readonly mode of operation. + +It works like this: + +* When a key's content is stored on the remote, use SETURLPRESENT to + tell git-annex the public url from which it can be downloaded. +* When a key's content is removed from the remote, use SETURLMISSING. +* Document that this external special remote can be used in readonly mode. + + The user doesn't even need to install your external special remote + program to use such a remote! All they need to do is run: + `git annex enableremote $remotename readonly=true` + +* The readonly=true parameter makes git-annex download content from the + urls recorded earlier by SETURLPRESENT. ## TODO diff --git a/doc/devblog/day_313__optimisation.mdwn b/doc/devblog/day_313__optimisation.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_313__optimisation.mdwn @@ -0,0 +1,7 @@ +Been doing a little bit of optimisation work. Which meant, first improving +the --debug output to show fractions of a second, and show when commands +exit. + +That let me measure what takes up time when downloading files from ssh remotes. +Found one place I could spawn a thread to run a cleanup action, and this simple +change reduced the non-data-transfer overhead to 1/6th of what it had been! diff --git a/doc/devblog/day_314__pre_trip_catchup.mdwn b/doc/devblog/day_314__pre_trip_catchup.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_314__pre_trip_catchup.mdwn @@ -0,0 +1,13 @@ +Did some work on Friday and Monday to let external special remotes be used +in a readonly mode. This lets files that are stored in the remote be +downloaded by git-annex without the user needing to install the +external special remote program. For this to work, the external special +remote just has to tell git-annex the urls to use. This was developed in +collaboration with Benjamin Gilbert, who is developing +[gcsannex](https://github.com/bgilbert/gcsannex), a Google Cloud Storage +special remote. + +Today, got caught up with recent traffic, including fixing a couple of +bugs. The backlog remains in the low 90's, which is a good place to be as I +prepare for my August vacation week in the SF Bay Area, followed by a week +for ICFP and the Haskell Symposium in Vancouver. diff --git a/doc/forum/Github_pull_request_with_git-annex.mdwn b/doc/forum/Github_pull_request_with_git-annex.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Github_pull_request_with_git-annex.mdwn @@ -0,0 +1,11 @@ +I would like to use git annex with github. Here what I did: + +1. Create github directory "a". Added all the special remote and git annex files. Pushed them to the "a" using "git annex sync." +2. Fork the repository on gihub to "b". Clone it and enable remote and I could pull annexed files. +3. Added repository "a" as "upstream" tracking remote to "b". +3. Created a second special remote for "b". Added new git annex files to it. Used "git annex sync origin" to push the changed files to the forked repository "b". +4. Now I would like to create a pull request. What are branches I should create pull request for? a:synced/git-annex vs b:synced/git-annex? Or, a:synced/git-annex vs b:git-annex? Do I need to create separate pull request for the "master"? + +Thanks for the help. + + diff --git a/doc/forum/HowTo_Conflict_Resolution.mdwn b/doc/forum/HowTo_Conflict_Resolution.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/HowTo_Conflict_Resolution.mdwn @@ -0,0 +1,66 @@ +Hello, + +I have two git annex repos, AA and BB in direct mode. Both have edited a binary file before syncing + +``` +~/Desktop/BB (git)-[annex/direct/master] % git annex sync +commit ok +pull origin +remote: Counting objects: 21, done. +remote: Compressing objects: 100% (17/17), done. +remote: Total 21 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (21/21), done. +From /home/florian/Desktop/AA + d55cfa2..b86e708 annex/direct/master -> origin/annex/direct/master + 83ff4c6..5cfbd34 git-annex -> origin/git-annex + d7b79e8..b86e708 master -> origin/master + d7b79e8..b86e708 synced/master -> origin/synced/master + +Auto-merging bin2 +CONFLICT (content): Merge conflict in bin2 +Auto-merging bin1 +CONFLICT (content): Merge conflict in bin1 +Automatic merge failed; fix conflicts and then commit the result. +bin2: needs merge +bin1: needs merge +(recording state in git...) + + Merge conflict was automatically resolved; you may want to examine the result. + +ok +(merging origin/git-annex into git-annex...) +(recording state in git...) +push origin +Counting objects: 31, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (25/25), done. +Writing objects: 100% (31/31), 3.04 KiB | 0 bytes/s, done. +Total 31 (delta 6), reused 0 (delta 0) +To /home/florian/Desktop/AA + 83ff4c6..7df9834 git-annex -> synced/git-annex + b86e708..59a1240 annex/direct/master -> synced/master +ok +``` + +``` +florian@horus ~/Desktop/BB (git)-[annex/direct/master] % git annex get . +get bin1.variant-c4b6 (from origin...) ok +get bin2.variant-f16a (from origin...) ok +(recording state in git...) +florian@horus ~/Desktop/BB (git)-[annex/direct/master] % ll +insgesamt 5824 +-rw-r----- 1 florian florian 2863795 24. Aug 21:32 bin1.variant-1696 +-rw-r----- 1 florian florian 2841749 24. Aug 21:30 bin1.variant-c4b6 +-rw-r----- 1 florian florian 125612 24. Aug 21:32 bin2.variant-0efa +-rw-r----- 1 florian florian 126067 24. Aug 21:31 bin2.variant-f16a + +``` +How can I resolve this conflict now? Is there a way to tell which bin1 / bin2 is from AA, which from BB? Is there a way to tell git annex to completely take the data from AA or BB? + +This case might appear somehow artifical but I was caught in that various times when syncing my music database. Actually I didn't care which version of the database it took, but I was unable to produce a coherent data set, so that the database files only come from one sync partner. + +I did the git annex get after syncing, becaue usually I sync using --content. + +Thanks, +Florian + diff --git a/doc/forum/Searching_metadata_and_file_content__63__.mdwn b/doc/forum/Searching_metadata_and_file_content__63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Searching_metadata_and_file_content__63__.mdwn @@ -0,0 +1,11 @@ +Hi + +Is there any (built-in or otherwise) way to search git-annex metadata and file content? Ideally I think something that knows about git-annex would be helpful because of files moving around / going away due to metadata driven views, dangling symlinks etc. + +I'm imagining: + +* Something based on Lucene (solr/elasticsearch) or Xapian for fast searches +* Probably ideally using git-annex metadata to track which files move where on disk +* Maybe use of inotify to let it know when git annex has moved file content around or added/removed it from a working tree + +Thanks everybody, and Joey for making git annex and being an inspiration diff --git a/doc/forum/mesh_configurations.mdwn b/doc/forum/mesh_configurations.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/mesh_configurations.mdwn @@ -0,0 +1,181 @@ +I have a setup where a *source* repository `a` is connected to a *source* repository `b` (through SSH) which is then connected to *backup* repository `c` (on amazon S3). I was expecting a file added on `a` to be moved to `c` *through* `b`, but that doesn't seem to be happening... + +I tried to reproduce with this basic setup: + +
+[1009]anarcat@angela:g-a$ git init a
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/a/.git/
+[1010]anarcat@angela:g-a$ git init b
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/b/.git/
+[1011]anarcat@angela:g-a$ git init c
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/c/.git/
+[1012]anarcat@angela:g-a$ cd a/
+[1013]anarcat@angela:a$ git annex init
+init  ok
+(Recording state in git...)
+[1014]anarcat@angela:a$ git annex group . source
+group . ok
+(Recording state in git...)
+[1015]anarcat@angela:a$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+[1036]anarcat@angela:a$ git remote add origin ../b
+[1016]anarcat@angela:a$ cd ../b
+[1025]anarcat@angela:b$ git annex init
+init  ok
+(Recording state in git...)
+[1026]anarcat@angela:b$ git annex group . source
+group . ok
+(Recording state in git...)
+[1027]anarcat@angela:b$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+[1038]anarcat@angela:b$ git remote add origin ../c
+[1019]anarcat@angela:b$ cd ../c
+[1021]anarcat@angela:c$ git annex init
+init  ok
+(Recording state in git...)
+[1022]anarcat@angela:c$ git annex group . backup
+group . ok
+(Recording state in git...)
+[1023]anarcat@angela:c$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+anarcat@angela:c$ cd ../a
+[1041]anarcat@angela:a$ git annex sync
+commit  ok
+pull origin
+warning: no common commits
+remote: Décompte des objets: 11, fait.
+remote: Compression des objets: 100% (9/9), fait.
+remote: Total 11 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (11/11), fait.
+Depuis ../b
+ * [nouvelle branche] git-annex  -> origin/git-annex
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+git-annex: sync: 1 failed
+[1042]anarcat@angela:a1$ cd ../b
+[1043]anarcat@angela:b$ git annex sync
+commit  ok
+pull origin
+warning: no common commits
+remote: Décompte des objets: 11, fait.
+remote: Compression des objets: 100% (9/9), fait.
+remote: Total 11 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (11/11), fait.
+Depuis ../c
+ * [nouvelle branche] git-annex  -> origin/git-annex
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+git-annex: sync: 1 failed
+[1063]anarcat@angela:b$ touch bar
+[1064]anarcat@angela:b$ ls
+bar
+[1065]anarcat@angela:b$ ls -al
+total 16K
+drwxr-xr-x 3 anarcat anarcat 4096 aoû 18 14:41 .
+drwxr-xr-x 5 anarcat anarcat 4096 aoû 18 14:33 ..
+lrwxrwxrwx 1 anarcat anarcat  178 aoû 18 14:41 bar -> .git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+drwxr-xr-x 9 anarcat anarcat 4096 aoû 18 14:41 .git
+[1066]anarcat@angela:b$ git annex sync
+commit  ok
+pull origin
+ok
+push origin
+Décompte des objets: 26, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (22/22), fait.
+Écriture des objets: 100% (26/26), 2.47 KiB | 0 bytes/s, fait.
+Total 26 (delta 5), reused 0 (delta 0)
+To ../c
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+ok
+[1067]anarcat@angela:b$ cd ../a
+[1068]anarcat@angela:a$ git annex sync
+commit  ok
+pull origin
+remote: Décompte des objets: 8, fait.
+remote: Compression des objets: 100% (6/6), fait.
+remote: Total 8 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (8/8), fait.
+Depuis ../b
+   5d3090f..9e345e6  git-annex  -> origin/git-annex
+ * [nouvelle branche] master     -> origin/master
+ * [nouvelle branche] synced/master -> origin/synced/master
+
+Merge made by the 'recursive' strategy.
+ bar | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 120000 bar
+
+Already up-to-date.
+ok
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+push origin
+Décompte des objets: 41, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (36/36), fait.
+Écriture des objets: 100% (41/41), 3.50 KiB | 0 bytes/s, fait.
+Total 41 (delta 20), reused 0 (delta 0)
+To ../b
+   6019ab8..368ca15  master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+ok
+[1069]anarcat@angela:a$ touch quu^C
+[1069]anarcat@angela:a130$ echo foo > quux
+[1070]anarcat@angela:a$ cd ../b
+[1071]anarcat@angela:b$ ls
+bar  foo
+[1072]anarcat@angela:b$ cd ..
+[1073]anarcat@angela:g-a$ cd a
+[1074]anarcat@angela:a$ git annex list
+here
+|origin
+||web
+|||
+XX_ bar
+XX_ foo
+X__ quux
+[1075]anarcat@angela:a$ git annex list --help
+git-annex: unrecognized option `--help'
+
+Usage: git-annex list [PATH ...] [option ...]
+    --allrepos  show all repositories, not only remotes
+
+To see additional options common to all commands, run: git annex help options
+
+
+[1076]anarcat@angela:a1$ git annex list --allrepos
+here-
+|origin
+||web
+|||anarcat@angela:~/test/g-a/c
+||||
+XX__ bar
+XX__ foo
+X___ quux
+
+ +why don't the files get copied over to the backup repo by the assistant? + +i somewhat understand that files don't get sent from `a` to `b`, but why doesn't the assistant copy the files from `b` to `c`? + +i have tried using `required` instead of `wanted` and it doesn't work much better. + +tested with `5.20150610+gitg608172f-1~ndall+1` (prod) and `5.20141125` (the above test). --[[anarcat]] diff --git a/doc/git-annex-addurl.mdwn b/doc/git-annex-addurl.mdwn --- a/doc/git-annex-addurl.mdwn +++ b/doc/git-annex-addurl.mdwn @@ -25,7 +25,8 @@ * `--fast` - Avoid immediately downloading the url. + Avoid immediately downloading the url. The url is still checked + (via HEAD) to verify that it exists, and to get its size if possible. * `--relaxed` diff --git a/doc/git-annex-required.mdwn b/doc/git-annex-required.mdwn --- a/doc/git-annex-required.mdwn +++ b/doc/git-annex-required.mdwn @@ -24,6 +24,10 @@ annex drop`, but if that file is `required`, it would need to be removed with `git annex drop --force`. +# NOTES + +The `required` command was added in git-annex 5.20150420. + # SEE ALSO [[git-annex]](1) diff --git a/doc/git-annex-sync.mdwn b/doc/git-annex-sync.mdwn --- a/doc/git-annex-sync.mdwn +++ b/doc/git-annex-sync.mdwn @@ -65,6 +65,16 @@ will only match the version of files currently in the work tree, but not past versions of files. +* `--jobs=N` `-JN` + + Enables parallel syncing with up to the specified number of jobs + running at once. For example: `-J10` + + When there are multiple git remotes, pushes will be made to them in + parallel. Pulls are not done in parallel because that tends to be + less efficient. When --content is synced, the files are processed + in parallel as well. + # SEE ALSO [[git-annex]](1) diff --git a/doc/news/version_5.20150710.mdwn b/doc/news/version_5.20150710.mdwn deleted file mode 100644 --- a/doc/news/version_5.20150710.mdwn +++ /dev/null @@ -1,34 +0,0 @@ -git-annex 5.20150710 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * add: Stage symlinks the same as git add would, even if they are not a - link to annexed content. - * sync: When annex.autocommit=false, avoid making any commit of local - changes, while still merging with remote to the extent possible. - * unused: --used-refspec can now be configured to look at refs in the - reflog. This provides a way to not consider old versions of files to be - unused after they have reached a specified age, when the old refs in - the reflog expire. - * log: Fix reversion introduced in version 5.20150528 that broke this command. - * assistant --autostart: First stop any daemons that are already running, - which might be left over from a previous login session and so unable to - use the ssh agent of a new login session. - * assistant: Fix local pairing to not include newline in ssh pubkey, - which is rejected on the other end for security reasons. - * assistant: Fix ANNEX\_SHELL\_DIR written to ~/.ssh/authorized\_keys - in local pairing to be the absolute path to the repository, not "." - This was a reversion caused by the relative path changes in 5.20150113. - * Brought back the setkey plumbing command that was removed in 2011, since - we found a use case for it. Note that the command's syntax was changed - for consistency. - * bugfix: Pass --full-tree when using git ls-files to get a list of files - on the git-annex branch, so it works when run in a subdirectory. - This bug affected git-annex unused, and potentially also transitions - running code and other things. - * Support git's undocumented core.sharedRepository=2 value, which - is equivalent to "world", and is set when a repo was created using - git init --shared=world. - * When building on linux, pass --as-needed to linker to avoid linking - with unused shared libraries including libyaml. - * import: Fix failure of cross-device import on Windows. - * merge: Avoid creating the synced/master branch. - * Removed support for optparse-applicative versions older than 0.10."""]] diff --git a/doc/news/version_5.20150824.mdwn b/doc/news/version_5.20150824.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20150824.mdwn @@ -0,0 +1,26 @@ +git-annex 5.20150824 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Sped up downloads of files from ssh remotes, reducing the + non-data-transfer overhead 6x. + * sync: Support --jobs + * sync --content: Avoid unnecessary second pull from remotes when + no file transfers are made. + * External special remotes can now be built that can be used in readonly + mode, where git-annex downloads content from the remote using regular + http. + * Added WHEREIS to external special remote protocol. + * importfeed --relaxed: Avoid hitting the urls of items in the feed. + * Fix reversion in init when ran as root, introduced in version 5.20150731. + * Reorder declaration to fix build with yesod-core > 1.4.13. + Thanks, Michael Alan Dorman. + * Fix building without quvi and without database. + Thanks, Ben Boeckel. + * Avoid building the assistant on the hurd, since an inotify equivalent + is not yet implemented in git-annex for the hurd. + * --debug log messages are now timestamped with fractional seconds. + * --debug is passed along to git-annex-shell when git-annex is in debug mode. + * Makefile: Pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc and on to + ld, cc, and cpp. + * As a result of the Makefile changes, the Debian package is built + with various hardening options. Although their benefit to a largely + haskell program is unknown."""]] diff --git a/doc/not.mdwn b/doc/not.mdwn --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -14,6 +14,16 @@ too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. +* git-annex is also not a folder mirroring system like [syncthing](https://syncthing.net/) + (although [syncthing could be supported as a special remote](todo/syncthing_special_remote/)) + or [gut](https://github.com/tillberg/gut), but it can be used to sync + files such a way, with certain limitations (for example, it doesn't + like syncing `.git` directories so much). + +* git-annex is also not a distributed file system like Bittorrent or [ipfs](http://ipfs.io) + but both are supported as special remotes with more work in making + [[git-annex more distributed underway|todo/Bittorrent-like_features/]]. + * git-annex is more than just a workaround for git scalability limitations that might eventually be fixed by efforts like [git-bigfiles](http://caca.zoy.org/wiki/git-bigfiles). In particular, @@ -56,4 +66,5 @@ Although mercurial and git have some of the same problems around large files, and both try to solve them in similar ways (standin files using mostly hashes of the real content). - + +See also the [[related_software]] page for software that git-annex *is* similar to. diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -36,3 +36,5 @@ * [git-annex-watcher](https://github.com/rubiojr/git-annex-watcher) is a status icon for your desktop. + +See also [[not]] for software that is *not* related to git-annex, but similar. diff --git a/doc/special_remotes/bup/comment_13_ce960bc69b27dfc2a233c9baa5b2cd2b._comment b/doc/special_remotes/bup/comment_13_ce960bc69b27dfc2a233c9baa5b2cd2b._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/bup/comment_13_ce960bc69b27dfc2a233c9baa5b2cd2b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="darkfeline@e6d098788a13ce41f3141a2dfc1bd31b401e83f0" + nickname="darkfeline" + subject="buprepo doesn't work properly" + date="2015-08-17T12:59:25Z" + content=""" +The buprepo parameter doesn't seem to work properly, at least for local repos. For example, I added a bup remote with buprepo=/media/hdd/bup, yet when I try to move files onto it, it still tries to use /home/foo/.bup (the default path). I suppose git-annex should be setting the envvar BUP_DIR before calling bup? + +I can do it manually like BUP_DIR=/media/hdd/bup git annex move --to hdd foo, but that almost seems to defeat the purpose... +"""]] diff --git a/doc/special_remotes/bup/comment_14_940b778f97377e83dc16439c0c7e5b38._comment b/doc/special_remotes/bup/comment_14_940b778f97377e83dc16439c0c7e5b38._comment new file mode 100644 --- /dev/null +++ b/doc/special_remotes/bup/comment_14_940b778f97377e83dc16439c0c7e5b38._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 14""" + date="2015-08-19T18:30:16Z" + content=""" +@darkfeline, setting buprepo= causes git-annex to run bup with -r. +You can verify this by using the --debug switch. + +IIRC, bup still creates ~/.bup when used this way, but doesn't store the +contents of annexed files there. It uses it only to store some small index +files, which are also stored in the repo specified with -r. This seems +weird, but I don't think this is a bug on bup's part; it seems to +intentionally do that, using path names in ~/.bup that are constructed to +not conflict when -r is used with different repositories. +I suppose bup has a good reason to do this, though I don't know what the +reason is. + +I can blow ~/.bup away, run "bup init" to make a fresh clean ~/.bup, +and then git-annex can still get the content of files from the buprepo= +repository. So, it seems that buprepo= is working ok. +"""]] diff --git a/doc/todo/Support_--jobs_option_for___39__sync_--content__39__.mdwn b/doc/todo/Support_--jobs_option_for___39__sync_--content__39__.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/Support_--jobs_option_for___39__sync_--content__39__.mdwn @@ -0,0 +1,14 @@ +As the subject says. I mostly use `git annex sync --content` to transfer +files between repositories, as its easier than running `git annex sync`, a +bunch of `git annex copy`s and then a `git annex get` to make sure I have +all the files I should have. It would be good if the shortcut could also +work in parallel. + +> It also can be faster to push concurrent. OTOH, concurrent pulls +> can lead to the same git objects being downloaded redundantly, so best to +> avoid those I think. +> +> I've implemented this. It suffers from the same +> lack of support for displaying progress when running it parallel as +> documented on [[parallel_get]]. Other than that wart, this is [[done]]. +> --[[Joey]] diff --git a/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn b/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn @@ -0,0 +1,7 @@ +Feature Request. Instead of (or perhaps in addition to) setting +`annex.diskreserve`, I'd like to be able to tell git-annex "use up to 2TB +of disk space". + +> This would need changes for git-annex to keep a running total of the +> space it's using, which becomes a little hairy in light of concurrency, +> possibly manual changes to the object store, etc. --[[Joey]] diff --git a/doc/todo/subsecond_timestamping_of_the_--debug_output.mdwn b/doc/todo/subsecond_timestamping_of_the_--debug_output.mdwn --- a/doc/todo/subsecond_timestamping_of_the_--debug_output.mdwn +++ b/doc/todo/subsecond_timestamping_of_the_--debug_output.mdwn @@ -1,1 +1,4 @@ ATM --debug uses timestamps at second precision. Would be nice (to see where time is spent) to have subsecond timing + +> [[done]], I was able to get fractional seconds down to 0.000001 +> in the debug output. --[[Joey]] diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20150812 +Version: 5.20150824 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess @@ -152,6 +152,9 @@ else Build-Depends: cryptohash (>= 0.11.0) + -- Fully optimize for production. + -- Parallel builds only when not building for production, + -- because ghc is known to not yield reproducible builds this way. if flag(Production) GHC-Options: -O2 diff --git a/man/git-annex-addurl.1 b/man/git-annex-addurl.1 --- a/man/git-annex-addurl.1 +++ b/man/git-annex-addurl.1 @@ -22,7 +22,8 @@ .SH OPTIONS .IP "\fB\-\-fast\fP" .IP -Avoid immediately downloading the url. +Avoid immediately downloading the url. The url is still checked +(via HEAD) to verify that it exists, and to get its size if possible. .IP .IP "\fB\-\-relaxed\fP" Avoid storing the size of the url's content, and accept whatever diff --git a/man/git-annex-required.1 b/man/git-annex-required.1 --- a/man/git-annex-required.1 +++ b/man/git-annex-required.1 @@ -22,6 +22,9 @@ annex drop\fB, but if that file is \fPrequired, it would need to be removed with \fBgit annex drop \-\-force\fP. .PP +.SH NOTES +The \fBrequired\fP command was added in git-annex 5.20150420. +.PP .SH SEE ALSO git-annex(1) .PP diff --git a/man/git-annex-sync.1 b/man/git-annex-sync.1 --- a/man/git-annex-sync.1 +++ b/man/git-annex-sync.1 @@ -59,6 +59,15 @@ will only match the version of files currently in the work tree, but not past versions of files. .IP +.IP "\fB\-\-jobs=N\fP \fB\-JN\fP" +Enables parallel syncing with up to the specified number of jobs +running at once. For example: \fB\-J10\fP +.IP +When there are multiple git remotes, pushes will be made to them in +parallel. Pulls are not done in parallel because that tends to be +less efficient. When \-\-content is synced, the files are processed +in parallel as well. +.IP .SH SEE ALSO git-annex(1) .PP