diff --git a/Annex.hs b/Annex.hs --- a/Annex.hs +++ b/Annex.hs @@ -10,7 +10,6 @@ module Annex ( Annex, AnnexState(..), - PreferredContentMap, new, run, eval, @@ -60,8 +59,8 @@ import Types.NumCopies import Types.LockPool import Types.MetaData +import Types.DesktopNotify import Types.CleanupActions -import qualified Utility.Matcher import qualified Data.Map as M import qualified Data.Set as S import Utility.Quvi (QuviVersion) @@ -80,9 +79,6 @@ Applicative ) -type Matcher a = Either [Utility.Matcher.Token a] (Utility.Matcher.Matcher a) -type PreferredContentMap = M.Map UUID (Utility.Matcher.Matcher (S.Set UUID -> MatchInfo -> Annex Bool)) - -- internal state storage data AnnexState = AnnexState { repo :: Git.Repo @@ -103,9 +99,10 @@ , forcebackend :: Maybe String , globalnumcopies :: Maybe NumCopies , forcenumcopies :: Maybe NumCopies - , limit :: Matcher (MatchInfo -> Annex Bool) + , limit :: ExpandableMatcher Annex , uuidmap :: Maybe UUIDMap - , preferredcontentmap :: Maybe PreferredContentMap + , preferredcontentmap :: Maybe (FileMatcherMap Annex) + , requiredcontentmap :: Maybe (FileMatcherMap Annex) , shared :: Maybe SharedRepository , forcetrust :: TrustMap , trustmap :: Maybe TrustMap @@ -122,6 +119,7 @@ , unusedkeys :: Maybe (S.Set Key) , quviversion :: Maybe QuviVersion , existinghooks :: M.Map Git.Hook.Hook Bool + , desktopnotify :: DesktopNotify } newState :: GitConfig -> Git.Repo -> AnnexState @@ -144,9 +142,10 @@ , forcebackend = Nothing , globalnumcopies = Nothing , forcenumcopies = Nothing - , limit = Left [] + , limit = BuildingMatcher [] , uuidmap = Nothing , preferredcontentmap = Nothing + , requiredcontentmap = Nothing , shared = Nothing , forcetrust = M.empty , trustmap = Nothing @@ -163,6 +162,7 @@ , unusedkeys = Nothing , quviversion = Nothing , existinghooks = M.empty + , desktopnotify = mempty } {- Makes an Annex state object for the specified git repo. diff --git a/Annex/Branch/Transitions.hs b/Annex/Branch/Transitions.hs --- a/Annex/Branch/Transitions.hs +++ b/Annex/Branch/Transitions.hs @@ -32,8 +32,12 @@ dropDead :: FilePath -> String -> TrustMap -> FileTransition dropDead f content trustmap = case getLogVariety f of - Just UUIDBasedLog -> ChangeFile $ - UUIDBased.showLog id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLog Just content + Just UUIDBasedLog + -- Don't remove the dead repo from the trust log, + -- because git remotes may still exist, and they need + -- to still know it's dead. + | f == trustLog -> PreserveFile + | otherwise -> ChangeFile $ UUIDBased.showLog id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLog Just content Just NewUUIDBasedLog -> ChangeFile $ UUIDBased.showLogNew id $ dropDeadFromUUIDBasedLog trustmap $ UUIDBased.parseLogNew Just content Just (PresenceLog _) -> diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -13,7 +13,6 @@ import Limit import Utility.Matcher import Types.Group -import Types.Limit import Logs.Group import Logs.Remote import Annex.UUID @@ -25,12 +24,10 @@ import Data.Either import qualified Data.Set as S -type FileMatcher = Matcher MatchFiles - -checkFileMatcher :: FileMatcher -> FilePath -> Annex Bool +checkFileMatcher :: (FileMatcher Annex) -> FilePath -> Annex Bool checkFileMatcher matcher file = checkMatcher matcher Nothing (Just file) S.empty True -checkMatcher :: FileMatcher -> Maybe Key -> AssociatedFile -> AssumeNotPresent -> Bool -> Annex Bool +checkMatcher :: (FileMatcher Annex) -> Maybe Key -> AssociatedFile -> AssumeNotPresent -> Bool -> Annex Bool checkMatcher matcher mkey afile notpresent def | isEmpty matcher = return def | otherwise = case (mkey, afile) of @@ -48,15 +45,15 @@ , relFile = file } -matchAll :: FileMatcher +matchAll :: FileMatcher Annex matchAll = generate [] -parsedToMatcher :: [Either String (Token MatchFiles)] -> Either String FileMatcher +parsedToMatcher :: [Either String (Token (MatchFiles Annex))] -> Either String (FileMatcher Annex) parsedToMatcher parsed = case partitionEithers parsed of ([], vs) -> Right $ generate vs (es, _) -> Left $ unwords $ map ("Parse failure: " ++) es -exprParser :: FileMatcher -> FileMatcher -> GroupMap -> M.Map UUID RemoteConfig -> Maybe UUID -> String -> [Either String (Token MatchFiles)] +exprParser :: FileMatcher Annex -> FileMatcher Annex -> GroupMap -> M.Map UUID RemoteConfig -> Maybe UUID -> String -> [Either String (Token (MatchFiles Annex))] exprParser matchstandard matchgroupwanted groupmap configmap mu expr = map parse $ tokenizeMatcher expr where @@ -69,7 +66,7 @@ preferreddir = fromMaybe "public" $ M.lookup "preferreddir" =<< (`M.lookup` configmap) =<< mu -parseToken :: FileMatcher -> FileMatcher -> MkLimit -> MkLimit -> GroupMap -> String -> Either String (Token MatchFiles) +parseToken :: FileMatcher Annex -> FileMatcher Annex -> MkLimit Annex -> MkLimit Annex -> GroupMap -> String -> Either String (Token (MatchFiles Annex)) parseToken matchstandard matchgroupwanted checkpresent checkpreferreddir groupmap t | t `elem` tokens = Right $ token t | t == "standard" = call matchstandard @@ -106,7 +103,7 @@ {- Generates a matcher for files large enough (or meeting other criteria) - to be added to the annex, rather than directly to git. -} -largeFilesMatcher :: Annex FileMatcher +largeFilesMatcher :: Annex (FileMatcher Annex) largeFilesMatcher = go =<< annexLargeFiles <$> Annex.getGitConfig where go Nothing = return matchAll diff --git a/Annex/Notification.hs b/Annex/Notification.hs new file mode 100644 --- /dev/null +++ b/Annex/Notification.hs @@ -0,0 +1,81 @@ +{- git-annex desktop notifications + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Notification where + +import Common.Annex +import Logs.Transfer +#ifdef WITH_DBUS_NOTIFICATIONS +import qualified Annex +import Types.DesktopNotify +import qualified DBus.Notify as Notify +import qualified DBus.Client +#endif + +-- Witness that notification has happened. +data NotifyWitness = NotifyWitness + +{- Wrap around an action that performs a transfer, which may run multiple + - attempts. Displays notification when supported and when the user asked + - for it. -} +notifyTransfer :: Direction -> Maybe FilePath -> (NotifyWitness -> Annex Bool) -> Annex Bool +notifyTransfer _ Nothing a = a NotifyWitness +#ifdef WITH_DBUS_NOTIFICATIONS +notifyTransfer direction (Just f) a = do + wanted <- Annex.getState Annex.desktopnotify + let action = if direction == Upload then "uploading" else "downloading" + let basedesc = action ++ " " ++ f + let startdesc = "started " ++ basedesc + let enddesc ok = if ok + then "finished " ++ basedesc + else basedesc ++ " failed" + if (notifyStart wanted || notifyFinish wanted) + then do + client <- liftIO DBus.Client.connectSession + startnotification <- liftIO $ if notifyStart wanted + then Just <$> Notify.notify client (mkNote startdesc) + else pure Nothing + ok <- a NotifyWitness + when (notifyFinish wanted) $ liftIO $ void $ maybe + (Notify.notify client $ mkNote $ enddesc ok) + (\n -> Notify.replace client n $ mkNote $ enddesc ok) + startnotification + return ok + else a NotifyWitness +#else +notifyTransfer _ (Just _) a = do a NotifyWitness +#endif + +notifyDrop :: Maybe FilePath -> Bool -> Annex () +notifyDrop Nothing _ = noop +#ifdef WITH_DBUS_NOTIFICATIONS +notifyDrop (Just f) ok = do + wanted <- Annex.getState Annex.desktopnotify + when (notifyFinish wanted) $ liftIO $ do + client <- DBus.Client.connectSession + let msg = if ok + then "dropped " ++ f + else "failed to drop" ++ f + void $ Notify.notify client (mkNote msg) +#else +notifyDrop (Just _) _ = noop +#endif + +#ifdef WITH_DBUS_NOTIFICATIONS +mkNote :: String -> Notify.Note +mkNote desc = Notify.blankNote + { Notify.appName = "git-annex" + , Notify.body = Just $ Notify.Text desc + , Notify.hints = + [ Notify.Category Notify.Transfer + , Notify.Urgency Notify.Low + , Notify.SuppressSound True + ] + } +#endif diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs new file mode 100644 --- /dev/null +++ b/Annex/Transfer.hs @@ -0,0 +1,131 @@ +{- git-annex transfers + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Transfer ( + module X, + upload, + download, + runTransfer, + noRetry, + forwardRetry, +) where + +import Common.Annex +import Logs.Transfer as X +import Annex.Notification as X +import Annex.Perms +import Annex.Exception +import Utility.Metered +#ifdef mingw32_HOST_OS +import Utility.WinLock +#endif + +import Control.Concurrent + +upload :: UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex Bool) -> NotifyWitness -> Annex Bool +upload u key f d a _witness = runTransfer (Transfer Upload u key) f d a + +download :: UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex Bool) -> NotifyWitness -> Annex Bool +download u key f d a _witness = runTransfer (Transfer Download u key) f d a + +{- Runs a transfer action. Creates and locks the lock file while the + - action is running, and stores info in the transfer information + - file. + - + - If the transfer action returns False, the transfer info is + - left in the failedTransferDir. + - + - If the transfer is already in progress, returns False. + - + - An upload can be run from a read-only filesystem, and in this case + - no transfer information or lock file is used. + -} +runTransfer :: Transfer -> Maybe FilePath -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool +runTransfer t file shouldretry a = do + info <- liftIO $ startTransferInfo file + (meter, tfile, metervar) <- mkProgressUpdater t info + mode <- annexFileMode + (fd, inprogress) <- liftIO $ prep tfile mode info + if inprogress + then do + showNote "transfer already in progress" + return False + else do + ok <- retry info metervar $ + bracketIO (return fd) (cleanup tfile) (const $ a meter) + unless ok $ recordFailedTransfer t info + return ok + where +#ifndef mingw32_HOST_OS + prep tfile mode info = do + mfd <- catchMaybeIO $ + openFd (transferLockFile tfile) ReadWrite (Just mode) + defaultFileFlags { trunc = True } + case mfd of + Nothing -> return (Nothing, False) + Just fd -> do + locked <- catchMaybeIO $ + setLock fd (WriteLock, AbsoluteSeek, 0, 0) + if isNothing locked + then return (Nothing, True) + else do + void $ tryIO $ writeTransferInfoFile info tfile + return (mfd, False) +#else + prep tfile _mode info = do + v <- catchMaybeIO $ lockExclusive (transferLockFile tfile) + case v of + Nothing -> return (Nothing, False) + Just Nothing -> return (Nothing, True) + Just (Just lockhandle) -> do + void $ tryIO $ writeTransferInfoFile info tfile + return (Just lockhandle, False) +#endif + cleanup _ Nothing = noop + cleanup tfile (Just lockhandle) = do + void $ tryIO $ removeFile tfile +#ifndef mingw32_HOST_OS + void $ tryIO $ removeFile $ transferLockFile tfile + closeFd lockhandle +#else + {- Windows cannot delete the lockfile until the lock + - is closed. So it's possible to race with another + - process that takes the lock before it's removed, + - so ignore failure to remove. + -} + dropLock lockhandle + void $ tryIO $ removeFile $ transferLockFile tfile +#endif + retry oldinfo metervar run = do + v <- tryAnnex run + case v of + Right b -> return b + Left _ -> do + b <- getbytescomplete metervar + let newinfo = oldinfo { bytesComplete = Just b } + if shouldretry oldinfo newinfo + then retry newinfo metervar run + else return False + getbytescomplete metervar + | transferDirection t == Upload = + liftIO $ readMVar metervar + | otherwise = do + f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t) + liftIO $ catchDefaultIO 0 $ + fromIntegral . fileSize <$> getFileStatus f + +type RetryDecider = TransferInfo -> TransferInfo -> Bool + +noRetry :: RetryDecider +noRetry _ _ = False + +{- Retries a transfer when it fails, as long as the failed transfer managed + - to send some data. -} +forwardRetry :: RetryDecider +forwardRetry old new = bytesComplete old < bytesComplete new diff --git a/Assistant/Alert/Utility.hs b/Assistant/Alert/Utility.hs --- a/Assistant/Alert/Utility.hs +++ b/Assistant/Alert/Utility.hs @@ -14,7 +14,6 @@ import qualified Data.Text as T import Data.Text (Text) import qualified Data.Map as M -import Data.Monoid {- This is as many alerts as it makes sense to display at a time. - A display might be smaller, or larger, the point is to not overwhelm the diff --git a/Assistant/Install.hs b/Assistant/Install.hs --- a/Assistant/Install.hs +++ b/Assistant/Install.hs @@ -35,11 +35,14 @@ - - Note that this is done every time it's started, so if the user moves - it around, the paths this sets up won't break. + - + - Nautilus hook script installation is done even for packaged apps, + - since it has to go into the user's home directory. -} ensureInstalled :: IO () ensureInstalled = go =<< standaloneAppBase where - go Nothing = noop + go Nothing = installNautilus "git-annex" go (Just base) = do let program = base "git-annex" programfile <- programFile @@ -77,6 +80,32 @@ createDirectoryIfMissing True (parentDir shim) viaTmp writeFile shim content modifyFileMode shim $ addModes [ownerExecuteMode] + + installNautilus program + +installNautilus :: FilePath -> IO () +#ifdef linux_HOST_OS +installNautilus program = do + scriptdir <- (\d -> d "nautilus" "scripts") <$> userDataDir + genscript scriptdir "get" + genscript scriptdir "drop" + where + genscript scriptdir action = + installscript (scriptdir scriptname action) $ unlines + [ shebang_local + , autoaddedcomment + , "exec " ++ program ++ " " ++ action ++ " --notify-start --notify-finish -- \"$@\"" + ] + scriptname action = "git-annex " ++ action + installscript f c = whenM (safetoinstallscript f) $ do + writeFile f c + modifyFileMode f $ addModes [ownerExecuteMode] + safetoinstallscript f = catchDefaultIO True $ + elem autoaddedcomment . lines <$> readFileStrict f + autoaddedcomment = "# Automatically added by git-annex, do not edit. (To disable, chmod 600 this file.)" +#else +installNautilus _ = noop +#endif {- Returns a cleaned up environment that lacks settings used to make the - standalone builds use their bundled libraries and programs. diff --git a/Assistant/Ssh.hs b/Assistant/Ssh.hs --- a/Assistant/Ssh.hs +++ b/Assistant/Ssh.hs @@ -197,7 +197,7 @@ - long perl script. -} | otherwise = pubkey where - limitcommand = "command=\"GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty " + limitcommand = "command=\"env GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty " {- Generates a ssh key pair. -} genSshKeyPair :: IO SshKeyPair diff --git a/Assistant/Threads/ConfigMonitor.hs b/Assistant/Threads/ConfigMonitor.hs --- a/Assistant/Threads/ConfigMonitor.hs +++ b/Assistant/Threads/ConfigMonitor.hs @@ -62,15 +62,17 @@ , (groupLog, void $ liftAnnex groupMapLoad) , (numcopiesLog, void $ liftAnnex globalNumCopiesLoad) , (scheduleLog, void updateScheduleLog) - -- Preferred content settings depend on most of the other configs, - -- so will be reloaded whenever any configs change. + -- Preferred and required content settings depend on most of the + -- other configs, so will be reloaded whenever any configs change. , (preferredContentLog, noop) + , (requiredContentLog, noop) + , (groupPreferredContentLog, noop) ] reloadConfigs :: Configs -> Assistant () reloadConfigs changedconfigs = do sequence_ as - void $ liftAnnex preferredContentMapLoad + void $ liftAnnex preferredRequiredMapsLoad {- Changes to the remote log, or the trust log, can affect the - syncRemotes list. Changes to the uuid log may affect its - display so are also included. -} diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -35,6 +35,7 @@ import Annex.CheckIgnore import Annex.Link import Annex.FileMatcher +import Types.FileMatcher import Annex.ReplaceFile import Git.Types import Config @@ -196,7 +197,7 @@ | otherwise = f {- Small files are added to git as-is, while large ones go into the annex. -} -add :: FileMatcher -> FilePath -> Assistant (Maybe Change) +add :: FileMatcher Annex -> FilePath -> Assistant (Maybe Change) add bigfilematcher file = ifM (liftAnnex $ checkFileMatcher bigfilematcher file) ( pendingAddChange file , do @@ -205,7 +206,7 @@ madeChange file AddFileChange ) -onAdd :: FileMatcher -> Handler +onAdd :: FileMatcher Annex -> Handler onAdd matcher file filestatus | maybe False isRegularFile filestatus = unlessIgnored file $ @@ -218,7 +219,7 @@ {- In direct mode, add events are received for both new files, and - modified existing files. -} -onAddDirect :: Bool -> FileMatcher -> Handler +onAddDirect :: Bool -> FileMatcher Annex -> Handler onAddDirect symlinkssupported matcher file fs = do v <- liftAnnex $ catKeyFile file case (v, fs) of diff --git a/BuildFlags.hs b/BuildFlags.hs --- a/BuildFlags.hs +++ b/BuildFlags.hs @@ -57,6 +57,9 @@ #ifdef WITH_DBUS , "DBus" #endif +#ifdef WITH_DESKTOP_NOTIFY + , "DesktopNotify" +#endif #ifdef WITH_XMPP , "XMPP" #else diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,40 @@ +git-annex (5.20140402) unstable; urgency=medium + + * unannex, uninit: Avoid committing after every file is unannexed, + for massive speedup. + * --notify-finish switch will cause desktop notifications after each + file upload/download/drop completes + (using the dbus Desktop Notifications Specification) + * --notify-start switch will show desktop notifications when each + file upload/download starts. + * webapp: Automatically install Nautilus integration scripts + to get and drop files. + * tahoe: Pass -d parameter before subcommand; putting it after + the subcommand no longer works with tahoe-lafs version 1.10. + (Thanks, Alberto Berti) + * forget --drop-dead: Avoid removing the dead remote from the trust.log, + so that if git remotes for it still exist anywhere, git annex info + will still know it's dead and not show it. + * git-annex-shell: Make configlist automatically initialize + a remote git repository, as long as a git-annex branch has + been pushed to it, to simplify setup of remote git repositories, + including via gitolite. + * add --include-dotfiles: New option, perhaps useful for backups. + * Version 5.20140227 broke creation of glacier repositories, + not including the datacenter and vault in their configuration. + This bug is fixed, but glacier repositories set up with the broken + version of git-annex need to have the datacenter and vault set + in order to be usable. This can be done using git annex enableremote + to add the missing settings. For details, see + http://git-annex.branchable.com/bugs/problems_with_glacier/ + * Added required content configuration. + * assistant: Improve ssh authorized keys line generated in local pairing + or for a remote ssh server to set environment variables in an + alternative way that works with the non-POSIX fish shell, as well + as POSIX shells. + + -- Joey Hess Wed, 02 Apr 2014 16:42:53 -0400 + git-annex (5.20140320) unstable; urgency=medium * Fix zombie leak and general inneficiency when copying files to a diff --git a/CmdLine/Option.hs b/CmdLine/Option.hs --- a/CmdLine/Option.hs +++ b/CmdLine/Option.hs @@ -20,6 +20,7 @@ import Common.Annex import qualified Annex import Types.Messages +import Types.DesktopNotify import Limit import CmdLine.Usage @@ -41,6 +42,10 @@ "don't show debug messages" , Option ['b'] ["backend"] (ReqArg setforcebackend paramName) "specify key-value backend to use" + , Option [] ["notify-finish"] (NoArg (setdesktopnotify mkNotifyFinish)) + "show desktop notification after transfer finishes" + , Option [] ["notify-start"] (NoArg (setdesktopnotify mkNotifyStart)) + "show desktop notification after transfer completes" ] where setforce v = Annex.changeState $ \s -> s { Annex.force = v } @@ -49,6 +54,7 @@ setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v } setdebug = Annex.changeGitConfig $ \c -> c { annexDebug = True } unsetdebug = Annex.changeGitConfig $ \c -> c { annexDebug = False } + setdesktopnotify v = Annex.changeState $ \s -> s { Annex.desktopnotify = Annex.desktopnotify s <> v } matcherOptions :: [Option] matcherOptions = diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -30,14 +30,15 @@ withFilesInGit a params = seekActions $ prepFiltered a $ seekHelper LsFiles.inRepo params -withFilesNotInGit :: (FilePath -> CommandStart) -> CommandSeek -withFilesNotInGit a params = do - {- dotfiles are not acted on unless explicitly listed -} - files <- filter (not . dotfile) <$> - seekunless (null ps && not (null params)) ps - dotfiles <- seekunless (null dotps) dotps - seekActions $ prepFiltered a $ - return $ concat $ segmentPaths params (files++dotfiles) +withFilesNotInGit :: Bool -> (FilePath -> CommandStart) -> CommandSeek +withFilesNotInGit skipdotfiles a params + | skipdotfiles = do + {- dotfiles are not acted on unless explicitly listed -} + files <- filter (not . dotfile) <$> + seekunless (null ps && not (null params)) ps + dotfiles <- seekunless (null dotps) dotps + go (files++dotfiles) + | otherwise = go =<< seekunless False params where (dotps, ps) = partition dotfile params seekunless True _ = return [] @@ -45,6 +46,8 @@ force <- Annex.getState Annex.force g <- gitRepo liftIO $ Git.Command.leaveZombie <$> LsFiles.notInRepo force l g + go l = seekActions $ prepFiltered a $ + return $ concat $ segmentPaths params l withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek withPathContents a params = seekActions $ diff --git a/Command/Add.hs b/Command/Add.hs --- a/Command/Add.hs +++ b/Command/Add.hs @@ -34,9 +34,13 @@ import Utility.Tmp def :: [Command] -def = [notBareRepo $ command "add" paramPaths seek SectionCommon - "add files to annex"] +def = [notBareRepo $ withOptions [includeDotFilesOption] $ + command "add" paramPaths seek SectionCommon + "add files to annex"] +includeDotFilesOption :: Option +includeDotFilesOption = flagOption [] "include-dotfiles" "don't skip dotfiles" + {- Add acts on both files not checked into git yet, and unlocked files. - - In direct mode, it acts on any files that have changed. -} @@ -47,7 +51,8 @@ ( start file , stop ) - go withFilesNotInGit + skipdotfiles <- not <$> Annex.getFlag (optionName includeDotFilesOption) + go $ withFilesNotInGit skipdotfiles ifM isDirect ( go withFilesMaybeModified , go withFilesUnlocked diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -26,7 +26,7 @@ import Config import Annex.Content.Direct import Logs.Location -import qualified Logs.Transfer as Transfer +import qualified Annex.Transfer as Transfer #ifdef WITH_QUVI import Annex.Quvi import qualified Utility.Quvi as Quvi @@ -116,9 +116,10 @@ prepGetViaTmpChecked sizedkey $ do tmp <- fromRepo $ gitAnnexTmpObjectLocation key showOutput - ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do - liftIO $ createDirectoryIfMissing True (parentDir tmp) - downloadUrl [videourl] tmp + ok <- Transfer.notifyTransfer Transfer.Download (Just file) $ + Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do + liftIO $ createDirectoryIfMissing True (parentDir tmp) + downloadUrl [videourl] tmp if ok then cleanup quviurl file key (Just tmp) else return False @@ -133,17 +134,20 @@ | relaxed = do setUrlPresent key url next $ return True - | otherwise = do - (exists, samesize) <- Url.withUrlOptions $ Url.check url (keySize key) - if exists && samesize - then do - setUrlPresent key url - next $ return True - else do - warning $ if exists - then "url does not have expected file size (use --relaxed to bypass this check) " ++ url - else "failed to verify url exists: " ++ url - stop + | otherwise = ifM (elem url <$> getUrls key) + ( stop + , do + (exists, samesize) <- Url.withUrlOptions $ Url.check url (keySize key) + if exists && samesize + then do + setUrlPresent key url + next $ return True + else do + warning $ "while adding a new url to an already annexed file, " ++ if exists + then "url does not have expected file size (use --relaxed to bypass this check) " ++ url + else "failed to verify url exists: " ++ url + stop + ) addUrlFile :: Bool -> URLString -> FilePath -> Annex Bool addUrlFile relaxed url file = do @@ -179,7 +183,7 @@ , return False ) where - runtransfer dummykey tmp = + runtransfer dummykey tmp = Transfer.notifyTransfer Transfer.Download (Just file) $ Transfer.download webUUID dummykey (Just file) Transfer.forwardRetry $ const $ do liftIO $ createDirectoryIfMissing True (parentDir tmp) downloadUrl [url] tmp diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs --- a/Command/ConfigList.hs +++ b/Command/ConfigList.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010 Joey Hess + - Copyright 2010-2014 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -10,6 +10,8 @@ import Common.Annex import Command import Annex.UUID +import Annex.Init +import qualified Annex.Branch import qualified Git.Config import Remote.GCrypt (coreGCryptId) @@ -22,9 +24,23 @@ start :: CommandStart start = do - u <- getUUID + u <- findOrGenUUID showConfig "annex.uuid" $ fromUUID u showConfig coreGCryptId =<< fromRepo (Git.Config.get coreGCryptId "") stop where showConfig k v = liftIO $ putStrLn $ k ++ "=" ++ v + +{- The repository may not yet have a UUID; automatically initialize it + - when there's a git-annex branch available. -} +findOrGenUUID :: Annex UUID +findOrGenUUID = do + u <- getUUID + if u /= NoUUID + then return u + else ifM Annex.Branch.hasSibling + ( do + initialize Nothing + getUUID + , return NoUUID + ) diff --git a/Command/Drop.hs b/Command/Drop.hs --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -14,10 +14,14 @@ import Annex.UUID import Logs.Location import Logs.Trust +import Logs.PreferredContent import Config.NumCopies import Annex.Content import Annex.Wanted +import Annex.Notification +import qualified Data.Set as S + def :: [Command] def = [withOptions [dropFromOption] $ command "drop" paramPaths seek SectionCommon "indicate content of files not currently wanted"] @@ -44,27 +48,34 @@ startLocal :: AssociatedFile -> NumCopies -> Key -> Maybe Remote -> CommandStart startLocal afile numcopies key knownpresentremote = stopUnless (inAnnex key) $ do showStart' "drop" key afile - next $ performLocal key numcopies knownpresentremote + next $ performLocal key afile numcopies knownpresentremote startRemote :: AssociatedFile -> NumCopies -> Key -> Remote -> CommandStart startRemote afile numcopies key remote = do showStart' ("drop " ++ Remote.name remote) key afile - next $ performRemote key numcopies remote + next $ performRemote key afile numcopies remote -performLocal :: Key -> NumCopies -> Maybe Remote -> CommandPerform -performLocal key numcopies knownpresentremote = lockContent key $ do +performLocal :: Key -> AssociatedFile -> NumCopies -> Maybe Remote -> CommandPerform +performLocal key afile numcopies knownpresentremote = lockContent key $ do (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key let trusteduuids' = case knownpresentremote of Nothing -> trusteduuids Just r -> nub (Remote.uuid r:trusteduuids) untrusteduuids <- trustGet UnTrusted let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids'++untrusteduuids) - stopUnless (canDropKey key numcopies trusteduuids' tocheck []) $ do - removeAnnex key - next $ cleanupLocal key + u <- getUUID + ifM (canDrop u key afile numcopies trusteduuids' tocheck []) + ( do + removeAnnex key + notifyDrop afile True + next $ cleanupLocal key + , do + notifyDrop afile False + stop + ) -performRemote :: Key -> NumCopies -> Remote -> CommandPerform -performRemote key numcopies remote = lockContent key $ do +performRemote :: Key -> AssociatedFile -> NumCopies -> Remote -> CommandPerform +performRemote key afile numcopies remote = lockContent key $ do -- Filter the remote it's being dropped from out of the lists of -- places assumed to have the key, and places to check. -- When the local repo has the key, that's one additional copy. @@ -76,7 +87,7 @@ untrusteduuids <- trustGet UnTrusted let tocheck = filter (/= remote) $ Remote.remotesWithoutUUID remotes (have++untrusteduuids) - stopUnless (canDropKey key numcopies have tocheck [uuid]) $ do + stopUnless (canDrop uuid key afile numcopies have tocheck [uuid]) $ do ok <- Remote.removeKey remote key next $ cleanupRemote key remote ok where @@ -95,13 +106,19 @@ {- Checks specified remotes to verify that enough copies of a key exist to - allow it to be safely removed (with no data loss). Can be provided with - - some locations where the key is known/assumed to be present. -} -canDropKey :: Key -> NumCopies -> [UUID] -> [Remote] -> [UUID] -> Annex Bool -canDropKey key numcopies have check skip = do - force <- Annex.getState Annex.force - if force || numcopies == NumCopies 0 - then return True - else findCopies key numcopies skip have check + - some locations where the key is known/assumed to be present. + - + - Also checks if it's required content, and refuses to drop if so. + - + - --force overrides and always allows dropping. + -} +canDrop :: UUID -> Key -> AssociatedFile -> NumCopies -> [UUID] -> [Remote] -> [UUID] -> Annex Bool +canDrop dropfrom key afile numcopies have check skip = ifM (Annex.getState Annex.force) + ( return True + , checkRequiredContent dropfrom key afile + <&&> + findCopies key numcopies skip have check + ) findCopies :: Key -> NumCopies -> [UUID] -> [UUID] -> [Remote] -> Annex Bool findCopies key need skip = helper [] [] @@ -136,6 +153,19 @@ where unsafe = showNote "unsafe" hint = showLongNote "(Use --force to override this check, or adjust numcopies.)" + +checkRequiredContent :: UUID -> Key -> AssociatedFile -> Annex Bool +checkRequiredContent u k afile = + ifM (isRequiredContent (Just u) S.empty (Just k) afile False) + ( requiredContent + , return True + ) + +requiredContent :: Annex Bool +requiredContent = do + showLongNote "That file is required content, it cannot be dropped!" + showLongNote "(Use --force to override this check, or adjust required content configuration.)" + return False {- In auto mode, only runs the action if there are enough - copies on other semitrusted repositories. -} diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -34,8 +34,8 @@ where dropremote r = do showAction $ "from " ++ Remote.name r - Command.Drop.performRemote key numcopies r - droplocal = Command.Drop.performLocal key numcopies Nothing + Command.Drop.performRemote key Nothing numcopies r + droplocal = Command.Drop.performLocal key Nothing numcopies Nothing from = Annex.getField $ optionName Command.Drop.dropFromOption performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform diff --git a/Command/Get.hs b/Command/Get.hs --- a/Command/Get.hs +++ b/Command/Get.hs @@ -11,7 +11,7 @@ import Command import qualified Remote import Annex.Content -import Logs.Transfer +import Annex.Transfer import Config.NumCopies import Annex.Wanted import qualified Command.Move @@ -69,15 +69,15 @@ showNote "not available" showlocs return False - dispatch remotes = trycopy remotes remotes - trycopy full [] = do + dispatch remotes = notifyTransfer Download afile $ trycopy remotes remotes + trycopy full [] _ = do Remote.showTriedRemotes full showlocs return False - trycopy full (r:rs) = + trycopy full (r:rs) witness = ifM (probablyPresent r) - ( docopy r (trycopy full rs) - , trycopy full rs + ( docopy r witness <||> trycopy full rs witness + , trycopy full rs witness ) showlocs = Remote.showLocations key [] "No other repository is known to contain the file." @@ -87,8 +87,6 @@ | Remote.hasKeyCheap r = either (const False) id <$> Remote.hasKey r key | otherwise = return True - docopy r continue = do - ok <- download (Remote.uuid r) key afile noRetry $ \p -> do - showAction $ "from " ++ Remote.name r - Remote.retrieveKeyFile r key afile dest p - if ok then return ok else continue + docopy r = download (Remote.uuid r) key afile noRetry $ \p -> do + showAction $ "from " ++ Remote.name r + Remote.retrieveKeyFile r key afile dest p diff --git a/Command/List.hs b/Command/List.hs --- a/Command/List.hs +++ b/Command/List.hs @@ -38,7 +38,7 @@ getList :: Annex [(UUID, RemoteName, TrustLevel)] getList = ifM (Annex.getFlag $ optionName allrepos) - ( nubBy ((==) `on` fst3) <$> ((++) <$> getRemotes <*> getAll) + ( nubBy ((==) `on` fst3) <$> ((++) <$> getRemotes <*> getAllUUIDs) , getRemotes ) where @@ -48,7 +48,7 @@ hereu <- getUUID heretrust <- lookupTrust hereu return $ (hereu, "here", heretrust) : zip3 (map uuid rs) (map name rs) ts - getAll = do + getAllUUIDs = do rs <- M.toList <$> uuidMap rs3 <- forM rs $ \(u, n) -> (,,) <$> pure u diff --git a/Command/Move.hs b/Command/Move.hs --- a/Command/Move.hs +++ b/Command/Move.hs @@ -14,8 +14,8 @@ import Annex.Content import qualified Remote import Annex.UUID +import Annex.Transfer import Logs.Presence -import Logs.Transfer def :: [Command] def = [withOptions moveOptions $ command "move" paramPaths seek @@ -98,8 +98,9 @@ stop Right False -> do showAction $ "to " ++ Remote.name dest - ok <- upload (Remote.uuid dest) key afile noRetry $ - Remote.storeKey dest key afile + ok <- notifyTransfer Upload afile $ + upload (Remote.uuid dest) key afile noRetry $ + Remote.storeKey dest key afile if ok then do Remote.logStatus dest key InfoPresent @@ -155,9 +156,10 @@ , handle move =<< go ) where - go = download (Remote.uuid src) key afile noRetry $ \p -> do - showAction $ "from " ++ Remote.name src - getViaTmp key $ \t -> Remote.retrieveKeyFile src key afile t p + go = notifyTransfer Download afile $ + download (Remote.uuid src) key afile noRetry $ \p -> do + showAction $ "from " ++ Remote.name src + getViaTmp key $ \t -> Remote.retrieveKeyFile src key afile t p handle _ False = stop -- failed handle False True = next $ return True -- copy complete handle True True = do -- finish moving diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -5,6 +5,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE CPP #-} + module Command.PreCommit where import Common.Annex @@ -16,11 +18,17 @@ import Annex.Hook import Annex.View import Annex.View.ViewedFile +import Annex.Perms +import Annex.Exception import Logs.View import Logs.MetaData import Types.View import Types.MetaData +#ifdef mingw32_HOST_OS +import Utility.WinLock +#endif + import qualified Data.Set as S def :: [Command] @@ -28,7 +36,7 @@ "run by git pre-commit hook"] seek :: CommandSeek -seek ps = ifM isDirect +seek ps = lockPreCommitHook $ ifM isDirect ( do -- update direct mode mappings for committed files withWords startDirect ps @@ -82,3 +90,22 @@ showset v | isSet v = "+" | otherwise = "-" + +{- Takes exclusive lock; blocks until available. -} +lockPreCommitHook :: Annex a -> Annex a +lockPreCommitHook a = do + lockfile <- fromRepo gitAnnexPreCommitLock + createAnnexDirectory $ takeDirectory lockfile + mode <- annexFileMode + bracketIO (lock lockfile mode) unlock (const a) + where +#ifndef mingw32_HOST_OS + lock lockfile mode = do + l <- liftIO $ noUmask mode $ createFile lockfile mode + liftIO $ waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0) + return l + unlock = closeFd +#else + lock lockfile _mode = liftIO $ waitToLock $ lockExclusive lockfile + unlock = dropLock +#endif diff --git a/Command/SendKey.hs b/Command/SendKey.hs --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -12,7 +12,7 @@ import Annex.Content import Annex import Utility.Rsync -import Logs.Transfer +import Annex.Transfer import qualified CmdLine.GitAnnexShell.Fields as Fields import Utility.Metered diff --git a/Command/TransferKey.hs b/Command/TransferKey.hs --- a/Command/TransferKey.hs +++ b/Command/TransferKey.hs @@ -11,7 +11,7 @@ import Command import Annex.Content import Logs.Location -import Logs.Transfer +import Annex.Transfer import qualified Remote import Types.Remote @@ -41,7 +41,7 @@ _ -> error "specify either --from or --to" toPerform :: Remote -> Key -> AssociatedFile -> CommandPerform -toPerform remote key file = go $ +toPerform remote key file = go Upload file $ upload (uuid remote) key file forwardRetry $ \p -> do ok <- Remote.storeKey remote key file p when ok $ @@ -49,9 +49,9 @@ return ok fromPerform :: Remote -> Key -> AssociatedFile -> CommandPerform -fromPerform remote key file = go $ +fromPerform remote key file = go Upload file $ download (uuid remote) key file forwardRetry $ \p -> getViaTmp key $ \t -> Remote.retrieveKeyFile remote key file t p -go :: Annex Bool -> CommandPerform -go a = a >>= liftIO . exitBool +go :: Direction -> AssociatedFile -> (NotifyWitness -> Annex Bool) -> CommandPerform +go direction file a = notifyTransfer direction file a >>= liftIO . exitBool diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs --- a/Command/TransferKeys.hs +++ b/Command/TransferKeys.hs @@ -13,7 +13,7 @@ import Command import Annex.Content import Logs.Location -import Logs.Transfer +import Annex.Transfer import qualified Remote import Types.Key @@ -34,14 +34,15 @@ stop where runner (TransferRequest direction remote key file) - | direction == Upload = + | direction == Upload = notifyTransfer direction file $ upload (Remote.uuid remote) key file forwardRetry $ \p -> do ok <- Remote.storeKey remote key file p when ok $ Remote.logStatus remote key InfoPresent return ok - | otherwise = download (Remote.uuid remote) key file forwardRetry $ \p -> - getViaTmp key $ \t -> Remote.retrieveKeyFile remote key file t p + | otherwise = notifyTransfer direction file $ + download (Remote.uuid remote) key file forwardRetry $ \p -> + getViaTmp key $ \t -> Remote.retrieveKeyFile remote key file t p {- stdin and stdout are connected with the caller, to be used for - communication with it. But doing a transfer might involve something diff --git a/Command/Unannex.hs b/Command/Unannex.hs --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -16,16 +16,48 @@ import Annex.Content import Annex.Content.Direct import qualified Git.Command -import qualified Git.LsFiles as LsFiles +import qualified Git.Ref +import qualified Git.DiffTree as DiffTree import Utility.CopyFile +import Command.PreCommit (lockPreCommitHook) def :: [Command] def = [command "unannex" paramPaths seek SectionUtility "undo accidential add command"] seek :: CommandSeek -seek = withFilesInGit $ whenAnnexed start +seek = wrapUnannex . (withFilesInGit $ whenAnnexed start) +wrapUnannex :: Annex a -> Annex a +wrapUnannex a = ifM isDirect + ( a + {- Run with the pre-commit hook disabled, to avoid confusing + - behavior if an unannexed file is added back to git as + - a normal, non-annexed file and then committed. + - Otherwise, the pre-commit hook would think that the file + - has been unlocked and needs to be re-annexed. + - + - At the end, make a commit removing the unannexed files. + -} + , ifM cleanindex + ( lockPreCommitHook $ commit `after` a + , error "Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit" + ) + ) + where + commit = inRepo $ Git.Command.run + [ Param "commit" + , Param "-q" + , Param "--allow-empty" + , Param "--no-verify" + , Param "-m", Param "content removed from git annex" + ] + cleanindex = do + (diff, cleanup) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef + if null diff + then void (liftIO cleanup) >> return True + else void (liftIO cleanup) >> return False + start :: FilePath -> (Key, Backend) -> CommandStart start file (key, _) = stopUnless (inAnnex key) $ do showStart "unannex" file @@ -36,26 +68,7 @@ performIndirect :: FilePath -> Key -> CommandPerform performIndirect file key = do liftIO $ removeFile file - - -- git rm deletes empty directory without --cached inRepo $ Git.Command.run [Params "rm --cached --force --quiet --", File file] - - -- If the file was already committed, it is now staged for removal. - -- Commit that removal now, to avoid later confusing the - -- pre-commit hook, if this file is later added back to - -- git as a normal non-annexed file, to thinking that the - -- file has been unlocked and needs to be re-annexed. - (s, reap) <- inRepo $ LsFiles.staged [file] - unless (null s) $ - inRepo $ Git.Command.run - [ Param "commit" - , Param "-q" - , Param "--no-verify" - , Param "-m", Param "content removed from git annex" - , Param "--", File file - ] - void $ liftIO reap - next $ cleanupIndirect file key cleanupIndirect :: FilePath -> Key -> CommandCleanup diff --git a/Command/Uninit.hs b/Command/Uninit.hs --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -36,7 +36,7 @@ seek :: CommandSeek seek ps = do - withFilesNotInGit (whenAnnexed startCheckIncomplete) ps + withFilesNotInGit False (whenAnnexed startCheckIncomplete) ps withFilesInGit (whenAnnexed Command.Unannex.start) ps finish diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs --- a/Command/Vicfg.hs +++ b/Command/Vicfg.hs @@ -61,6 +61,7 @@ { cfgTrustMap :: TrustMap , cfgGroupMap :: M.Map UUID (S.Set Group) , cfgPreferredContentMap :: M.Map UUID PreferredContentExpression + , cfgRequiredContentMap :: M.Map UUID PreferredContentExpression , cfgGroupPreferredContentMap :: M.Map Group PreferredContentExpression , cfgScheduleMap :: M.Map UUID [ScheduledActivity] } @@ -70,6 +71,7 @@ <$> trustMapRaw -- without local trust overrides <*> (groupsByUUID <$> groupMap) <*> preferredContentMapRaw + <*> requiredContentMapRaw <*> groupPreferredContentMapRaw <*> scheduleMap @@ -79,6 +81,7 @@ mapM_ (uncurry trustSet) $ M.toList $ cfgTrustMap diff mapM_ (uncurry groupSet) $ M.toList $ cfgGroupMap diff mapM_ (uncurry preferredContentSet) $ M.toList $ cfgPreferredContentMap diff + mapM_ (uncurry requiredContentSet) $ M.toList $ cfgRequiredContentMap diff mapM_ (uncurry groupPreferredContentSet) $ M.toList $ cfgGroupPreferredContentMap diff mapM_ (uncurry scheduleSet) $ M.toList $ cfgScheduleMap diff @@ -87,6 +90,7 @@ { cfgTrustMap = diff cfgTrustMap , cfgGroupMap = diff cfgGroupMap , cfgPreferredContentMap = diff cfgPreferredContentMap + , cfgRequiredContentMap = diff cfgRequiredContentMap , cfgGroupPreferredContentMap = diff cfgGroupPreferredContentMap , cfgScheduleMap = diff cfgScheduleMap } @@ -102,6 +106,7 @@ , preferredcontent , grouppreferredcontent , standardgroups + , requiredcontent , schedule ] where @@ -137,6 +142,11 @@ [ com "Repository preferred contents" ] (\(s, u) -> line "wanted" u s) (\u -> line "wanted" u "standard") + + requiredcontent = settings cfg descs cfgRequiredContentMap + [ com "Repository required contents" ] + (\(s, u) -> line "required" u s) + (\u -> line "required" u "") grouppreferredcontent = settings' cfg allgroups cfgGroupPreferredContentMap [ com "Group preferred contents" @@ -228,6 +238,12 @@ Nothing -> let m = M.insert u value (cfgPreferredContentMap cfg) in Right $ cfg { cfgPreferredContentMap = m } + | setting == "required" = + case checkPreferredContentExpression value of + Just e -> Left e + Nothing -> + let m = M.insert u value (cfgRequiredContentMap cfg) + in Right $ cfg { cfgRequiredContentMap = m } | setting == "groupwanted" = case checkPreferredContentExpression value of Just e -> Left e @@ -255,7 +271,6 @@ [ com "** There was a problem parsing your input!" , com "** Search for \"Parse error\" to find the bad lines." , com "** Either fix the bad lines, or delete them (to discard your changes)." - , "" ] parseerr = com "** Parse error in next line: " diff --git a/Common.hs b/Common.hs --- a/Common.hs +++ b/Common.hs @@ -11,6 +11,7 @@ import Data.Maybe as X import Data.List as X hiding (head, tail, init, last) import Data.String.Utils as X hiding (join) +import Data.Monoid as X import System.FilePath as X import System.Directory as X diff --git a/INSTALL b/INSTALL --- a/INSTALL +++ b/INSTALL @@ -14,6 +14,7 @@   [[Gentoo]] | `emerge git-annex`   [[ScientificLinux5]] |   [[openSUSE]] | +  [[Docker]] | [[Windows]] | [download installer](http://downloads.kitenet.net/git-annex/windows/current/) **alpha** """]] diff --git a/Limit.hs b/Limit.hs --- a/Limit.hs +++ b/Limit.hs @@ -20,7 +20,6 @@ import Types.Key import Types.Group import Types.FileMatcher -import Types.Limit import Types.MetaData import Logs.MetaData import Logs.Group @@ -45,21 +44,20 @@ getMatcher = Utility.Matcher.matchM <$> getMatcher' getMatcher' :: Annex (Utility.Matcher.Matcher (MatchInfo -> Annex Bool)) -getMatcher' = do - m <- Annex.getState Annex.limit - case m of - Right r -> return r - Left l -> do - let matcher = Utility.Matcher.generate (reverse l) - Annex.changeState $ \s -> - s { Annex.limit = Right matcher } - return matcher +getMatcher' = go =<< Annex.getState Annex.limit + where + go (CompleteMatcher matcher) = return matcher + go (BuildingMatcher l) = do + let matcher = Utility.Matcher.generate (reverse l) + Annex.changeState $ \s -> + s { Annex.limit = CompleteMatcher matcher } + return matcher {- Adds something to the limit list, which is built up reversed. -} add :: Utility.Matcher.Token (MatchInfo -> Annex Bool) -> Annex () add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s } where - prepend (Left ls) = Left $ l:ls + prepend (BuildingMatcher ls) = BuildingMatcher $ l:ls prepend _ = error "internal" {- Adds a new token. -} @@ -67,21 +65,21 @@ addToken = add . Utility.Matcher.token {- Adds a new limit. -} -addLimit :: Either String MatchFiles -> Annex () +addLimit :: Either String (MatchFiles Annex) -> Annex () addLimit = either error (\l -> add $ Utility.Matcher.Operation $ l S.empty) {- Add a limit to skip files that do not match the glob. -} addInclude :: String -> Annex () addInclude = addLimit . limitInclude -limitInclude :: MkLimit +limitInclude :: MkLimit Annex limitInclude glob = Right $ const $ return . matchGlobFile glob {- Add a limit to skip files that match the glob. -} addExclude :: String -> Annex () addExclude = addLimit . limitExclude -limitExclude :: MkLimit +limitExclude :: MkLimit Annex limitExclude glob = Right $ const $ return . not . matchGlobFile glob matchGlobFile :: String -> (MatchInfo -> Bool) @@ -119,10 +117,10 @@ else inAnnex key {- Limit to content that is currently present on a uuid. -} -limitPresent :: Maybe UUID -> MkLimit +limitPresent :: Maybe UUID -> MkLimit Annex limitPresent u _ = Right $ matchPresent u -matchPresent :: Maybe UUID -> MatchFiles +matchPresent :: Maybe UUID -> MatchFiles Annex matchPresent u _ = checkKey $ \key -> do hereu <- getUUID if u == Just hereu || isNothing u @@ -132,7 +130,7 @@ return $ maybe False (`elem` us) u {- Limit to content that is in a directory, anywhere in the repository tree -} -limitInDir :: FilePath -> MkLimit +limitInDir :: FilePath -> MkLimit Annex limitInDir dir = const $ Right $ const go where go (MatchingFile fi) = return $ elem dir $ splitPath $ takeDirectory $ matchFile fi @@ -143,7 +141,7 @@ addCopies :: String -> Annex () addCopies = addLimit . limitCopies -limitCopies :: MkLimit +limitCopies :: MkLimit Annex limitCopies want = case split ":" want of [v, n] -> case parsetrustspec v of Just checker -> go n $ checktrust checker @@ -169,7 +167,7 @@ addLackingCopies :: Bool -> String -> Annex () addLackingCopies approx = addLimit . limitLackingCopies approx -limitLackingCopies :: Bool -> MkLimit +limitLackingCopies :: Bool -> MkLimit Annex limitLackingCopies approx want = case readish want of Just needed -> Right $ \notpresent mi -> flip checkKey mi $ handle mi needed notpresent @@ -191,7 +189,7 @@ - This has a nice optimisation: When a file exists, - its key is obviously not unused. -} -limitUnused :: MatchFiles +limitUnused :: MatchFiles Annex limitUnused _ (MatchingFile _) = return False limitUnused _ (MatchingKey k) = S.member k <$> unusedKeys @@ -202,7 +200,7 @@ m <- groupMap addLimit $ limitInAllGroup m groupname -limitInAllGroup :: GroupMap -> MkLimit +limitInAllGroup :: GroupMap -> MkLimit Annex limitInAllGroup m groupname | S.null want = Right $ const $ const $ return True | otherwise = Right $ \notpresent -> checkKey $ check notpresent @@ -219,7 +217,7 @@ addInBackend :: String -> Annex () addInBackend = addLimit . limitInBackend -limitInBackend :: MkLimit +limitInBackend :: MkLimit Annex limitInBackend name = Right $ const $ checkKey check where check key = pure $ keyBackendName key == name @@ -231,7 +229,7 @@ addSmallerThan :: String -> Annex () addSmallerThan = addLimit . limitSize (<) -limitSize :: (Maybe Integer -> Maybe Integer -> Bool) -> MkLimit +limitSize :: (Maybe Integer -> Maybe Integer -> Bool) -> MkLimit Annex limitSize vs s = case readSize dataUnits s of Nothing -> Left "bad size" Just sz -> Right $ go sz @@ -249,7 +247,7 @@ addMetaData :: String -> Annex () addMetaData = addLimit . limitMetaData -limitMetaData :: MkLimit +limitMetaData :: MkLimit Annex limitMetaData s = case parseMetaData s of Left e -> Left e Right (f, v) -> diff --git a/Locations.hs b/Locations.hs --- a/Locations.hs +++ b/Locations.hs @@ -41,6 +41,7 @@ gitAnnexMergeDir, gitAnnexJournalDir, gitAnnexJournalLock, + gitAnnexPreCommitLock, gitAnnexIndex, gitAnnexIndexStatus, gitAnnexViewIndex, @@ -256,6 +257,10 @@ {- Lock file for the journal. -} gitAnnexJournalLock :: Git.Repo -> FilePath gitAnnexJournalLock r = gitAnnexDir r "journal.lck" + +{- Lock file for the pre-commit hook. -} +gitAnnexPreCommitLock :: Git.Repo -> FilePath +gitAnnexPreCommitLock r = gitAnnexDir r "precommit.lck" {- .git/annex/index is used to stage changes to the git-annex branch -} gitAnnexIndex :: Git.Repo -> FilePath diff --git a/Logs.hs b/Logs.hs --- a/Logs.hs +++ b/Logs.hs @@ -35,6 +35,7 @@ , trustLog , groupLog , preferredContentLog + , requiredContentLog , scheduleLog ] @@ -69,6 +70,9 @@ preferredContentLog :: FilePath preferredContentLog = "preferred-content.log" + +requiredContentLog :: FilePath +requiredContentLog = "required-content.log" groupPreferredContentLog :: FilePath groupPreferredContentLog = "group-preferred-content.log" diff --git a/Logs/PreferredContent.hs b/Logs/PreferredContent.hs --- a/Logs/PreferredContent.hs +++ b/Logs/PreferredContent.hs @@ -6,16 +6,19 @@ -} module Logs.PreferredContent ( - preferredContentLog, preferredContentSet, + requiredContentSet, groupPreferredContentSet, isPreferredContent, + isRequiredContent, preferredContentMap, - preferredContentMapLoad, preferredContentMapRaw, + requiredContentMap, + requiredContentMapRaw, groupPreferredContentMapRaw, checkPreferredContentExpression, setStandardGroup, + preferredRequiredMapsLoad, ) where import qualified Data.Map as M @@ -28,43 +31,57 @@ import qualified Annex import Logs import Logs.UUIDBased -import qualified Utility.Matcher +import Utility.Matcher hiding (tokens) import Annex.FileMatcher import Annex.UUID -import Types.Limit import Types.Group import Types.Remote (RemoteConfig) import Logs.Group import Logs.Remote +import Types.FileMatcher import Types.StandardGroups import Limit {- Checks if a file is preferred content for the specified repository - (or the current repository if none is specified). -} isPreferredContent :: Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool -isPreferredContent mu notpresent mkey afile def = do +isPreferredContent = checkMap preferredContentMap + +isRequiredContent :: Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool +isRequiredContent = checkMap requiredContentMap + +checkMap :: Annex (FileMatcherMap Annex) -> Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool +checkMap getmap mu notpresent mkey afile def = do u <- maybe getUUID return mu - m <- preferredContentMap + m <- getmap case M.lookup u m of Nothing -> return def Just matcher -> checkMatcher matcher mkey afile notpresent def -{- The map is cached for speed. -} -preferredContentMap :: Annex Annex.PreferredContentMap -preferredContentMap = maybe preferredContentMapLoad return +preferredContentMap :: Annex (FileMatcherMap Annex) +preferredContentMap = maybe (fst <$> preferredRequiredMapsLoad) return =<< Annex.getState Annex.preferredcontentmap -{- Loads the map, updating the cache. -} -preferredContentMapLoad :: Annex Annex.PreferredContentMap -preferredContentMapLoad = do +requiredContentMap :: Annex (FileMatcherMap Annex) +requiredContentMap = maybe (snd <$> preferredRequiredMapsLoad) return + =<< Annex.getState Annex.requiredcontentmap + +preferredRequiredMapsLoad :: Annex (FileMatcherMap Annex, FileMatcherMap Annex) +preferredRequiredMapsLoad = do groupmap <- groupMap configmap <- readRemoteLog - groupwantedmap <- groupPreferredContentMapRaw - m <- simpleMap - . parseLogWithUUID ((Just .) . makeMatcher groupmap configmap groupwantedmap) - <$> Annex.Branch.get preferredContentLog - Annex.changeState $ \s -> s { Annex.preferredcontentmap = Just m } - return m + let genmap l gm = simpleMap + . parseLogWithUUID ((Just .) . makeMatcher groupmap configmap gm) + <$> Annex.Branch.get l + pc <- genmap preferredContentLog =<< groupPreferredContentMapRaw + rc <- genmap requiredContentLog M.empty + -- Required content is implicitly also preferred content, so OR + let m = M.unionWith MOr pc rc + Annex.changeState $ \s -> s + { Annex.preferredcontentmap = Just m + , Annex.requiredcontentmap = Just rc + } + return (m, rc) {- This intentionally never fails, even on unparsable expressions, - because the configuration is shared among repositories and newer @@ -75,11 +92,11 @@ -> M.Map Group PreferredContentExpression -> UUID -> PreferredContentExpression - -> FileMatcher + -> FileMatcher Annex makeMatcher groupmap configmap groupwantedmap u = go True True where go expandstandard expandgroupwanted expr - | null (lefts tokens) = Utility.Matcher.generate $ rights tokens + | null (lefts tokens) = generate $ rights tokens | otherwise = unknownMatcher u where tokens = exprParser matchstandard matchgroupwanted groupmap configmap (Just u) expr @@ -102,10 +119,10 @@ - - This avoid unwanted/expensive changes to the content, until the problem - is resolved. -} -unknownMatcher :: UUID -> FileMatcher -unknownMatcher u = Utility.Matcher.generate [present] +unknownMatcher :: UUID -> FileMatcher Annex +unknownMatcher u = generate [present] where - present = Utility.Matcher.Operation $ matchPresent (Just u) + present = Operation $ matchPresent (Just u) {- Checks if an expression can be parsed, if not returns Just error -} checkPreferredContentExpression :: PreferredContentExpression -> Maybe String diff --git a/Logs/PreferredContent/Raw.hs b/Logs/PreferredContent/Raw.hs --- a/Logs/PreferredContent/Raw.hs +++ b/Logs/PreferredContent/Raw.hs @@ -21,14 +21,23 @@ {- Changes the preferred content configuration of a remote. -} preferredContentSet :: UUID -> PreferredContentExpression -> Annex () -preferredContentSet uuid@(UUID _) val = do +preferredContentSet = setLog preferredContentLog + +requiredContentSet :: UUID -> PreferredContentExpression -> Annex () +requiredContentSet = setLog requiredContentLog + +setLog :: FilePath -> UUID -> PreferredContentExpression -> Annex () +setLog logfile uuid@(UUID _) val = do ts <- liftIO getPOSIXTime - Annex.Branch.change preferredContentLog $ + Annex.Branch.change logfile $ showLog id . changeLog ts uuid val . parseLog Just - Annex.changeState $ \s -> s { Annex.preferredcontentmap = Nothing } -preferredContentSet NoUUID _ = error "unknown UUID; cannot modify" + Annex.changeState $ \s -> s + { Annex.preferredcontentmap = Nothing + , Annex.requiredcontentmap = Nothing + } +setLog _ NoUUID _ = error "unknown UUID; cannot modify" {- Changes the preferred content configuration of a group. -} groupPreferredContentSet :: Group -> PreferredContentExpression -> Annex () @@ -43,6 +52,10 @@ preferredContentMapRaw :: Annex (M.Map UUID PreferredContentExpression) preferredContentMapRaw = simpleMap . parseLog Just <$> Annex.Branch.get preferredContentLog + +requiredContentMapRaw :: Annex (M.Map UUID PreferredContentExpression) +requiredContentMapRaw = simpleMap . parseLog Just + <$> Annex.Branch.get requiredContentLog groupPreferredContentMapRaw :: Annex (M.Map Group PreferredContentExpression) groupPreferredContentMapRaw = simpleMap . parseMapLog Just Just diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -88,108 +88,6 @@ percentComplete (Transfer { transferKey = key }) info = percentage <$> keySize key <*> Just (fromMaybe 0 $ bytesComplete info) -type RetryDecider = TransferInfo -> TransferInfo -> Bool - -noRetry :: RetryDecider -noRetry _ _ = False - -{- Retries a transfer when it fails, as long as the failed transfer managed - - to send some data. -} -forwardRetry :: RetryDecider -forwardRetry old new = bytesComplete old < bytesComplete new - -upload :: UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool -upload u key = runTransfer (Transfer Upload u key) - -download :: UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool -download u key = runTransfer (Transfer Download u key) - -{- Runs a transfer action. Creates and locks the lock file while the - - action is running, and stores info in the transfer information - - file. - - - - If the transfer action returns False, the transfer info is - - left in the failedTransferDir. - - - - If the transfer is already in progress, returns False. - - - - An upload can be run from a read-only filesystem, and in this case - - no transfer information or lock file is used. - -} -runTransfer :: Transfer -> Maybe FilePath -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool -runTransfer t file shouldretry a = do - info <- liftIO $ startTransferInfo file - (meter, tfile, metervar) <- mkProgressUpdater t info - mode <- annexFileMode - (fd, inprogress) <- liftIO $ prep tfile mode info - if inprogress - then do - showNote "transfer already in progress" - return False - else do - ok <- retry info metervar $ - bracketIO (return fd) (cleanup tfile) (const $ a meter) - unless ok $ recordFailedTransfer t info - return ok - where -#ifndef mingw32_HOST_OS - prep tfile mode info = do - mfd <- catchMaybeIO $ - openFd (transferLockFile tfile) ReadWrite (Just mode) - defaultFileFlags { trunc = True } - case mfd of - Nothing -> return (Nothing, False) - Just fd -> do - locked <- catchMaybeIO $ - setLock fd (WriteLock, AbsoluteSeek, 0, 0) - if isNothing locked - then return (Nothing, True) - else do - void $ tryIO $ writeTransferInfoFile info tfile - return (mfd, False) -#else - prep tfile _mode info = do - v <- catchMaybeIO $ lockExclusive (transferLockFile tfile) - case v of - Nothing -> return (Nothing, False) - Just Nothing -> return (Nothing, True) - Just (Just lockhandle) -> do - void $ tryIO $ writeTransferInfoFile info tfile - return (Just lockhandle, False) -#endif - cleanup _ Nothing = noop - cleanup tfile (Just lockhandle) = do - void $ tryIO $ removeFile tfile -#ifndef mingw32_HOST_OS - void $ tryIO $ removeFile $ transferLockFile tfile - closeFd lockhandle -#else - {- Windows cannot delete the lockfile until the lock - - is closed. So it's possible to race with another - - process that takes the lock before it's removed, - - so ignore failure to remove. - -} - dropLock lockhandle - void $ tryIO $ removeFile $ transferLockFile tfile -#endif - retry oldinfo metervar run = do - v <- tryAnnex run - case v of - Right b -> return b - Left _ -> do - b <- getbytescomplete metervar - let newinfo = oldinfo { bytesComplete = Just b } - if shouldretry oldinfo newinfo - then retry newinfo metervar run - else return False - getbytescomplete metervar - | transferDirection t == Upload = - liftIO $ readMVar metervar - | otherwise = do - f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t) - liftIO $ catchDefaultIO 0 $ - fromIntegral . fileSize <$> getFileStatus f - {- Generates a callback that can be called as transfer progresses to update - the transfer info file. Also returns the file it'll be updating, and a - MVar that can be used to read the number of bytesComplete. -} diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -24,7 +24,7 @@ import qualified Git.GCrypt import qualified Annex import Logs.Presence -import Logs.Transfer +import Annex.Transfer import Annex.UUID import Annex.Exception import qualified Annex.Content @@ -321,7 +321,7 @@ case v of Nothing -> return False Just (object, checksuccess) -> - upload u key file noRetry + runTransfer (Transfer Download u key) file noRetry (rsyncOrCopyFile params object dest) <&&> checksuccess | Git.repoIsSsh (repo r) = feedprogressback $ \feeder -> do @@ -418,7 +418,7 @@ ( return True , do ensureInitialized - download u key file noRetry $ const $ + runTransfer (Transfer Download u key) file noRetry $ const $ Annex.Content.saveState True `after` Annex.Content.getViaTmpChecked (liftIO checksuccessio) key (\d -> rsyncOrCopyFile params object d p) diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -82,7 +82,7 @@ unless enabling $ genVault fullconfig u gitConfigSpecialRemote u fullconfig "glacier" "true" - return (c', u) + return (fullconfig, u) where remotename = fromJust (M.lookup "name" c) defvault = remotename ++ "-" ++ fromUUID u @@ -225,7 +225,8 @@ glacierParams c params = datacenter:params where datacenter = Param $ "--region=" ++ - fromJust (M.lookup "datacenter" c) + fromMaybe (error "Missing datacenter configuration") + (M.lookup "datacenter" c) glacierEnv :: RemoteConfig -> UUID -> Annex (Maybe [(String, String)]) glacierEnv c u = go =<< getRemoteCredPairFor "glacier" c creds @@ -239,7 +240,8 @@ (uk, pk) = credPairEnvironment creds getVault :: RemoteConfig -> Vault -getVault = fromJust . M.lookup "vault" +getVault = fromMaybe (error "Missing vault configuration") + . M.lookup "vault" archive :: Remote -> Key -> Archive archive r k = fileprefix ++ key2file k diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs --- a/Remote/Tahoe.hs +++ b/Remote/Tahoe.hs @@ -216,7 +216,7 @@ tahoeParams :: TahoeConfigDir -> String -> [CommandParam] -> [CommandParam] tahoeParams configdir command params = - Param command : Param "-d" : File configdir : params + Param "-d" : File configdir : Param command : params storeCapability :: UUID -> Key -> Capability -> Annex () storeCapability u k cap = setRemoteState u k cap diff --git a/Types/DesktopNotify.hs b/Types/DesktopNotify.hs new file mode 100644 --- /dev/null +++ b/Types/DesktopNotify.hs @@ -0,0 +1,27 @@ +{- git-annex DesktopNotify type + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.DesktopNotify where + +import Data.Monoid + +data DesktopNotify = DesktopNotify + { notifyStart :: Bool + , notifyFinish :: Bool + } + deriving (Show) + +instance Monoid DesktopNotify where + mempty = DesktopNotify False False + mappend (DesktopNotify s1 f1) (DesktopNotify s2 f2) = + DesktopNotify (s1 || s2) (f1 || f2) + +mkNotifyStart :: DesktopNotify +mkNotifyStart = DesktopNotify True False + +mkNotifyFinish :: DesktopNotify +mkNotifyFinish = DesktopNotify False True diff --git a/Types/FileMatcher.hs b/Types/FileMatcher.hs --- a/Types/FileMatcher.hs +++ b/Types/FileMatcher.hs @@ -7,8 +7,13 @@ module Types.FileMatcher where +import Types.UUID (UUID) import Types.Key (Key) +import Utility.Matcher (Matcher, Token) +import qualified Data.Map as M +import qualified Data.Set as S + data MatchInfo = MatchingFile FileInfo | MatchingKey Key @@ -17,3 +22,19 @@ { relFile :: FilePath -- may be relative to cwd , matchFile :: FilePath -- filepath to match on; may be relative to top } + +type FileMatcherMap a = M.Map UUID (Utility.Matcher.Matcher (S.Set UUID -> MatchInfo -> a Bool)) + +type MkLimit a = String -> Either String (MatchFiles a) + +type AssumeNotPresent = S.Set UUID + +type MatchFiles a = AssumeNotPresent -> MatchInfo -> a Bool + +type FileMatcher a = Matcher (MatchFiles a) + +-- This is a matcher that can have tokens added to it while it's being +-- built, and once complete is compiled to an unchangable matcher. +data ExpandableMatcher a + = BuildingMatcher [Token (MatchInfo -> a Bool)] + | CompleteMatcher (Matcher (MatchInfo -> a Bool)) diff --git a/Types/Limit.hs b/Types/Limit.hs deleted file mode 100644 --- a/Types/Limit.hs +++ /dev/null @@ -1,20 +0,0 @@ -{- types for limits - - - - Copyright 2013 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -{-# LANGUAGE CPP #-} - -module Types.Limit where - -import Common.Annex -import Types.FileMatcher - -import qualified Data.Set as S - -type MkLimit = String -> Either String MatchFiles - -type AssumeNotPresent = S.Set UUID -type MatchFiles = AssumeNotPresent -> MatchInfo -> Annex Bool diff --git a/Types/MetaData.hs b/Types/MetaData.hs --- a/Types/MetaData.hs +++ b/Types/MetaData.hs @@ -264,7 +264,9 @@ instance Arbitrary MetaData where arbitrary = do size <- arbitrarySizedBoundedIntegral `suchThat` (< 500) - MetaData . M.fromList <$> vector size + MetaData . M.filterWithKey legal . M.fromList <$> vector size + where + legal k _v = legalField $ fromMetaField k instance Arbitrary MetaValue where arbitrary = MetaValue <$> arbitrary <*> arbitrary diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs --- a/Utility/FileMode.hs +++ b/Utility/FileMode.hs @@ -9,15 +9,18 @@ module Utility.FileMode where -import Common - +import System.IO +import Control.Monad import Control.Exception (bracket) import System.PosixCompat.Types +import Utility.PosixFiles #ifndef mingw32_HOST_OS import System.Posix.Files #endif import Foreign (complement) +import Utility.Exception + {- Applies a conversion function to a file's mode. -} modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO () modifyFileMode f convert = void $ modifyFileMode' f convert @@ -56,6 +59,12 @@ executeModes :: [FileMode] executeModes = [ownerExecuteMode, groupExecuteMode, otherExecuteMode] +otherGroupModes :: [FileMode] +otherGroupModes = + [ groupReadMode, otherReadMode + , groupWriteMode, otherWriteMode + ] + {- Removes the write bits from a file. -} preventWrite :: FilePath -> IO () preventWrite f = modifyFileMode f $ removeModes writeModes @@ -145,9 +154,5 @@ writeFileProtected :: FilePath -> String -> IO () writeFileProtected file content = withUmask 0o0077 $ withFile file WriteMode $ \h -> do - void $ tryIO $ modifyFileMode file $ - removeModes - [ groupReadMode, otherReadMode - , groupWriteMode, otherWriteMode - ] + void $ tryIO $ modifyFileMode file $ removeModes otherGroupModes hPutStr h content diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs --- a/Utility/Matcher.hs +++ b/Utility/Matcher.hs @@ -19,7 +19,7 @@ module Utility.Matcher ( Token(..), - Matcher, + Matcher(..), token, tokens, generate, diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs --- a/Utility/ThreadScheduler.hs +++ b/Utility/ThreadScheduler.hs @@ -10,9 +10,12 @@ module Utility.ThreadScheduler where -import Common - +import Control.Monad import Control.Concurrent +#ifndef mingw32_HOST_OS +import Control.Monad.IfElse +import System.Posix.IO +#endif #ifndef mingw32_HOST_OS import System.Posix.Signals #ifndef __ANDROID__ diff --git a/Utility/Url.hs b/Utility/Url.hs --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -77,7 +77,8 @@ Nothing -> dne | otherwise -> if Build.SysConfig.curl then do - output <- readProcess "curl" $ toCommand curlparams + output <- catchDefaultIO "" $ + readProcess "curl" $ toCommand curlparams case lastMaybe (lines output) of Just ('2':_:_) -> return (True, extractsize output) _ -> dne diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -33,7 +33,6 @@ import qualified Data.Text.Encoding as TE import Blaze.ByteString.Builder.Char.Utf8 (fromText) import Blaze.ByteString.Builder (Builder) -import Data.Monoid import Control.Arrow ((***)) import Control.Concurrent #ifdef WITH_WEBAPP_SECURE diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,40 @@ +git-annex (5.20140402) unstable; urgency=medium + + * unannex, uninit: Avoid committing after every file is unannexed, + for massive speedup. + * --notify-finish switch will cause desktop notifications after each + file upload/download/drop completes + (using the dbus Desktop Notifications Specification) + * --notify-start switch will show desktop notifications when each + file upload/download starts. + * webapp: Automatically install Nautilus integration scripts + to get and drop files. + * tahoe: Pass -d parameter before subcommand; putting it after + the subcommand no longer works with tahoe-lafs version 1.10. + (Thanks, Alberto Berti) + * forget --drop-dead: Avoid removing the dead remote from the trust.log, + so that if git remotes for it still exist anywhere, git annex info + will still know it's dead and not show it. + * git-annex-shell: Make configlist automatically initialize + a remote git repository, as long as a git-annex branch has + been pushed to it, to simplify setup of remote git repositories, + including via gitolite. + * add --include-dotfiles: New option, perhaps useful for backups. + * Version 5.20140227 broke creation of glacier repositories, + not including the datacenter and vault in their configuration. + This bug is fixed, but glacier repositories set up with the broken + version of git-annex need to have the datacenter and vault set + in order to be usable. This can be done using git annex enableremote + to add the missing settings. For details, see + http://git-annex.branchable.com/bugs/problems_with_glacier/ + * Added required content configuration. + * assistant: Improve ssh authorized keys line generated in local pairing + or for a remote ssh server to set environment variables in an + alternative way that works with the non-POSIX fish shell, as well + as POSIX shells. + + -- Joey Hess Wed, 02 Apr 2014 16:42:53 -0400 + git-annex (5.20140320) unstable; urgency=medium * Fix zombie leak and general inneficiency when copying files to a diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -30,6 +30,7 @@ libghc-hinotify-dev [linux-any], libghc-stm-dev (>= 2.3), libghc-dbus-dev (>= 0.10.3) [linux-any], + libghc-fdo-notify-dev (>= 0.3) [linux-any], libghc-yesod-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-yesod-static-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64 powerpc sparc], libghc-yesod-default-dev [i386 amd64 kfreebsd-amd64 powerpc sparc], diff --git a/doc/assistant/downloadnotification.png b/doc/assistant/downloadnotification.png new file mode 100644 Binary files /dev/null and b/doc/assistant/downloadnotification.png differ diff --git a/doc/assistant/nautilusmenu.png b/doc/assistant/nautilusmenu.png new file mode 100644 Binary files /dev/null and b/doc/assistant/nautilusmenu.png differ diff --git a/doc/bugs/Assistant_lost_dbus_connection_spamming_log.mdwn b/doc/bugs/Assistant_lost_dbus_connection_spamming_log.mdwn --- a/doc/bugs/Assistant_lost_dbus_connection_spamming_log.mdwn +++ b/doc/bugs/Assistant_lost_dbus_connection_spamming_log.mdwn @@ -76,3 +76,13 @@ lost dbus connection; falling back to polling (SocketError {socketErrorMessage = "connect: does not exist (No such file or directory)", socketErrorFatal = True, socketErrorAddress = Just (Address "unix:path=/var/run/dbus/system_bus_socket")}) """]] + +> [[done]]; This turned out to not be dbus related, but the http server failing, +> and I fixed that bug. +> +> AFAICS the user running git-annex did not have their own dbus daemon +> running, and that's why the low-volume dbus messages come up. +> Probably because this is an embedded device, and so no desktop +> environment. git-annex only uses dbus for detecting network connection +> changes and removable media mounts. None of which probably matter in an +> embedded environment. --[[Joey]] diff --git a/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. +The watcher crashes. + +I only need to restart the thread in the pop-up to get everything to work again, but I'm reporting just in case that this issue has any other implications. + + +### What steps will reproduce the problem? +I open the webapp and in the minutes before it starts syncing (syncing is enabled) I disable it (clicking in the 'syncing enabled' text). + +This produces a crash every time. + + +### What version of git-annex are you using? On what operating system? +5.20140320 in Debian sid and testing + + +### Please provide any additional information below. +This is all I can see in the logs + +[[!format sh """ +Watcher crashed: PauseWatcher +[2014-03-26 08:54:57 CET] Watcher: warning Watcher crashed: PauseWatcher +"""]] diff --git a/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn @@ -0,0 +1,195 @@ +Addurl can fail due to an apparent race condition when watch or assistant is running and the repository is in direct mode. The following stress test script encounters the bug consistently on my system. I am running git-annex 5.20140320 on on Ubuntu 13.10. + +[[!format sh """ +#!/bin/sh +set -eu + +cleanup() { + local dir + dir="$1"; shift + if [ -d "$dir" ]; then + ( + set -x + fuser -k -w "$dir/annex/.git/annex/daemon.log" || : + find "$dir" -type d -exec chmod 700 '{}' '+' + find "$dir" -type f -exec chmod 600 '{}' '+' + rm -fr "$dir" + ) + fi +} + +go() { + local dir + dir="$(mktemp -d "${TMP:-/tmp}/stress-annex.XXXXXXXXXX")" + trap "cleanup '$dir'" 0 1 2 13 15 + + ( + cd "$dir" + mkdir annex + cd annex + set -x + + git init + git annex init + git annex direct + git annex watch + + for n in $(seq 100); do + git annex addurl --file=foo http://heh.fi/robots.txt + git annex sync + rm -f foo + git annex sync + done + + git annex watch --stop + git annex uninit + ) + + cleanup "$dir" + trap - 0 1 2 13 14 +} + +go +"""]] + +Script output: + +[[!format sh """ +% ./stress-annex ++ git init +Initialized empty Git repository in /tmp/stress-annex.OKj6D8kVmV/annex/.git/ ++ git annex init +init ok +(Recording state in git...) ++ git annex direct +commit +On branch master + +Initial commit + +nothing to commit +ok +direct ok ++ git annex watch ++ seq 100 ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + + +git-annex: /tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/: openTempFile: permission denied (Permission denied) +failed +git-annex: addurl: 1 failed ++ fuser -k -w /tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/daemon.log +/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/daemon.log: 30704 30709 30735 30738 30778 ++ find /tmp/stress-annex.OKj6D8kVmV -type d -exec chmod 700 {} + ++ find /tmp/stress-annex.OKj6D8kVmV -type f -exec chmod 600 {} + ++ rm -fr /tmp/stress-annex.OKj6D8kVmV +"""]] + +The script also seems to encounter another issue. The output when seq 100 is changed to seq 1 and addurl happens to succeed: + +[[!format sh """ ++ git init +Initialized empty Git repository in /tmp/stress-annex.QEs0pNyS9z/annex/.git/ ++ git annex init +init ok +(Recording state in git...) ++ git annex direct +commit +On branch master + +Initial commit + +nothing to commit +ok +direct ok ++ git annex watch ++ seq 1 ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:17:20-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:17:20 (0.00 B/s) - ‘/tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex watch --stop ++ git annex uninit +git-annex: /tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.map: removeLink: permission denied (Permission denied) ++ fuser -k -w /tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/daemon.log ++ : ++ find /tmp/stress-annex.QEs0pNyS9z -type d -exec chmod 700 {} + ++ find /tmp/stress-annex.QEs0pNyS9z -type f -exec chmod 600 {} + ++ rm -fr /tmp/stress-annex.QEs0pNyS9z +"""]] diff --git a/doc/bugs/git-annex_fails_to_initialize_under_Windows.mdwn b/doc/bugs/git-annex_fails_to_initialize_under_Windows.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git-annex_fails_to_initialize_under_Windows.mdwn @@ -0,0 +1,212 @@ +### Please describe the problem. +Git-annex fails to initialize and fails tests. + +### What steps will reproduce the problem? +Attempted initialization: + + C:\Users\Andrew\Documents\GitHub\git-annex-test [master]> git annex init + init + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. + fatal: index file open failed: Invalid argument + git-annex: git [Param "checkout",Param "-q",Param "-B",Param "annex/direct/master"] failed + +Tests: + + C:\Users\Andrew\Documents\GitHub\git-annex-test [master]> git annex test + Tests + QuickCheck + prop_idempotent_deencode_git: OK + +++ OK, passed 1000 tests. + prop_idempotent_deencode: OK + +++ OK, passed 1000 tests. + prop_idempotent_fileKey: OK + +++ OK, passed 1000 tests. + prop_idempotent_key_encode: OK + +++ OK, passed 1000 tests. + prop_idempotent_key_decode: OK + +++ OK, passed 1000 tests. + prop_idempotent_shellEscape: OK + +++ OK, passed 1000 tests. + prop_idempotent_shellEscape_multiword: OK + +++ OK, passed 1000 tests. + prop_logs_sane: OK + +++ OK, passed 1000 tests. + prop_idempotent_configEscape: OK + +++ OK, passed 1000 tests. + prop_parse_show_Config: OK + +++ OK, passed 1000 tests. + prop_parentDir_basics: OK + +++ OK, passed 1000 tests. + prop_relPathDirToFile_basics: OK + +++ OK, passed 1000 tests. + prop_relPathDirToFile_regressionTest: OK + +++ OK, passed 1000 tests. + prop_cost_sane: OK + +++ OK, passed 1000 tests. + prop_matcher_sane: OK + +++ OK, passed 1000 tests. + prop_HmacSha1WithCipher_sane: OK + +++ OK, passed 1000 tests. + prop_TimeStamp_sane: OK + +++ OK, passed 1000 tests. + prop_addLog_sane: OK + +++ OK, passed 1000 tests. + prop_verifiable_sane: OK + +++ OK, passed 1000 tests. + prop_segment_regressionTest: OK + +++ OK, passed 1000 tests. + prop_read_write_transferinfo: OK + +++ OK, passed 1000 tests. + prop_read_show_inodecache: OK + +++ OK, passed 1000 tests. + prop_parse_show_log: OK + +++ OK, passed 1000 tests. + prop_read_show_TrustLevel: OK + +++ OK, passed 1000 tests. + prop_parse_show_TrustLog: OK + +++ OK, passed 1000 tests. + prop_hashes_stable: OK + +++ OK, passed 1000 tests. + prop_schedule_roundtrips: OK + +++ OK, passed 1000 tests. + prop_duration_roundtrips: OK + +++ OK, passed 1000 tests. + prop_metadata_sane: OK + +++ OK, passed 1000 tests. + prop_metadata_serialize: OK + +++ OK, passed 1000 tests. + prop_branchView_legal: OK + +++ OK, passed 1000 tests. + prop_view_roundtrips: OK + +++ OK, passed 1000 tests. + prop_viewedFile_rountrips: I n i t TOeKs + ts + i n+i+t+: OK, passed 1000 tests. + Unit Tests + add sha1dup: git-annex: System.PosixCompat.User.getEffectiveUserID: not support + ed: illegal operation + FAIL + init failed + add: git-annex: System.PosixCompat.User.getEffectiveUserID: not supported: illegal operation + FAIL + add failed + + 2 out of 2 tests failed + FAIL + Exception: init tests failed! cannot continue + add extras: FAIL + Exception: init tests failed! cannot continue + reinject: FAIL + Exception: init tests failed! cannot continue + unannex (no copy): FAIL + Exception: init tests failed! cannot continue + unannex (with copy): FAIL + Exception: init tests failed! cannot continue + drop (no remote): FAIL + Exception: init tests failed! cannot continue + drop (with remote): FAIL + Exception: init tests failed! cannot continue + drop (untrusted remote): FAIL + Exception: init tests failed! cannot continue + get: FAIL + Exception: init tests failed! cannot continue + move: FAIL + Exception: init tests failed! cannot continue + copy: FAIL + Exception: init tests failed! cannot continue + lock: FAIL + Exception: init tests failed! cannot continue + edit (no pre-commit): FAIL + Exception: init tests failed! cannot continue + edit (pre-commit): FAIL + Exception: init tests failed! cannot continue + fix: FAIL + Exception: init tests failed! cannot continue + trust: FAIL + Exception: init tests failed! cannot continue + fsck (basics): FAIL + Exception: init tests failed! cannot continue + fsck (bare): FAIL + Exception: init tests failed! cannot continue + fsck (local untrusted): FAIL + Exception: init tests failed! cannot continue + fsck (remote untrusted): FAIL + Exception: init tests failed! cannot continue + migrate: FAIL + Exception: init tests failed! cannot continue + migrate (via gitattributes): FAIL + Exception: init tests failed! cannot continue + unused: FAIL + Exception: init tests failed! cannot continue + describe: FAIL + Exception: init tests failed! cannot continue + find: FAIL + Exception: init tests failed! cannot continue + merge: FAIL + Exception: init tests failed! cannot continue + info: FAIL + Exception: init tests failed! cannot continue + version: FAIL + Exception: init tests failed! cannot continue + sync: FAIL + Exception: init tests failed! cannot continue + union merge regression: FAIL + Exception: init tests failed! cannot continue + conflict resolution: FAIL + Exception: init tests failed! cannot continue + conflict_resolution (mixed directory and file): FAIL + Exception: init tests failed! cannot continue + conflict_resolution (mixed directory and file) 2: FAIL + Exception: init tests failed! cannot continue + map: FAIL + Exception: init tests failed! cannot continue + uninit: FAIL + Exception: init tests failed! cannot continue + uninit (in git-annex branch): FAIL + Exception: init tests failed! cannot continue + upgrade: FAIL + Exception: init tests failed! cannot continue + whereis: FAIL + Exception: init tests failed! cannot continue + hook remote: FAIL + Exception: init tests failed! cannot continue + directory remote: FAIL + Exception: init tests failed! cannot continue + rsync remote: FAIL + Exception: init tests failed! cannot continue + bup remote: FAIL + Exception: init tests failed! cannot continue + crypto: FAIL + Exception: init tests failed! cannot continue + preferred content: FAIL + Exception: init tests failed! cannot continue + add subdirs: FAIL + Exception: init tests failed! cannot continue + + 45 out of 78 tests failed + (This could be due to a bug in git-annex, or an incompatability + with utilities, such as git, installed on this system.) + +### What version of git-annex are you using? On what operating system? + C:\Users\Andrew\Documents\GitHub\git-annex-test [master]> git --version + git version 1.8.4.msysgit.0 + C:\Users\Andrew\Documents\GitHub\git-annex-test [master]> git annex version + git-annex version: 5.20140227-gd872677 + build flags: Assistant Webapp Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 + SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 2 3 4 + C:\Users\Andrew\Documents\GitHub\git-annex-test [master]> (Get-WmiObject -class Win32_OperatingSystem).Caption + Microsoft Windows 8.1 + +### Please provide any additional information below. +^^^ See above diff --git a/doc/bugs/git_annex_test_under_windows_8.1.mdwn b/doc/bugs/git_annex_test_under_windows_8.1.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/git_annex_test_under_windows_8.1.mdwn @@ -0,0 +1,67 @@ +### Please describe the problem. +I installed git and git annex under Windows (8.1) and ran git annex test. All except one tests passed with "ok" + +### What steps will reproduce the problem? +git annex test +under Windows 8.1 + +### What version of git-annex are you using? On what operating system? +$ git --version +git version 1.9.0.msysgit.0 + +$ git annex version +git-annex version: 5.20140320-g63535e3 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external local repository version: 5 supported repository version: 5 upgrade supported from repository versions: 2 3 4 + +Windows 8.1 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + prop_view_roundtrips: FAIL + *** Failed! Falsifiable (after 814 tests and 5 shrinks): + "a" + IMneitta DTaetsat s( +fr o miLniistt: [(MetaField "1\194",fromList [MetaValue (CurrentlySet True) "\r ++\231Gb\157\227\ETB\bG",MetaValue (CurrentlySet True) "\DEL\239~\243_p\DC2."]),( +MetaField "EG",fromList [MetaValue (CurrentlySet True) "",MetaValue (CurrentlySe +t True) "\v\205] .T(",MetaValue (CurrentlySet False) "\NAK\128lo\169w",MetaValue + (CurrentlySet True) "\SYN\STX\ENQ\n#u\ETXv\CANP\128\US~p",MetaValue (CurrentlySet False) "\250C\b\DC1\17 +6\154KT\191\SOf?\SI"]),(MetaField "\225a",fromList [MetaValue (CurrentlySet True +) "",MetaValue (CurrentlySet True) "\b\ETB\b",MetaValue (CurrentlySet True) "\f\ +161\FS\176h-\ta\169\t",MetaValue (CurrentlySet False) "4",MetaValue (CurrentlySe +t True) "A\FS\244V:\249kl5\ETX\SOH\SI)",MetaValue (CurrentlySet False) "Z",MetaV +alue (CurrentlySet True) "\\Lt~\235v\"\211\DLE\NAK\210",MetaValue (CurrentlySet +False) "a\SYNN",MetaValue (CurrentlySet True) "g:init test repo U5j\167G\ap-\ETX +",MetaValue (CurrentlySet False) "l\NULoW\238rD",MetaValue (CurrentlySet True) " +}\202\141\183Nxr",MetaValue (CurrentlySet False) "\170=\216S\ETB\187\SUB+!\DC3", +MetaValue (CurrentlySet True) "\240H\GS\NAK\ETB\SYNRq\153\&4\204\EOT"])]) + True + Use --quickcheck-replay '13 347062936 40785707' to reproduce. + prop_viewedFile_rountrips: OK + +++ OK, passed 1000 t +e s tDse.t + +# End of transcript or log. +"""]] + +> A sort of windows-specific bug in the test suite. I've fixed it. [[done]] +> --[[Joey]] diff --git a/doc/bugs/issues_with_non-posix_compatible_shells.mdwn b/doc/bugs/issues_with_non-posix_compatible_shells.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/issues_with_non-posix_compatible_shells.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. +Some internals of git annex does not check if the shell it is running is Posix-compatible, ie. bash. + +I am using fish, and after setting up local pairing, and working, I switched back the login-shell to fish, and when syncing a file, I got this error, read from daemon.log: + +fish: Unknown command 'GIT_ANNEX_SHELL_DIRECTORY=/home/s/annex'. Did you mean to run ~/.ssh/git-annex-shell with a modified environment? Try 'env GIT_ANNEX_SHELL_DIRECTORY=/home/s/annex ~/.ssh/git-annex-shell...'. See the help section on the set command by typing 'help set'. +Standard input: GIT_ANNEX_SHELL_DIRECTORY='/home/s/annex' ~/.ssh/git-annex-shell + ^ +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. + +### What steps will reproduce the problem? +Set up local pairing ( I believe having sh/bash as login terminal is necessary for this). +Switch back to fish as login-shell with chsh -s /usr/bin/fish +Add a file to either repository. + +### What version of git-annex are you using? On what operating system? +[s@b ~]$ git annex version +git-annex version: 5.20140320-g63535e3 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> [[fixed|done]] so +> +> I have not tried to make the assistant go back and fix up existing +> `authorized_keys` lines. So if someone had been using a posix shell and +> switched to fish, they'll hit this and need to fix it themselves. I judge +> this is pretty small number of users. --[[Joey]] diff --git a/doc/bugs/problem_to_addurl_--file_with_ftp.mdwn b/doc/bugs/problem_to_addurl_--file_with_ftp.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/problem_to_addurl_--file_with_ftp.mdwn @@ -0,0 +1,67 @@ +### Please describe the problem. +I want to addurl using ftp protocol. +`git annex addurl ftp://...` works fine, but `git annex addurl --file` fails with an error "failed to verify url exists". + +### What steps will reproduce the problem? + +setting up a new repo + + % alias ga + ga=/home/applis/git-annex.linux/git-annex + % ga init + init ok + (Recording state in git...) + +addurl --file works with http + + % wget http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz + [...] + 2014-03-27 15:25:06 (10,1 MB/s) - ‘git-annex-standalone-amd64.tar.gz’ saved [30689438/30689438] + % ga add git-annex-standalone-amd64.tar.gz + add git-annex-standalone-amd64.tar.gz ok + (Recording state in git...) + % ga addurl http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz --file git-annex-standalone-amd64.tar.gz + addurl git-annex-standalone-amd64.tar.gz ok + (Recording state in git...) + +addurl works with ftp: + + % ga addurl ftp://ftp.belnet.be/debian-cd/7.4.0-live/i386/iso-hybrid/debian-live-7.4-i386-lxde-desktop.iso.log + addurl ftp.belnet.be_debian_cd_7.4.0_live_i386_iso_hybrid_debian_live_7.4_i386_lxde_desktop.iso.log (downloading ftp://ftp.belnet.be/debian-cd/7.4.0-live/i386/iso-hybrid/debian-live-7.4-i386-lxde-desktop.iso.log ...) + [...] + 2014-03-27 15:27:47 (11,1 MB/s) - ‘/data/annex/.git/annex/tmp/URL--ftp&c%%ftp.belnet.be%debian-cd%7.4.0-live%i386%iso-hybrid%debian-live-7.4-i386-lxde-desktop.iso.log’ saved [1235181] + ok + (Recording state in git...) + +addurl --file doesn't work with ftp + + % wget ftp://ftp.belnet.be/debian-cd/7.4.0-live/i386/iso-hybrid/debian-live-7.4-i386-standard.iso.zsync + [...] + 2014-03-27 15:29:32 (19,4 MB/s) - ‘debian-live-7.4-i386-standard.iso.zsync’ saved [1932014] + % ga add debian-live-7.4-i386-standard.iso.zsync + add debian-live-7.4-i386-standard.iso.zsync ok + (Recording state in git...) + % ga addurl ftp://ftp.belnet.be/debian-cd/7.4.0-live/i386/iso-hybrid/debian-live-7.4-i386-standard.iso.zsync --file debian-live-7.4-i386-standard.iso.zsync + addurl debian-live-7.4-i386-standard.iso.zsync + failed to verify url exists: ftp://ftp.belnet.be/debian-cd/7.4.0-live/i386/iso-hybrid/debian-live-7.4-i386-standard.iso.zsync + failed + git-annex: addurl: 1 failed + +### What version of git-annex are you using? On what operating system? + +I am using current git-annex binary linux version on Fedora 19. + + % which git ; git --version + /usr/bin/git + git version 1.8.3.1 + % which ga ; ga version + ga=/home/applis/git-annex.linux/git-annex + git-annex version: 5.20140320-g63535e3 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +> [[done]] --[[Joey]] diff --git a/doc/bugs/problems_with_glacier.mdwn b/doc/bugs/problems_with_glacier.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/problems_with_glacier.mdwn @@ -0,0 +1,65 @@ +### Please describe the problem. +Annex errors when copying to glacier. + +### What version of git-annex are you using? On what operating system? + +OS X 10.9.2 Build 13C64 + + git-annex version: 5.20140318-gdcf93d0 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +> git annex initremote glacier type=glacier encryption=hybrid keyid=E9053BDA datacenter=us-west-1 ║██████████╠ ∞ ∞ +initremote glacier (encryption setup) (hybrid cipher with gpg key B608B8F6E9053BDA) ok +(Recording state in git...) +> git annex copy Cobalt\ Strike\ Tradecraft --to=glacier --debug +[2014-03-27 07:27:39 PDT] read: git ["--git-dir=/Users/akraut/Desktop/annexes/media/.git","--work-tree=/Users/akraut/Desktop/annexes/media","show-ref","git-annex"] +[2014-03-27 07:27:39 PDT] read: git ["--git-dir=/Users/akraut/Desktop/annexes/media/.git","--work-tree=/Users/akraut/Desktop/annexes/media","show-ref","--hash","refs/heads/git-annex"] +[2014-03-27 07:27:39 PDT] read: git ["--git-dir=/Users/akraut/Desktop/annexes/media/.git","--work-tree=/Users/akraut/Desktop/annexes/media","log","refs/heads/git-annex..9f59057d857784e6ae6b3dcd6793092264375913","--oneline","-n1"] +[2014-03-27 07:27:39 PDT] chat: git ["--git-dir=/Users/akraut/Desktop/annexes/media/.git","--work-tree=/Users/akraut/Desktop/annexes/media","cat-file","--batch"] +[2014-03-27 07:27:39 PDT] read: git ["config","--null","--list"] +[2014-03-27 07:27:39 PDT] read: git ["--git-dir=/Users/akraut/Desktop/annexes/media/.git","--work-tree=/Users/akraut/Desktop/annexes/media","ls-files","--cached","-z","--","Cobalt Strike Tradecraft"] +copy Cobalt Strike Tradecraft/Tradecraft__1_of_9____Introduction.mp4 (gpg) [2014-03-27 07:27:39 PDT] chat: gpg ["--quiet","--trust-model","always","--decrypt"] + +You need a passphrase to unlock the secret key for +user: "Andrew Mark Kraut " +4096-bit ELG-E key, ID 353E49B9, created 2008-11-11 (main key ID E9053BDA) + +(checking glacier...) [2014-03-27 07:27:46 PDT] read: glacier ["--region=us-west-1","archive","checkpresent","git-annex: Maybe.fromJust: Nothing + +# End of transcript or log. +"""]] + +> This was a bug introduced last month, it forgot to receord the +> datacenter and vault used when initializing the glacier repository. +> +> I've fixed the bug, but this does not fix repositories created with +> the broken version. I considered just making it use the default +> datacenter and vault for such a repository, but +> a) those may change in the future +> and I don't want to have to worry about breaking such a repository +> going forward and b) someone may have overridden it to use another +> datacenter or vault name and so it shouldn't blindly assume the defaults. +> +> Instead, there's a manual fix up step you need to do. Luckily quite easy. +> For example: +> +> git annex enableremote myglacier datacenter=us-east-1 vault=myglacier-fae9be57-8eb4-47af-932f-136b9b40e669 +> +> The default datacenter is us-east-1, and the default vault name is +> "$remotename-$uuid". So you just have to tell it these values +> once with an enableremote command, and it will then work. + +> You don't even need to get the fixed version of git-annex to work +> around the bug this way.. Although it does have better error messages +> too. [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/set_metadata_on_wrong_files.mdwn b/doc/bugs/set_metadata_on_wrong_files.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/set_metadata_on_wrong_files.mdwn @@ -0,0 +1,90 @@ +### Please describe the problem. + +For an example I wanted to add different metadata to some test files, +but the outcome is that the last metadata gets applied to all three files. see transcript below. + + + +### What steps will reproduce the problem? + +1. Create a git annex repository +2. add a few files +3. add some metadata to the files, same keys, differnt values +4. watch the metadata, only the last added one is shown for all files + + +### What version of git-annex are you using? On what operating system? + $cat /etc/debian_version; uname -a; git annex version + 7.4 + Linux jupiter 3.13.0ct #33 SMP PREEMPT Tue Jan 21 05:04:01 CET 2014 x86_64 GNU/Linux + git-annex version: 5.20140306~bpo70+1 + build flags: Assistant Webapp Pairing S3 Inotify DBus XMPP Feeds Quvi TDFA + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + remote types: git gcrypt S3 bup directory rsync web tahoe glacier hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + + +### Please provide any additional information below. + +Debian/Wheezy with git annex from backports. The test was done in /tmp which is a tmpfs. + + +[[!format sh """ +$export LC_ALL=C +$mkdir /tmp/annextest +$cd /tmp/annextest +$git init +Initialized empty Git repository in /tmp/annextest/.git/ +$git annex init +init ok +(Recording state in git...) + +$touch a.txt b.txt c.txt +$git annex add a.txt b.txt c.txt +add a.txt ok +add b.txt ok +add c.txt ok +(Recording state in git...) +$git commit -m init +[master (root-commit) 5470bdb] init + 3 files changed, 3 insertions(+) + create mode 120000 a.txt + create mode 120000 b.txt + create mode 120000 c.txt + +$git annex metadata a.txt -s foo=bar -s num=1 +metadata a.txt + foo=bar + num=1 +ok +(Recording state in git...) +$git annex metadata b.txt -s foo=baz -s num=2 +metadata b.txt + foo=baz + num=2 +ok +(Recording state in git...) +$git annex metadata c.txt -s foo=barf -s num=3 +metadata c.txt + foo=barf + num=3 +ok +(Recording state in git...) +$git annex metadata +metadata a.txt + foo=barf + num=3 +ok +metadata b.txt + foo=barf + num=3 +ok +metadata c.txt + foo=barf + num=3 +ok +"""]] + +> [[fixed|done]]; documentation improved --[[Joey]] diff --git a/doc/copies.mdwn b/doc/copies.mdwn --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -30,3 +30,6 @@ With N=2, in order to drop the file content from Laptop, it would need access to both USB and Server. + +For more complicated requirements about which repositories contain which +content, see [[required_content]]. diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 16 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 9 "Box.com (done)" 71 "My phone (or MP3 player)" 25 "Tahoe-LAFS" 10 "OpenStack SWIFT" 31 "Google Drive"]] +[[!poll open=yes 16 "Amazon S3 (done)" 12 "Amazon Glacier (done)" 9 "Box.com (done)" 71 "My phone (or MP3 player)" 25 "Tahoe-LAFS" 10 "OpenStack SWIFT" 33 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn --- a/doc/design/assistant/telehash.mdwn +++ b/doc/design/assistant/telehash.mdwn @@ -11,6 +11,9 @@ * Rapid development, situation may change in a month or 2. * Is it secure? A security review should be done by competant people (not Joey). See +* **Haskell version** + + Development on v2 in haskell is just starting up! ## implementation basics diff --git a/doc/design/roadmap.mdwn b/doc/design/roadmap.mdwn --- a/doc/design/roadmap.mdwn +++ b/doc/design/roadmap.mdwn @@ -10,8 +10,8 @@ * Month 4 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, [[!traillink todo/support_for_writing_external_special_remotes]] * Month 5 user-driven features and polishing * Month 6 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]] -* **Month 7 user-driven features and polishing** -* Month 8 [[!traillink assistant/telehash]] +* Month 7 user-driven features and polishing +* **Month 8 [[!traillink assistant/telehash]]** * Month 9 [[!traillink assistant/gpgkeys]] [[!traillink assistant/sshpassword]] * Month 10 get [[assistant/Android]] out of beta * Month 11 [[!traillink assistant/chunks]] [[!traillink assistant/deltas]] diff --git a/doc/devblog/day_139-140__traveling.mdwn b/doc/devblog/day_139-140__traveling.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_139-140__traveling.mdwn @@ -0,0 +1,17 @@ +Yesterday coded up one nice improvement on the plane -- `git annex unannex` +(and `uninit`) is now tons faster. Before it did a git commit after every +file processed, now there's just 1 commit at the end. This required using +some locking to prevent the `pre-commit` hook from running in a confusing +state. + +Today. LibrePlanet and a surprising amount of development. I've +added [[tips/file_manager_integration]], only for Nautilus so far. +The main part of this was adding --notify-start and --notify-finish, which +use dbus desktop notifications to provide feedback. + +(Made possible thanks to Max Rabkin for updating +[fdo-notify](http://hackage.haskell.org/package/fdo-notify) to use the +new dbus library, and ion for developing the initial Nautilus integration +scripts.) + +Today's work and LibrePlanet visit was sponsored by Jürgen Lüters. diff --git a/doc/devblog/day_141__f-droid_sprint.mdwn b/doc/devblog/day_141__f-droid_sprint.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_141__f-droid_sprint.mdwn @@ -0,0 +1,3 @@ +Attended at the f-droid sprint at LibrePlanet, and have been getting a +handle on how their build server works with an eye toward adding git-annex +to it. Not entirely successful getting vagrant to build an image yet. diff --git a/doc/devblog/day_142__digging_out.mdwn b/doc/devblog/day_142__digging_out.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_142__digging_out.mdwn @@ -0,0 +1,13 @@ +Catching up on conference backlog. 36 messages backlog remains. + +Fixed `git-annex-shell configlist` to automatically initialize a +git remote when a git-annex branch had been pushed to it. This is necessary +for gitolite to be easy to use, and I'm sure it used to work. + +Updated the Debian backport and made a Debian package of the +fdo-notify haskell library used for notifications. + +Applied a patch from Alberto Berti to fix support for tahoe-lafs +1.10. + +And various other bug fixes and small improvements. diff --git a/doc/devblog/day_143__foolish_hiatus.mdwn b/doc/devblog/day_143__foolish_hiatus.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_143__foolish_hiatus.mdwn @@ -0,0 +1,20 @@ +Last week's trip was productive, but I came home more tired than I +realized. Found myself being snappy & stressed, so I have been on break. + +I did do a little git-annex dev in the past 5 days. On Saturday I +implemented [[todo/preferred_content]] (although without the active checks +I think it probably ought to have.) Yesterday I had a long conversation +with the Tahoe developers about improving git-annex's tahoe integration. + +Today, I have been wrapping up [building propellor](http://joeyh.name/code/propellor/). +To test its docker support, I used propellor to build and deploy a +container that is a git-annex autobuilder. I'll be replacing the old +autobuilder setup with this shortly, and expect to also publish docker +images for git-annex autobuilders, so anyone who wants to can run their +own autobuilder really easily. + +--- + +I have April penciled in on the roadmap as the month to do telehash. +I don't know if telehash-c is ready for me yet, but it has had a lot of +activity lately, so this schedule may still work out! diff --git a/doc/devblog/day_144__catching_up.mdwn b/doc/devblog/day_144__catching_up.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_144__catching_up.mdwn @@ -0,0 +1,12 @@ +Got caught up on all recent bugs and questions, although I still have a +backlog of 27 older things that I really should find time for. + +Fixed a couple of bugs. One was that the assistant set up ssh +`authorized_keys` that didn't work with the fish shell. + +Also got caught up on the current state of telehash-c. Have not quite +gotten it to work, but it seems pretty close to being able to see it do +something useful for the first time. + +Pushing out a release this evening with a good number of changes left over +from March. diff --git a/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn b/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn @@ -0,0 +1,1 @@ +Is it possible to generate a view of files currently available on the annex? My use case is that I have pretty large repo (couple of TBs) and I have partial checkouts on multiple machines instead of seeing 100s of broken symlinks I would like to just filter filter files that are present on the machine? diff --git a/doc/forum/faking_location_information.mdwn b/doc/forum/faking_location_information.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/faking_location_information.mdwn @@ -0,0 +1,19 @@ +Hi + +I am using git-annex even if people I exchange data with (currently) don‘t use it for there data. My idea behind this is that I would like to know from where I got a file, whom I gave a file and who does (probably) still have a copy of the file. To do this you need to trick git-annex location tracking feature a bit. I successfully managed to achieve this in a simple data exchange which only consisted of me coping over files to one of my git-annex repos. I did the following to make git-annex believe that the files are in two repos without the need to *copy* them around the repos. + +This is what I did in this simple case: + +1. mounted the drive from someone +2. made a clone of my git-annex repo on the filesystem which should hold the copy of the data +3. initialized the cloned repo with `git annex init "Drive from person X"` +4. imported the files to the cloned repo with `git annex import --duplicate $path_to_files_from_person_x` +5. `git annex sync` in the cloned repo +6. `git annex sync` in main repo +7. `git annex move . --to origin` in the cloned repo + +The impotent part (and the limit) here was that you can not sync these two repos after you moved files to the main repo. The problem is that there will be situations where I will have to sync them also after moving files around (for example if I want to store new files in multiple repos (and not just one main repo), or if I also want to copy files over to drives from someone). + +Note: I have also worked out a solution to allow someone to choose which files he/she would like to get as described [on superuser.com](http://superuser.com/a/717689). + +Are there better ways to fake location information then the thing I came up with (except making multiple repos for one person/drive)? Can multiple remotes be merged to one remote? diff --git a/doc/forum/ignore_changes_made_by_a_remote.mdwn b/doc/forum/ignore_changes_made_by_a_remote.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/ignore_changes_made_by_a_remote.mdwn @@ -0,0 +1,8 @@ +Hi, + +I have two repo one in direct (on windows) and one in indirect mode. From time to time the files in the direct repo are replaced by empty files however running git annex fsck always solves it. +The problem is that today I did run git annex sync before running git annex fsck and git annex has then created two -variants for each of my files one empty and one with the content. +I guess the easier for me is to just scrap that repo and make a new one however how do I prevent the changes to propagate now? I guess that if I now run git annex sync on my other repo all those small files are going to have linked created for them there as well. + +I hope this is clear, +Thanks in advance. diff --git a/doc/forum/telehash_syncing.mdwn b/doc/forum/telehash_syncing.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/telehash_syncing.mdwn @@ -0,0 +1,10 @@ +Hi + +I have read some info about telehash. It looks verry promising. I was wondering though how syncing will work. For example. I have 2 computers. Normal PC and a laptop. Mostly only one is on at a time. + +* Sync messages will be sent over telehash protocoll ? +* What if I push some changes (they will be synced to a shared repository) and laptop is not online. How will git-annex know what to sync from a shared repository ? +* Do you plan to send files/commits directly to online clients ? If 2 friends are online at the same time. +* What will happen with data on a shared repository if all clients have synced content ? Will it be deleted since it is not longer needed ? + +I was thinking of a model where you sync directly (if possible), and just drop shared content to repo for offline users. Whan everyone have pulled content it may be removed from shared repo. diff --git a/doc/forum/unrelated_repositories_sync.mdwn b/doc/forum/unrelated_repositories_sync.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/unrelated_repositories_sync.mdwn @@ -0,0 +1,15 @@ +I have no idea how to search for this here, so I'll just go the "lazy web" approach and just ask. + +Say I have two "conference" repos. One is the famous [conference procedings](https://github.com/RichiH/conference_proceedings) repo, and another one is a totally unrelated repo of local conferences that are not of world-wide significance. Let's call this second repo `presentations`. + +I would like to have my videos of both repos in a single repo. + +Can I add the `conference procedings` repo as a git remote to the `presentations` repo and have it do the right thing? + +In fact, I'm not even sure what the right thing would be here, I guess that's the first thing I would like to clear up. But I would like to do things like what the new [[metadata]] system does. For example, I would have only the "Debian" directory from `conference procedings` in my `presentations` repo. + +How would that work? Would I need to do some [subtree merging](http://git-scm.com/book/ch6-7.html) magic? or `git subtree`? or submodules? or should i just use myrepos and pretend I never brought up this idea? + +thanks! -- [[anarcat]] + +related: [[tips/migrating_two_seperate_disconnected_directories_to_git_annex/]] - but that creates a merged repo... diff --git a/doc/forum/view_from_numeric_values.mdwn b/doc/forum/view_from_numeric_values.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/view_from_numeric_values.mdwn @@ -0,0 +1,9 @@ +Hi Joey, + +it would be nice when views could take numeric comparisons as filters. + + git annex metadata -s length=273.0 john_cage_4_33.mp3 + + git annex view length<=300 + +... here is the catch, < and > don't work well in shell, this needs some other Syntax. I think the underlying machinery (using numeric comparisons instead globs) should be quite trivial. Any Ideas about a Syntax? diff --git a/doc/forum/view_including_files_with_no_tags.mdwn b/doc/forum/view_including_files_with_no_tags.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/view_including_files_with_no_tags.mdwn @@ -0,0 +1,5 @@ +Hi + +Is it possible to create a view which also includes files with no tag? + +I use something like `git annex view 'rating=*'` to view files sorted by rating but this view does not include files which don‘t have a rating yet. What I was looking for is a way to show tagged files and untagged files in one view. diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -26,7 +26,12 @@ * configlist directory This outputs a subset of the git configuration, in the same form as - `git config --list` + `git config --list`. This is used to get the annex.uuid of the remote + repository. + + When run in a repository that does not yet have an annex.uuid, one + will be created, as long as a git-annex branch has already been pushed to + the repository. * inannex directory [key ...] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -62,11 +62,15 @@ * `add [path ...]` - Adds files in the path to the annex. Files that are already checked into - git, or that git has been configured to ignore will be silently skipped. - (Use `--force` to add ignored files.) Dotfiles are skipped unless explicitly - listed. + Adds files in the path to the annex. If no path is specified, adds + files from the current directory and below. + + Files that are already checked into git, or that git has been configured + to ignore will be silently skipped. (Use `--force` to add ignored files.) + Dotfiles are skipped unless explicitly listed, or the --include-dotfiles + option is used. + * `get [path ...]` Makes the content of annexed files available in this repository. This @@ -707,8 +711,9 @@ * `metadata [path ...] [-s field=value -s field+=value -s field-=value ...] [-g field]` - Each file can have any number of metadata fields attached to it, - which each in turn have any number of values. + The content of a file can have any number of metadata fields + attached to it to describe it. Each metadata field can in turn + have any number of values. This command can be used to set metadata, or show the currently set metadata. @@ -1053,6 +1058,19 @@ Overrides the User-Agent to use when downloading files from the web. +* `--notify-finish` + + Caused a desktop notification to be displayed after each successful + file download and upload. + + (Only supported on some platforms, eg Linux with dbus. A no-op when + not supported.) + +* `--notify-start` + + Caused a desktop notification to be displayed when a file upload + or download has started, or when a file is dropped. + * `-c name=value` Overrides git configuration settings. May be specified multiple times. @@ -1697,7 +1715,7 @@ `~/.config/git-annex/autostart` is a list of git repositories to start the git-annex assistant in. -`.git/hooks/pre-commit-annex` in your git repsitory will be run whenever +`.git/hooks/pre-commit-annex` in your git repository will be run whenever a commit is made, either by git commit, git-annex sync, or the git-annex assistant. diff --git a/doc/install.mdwn b/doc/install.mdwn --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -14,6 +14,7 @@   [[Gentoo]] | `emerge git-annex`   [[ScientificLinux5]] |   [[openSUSE]] | +  [[Docker]] | [[Windows]] | [download installer](http://downloads.kitenet.net/git-annex/windows/current/) **alpha** """]] diff --git a/doc/install/ArchLinux/comment_6_1d597d6a95f9c2df7dae6e98813e4865._comment b/doc/install/ArchLinux/comment_6_1d597d6a95f9c2df7dae6e98813e4865._comment new file mode 100644 --- /dev/null +++ b/doc/install/ArchLinux/comment_6_1d597d6a95f9c2df7dae6e98813e4865._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqWbWVRH2k9spSMqKfIXBP1G3ekkj9Igg" + nickname="Rado" + subject="problem installing using cabal: language-javascript missing" + date="2014-03-28T22:38:04Z" + content=""" +Configuring gnuidn-0.2.1... +cabal: The program c2hs is required but it could not be found. +Failed to install gnuidn-0.2.1 +Configuring language-javascript-0.5.9... +cabal: The program happy version >=1.18.5 is required but it could not be +found. +Failed to install language-javascript-0.5.9 +cabal: Error: some packages failed to install: +git-annex-5.20140320 depends on language-javascript-0.5.9 which failed to +install. +gnuidn-0.2.1 failed during the configure step. The exception was: +ExitFailure 1 +hjsmin-0.1.4.6 depends on language-javascript-0.5.9 which failed to install. +language-javascript-0.5.9 failed during the configure step. The exception was: +ExitFailure 1 +network-protocol-xmpp-0.4.6 depends on gnuidn-0.2.1 which failed to install. +yesod-static-1.2.2.4 depends on language-javascript-0.5.9 which failed to +install. +[r-c@rc-laptop ~]$ cabal install language-javascript +Resolving dependencies... +Configuring language-javascript-0.5.9... +cabal: The program happy version >=1.18.5 is required but it could not be +found. +Failed to install language-javascript-0.5.9 +cabal: Error: some packages failed to install: +language-javascript-0.5.9 failed during the configure step. The exception was: +ExitFailure 1 + +Can you help how to solve? +"""]] diff --git a/doc/install/ArchLinux/comment_7_2d708977e2fad6b68803494576382df5._comment b/doc/install/ArchLinux/comment_7_2d708977e2fad6b68803494576382df5._comment new file mode 100644 --- /dev/null +++ b/doc/install/ArchLinux/comment_7_2d708977e2fad6b68803494576382df5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://alerque.com/" + nickname="Caleb" + subject="dep problems" + date="2014-03-28T22:50:37Z" + content=""" +@rado The Haskel dependencies can be a nightmare to sort out for the un-initiated. You can side-step the whole issue by uninstalling the pre-built version that that has all the dependencies built in out of the box. + +Just grab the git-annex-bin package from the AUR and be done with it. (The -bin and -standalone packages recently merged so there is just -bin now). +"""]] diff --git a/doc/install/ArchLinux/comment_8_5b5f5e0b64e5bfb1ea12e8b251c6fb5f._comment b/doc/install/ArchLinux/comment_8_5b5f5e0b64e5bfb1ea12e8b251c6fb5f._comment new file mode 100644 --- /dev/null +++ b/doc/install/ArchLinux/comment_8_5b5f5e0b64e5bfb1ea12e8b251c6fb5f._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqWbWVRH2k9spSMqKfIXBP1G3ekkj9Igg" + nickname="Rado" + subject="I solved it installing dependencies....but dont know hot to start it..." + date="2014-03-29T07:45:19Z" + content=""" +cabal install gsasl +cabal install happy +cabal install language-javascript +cabal install alex +cabal install c2hs + +after installing writing in terminal: git-annex, git-annex webapp does nothing... +can you help how to start git-annex? +"""]] diff --git a/doc/install/Docker.mdwn b/doc/install/Docker.mdwn new file mode 100644 --- /dev/null +++ b/doc/install/Docker.mdwn @@ -0,0 +1,27 @@ +There is not yet a pre-built Docker image for git-annex. However, it's +easy to add it to an image. + +For example: + + docker run -i -t joeyh/debian-unstable apt-get install git-annex + +# autobuilders + +The git-annex Linux autobuilds are built using a Docker container. +If you'd like to set up your own autobuilder in a Docker container, +the image that is used is not currently published, but you can build +a new image using [Propellor](http://joeyh.name/code/propellor). Just +install Propellor and add this to its `config.hs`: + +[[!format haskell """ +host hostname@"your.machine.net" = Just $ props + & Docker.configured + & Docker.docked container hostname "amd64-git-annex-builder" + +container _ "amd64-git-annex-builder" = in Just $ Docker.containerFrom + (image $ System (Debian Unstable) "amd64") + [ Docker.inside $ props & GitAnnexBuilder.builder "amd64" "15 * * * *" False ] +"""]] + +This will autobuild every hour at :15, and the autobuilt image will be +left inside the container in /home/builder/gitbuilder/out/ diff --git a/doc/internals.mdwn b/doc/internals.mdwn --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -150,6 +150,13 @@ repository, while files not matching it are preferred to be stored somewhere else. +## `required-content.log` + +Used to indicate which repositories are required to contain which file +contents. + +File format is identical to preferred-content.log. + ## `group-preferred-content.log` Contains standard preferred content settings for groups. (Overriding or diff --git a/doc/metadata.mdwn b/doc/metadata.mdwn --- a/doc/metadata.mdwn +++ b/doc/metadata.mdwn @@ -1,7 +1,7 @@ -git-annex allows you to store arbitrary metadata about files stored in the -git-annex repository. The metadata is stored in the `git-annex` branch, and -so is automatically kept in sync with the rest of git-annex's state, such -as [[location_tracking]] information. +git-annex allows you to store arbitrary metadata about the content of files +stored in the git-annex repository. The metadata is stored in the +`git-annex` branch, and so is automatically kept in sync with the rest of +git-annex's state, such as [[location_tracking]] information. Some of the things you can do with metadata include: diff --git a/doc/news/version_5.20140402.mdwn b/doc/news/version_5.20140402.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_5.20140402.mdwn @@ -0,0 +1,34 @@ +git-annex 5.20140402 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * unannex, uninit: Avoid committing after every file is unannexed, + for massive speedup. + * --notify-finish switch will cause desktop notifications after each + file upload/download/drop completes + (using the dbus Desktop Notifications Specification) + * --notify-start switch will show desktop notifications when each + file upload/download starts. + * webapp: Automatically install Nautilus integration scripts + to get and drop files. + * tahoe: Pass -d parameter before subcommand; putting it after + the subcommand no longer works with tahoe-lafs version 1.10. + (Thanks, Alberto Berti) + * forget --drop-dead: Avoid removing the dead remote from the trust.log, + so that if git remotes for it still exist anywhere, git annex info + will still know it's dead and not show it. + * git-annex-shell: Make configlist automatically initialize + a remote git repository, as long as a git-annex branch has + been pushed to it, to simplify setup of remote git repositories, + including via gitolite. + * add --include-dotfiles: New option, perhaps useful for backups. + * Version 5.20140227 broke creation of glacier repositories, + not including the datacenter and vault in their configuration. + This bug is fixed, but glacier repositories set up with the broken + version of git-annex need to have the datacenter and vault set + in order to be usable. This can be done using git annex enableremote + to add the missing settings. For details, see + http://git-annex.branchable.com/bugs/problems\_with\_glacier/ + * Added required content configuration. + * assistant: Improve ssh authorized keys line generated in local pairing + or for a remote ssh server to set environment variables in an + alternative way that works with the non-POSIX fish shell, as well + as POSIX shells."""]] diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -1,7 +1,7 @@ git-annex tries to ensure that the configured number of [[copies]] of your data always exist, and leaves it up to you to use commands like `git annex get` and `git annex drop` to move the content to the repositories you want -to contain it. But sometimes, it can be good to have more fine-grained +to contain it. But often, it can be good to have more fine-grained control over which content is wanted by which repositories. Configuring this allows the git-annex assistant as well as `git annex get --auto`, `git annex drop --auto`, `git annex sync --content`, @@ -33,9 +33,9 @@ To check at the command line which files are matched by preferred content settings, you can use the --want-get and --want-drop options. -For example, "git annex find --want-get --not --in ." will find all the -files that "git annex get --auto" will want to get, and "git annex find ---want-drop --in ." will find all the files that "git annex drop --auto" +For example, `git annex find --want-get --not --in .` will find all the +files that `git annex get --auto` will want to get, and `git annex find +--want-drop --in .` will find all the files that `git annex drop --auto` will want to drop. The expressions are very similar to the matching options documented diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -11,4 +11,5 @@ utility, with a `-A` switch that enables git-annex support. * Emacs Org mode can auto-commit attached files to git-annex. * [git annex darktable integration](https://github.com/xxv/darktable-git-annex) -* [Nautilus file manager ingegration](https://gist.github.com/ion1/9660286) +* Emacs's [Magit mode](http://www.emacswiki.org/emacs/Magit) has + [magit integration](http://melpa.milkbox.net/?utm_source=dlvr.it&utm_medium=twitter#/magit-annex) diff --git a/doc/required_content.mdwn b/doc/required_content.mdwn new file mode 100644 --- /dev/null +++ b/doc/required_content.mdwn @@ -0,0 +1,17 @@ +Required content settings can be configured to do more complicated +things than just setting the required number of [[copies]] of your data. +For example, you could require that data be archived in at least two +archival repositories, and also require that one copy be stored offsite. + +The format of required content expressions is the same as +[[preferred_content]] expressions. + +Required content settings can be edited using `git annex vicfg`. +Each repository can have its own settings, and other repositories will +try to honor those settings when interacting with it. + +While [[preferred_content]] expresses a preference, it can be overridden +by simply using `git annex drop`. On the other hand, required content +settings are enforced; `git annex drop` will refuse to drop a file if +doing so would violate its required content settings. +(Although even this can be overridden using `--force`). diff --git a/doc/special_remotes/tahoe.mdwn b/doc/special_remotes/tahoe.mdwn --- a/doc/special_remotes/tahoe.mdwn +++ b/doc/special_remotes/tahoe.mdwn @@ -22,8 +22,12 @@ These parameters can be passed to `git annex initremote` to configure the tahoe remote. +* `shared-convergence-secret` - Optional. Can be useful to set to + allow tahoe to deduplicate information. By default, a new + shared-convergence-secret is created for each tahoe remote. + * `embedcreds` - Optional. Set to "yes" embed the tahoe credentials - (specifically the introducer furl and shared-convergence-secret) + (specifically the introducer-furl and shared-convergence-secret) inside the git repository, which allows other clones to also use them in order to access the tahoe grid. diff --git a/doc/tips/automatically_adding_metadata.mdwn b/doc/tips/automatically_adding_metadata.mdwn --- a/doc/tips/automatically_adding_metadata.mdwn +++ b/doc/tips/automatically_adding_metadata.mdwn @@ -17,7 +17,7 @@ Now any fields you list in metadata.extract to will be extracted and stored when files are committed. -To get a list of all possible fields, run: `extract -L | sed ' ' _` +To get a list of all possible fields, run: `extract -L | sed 's/ /_/g'` By default, if a git-annex already has a metadata field for a file, its value will not be overwritten with metadata taken from files. diff --git a/doc/tips/file_manager_integration.mdwn b/doc/tips/file_manager_integration.mdwn new file mode 100644 --- /dev/null +++ b/doc/tips/file_manager_integration.mdwn @@ -0,0 +1,100 @@ +Integrating git-annex and your file manager provides an easy way to select +annexed files to get or drop. + +[[!toc]] + +## GNOME (nautilus) + +Recent git-annex comes with built-in nautilus integration. Just pick the +action from the menu. + +[[!img assistant/nautilusmenu.png]] + +[[!img assistant/downloadnotification.png]] + +This is set up by making simple scripts in +`~/.local/share/nautilus/scripts`, with names like "git-annex get" + +## KDE (Dolphin/Konqueror) + +Create a file `~/.kde4/share/kde4/services/ServiceMenus/git-annex.desktop` with the following contents: + + [Desktop Entry] + Type=Service + ServiceTypes=all/allfiles + MimeType=all/all; + Actions=GitAnnexGet;GitAnnexDrop; + X-KDE-Priority=TopLevel + X-KDE-Submenu=Git-Annex + X-KDE-Icon=git-annex + X-KDE-ServiceTypes=KonqPopupMenu/Plugin + + [Desktop Action GitAnnexGet] + Name=Get + Icon=git-annex + Exec=git-annex get --notify-start --notify-finish -- %U + + [Desktop Action GitAnnexDrop] + Name=Drop + Icon=git-annex + Exec=git-annex drop --notify-start --notify-finish -- %U + +## XFCE (Thunar) + +XFCE uses the Thunar file manager, which can also be easily configured to allow for custom actions. Just go to the "Configure custom actions..." item in the "Edit" menu, and create a custom action for get and drop with the following commands: + + git-annex drop --notify-start --notify-finish -- %F + +for drop, and for get: + + git-annex drop --notify-start --notify-finish -- %F + +This gives me the resulting config on disk, in `.config/Thunar/uca.xml`: + + + git-annex + git-annex get + 1396278104182858-3 + git-annex get --notify-start --notify-finish -- %F + get the files from a remote git annex repository + * + + + + + + + + + git-annex + git-annex drop + 1396278093174843-2 + git-annex drop --notify-start --notify-finish -- %F + drop the files from the local repository + * + + + + + + + + +The complete instructions on how to setup actions is [in the XFCE documentation](http://docs.xfce.org/xfce/thunar/custom-actions). + +## your file manager here + +Edit this page and add instructions! + +## general + +If your file manager can run a command on a file, it should be easy to +integrate git-annex with it. A simple script will suffice: + + #!/bun/sh + git-annex get --notify-start --notify-finish -- "$@" + +The --notify-start and --notify-stop options make git-annex display a +desktop notification. This is useful to give the user an indication that +their action took effect. Desktop notifications are currently only +implenented for Linux. diff --git a/doc/todo.mdwn b/doc/todo.mdwn --- a/doc/todo.mdwn +++ b/doc/todo.mdwn @@ -1,4 +1,4 @@ -This is git-annex's todo list. Link items to [[todo/done]] when done. +This is git-annex's todo list. Link items to [[todo/done]] when done. A more complete [[design/roadmap/]] is also available. [[!inline pages="./todo/* and !./todo/done and !link(done) and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/todo/Bittorrent-like_features.mdwn b/doc/todo/Bittorrent-like_features.mdwn --- a/doc/todo/Bittorrent-like_features.mdwn +++ b/doc/todo/Bittorrent-like_features.mdwn @@ -29,6 +29,8 @@ This was originally posted [[as a forum post|forum/Wishlist:_Bittorrent-like_transfers]] by [[users/GLITTAH]]. +Update: note how [[design/assistant/telehash/]] may be able to answer this specific use case. + Using an external client (addurl torrent support) ================================================= diff --git a/doc/todo/New_special_remote_suggeston_-_clean_directory.mdwn b/doc/todo/New_special_remote_suggeston_-_clean_directory.mdwn --- a/doc/todo/New_special_remote_suggeston_-_clean_directory.mdwn +++ b/doc/todo/New_special_remote_suggeston_-_clean_directory.mdwn @@ -13,3 +13,11 @@ First and foremost, this can't be (really really shouldn't be) a trusted remote; my wife could accidentally delete all files on the NAS while I am away. So my local git-annex shouldn't assume the NAS counts towards numcopies (unless I'm a real masochist). Secondly, what to do when files change/are added/removed on the special remote? Probably the same thing that the assistant does with everything. The only thing special is that new/modified files will need to be copied locally from this special remote before being added to the annex (to get hash and such). + +> This is not feaisble given git-annex's design. If I wanted to +> make something completely unlike git-annex, I suppose it could be done, +> but it's off topic here. [[wontfix|done]]. +> +> If you want to use git-annex on a Synology NAS, the arm standalone build +> will work, and then you can use the command-line, or the assistant +> to maintain a git repository that contains your files as desired. --[[Joey]] diff --git a/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn b/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn @@ -0,0 +1,11 @@ +## Use case: + +You have a few annexes that the assistant is watching for you. You're somewhere with poor wifi speed. You also just added a bunch of big files to a few annexes. Now all of a sudden your connection suffers and you want an easy way to pause all transfers until you're on a faster connection without losing the automatic 'add' and such of the assistant (iow: without having to shutdown the daemon). + +## Proposal: + +A "Pause all transfers" button in the webapp that pauses all transfers from all annexes the assistant is watching. + +It should toggle to "Resume all transfers" when pushed so you can also easily start the transfers again when you get somewhere else. + +This may or may not make more sense if the webapp showed all watched repos in a single view (instead of the separate pages/views as it is now). diff --git a/doc/todo/Views_Demo.mdwn b/doc/todo/Views_Demo.mdwn --- a/doc/todo/Views_Demo.mdwn +++ b/doc/todo/Views_Demo.mdwn @@ -11,3 +11,5 @@ FWIW, Bob + +> [[closing|done]]; requested feature was already present --[[Joey]] diff --git a/doc/todo/clear_file_names_in_special_remotes.mdwn b/doc/todo/clear_file_names_in_special_remotes.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/clear_file_names_in_special_remotes.mdwn @@ -0,0 +1,13 @@ +To properly use amazon AWS S3 for CDN, we need to publish videos to S3. Ideally, we would like to do this via git-annex as the back-end of video.debian.net is being migrated to git-annex by me, atm. + +Obviously, we will need clear text names and proper directory structure, not SHA512E file names. This would need to be supported by the S3 special remote. + +I talked to TobiasTheViking in the past and he hinted at a reasonably clean way to do this, but that a clean solution would need support from git-annex. I will link him to this page and ask him to supply whatever info is needed. + + +Thanks, +Richard + +> This is not feaisble given git-annex's design. If I wanted to +> make something completely unlike git-annex, I suppose it could be done, +> but it's off topic here. [[wontfix|done]] --[[Joey]] diff --git a/doc/todo/required_content.mdwn b/doc/todo/required_content.mdwn --- a/doc/todo/required_content.mdwn +++ b/doc/todo/required_content.mdwn @@ -5,3 +5,19 @@ For example, I might want a repository that is required to contain `*.jpeg`. This would make get --auto get it (it's implicitly part of the preferred content), and would make drop refuse to drop it. + +> I've implemented the basic required content. Currently only configurable +> via `vicfg`, because I don't think a lot of people are going to want to +> use it. +> +> Note that I did not yet add the active verification discussed below. +> So if required content is set to `not inallgroup=backup`, or +> `not copies=10`, trying to drop a file will not go off and prove +> that there are 10 copies or that the file is in every repository in +> the backup group. It will assume that the location log is accurate +> and go by that. +> +> I think this is enough to cover Richard's case, at least. +> In his example, A B and C are in group anchor and have required +> content set to `include=*`, and D E F have it set to +> `not inallgroup=anchor`. --[[Joey]] diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn --- a/doc/todo/tahoe_lfs_for_reals.mdwn +++ b/doc/todo/tahoe_lfs_for_reals.mdwn @@ -12,10 +12,12 @@ To support a special remote, a mapping is needed from git-annex keys to Tahoe keys, stored in the git-annex branch. -> This is now done, however, there are 2 known +> This is now done, however, there are 3 known > problems: > > * tahoe start run unncessarily > * web.port can conflict -> +> * Nothing renews leases, which is a problem on grids that expire. +> + > --[[Joey]] diff --git a/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn b/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn @@ -0,0 +1,12 @@ +Having the option of choosing for every file if we want it in our repository or not would be a great feature. It is currently possible using the wanted expression, but it is not very flexible, or it becomes unmaintainable. + +I tried with two repositories a and b, with the following wanted expressions : + +* for a: `not metadata=unwanted=` +* for b: `not metadata=unwanted=` + +I think those expressions should be included in standard wanted expressions. + +Also, to improbe the feature, it should be possible to set (or remove) metadata in directories, and those should automatically affect their content. + +And we could imagine a `git annex unwant` command that would add the unwanted metadata to a file, copy it to other repositories, and attempt to drop it. diff --git a/git-annex-shell.1 b/git-annex-shell.1 --- a/git-annex-shell.1 +++ b/git-annex-shell.1 @@ -22,7 +22,12 @@ .PP .IP "configlist directory" This outputs a subset of the git configuration, in the same form as -\fBgit config \-\-list\fP +\fBgit config \-\-list\fP. This is used to get the annex.uuid of the remote +repository. +.IP +When run in a repository that does not yet have an annex.uuid, one +will be created, as long as a git\-annex branch has already been pushed to +the repository. .IP .IP "inannex directory [key ...]" This checks if all specified keys are present in the annex, diff --git a/git-annex.1 b/git-annex.1 --- a/git-annex.1 +++ b/git-annex.1 @@ -57,11 +57,15 @@ subdirectories). .PP .IP "\fBadd [path ...]\fP" -Adds files in the path to the annex. Files that are already checked into -git, or that git has been configured to ignore will be silently skipped. -(Use \fB\-\-force\fP to add ignored files.) Dotfiles are skipped unless explicitly -listed. +Adds files in the path to the annex. If no path is specified, adds +files from the current directory and below. .IP +Files that are already checked into git, or that git has been configured +to ignore will be silently skipped. (Use \fB\-\-force\fP to add ignored files.) +.IP +Dotfiles are skipped unless explicitly listed, or the \-\-include\-dotfiles +option is used. +.IP .IP "\fBget [path ...]\fP" Makes the content of annexed files available in this repository. This will involve copying them from another repository, or downloading them, @@ -652,8 +656,9 @@ .SH METADATA COMMANDS .IP "\fBmetadata [path ...] [\-s field=value \-s field+=value \-s field\-=value ...] [\-g field]\fP" .IP -Each file can have any number of metadata fields attached to it, -which each in turn have any number of values. +The content of a file can have any number of metadata fields +attached to it to describe it. Each metadata field can in turn +have any number of values. .IP This command can be used to set metadata, or show the currently set metadata. @@ -959,6 +964,17 @@ .IP "\fB\-\-user\-agent=value\fP" Overrides the User\-Agent to use when downloading files from the web. .IP +.IP "\fB\-\-notify\-finish\fP" +Caused a desktop notification to be displayed after each successful +file download and upload. +.IP +(Only supported on some platforms, eg Linux with dbus. A no\-op when +not supported.) +.IP +.IP "\fB\-\-notify\-start\fP" +Caused a desktop notification to be displayed when a file upload +or download has started, or when a file is dropped. +.IP .IP "\fB\-c name=value\fP" Overrides git configuration settings. May be specified multiple times. .IP @@ -1519,7 +1535,7 @@ \fB~/.config/git\-annex/autostart\fP is a list of git repositories to start the git\-annex assistant in. .PP -\fB.git/hooks/pre\-commit\-annex\fP in your git repsitory will be run whenever +\fB.git/hooks/pre\-commit\-annex\fP in your git repository will be run whenever a commit is made, either by git commit, git\-annex sync, or the git\-annex assistant. .PP diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20140320 +Version: 5.20140402 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess @@ -85,6 +85,9 @@ Flag CryptoHash Description: Enable use of cryptohash for checksumming +Flag DesktopNotify + Description: Enable desktop environment notifications + Flag EKG Description: Enable use of EKG to monitor git-annex as it runs (at http://localhost:4242/) Default: False @@ -167,10 +170,16 @@ CPP-Options: -DWITH_KQUEUE C-Sources: Utility/libkqueue.c - if os(linux) && flag(Dbus) - Build-Depends: dbus (>= 0.10.3) - CPP-Options: -DWITH_DBUS + if (os(linux)) + if flag(Dbus) + Build-Depends: dbus (>= 0.10.3) + CPP-Options: -DWITH_DBUS + if flag(DesktopNotify) + if flag(Dbus) + Build-Depends: dbus (>= 0.10.3), fdo-notify (>= 0.3) + CPP-Options: -DWITH_DESKTOP_NOTIFY -DWITH_DBUS_NOTIFICATIONS + if flag(Android) Build-Depends: data-endian CPP-Options: -D__ANDROID__ -DANDROID_SPLICES -D__NO_TH__ @@ -182,11 +191,10 @@ yesod, yesod-default, yesod-static, yesod-form, yesod-core, http-types, transformers, wai, wai-logger, warp, warp-tls, blaze-builder, crypto-api, hamlet, clientsession, - template-haskell, data-default, aeson, network-conduit, - byteable + template-haskell, data-default, aeson, network-conduit CPP-Options: -DWITH_WEBAPP if flag(Webapp) && flag (Webapp-secure) - Build-Depends: warp-tls (>= 1.4), securemem + Build-Depends: warp-tls (>= 1.4), securemem, byteable CPP-Options: -DWITH_WEBAPP_SECURE if flag(Pairing) diff --git a/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch b/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch new file mode 100644 --- /dev/null +++ b/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch @@ -0,0 +1,25 @@ +From 0ad071f80ee72e7b8ca5b0b70dfae5bbf8677969 Mon Sep 17 00:00:00 2001 +From: Joey Hess +Date: Wed, 12 Mar 2014 12:18:17 -0400 +Subject: [PATCH] cross build + +--- + unbounded-delays.cabal | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/unbounded-delays.cabal b/unbounded-delays.cabal +index 76d0a50..0f27569 100644 +--- a/unbounded-delays.cabal ++++ b/unbounded-delays.cabal +@@ -1,7 +1,7 @@ + name: unbounded-delays + version: 0.1.0.6 + cabal-version: >= 1.6 +-build-type: Custom ++build-type: Simple + author: Bas van Dijk + Roel van Dijk + maintainer: Bas van Dijk +-- +1.7.10.4 + diff --git a/standalone/android/install-haskell-packages b/standalone/android/install-haskell-packages --- a/standalone/android/install-haskell-packages +++ b/standalone/android/install-haskell-packages @@ -108,6 +108,7 @@ patched gnutls patched libxml-sax patched network-protocol-xmpp + patched unbounded-delays cd ..