diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -13,6 +13,7 @@ import Limit import Utility.Matcher import Types.Group +import Types.Limit import Logs.Group import Logs.Remote import Annex.UUID diff --git a/Annex/Wanted.hs b/Annex/Wanted.hs --- a/Annex/Wanted.hs +++ b/Annex/Wanted.hs @@ -1,4 +1,4 @@ -{- git-annex control over whether content is wanted +{- git-annex checking whether content is wanted - - Copyright 2012 Joey Hess - diff --git a/Assistant.hs b/Assistant.hs --- a/Assistant.hs +++ b/Assistant.hs @@ -23,6 +23,7 @@ import Assistant.Threads.Transferrer import Assistant.Threads.SanityChecker import Assistant.Threads.Cronner +import Assistant.Threads.ProblemFixer #ifdef WITH_CLIBS import Assistant.Threads.MountWatcher #endif @@ -48,6 +49,7 @@ import qualified Utility.Daemon import Utility.LogFile import Utility.ThreadScheduler +import Utility.HumanTime import qualified Build.SysConfig as SysConfig import System.Log.Logger @@ -61,8 +63,8 @@ - - startbrowser is passed the url and html shim file, as well as the original - stdout and stderr descriptors. -} -startDaemon :: Bool -> Bool -> Maybe HostName -> Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex () -startDaemon assistant foreground listenhost startbrowser = do +startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe HostName -> Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex () +startDaemon assistant foreground startdelay listenhost startbrowser = do Annex.changeState $ \s -> s { Annex.daemon = True } pidfile <- fromRepo gitAnnexPidFile logfile <- fromRepo gitAnnexLogFile @@ -128,8 +130,9 @@ , assist $ daemonStatusThread , assist $ sanityCheckerDailyThread , assist $ sanityCheckerHourlyThread + , assist $ problemFixerThread urlrenderer #ifdef WITH_CLIBS - , assist $ mountWatcherThread + , assist $ mountWatcherThread urlrenderer #endif , assist $ netWatcherThread , assist $ netWatcherFallbackThread @@ -140,7 +143,7 @@ , watch $ watchThread -- must come last so that all threads that wait -- on it have already started waiting - , watch $ sanityCheckerStartupThread + , watch $ sanityCheckerStartupThread startdelay ] liftIO waitForTermination diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs --- a/Assistant/Alert.hs +++ b/Assistant/Alert.hs @@ -15,18 +15,19 @@ import qualified Remote import Utility.Tense import Logs.Transfer -import Git.Remote (RemoteName) import Data.String import qualified Data.Text as T +import qualified Control.Exception as E #ifdef WITH_WEBAPP -import Assistant.Monad import Assistant.DaemonStatus import Assistant.WebApp.Types -import Assistant.WebApp +import Assistant.WebApp (renderUrl) import Yesod #endif +import Assistant.Monad +import Assistant.Types.UrlRenderer {- Makes a button for an alert that opens a Route. - @@ -166,16 +167,62 @@ alerthead = "The daily sanity check found and fixed a problem:" alertfoot = "If these problems persist, consider filing a bug report." -fsckAlert :: AlertButton -> Maybe RemoteName -> Alert -fsckAlert button n = baseActivityAlert - { alertData = case n of +fsckingAlert :: AlertButton -> Maybe Remote -> Alert +fsckingAlert button mr = baseActivityAlert + { alertData = case mr of Nothing -> [ UnTensed $ T.pack $ "Consistency check in progress" ] - Just remotename -> [ UnTensed $ T.pack $ "Consistency check of " ++ remotename ++ " in progress"] + Just r -> [ UnTensed $ T.pack $ "Consistency check of " ++ Remote.name r ++ " in progress"] , alertButton = Just button } +showFscking :: UrlRenderer -> Maybe Remote -> IO (Either E.SomeException a) -> Assistant a +showFscking urlrenderer mr a = do +#ifdef WITH_WEBAPP + button <- mkAlertButton False (T.pack "Configure") urlrenderer ConfigFsckR + r <- alertDuring (fsckingAlert button mr) $ + liftIO a +#else + r <- liftIO a +#endif + either (liftIO . E.throwIO) return r + +notFsckedNudge :: UrlRenderer -> Maybe Remote -> Assistant () +#ifdef WITH_WEBAPP +notFsckedNudge urlrenderer mr = do + button <- mkAlertButton True (T.pack "Configure") urlrenderer ConfigFsckR + void $ addAlert (notFsckedAlert mr button) +#else +notFsckedNudge _ = noop +#endif + +notFsckedAlert :: Maybe Remote -> AlertButton -> Alert +notFsckedAlert mr button = Alert + { alertHeader = Just $ fromString $ concat + [ "You should enable consistency checking to protect your data" + , maybe "" (\r -> " in " ++ Remote.name r) mr + , "." + ] + , alertIcon = Just InfoIcon + , alertPriority = High + , alertButton = Just button + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just NotFsckedAlert + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + brokenRepositoryAlert :: AlertButton -> Alert brokenRepositoryAlert = errorAlert "Serious problems have been detected with your repository. This needs your immediate attention!" + +repairingAlert :: String -> Alert +repairingAlert repodesc = activityAlert Nothing + [ Tensed "Attempting to repair" "Repaired" + , UnTensed $ T.pack repodesc + ] pairingAlert :: AlertButton -> Alert pairingAlert button = baseActivityAlert diff --git a/Assistant/Fsck.hs b/Assistant/Fsck.hs new file mode 100644 --- /dev/null +++ b/Assistant/Fsck.hs @@ -0,0 +1,50 @@ +{- git-annex assistant fscking + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.Fsck where + +import Assistant.Common +import Types.ScheduledActivity +import qualified Types.Remote as Remote +import Annex.UUID +import Assistant.Alert +import Assistant.Types.UrlRenderer +import Logs.Schedule +import qualified Annex + +import qualified Data.Set as S + +{- Displays a nudge in the webapp if a fsck is not configured for + - the specified remote, or for the local repository. -} +fsckNudge :: UrlRenderer -> Maybe Remote -> Assistant () +fsckNudge urlrenderer mr + | maybe True fsckableRemote mr = + whenM (liftAnnex $ annexFsckNudge <$> Annex.getGitConfig) $ + unlessM (liftAnnex $ checkFscked mr) $ + notFsckedNudge urlrenderer mr + | otherwise = noop + +fsckableRemote :: Remote -> Bool +fsckableRemote = isJust . Remote.remoteFsck + +{- Checks if the remote, or the local repository, has a fsck scheduled. + - Only looks at fscks configured to run via the local repository, not + - other repositories. -} +checkFscked :: Maybe Remote -> Annex Bool +checkFscked mr = any wanted . S.toList <$> (scheduleGet =<< getUUID) + where + wanted = case mr of + Nothing -> isSelfFsck + Just r -> flip isFsckOf (Remote.uuid r) + +isSelfFsck :: ScheduledActivity -> Bool +isSelfFsck (ScheduledSelfFsck _ _) = True +isSelfFsck _ = False + +isFsckOf :: ScheduledActivity -> UUID -> Bool +isFsckOf (ScheduledRemoteFsck u _ _) u' = u == u' +isFsckOf _ _ = False diff --git a/Assistant/Monad.hs b/Assistant/Monad.hs --- a/Assistant/Monad.hs +++ b/Assistant/Monad.hs @@ -39,6 +39,7 @@ import Assistant.Types.BranchChange import Assistant.Types.Commits import Assistant.Types.Changes +import Assistant.Types.RepoProblem import Assistant.Types.Buddies import Assistant.Types.NetMessager import Assistant.Types.ThreadName @@ -63,6 +64,7 @@ , failedPushMap :: FailedPushMap , commitChan :: CommitChan , changePool :: ChangePool + , repoProblemChan :: RepoProblemChan , branchChangeHandle :: BranchChangeHandle , buddyList :: BuddyList , netMessager :: NetMessager @@ -80,6 +82,7 @@ <*> newFailedPushMap <*> newCommitChan <*> newChangePool + <*> newRepoProblemChan <*> newBranchChangeHandle <*> newBuddyList <*> newNetMessager diff --git a/Assistant/Repair.hs b/Assistant/Repair.hs new file mode 100644 --- /dev/null +++ b/Assistant/Repair.hs @@ -0,0 +1,153 @@ +{- git-annex assistant repository repair + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Repair where + +import Assistant.Common +import Command.Repair (repairAnnexBranch) +import Git.Fsck (FsckResults, foundBroken) +import Git.Repair (runRepairOf) +import qualified Git +import qualified Remote +import qualified Types.Remote as Remote +import Logs.FsckResults +import Annex.UUID +import Utility.Batch +import Config.Files +import Assistant.Sync +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.Types.UrlRenderer +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +import qualified Data.Text as T +#endif +import qualified Utility.Lsof as Lsof +import Utility.ThreadScheduler + +import Control.Concurrent.Async + +{- When the FsckResults require a repair, tries to do a non-destructive + - repair. If that fails, pops up an alert. -} +repairWhenNecessary :: UrlRenderer -> UUID -> Maybe Remote -> FsckResults -> Assistant Bool +repairWhenNecessary urlrenderer u mrmt fsckresults + | foundBroken fsckresults = do + liftAnnex $ writeFsckResults u fsckresults + repodesc <- liftAnnex $ Remote.prettyUUID u + ok <- alertWhile (repairingAlert repodesc) + (runRepair u mrmt False) +#ifdef WITH_WEBAPP + unless ok $ do + button <- mkAlertButton True (T.pack "Click Here") urlrenderer $ + RepairRepositoryR u + void $ addAlert $ brokenRepositoryAlert button +#endif + return ok + | otherwise = return False + +runRepair :: UUID -> Maybe Remote -> Bool -> Assistant Bool +runRepair u mrmt destructiverepair = do + fsckresults <- liftAnnex $ readFsckResults u + myu <- liftAnnex getUUID + ok <- if u == myu + then localrepair fsckresults + else remoterepair fsckresults + liftAnnex $ writeFsckResults u Nothing + debug [ "Repaired", show u, show ok ] + + return ok + where + localrepair fsckresults = do + -- Stop the watcher from running while running repairs. + changeSyncable Nothing False + + -- This intentionally runs the repair inside the Annex + -- monad, which is not strictly necessary, but keeps + -- other threads that might be trying to use the Annex + -- from running until it completes. + ok <- liftAnnex $ repair fsckresults Nothing + + -- Run a background fast fsck if a destructive repair had + -- to be done, to ensure that the git-annex branch + -- reflects the current state of the repo. + when destructiverepair $ + backgroundfsck [ Param "--fast" ] + + -- Start the watcher running again. This also triggers it to + -- do a startup scan, which is especially important if the + -- git repo repair removed files from the index file. Those + -- files will be seen as new, and re-added to the repository. + when (ok || destructiverepair) $ + changeSyncable Nothing True + + return ok + + remoterepair fsckresults = case Remote.repairRepo =<< mrmt of + Nothing -> return False + Just mkrepair -> do + thisrepopath <- liftIO . absPath + =<< liftAnnex (fromRepo Git.repoPath) + a <- liftAnnex $ mkrepair $ + repair fsckresults (Just thisrepopath) + liftIO $ catchBoolIO a + + repair fsckresults referencerepo = do + (ok, stillmissing, modifiedbranches) <- inRepo $ + runRepairOf fsckresults destructiverepair referencerepo + when destructiverepair $ + repairAnnexBranch stillmissing modifiedbranches + return ok + + backgroundfsck params = liftIO $ void $ async $ do + program <- readProgramFile + batchCommand program (Param "fsck" : params) + +{- Detect when a git lock file exists and has no git process currently + - writing to it. This strongly suggests it is a stale lock file. + - + - However, this could be on a network filesystem. Which is not very safe + - anyway (the assistant relies on being able to check when files have + - no writers to know when to commit them). Just in case, when the lock + - file appears stale, we delay for one minute, and check its size. If + - the size changed, delay for another minute, and so on. This will at + - least work to detect is another machine is writing out a new index + - file, since git does so by writing the new content to index.lock. + - + - Returns true if locks were cleaned up. + -} +repairStaleGitLocks :: Git.Repo -> Assistant Bool +repairStaleGitLocks r = do + lockfiles <- filter (not . isInfixOf "gc.pid") + . filter (".lock" `isSuffixOf`) + <$> liftIO (findgitfiles r) + repairStaleLocks lockfiles + return $ not $ null lockfiles + where + findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) . Git.localGitDir +repairStaleLocks :: [FilePath] -> Assistant () +repairStaleLocks lockfiles = go =<< getsizes + where + getsize lf = catchMaybeIO $ + (\s -> (lf, fileSize s)) <$> getFileStatus lf + getsizes = liftIO $ catMaybes <$> mapM getsize lockfiles + go [] = return () + go l = ifM (liftIO $ null <$> Lsof.query ("--" : map fst l)) + ( do + waitforit "to check stale git lock file" + l' <- getsizes + if l' == l + then liftIO $ mapM_ nukeFile (map fst l) + else go l' + , do + waitforit "for git lock file writer" + go =<< getsizes + ) + waitforit why = do + notice ["Waiting for 60 seconds", why] + liftIO $ threadDelaySeconds $ Seconds 60 diff --git a/Assistant/RepoProblem.hs b/Assistant/RepoProblem.hs new file mode 100644 --- /dev/null +++ b/Assistant/RepoProblem.hs @@ -0,0 +1,34 @@ +{- git-annex assistant remote problem handling + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.RepoProblem where + +import Assistant.Common +import Assistant.Types.RepoProblem +import Utility.TList + +import Control.Concurrent.STM + +{- Gets all repositories that have problems. Blocks until there is at + - least one. -} +getRepoProblems :: Assistant [RepoProblem] +getRepoProblems = nubBy sameRepoProblem + <$> (atomically . getTList) <<~ repoProblemChan + +{- Indicates that there was a problem with a repository, and the problem + - appears to not be a transient (eg network connection) problem. + - + - If the problem is able to be repaired, the passed action will be run. + - (However, if multiple problems are reported with a single repository, + - only a single action will be run.) + -} +repoHasProblem :: UUID -> Assistant () -> Assistant () +repoHasProblem u afterrepair = do + rp <- RepoProblem + <$> pure u + <*> asIO afterrepair + (atomically . flip consTList rp) <<~ repoProblemChan diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs --- a/Assistant/Sync.hs +++ b/Assistant/Sync.hs @@ -23,9 +23,18 @@ import qualified Git.Ref import qualified Remote import qualified Types.Remote as Remote +import qualified Remote.List as Remote import qualified Annex.Branch import Annex.UUID import Annex.TaggedPush +import qualified Config +import Git.Config +import Assistant.NamedThread +import Assistant.Threads.Watcher (watchThread, WatcherControl(..)) +import Assistant.TransferSlots +import Assistant.TransferQueue +import Assistant.RepoProblem +import Logs.Transfer import Data.Time.Clock import qualified Data.Map as M @@ -51,11 +60,14 @@ reconnectRemotes :: Bool -> [Remote] -> Assistant () reconnectRemotes _ [] = noop reconnectRemotes notifypushes rs = void $ do - rs' <- filterM (checkavailable . Remote.repo) rs + rs' <- liftIO $ filterM (Remote.checkAvailable True) rs unless (null rs') $ do modifyDaemonStatus_ $ \s -> s { desynced = S.union (S.fromList $ map Remote.uuid xmppremotes) (desynced s) } failedrs <- syncAction rs' (const go) + forM_ failedrs $ \r -> + whenM (liftIO $ Remote.checkAvailable False r) $ + repoHasProblem (Remote.uuid r) (syncRemote r) mapM_ signal $ filter (`notElem` failedrs) rs' where gitremotes = filter (notspecialremote . Remote.repo) rs @@ -82,10 +94,6 @@ signal r = liftIO . mapM_ (flip tryPutMVar ()) =<< fromMaybe [] . M.lookup (Remote.uuid r) . connectRemoteNotifiers <$> getDaemonStatus - checkavailable r - | Git.repoIsLocal r || Git.repoIsLocalUnknown r = - liftIO $ doesDirectoryExist $ Git.repoPath r - | otherwise = return True {- Updates the local sync branch, then pushes it to all remotes, in - parallel, along with the git-annex branch. This is the same @@ -233,3 +241,36 @@ reconnectRemotes False [remote] addScanRemotes True [remote] void $ liftIO $ forkIO $ thread + +{- Use Nothing to change autocommit setting; or a remote to change + - its sync setting. -} +changeSyncable :: Maybe Remote -> Bool -> Assistant () +changeSyncable Nothing enable = do + liftAnnex $ Config.setConfig key (boolConfig enable) + liftIO . maybe noop (`throwTo` signal) + =<< namedThreadId watchThread + where + key = Config.annexConfig "autocommit" + signal + | enable = ResumeWatcher + | otherwise = PauseWatcher +changeSyncable (Just r) True = do + liftAnnex $ changeSyncFlag r True + syncRemote r +changeSyncable (Just r) False = do + liftAnnex $ changeSyncFlag r False + updateSyncRemotes + {- Stop all transfers to or from this remote. + - XXX Can't stop any ongoing scan, or git syncs. -} + void $ dequeueTransfers tofrom + mapM_ (cancelTransfer False) =<< + filter tofrom . M.keys . currentTransfers <$> getDaemonStatus + where + tofrom t = transferUUID t == Remote.uuid r + +changeSyncFlag :: Remote -> Bool -> Annex () +changeSyncFlag r enabled = do + Config.setConfig key (boolConfig enabled) + void Remote.remoteListRefresh + where + key = Config.remoteConfig (Remote.repo r) "sync" diff --git a/Assistant/Threads/Cronner.hs b/Assistant/Threads/Cronner.hs --- a/Assistant/Threads/Cronner.hs +++ b/Assistant/Threads/Cronner.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE DeriveDataTypeable, CPP #-} +{-# LANGUAGE DeriveDataTypeable #-} module Assistant.Threads.Cronner ( cronnerThread @@ -28,12 +28,11 @@ import Assistant.Types.UrlRenderer import Assistant.Alert import Remote -#ifdef WITH_WEBAPP -import Assistant.WebApp.Types -#endif -import Git.Remote (RemoteName) +import qualified Types.Remote as Remote +import qualified Git import qualified Git.Fsck -import Logs.FsckResults +import Assistant.Fsck +import Assistant.Repair import Control.Concurrent.Async import Control.Concurrent.MVar @@ -41,8 +40,6 @@ import Data.Time.Clock import qualified Data.Map as M import qualified Data.Set as S -import qualified Control.Exception as E -import qualified Data.Text as T {- Loads schedules for this repository, and fires off one thread for each - scheduled event that runs on this repository. Each thread sleeps until @@ -59,6 +56,7 @@ - ones, and kill the threads for deleted ones. -} cronnerThread :: UrlRenderer -> NamedThread cronnerThread urlrenderer = namedThreadUnchecked "Cronner" $ do + fsckNudge urlrenderer Nothing dstatus <- getDaemonStatus h <- liftIO $ newNotificationHandle False (scheduleLogNotifier dstatus) go h M.empty M.empty @@ -186,48 +184,39 @@ program <- liftIO $ readProgramFile g <- liftAnnex gitRepo fsckresults <- showFscking urlrenderer Nothing $ tryNonAsync $ do - r <- Git.Fsck.findBroken True g void $ batchCommand program (Param "fsck" : annexFsckParams d) - return r - when (Git.Fsck.foundBroken fsckresults) $ do - u <- liftAnnex getUUID - liftAnnex $ writeFsckResults u fsckresults - button <- mkAlertButton True (T.pack "Click Here") urlrenderer $ - RepairRepositoryR u - void $ addAlert $ brokenRepositoryAlert button + Git.Fsck.findBroken True g + u <- liftAnnex getUUID + void $ repairWhenNecessary urlrenderer u Nothing fsckresults mapM_ reget =<< liftAnnex (dirKeys gitAnnexBadDir) where reget k = queueTransfers "fsck found bad file; redownloading" Next k Nothing Download -runActivity' urlrenderer (ScheduledRemoteFsck u s d) = go =<< liftAnnex (remoteFromUUID u) +runActivity' urlrenderer (ScheduledRemoteFsck u s d) = handle =<< liftAnnex (remoteFromUUID u) where - go (Just r) = void $ case Remote.remoteFsck r of - Nothing -> void $ showFscking urlrenderer (Just $ Remote.name r) $ tryNonAsync $ do + handle Nothing = debug ["skipping remote fsck of uuid without a configured remote", fromUUID u, fromSchedule s] + handle (Just rmt) = void $ case Remote.remoteFsck rmt of + Nothing -> go rmt $ do program <- readProgramFile - batchCommand program $ + void $ batchCommand program $ [ Param "fsck" -- avoid downloading files , Param "--fast" , Param "--from" - , Param $ Remote.name r + , Param $ Remote.name rmt ] ++ annexFsckParams d - Just mkfscker -> + Just mkfscker -> do {- Note that having mkfsker return an IO action - avoids running a long duration fsck in the - Annex monad. -} - void . showFscking urlrenderer (Just $ Remote.name r) . tryNonAsync - =<< liftAnnex (mkfscker (annexFsckParams d)) - go Nothing = debug ["skipping remote fsck of uuid without a configured remote", fromUUID u, fromSchedule s] - -showFscking :: UrlRenderer -> Maybe RemoteName -> IO (Either E.SomeException a) -> Assistant a -showFscking urlrenderer remotename a = do -#ifdef WITH_WEBAPP - button <- mkAlertButton False (T.pack "Configure") urlrenderer ConfigFsckR - r <- alertDuring (fsckAlert button remotename) $ - liftIO a - either (liftIO . E.throwIO) return r -#else - a -#endif + go rmt =<< liftAnnex (mkfscker (annexFsckParams d)) + go rmt annexfscker = do + fsckresults <- showFscking urlrenderer (Just rmt) $ tryNonAsync $ do + void annexfscker + let r = Remote.repo rmt + if Git.repoIsLocal r && not (Git.repoIsLocalUnknown r) + then Just <$> Git.Fsck.findBroken True r + else pure Nothing + maybe noop (void . repairWhenNecessary urlrenderer u (Just rmt)) fsckresults annexFsckParams :: Duration -> [CommandParam] annexFsckParams d = diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -19,6 +19,8 @@ import Utility.Mounts import Remote.List import qualified Types.Remote as Remote +import Assistant.Types.UrlRenderer +import Assistant.Fsck import qualified Data.Set as S @@ -33,18 +35,18 @@ #warning Building without dbus support; will use mtab polling #endif -mountWatcherThread :: NamedThread -mountWatcherThread = namedThread "MountWatcher" +mountWatcherThread :: UrlRenderer -> NamedThread +mountWatcherThread urlrenderer = namedThread "MountWatcher" $ #if WITH_DBUS - dbusThread + dbusThread urlrenderer #else - pollingThread + pollingThread urlrenderer #endif #if WITH_DBUS -dbusThread :: Assistant () -dbusThread = do +dbusThread :: UrlRenderer -> Assistant () +dbusThread urlrenderer = do runclient <- asIO1 go r <- liftIO $ E.try $ runClient getSessionAddress runclient either onerr (const noop) r @@ -59,13 +61,13 @@ handleevent <- asIO1 $ \_event -> do nowmounted <- liftIO $ currentMountPoints wasmounted <- liftIO $ swapMVar mvar nowmounted - handleMounts wasmounted nowmounted + handleMounts urlrenderer wasmounted nowmounted liftIO $ forM_ mountChanged $ \matcher -> listen client matcher handleevent , do liftAnnex $ warning "No known volume monitor available through dbus; falling back to mtab polling" - pollingThread + pollingThread urlrenderer ) onerr :: E.SomeException -> Assistant () onerr e = do @@ -76,7 +78,7 @@ - done in this situation. -} liftAnnex $ warning $ "dbus failed; falling back to mtab polling (" ++ show e ++ ")" - pollingThread + pollingThread urlrenderer {- Examine the list of services connected to dbus, to see if there - are any we can use to monitor mounts. If not, will attempt to start one. -} @@ -139,24 +141,25 @@ #endif -pollingThread :: Assistant () -pollingThread = go =<< liftIO currentMountPoints +pollingThread :: UrlRenderer -> Assistant () +pollingThread urlrenderer = go =<< liftIO currentMountPoints where go wasmounted = do liftIO $ threadDelaySeconds (Seconds 10) nowmounted <- liftIO currentMountPoints - handleMounts wasmounted nowmounted + handleMounts urlrenderer wasmounted nowmounted go nowmounted -handleMounts :: MountPoints -> MountPoints -> Assistant () -handleMounts wasmounted nowmounted = - mapM_ (handleMount . mnt_dir) $ +handleMounts :: UrlRenderer -> MountPoints -> MountPoints -> Assistant () +handleMounts urlrenderer wasmounted nowmounted = + mapM_ (handleMount urlrenderer . mnt_dir) $ S.toList $ newMountPoints wasmounted nowmounted -handleMount :: FilePath -> Assistant () -handleMount dir = do +handleMount :: UrlRenderer -> FilePath -> Assistant () +handleMount urlrenderer dir = do debug ["detected mount of", dir] rs <- filter (Git.repoIsLocal . Remote.repo) <$> remotesUnder dir + mapM_ (fsckNudge urlrenderer . Just) rs reconnectRemotes True rs {- Finds remotes located underneath the mount point. diff --git a/Assistant/Threads/ProblemFixer.hs b/Assistant/Threads/ProblemFixer.hs new file mode 100644 --- /dev/null +++ b/Assistant/Threads/ProblemFixer.hs @@ -0,0 +1,70 @@ +{- git-annex assistant thread to handle fixing problems with repositories + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.ProblemFixer ( + problemFixerThread +) where + +import Assistant.Common +import Assistant.Types.RepoProblem +import Assistant.RepoProblem +import Assistant.Types.UrlRenderer +import Assistant.Alert +import Remote +import qualified Types.Remote as Remote +import qualified Git.Fsck +import Assistant.Repair +import qualified Git +import Annex.UUID +import Utility.ThreadScheduler + +{- Waits for problems with a repo, and tries to fsck the repo and repair + - the problem. -} +problemFixerThread :: UrlRenderer -> NamedThread +problemFixerThread urlrenderer = namedThread "ProblemFixer" $ + go =<< getRepoProblems + where + go problems = do + mapM_ (handleProblem urlrenderer) problems + liftIO $ threadDelaySeconds (Seconds 60) + -- Problems may have been re-reported while they were being + -- fixed, so ignore those. If a new unique problem happened + -- 60 seconds after the last was fixed, we're unlikely + -- to do much good anyway. + go =<< filter (\p -> not (any (sameRepoProblem p) problems)) + <$> getRepoProblems + +handleProblem :: UrlRenderer -> RepoProblem -> Assistant () +handleProblem urlrenderer repoproblem = do + fixed <- ifM ((==) (problemUUID repoproblem) <$> liftAnnex getUUID) + ( handleLocalRepoProblem urlrenderer + , maybe (return False) (handleRemoteProblem urlrenderer) + =<< liftAnnex (remoteFromUUID $ problemUUID repoproblem) + ) + when fixed $ + liftIO $ afterFix repoproblem + +handleRemoteProblem :: UrlRenderer -> Remote -> Assistant Bool +handleRemoteProblem urlrenderer rmt + | Git.repoIsLocal r && not (Git.repoIsLocalUnknown r) = + ifM (liftIO $ checkAvailable True rmt) + ( do + fixedlocks <- repairStaleGitLocks r + fsckresults <- showFscking urlrenderer (Just rmt) $ tryNonAsync $ + Git.Fsck.findBroken True r + repaired <- repairWhenNecessary urlrenderer (Remote.uuid rmt) (Just rmt) fsckresults + return $ fixedlocks || repaired + , return False + ) + | otherwise = return False + where + r = Remote.repo rmt + +{- This is not yet used, and should probably do a fsck. -} +handleLocalRepoProblem :: UrlRenderer -> Assistant Bool +handleLocalRepoProblem _urlrenderer = do + repairStaleGitLocks =<< liftAnnex gitRepo diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -13,6 +13,7 @@ import Assistant.DaemonStatus import Assistant.Sync import Utility.ThreadScheduler +import qualified Remote import qualified Types.Remote as Remote {- This thread retries pushes that failed before. -} @@ -42,7 +43,7 @@ - to avoid ugly messages when a removable drive is not attached. -} pushTargets :: Assistant [Remote] -pushTargets = liftIO . filterM available =<< candidates <$> getDaemonStatus +pushTargets = liftIO . filterM (Remote.checkAvailable True) + =<< candidates <$> getDaemonStatus where candidates = filter (not . Remote.readonly) . syncGitRemotes - available = maybe (return True) doesDirectoryExist . Remote.localpath diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -14,6 +14,7 @@ import Assistant.Common import Assistant.DaemonStatus import Assistant.Alert +import Assistant.Repair import qualified Git.LsFiles import qualified Git.Command import qualified Git.Config @@ -23,18 +24,25 @@ import Utility.Batch import Utility.NotificationBroadcaster import Config -import qualified Git -import qualified Utility.Lsof as Lsof +import Utility.HumanTime import Data.Time.Clock.POSIX {- This thread runs once at startup, and most other threads wait for it - to finish. (However, the webapp thread does not, to prevent the UI - being nonresponsive.) -} -sanityCheckerStartupThread :: NamedThread -sanityCheckerStartupThread = namedThreadUnchecked "SanityCheckerStartup" $ - startupCheck +sanityCheckerStartupThread :: Maybe Duration -> NamedThread +sanityCheckerStartupThread startupdelay = namedThreadUnchecked "SanityCheckerStartup" $ do + {- Stale git locks can prevent commits from happening, etc. -} + void $ repairStaleGitLocks =<< liftAnnex gitRepo + {- If there's a startup delay, it's done here. -} + liftIO $ maybe noop (threadDelaySeconds . Seconds . fromIntegral . durationSeconds) startupdelay + + {- Notify other threads that the startup sanity check is done. -} + status <- getDaemonStatus + liftIO $ sendNotification $ startupSanityCheckNotifier status + {- This thread wakes up hourly for inxepensive frequent sanity checks. -} sanityCheckerHourlyThread :: NamedThread sanityCheckerHourlyThread = namedThread "SanityCheckerHourly" $ forever $ do @@ -80,14 +88,6 @@ oneDay - truncate (now - lastcheck) | otherwise = oneDay -startupCheck :: Assistant () -startupCheck = do - checkStaleGitLocks - - {- Notify other threads that the startup sanity check is done. -} - status <- getDaemonStatus - liftIO $ sendNotification $ startupSanityCheckNotifier status - {- It's important to stay out of the Annex monad as much as possible while - running potentially expensive parts of this check, since remaining in it - will block the watcher. -} @@ -146,46 +146,6 @@ checkLogSize $ n + 1 where filesize f = fromIntegral . fileSize <$> liftIO (getFileStatus f) - -{- Detect when a git lock file exists and has no git process currently - - writing to it. This strongly suggests it is a stale lock file. - - - - However, this could be on a network filesystem. Which is not very safe - - anyway (the assistant relies on being able to check when files have - - no writers to know when to commit them). Just in case, when the lock - - file appears stale, we delay for one minute, and check its size. If - - the size changed, delay for another minute, and so on. This will at - - least work to detect is another machine is writing out a new index - - file, since git does so by writing the new content to index.lock. - -} -checkStaleGitLocks :: Assistant () -checkStaleGitLocks = do - lockfiles <- filter (not . isInfixOf "gc.pid") - . filter (".lock" `isSuffixOf`) - <$> (liftIO . dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) - =<< liftAnnex (fromRepo Git.localGitDir)) - checkStaleLocks lockfiles -checkStaleLocks :: [FilePath] -> Assistant () -checkStaleLocks lockfiles = go =<< getsizes - where - getsize lf = catchMaybeIO $ - (\s -> (lf, fileSize s)) <$> getFileStatus lf - getsizes = liftIO $ catMaybes <$> mapM getsize lockfiles - go [] = return () - go l = ifM (liftIO $ null <$> Lsof.query ("--" : map fst l)) - ( do - waitforit "to check stale git lock file" - l' <- getsizes - if l' == l - then liftIO $ mapM_ nukeFile (map fst l) - else go l' - , do - waitforit "for git lock file writer" - go =<< getsizes - ) - waitforit why = do - notice ["Waiting for 60 seconds", why] - liftIO $ threadDelaySeconds $ Seconds 60 oneMegabyte :: Int oneMegabyte = 1000000 diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -9,9 +9,7 @@ import Assistant.Common import Assistant.DaemonStatus -import Assistant.TransferQueue -import Assistant.Drop -import Annex.Content +import Assistant.TransferSlots import Logs.Transfer import Utility.DirWatcher import Utility.DirWatcher.Types @@ -98,28 +96,3 @@ - runs. -} threadDelay 10000000 -- 10 seconds finished t minfo - -{- Queue uploads of files downloaded to us, spreading them - - out to other reachable remotes. - - - - Downloading a file may have caused a remote to not want it; - - so check for drops from remotes. - - - - Uploading a file may cause the local repo, or some other remote to not - - want it; handle that too. - -} -finishedTransfer :: Transfer -> Maybe TransferInfo -> Assistant () -finishedTransfer t (Just info) - | transferDirection t == Download = - whenM (liftAnnex $ inAnnex $ transferKey t) $ do - dodrops False - queueTransfersMatching (/= transferUUID t) - "newly received object" - Later (transferKey t) (associatedFile info) Upload - | otherwise = dodrops True - where - dodrops fromhere = handleDrops - ("drop wanted after " ++ describeTransfer t info) - fromhere (transferKey t) (associatedFile info) Nothing -finishedTransfer _ _ = noop - diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -8,23 +8,10 @@ module Assistant.Threads.Transferrer where import Assistant.Common -import Assistant.DaemonStatus import Assistant.TransferQueue import Assistant.TransferSlots -import Assistant.Alert -import Assistant.Alert.Utility -import Assistant.Commits -import Assistant.Drop -import Assistant.TransferrerPool import Logs.Transfer -import Logs.Location -import Annex.Content -import qualified Remote -import qualified Types.Remote as Remote -import qualified Git import Config.Files -import Assistant.Threads.TransferWatcher -import Annex.Wanted {- Dispatches transfers from the queue. -} transfererThread :: NamedThread @@ -36,105 +23,3 @@ where {- Skip transfers that are already running. -} notrunning = isNothing . startedTime - -{- By the time this is called, the daemonstatus's currentTransfers map should - - already have been updated to include the transfer. -} -genTransfer :: Transfer -> TransferInfo -> TransferGenerator -genTransfer t info = case (transferRemote info, associatedFile info) of - (Just remote, Just file) - | Git.repoIsLocalUnknown (Remote.repo remote) -> do - -- optimisation for removable drives not plugged in - liftAnnex $ recordFailedTransfer t info - void $ removeTransfer t - return Nothing - | otherwise -> ifM (liftAnnex $ shouldTransfer t info) - ( do - debug [ "Transferring:" , describeTransfer t info ] - notifyTransfer - return $ Just (t, info, go remote file) - , do - debug [ "Skipping unnecessary transfer:", - describeTransfer t info ] - void $ removeTransfer t - finishedTransfer t (Just info) - return Nothing - ) - _ -> return Nothing - where - direction = transferDirection t - isdownload = direction == Download - - {- Alerts are only shown for successful transfers. - - Transfers can temporarily fail for many reasons, - - so there's no point in bothering the user about - - those. The assistant should recover. - - - - After a successful upload, handle dropping it from - - here, if desired. In this case, the remote it was - - uploaded to is known to have it. - - - - Also, after a successful transfer, the location - - log has changed. Indicate that a commit has been - - made, in order to queue a push of the git-annex - - branch out to remotes that did not participate - - in the transfer. - - - - If the process failed, it could have crashed, - - so remove the transfer from the list of current - - transfers, just in case it didn't stop - - in a way that lets the TransferWatcher do its - - usual cleanup. However, first check if something else is - - running the transfer, to avoid removing active transfers. - -} - go remote file transferrer = ifM (liftIO $ performTransfer transferrer t $ associatedFile info) - ( do - void $ addAlert $ makeAlertFiller True $ - transferFileAlert direction True file - unless isdownload $ - handleDrops - ("object uploaded to " ++ show remote) - True (transferKey t) - (associatedFile info) - (Just remote) - void recordCommit - , whenM (liftAnnex $ isNothing <$> checkTransfer t) $ - void $ removeTransfer t - ) - -{- Called right before a transfer begins, this is a last chance to avoid - - unnecessary transfers. - - - - For downloads, we obviously don't need to download if the already - - have the object. - - - - Smilarly, for uploads, check if the remote is known to already have - - the object. - - - - Also, uploads get queued to all remotes, in order of cost. - - This may mean, for example, that an object is uploaded over the LAN - - to a locally paired client, and once that upload is done, a more - - expensive transfer remote no longer wants the object. (Since - - all the clients have it already.) So do one last check if this is still - - preferred content. - - - - We'll also do one last preferred content check for downloads. An - - example of a case where this could be needed is if a download is queued - - for a file that gets moved out of an archive directory -- but before - - that download can happen, the file is put back in the archive. - -} -shouldTransfer :: Transfer -> TransferInfo -> Annex Bool -shouldTransfer t info - | transferDirection t == Download = - (not <$> inAnnex key) <&&> wantGet True file - | transferDirection t == Upload = case transferRemote info of - Nothing -> return False - Just r -> notinremote r - <&&> wantSend True file (Remote.uuid r) - | otherwise = return False - where - key = transferKey t - file = associatedFile info - - {- Trust the location log to check if the remote already has - - the key. This avoids a roundtrip to the remote. -} - notinremote r = notElem (Remote.uuid r) <$> loggedLocations key diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -9,7 +9,7 @@ module Assistant.Threads.Watcher ( watchThread, - WatcherException(..), + WatcherControl(..), checkCanWatch, needLsof, onAddSymlink, @@ -64,10 +64,10 @@ ] {- A special exception that can be thrown to pause or resume the watcher. -} -data WatcherException = PauseWatcher | ResumeWatcher +data WatcherControl = PauseWatcher | ResumeWatcher deriving (Show, Eq, Typeable) -instance E.Exception WatcherException +instance E.Exception WatcherControl watchThread :: NamedThread watchThread = namedThread "Watcher" $ @@ -107,7 +107,7 @@ where hook a = Just <$> asIO2 (runHandler a) -waitFor :: WatcherException -> Assistant () -> Assistant () +waitFor :: WatcherControl -> Assistant () -> Assistant () waitFor sig next = do r <- liftIO (E.try pause :: IO (Either E.SomeException ())) case r of diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs --- a/Assistant/TransferSlots.hs +++ b/Assistant/TransferSlots.hs @@ -13,11 +13,27 @@ import Assistant.DaemonStatus import Assistant.TransferrerPool import Assistant.Types.TransferrerPool +import Assistant.Types.TransferQueue +import Assistant.TransferQueue +import Assistant.Alert +import Assistant.Alert.Utility +import Assistant.Commits +import Assistant.Drop import Logs.Transfer +import Logs.Location +import qualified Git +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Content +import Annex.Wanted +import Config.Files +import qualified Data.Map as M import qualified Control.Exception as E import Control.Concurrent import qualified Control.Concurrent.MSemN as MSemN +import System.Posix.Signals (signalProcessGroup, sigTERM, sigKILL) +import System.Posix.Process (getProcessGroupIDOf) type TransferGenerator = Assistant (Maybe (Transfer, TransferInfo, Transferrer -> Assistant ())) @@ -76,3 +92,186 @@ _ -> done done = runAssistant d $ flip MSemN.signal 1 <<~ transferSlots + +{- By the time this is called, the daemonstatus's currentTransfers map should + - already have been updated to include the transfer. -} +genTransfer :: Transfer -> TransferInfo -> TransferGenerator +genTransfer t info = case (transferRemote info, associatedFile info) of + (Just remote, Just file) + | Git.repoIsLocalUnknown (Remote.repo remote) -> do + -- optimisation for removable drives not plugged in + liftAnnex $ recordFailedTransfer t info + void $ removeTransfer t + return Nothing + | otherwise -> ifM (liftAnnex $ shouldTransfer t info) + ( do + debug [ "Transferring:" , describeTransfer t info ] + notifyTransfer + return $ Just (t, info, go remote file) + , do + debug [ "Skipping unnecessary transfer:", + describeTransfer t info ] + void $ removeTransfer t + finishedTransfer t (Just info) + return Nothing + ) + _ -> return Nothing + where + direction = transferDirection t + isdownload = direction == Download + + {- Alerts are only shown for successful transfers. + - Transfers can temporarily fail for many reasons, + - so there's no point in bothering the user about + - those. The assistant should recover. + - + - After a successful upload, handle dropping it from + - here, if desired. In this case, the remote it was + - uploaded to is known to have it. + - + - Also, after a successful transfer, the location + - log has changed. Indicate that a commit has been + - made, in order to queue a push of the git-annex + - branch out to remotes that did not participate + - in the transfer. + - + - If the process failed, it could have crashed, + - so remove the transfer from the list of current + - transfers, just in case it didn't stop + - in a way that lets the TransferWatcher do its + - usual cleanup. However, first check if something else is + - running the transfer, to avoid removing active transfers. + -} + go remote file transferrer = ifM (liftIO $ performTransfer transferrer t $ associatedFile info) + ( do + void $ addAlert $ makeAlertFiller True $ + transferFileAlert direction True file + unless isdownload $ + handleDrops + ("object uploaded to " ++ show remote) + True (transferKey t) + (associatedFile info) + (Just remote) + void recordCommit + , whenM (liftAnnex $ isNothing <$> checkTransfer t) $ + void $ removeTransfer t + ) + +{- Called right before a transfer begins, this is a last chance to avoid + - unnecessary transfers. + - + - For downloads, we obviously don't need to download if the already + - have the object. + - + - Smilarly, for uploads, check if the remote is known to already have + - the object. + - + - Also, uploads get queued to all remotes, in order of cost. + - This may mean, for example, that an object is uploaded over the LAN + - to a locally paired client, and once that upload is done, a more + - expensive transfer remote no longer wants the object. (Since + - all the clients have it already.) So do one last check if this is still + - preferred content. + - + - We'll also do one last preferred content check for downloads. An + - example of a case where this could be needed is if a download is queued + - for a file that gets moved out of an archive directory -- but before + - that download can happen, the file is put back in the archive. + -} +shouldTransfer :: Transfer -> TransferInfo -> Annex Bool +shouldTransfer t info + | transferDirection t == Download = + (not <$> inAnnex key) <&&> wantGet True file + | transferDirection t == Upload = case transferRemote info of + Nothing -> return False + Just r -> notinremote r + <&&> wantSend True file (Remote.uuid r) + | otherwise = return False + where + key = transferKey t + file = associatedFile info + + {- Trust the location log to check if the remote already has + - the key. This avoids a roundtrip to the remote. -} + notinremote r = notElem (Remote.uuid r) <$> loggedLocations key + +{- Queue uploads of files downloaded to us, spreading them + - out to other reachable remotes. + - + - Downloading a file may have caused a remote to not want it; + - so check for drops from remotes. + - + - Uploading a file may cause the local repo, or some other remote to not + - want it; handle that too. + -} +finishedTransfer :: Transfer -> Maybe TransferInfo -> Assistant () +finishedTransfer t (Just info) + | transferDirection t == Download = + whenM (liftAnnex $ inAnnex $ transferKey t) $ do + dodrops False + queueTransfersMatching (/= transferUUID t) + "newly received object" + Later (transferKey t) (associatedFile info) Upload + | otherwise = dodrops True + where + dodrops fromhere = handleDrops + ("drop wanted after " ++ describeTransfer t info) + fromhere (transferKey t) (associatedFile info) Nothing +finishedTransfer _ _ = noop + +{- Pause a running transfer. -} +pauseTransfer :: Transfer -> Assistant () +pauseTransfer = cancelTransfer True + +{- Cancel a running transfer. -} +cancelTransfer :: Bool -> Transfer -> Assistant () +cancelTransfer pause t = do + m <- getCurrentTransfers + unless pause $ + {- remove queued transfer -} + void $ dequeueTransfers $ equivilantTransfer t + {- stop running transfer -} + maybe noop stop (M.lookup t m) + where + stop info = do + {- When there's a thread associated with the + - transfer, it's signaled first, to avoid it + - displaying any alert about the transfer having + - failed when the transfer process is killed. -} + liftIO $ maybe noop signalthread $ transferTid info + liftIO $ maybe noop killproc $ transferPid info + if pause + then void $ alterTransferInfo t $ + \i -> i { transferPaused = True } + else void $ removeTransfer t + signalthread tid + | pause = throwTo tid PauseTransfer + | otherwise = killThread tid + {- In order to stop helper processes like rsync, + - kill the whole process group of the process running the transfer. -} + killproc pid = void $ tryIO $ do + g <- getProcessGroupIDOf pid + void $ tryIO $ signalProcessGroup sigTERM g + threadDelay 50000 -- 0.05 second grace period + void $ tryIO $ signalProcessGroup sigKILL g + +{- Start or resume a transfer. -} +startTransfer :: Transfer -> Assistant () +startTransfer t = do + m <- getCurrentTransfers + maybe startqueued go (M.lookup t m) + where + go info = maybe (start info) resume $ transferTid info + startqueued = do + is <- map snd <$> getMatchingTransfers (== t) + maybe noop start $ headMaybe is + resume tid = do + alterTransferInfo t $ \i -> i { transferPaused = False } + liftIO $ throwTo tid ResumeTransfer + start info = do + program <- liftIO readProgramFile + inImmediateTransferSlot program $ + genTransfer t info + +getCurrentTransfers :: Assistant TransferMap +getCurrentTransfers = currentTransfers <$> getDaemonStatus diff --git a/Assistant/Types/Alert.hs b/Assistant/Types/Alert.hs --- a/Assistant/Types/Alert.hs +++ b/Assistant/Types/Alert.hs @@ -30,6 +30,7 @@ | RemoteRemovalAlert String | CloudRepoNeededAlert | SyncAlert + | NotFsckedAlert deriving (Eq) {- The first alert is the new alert, the second is an old alert. diff --git a/Assistant/Types/RepoProblem.hs b/Assistant/Types/RepoProblem.hs new file mode 100644 --- /dev/null +++ b/Assistant/Types/RepoProblem.hs @@ -0,0 +1,28 @@ +{- git-annex assistant repository problem tracking + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.RepoProblem where + +import Types +import Utility.TList + +import Control.Concurrent.STM +import Data.Function + +data RepoProblem = RepoProblem + { problemUUID :: UUID + , afterFix :: IO () + } + +{- The afterFix actions are assumed to all be equivilant. -} +sameRepoProblem :: RepoProblem -> RepoProblem -> Bool +sameRepoProblem = (==) `on` problemUUID + +type RepoProblemChan = TList RepoProblem + +newRepoProblemChan :: IO RepoProblemChan +newRepoProblemChan = atomically newTList diff --git a/Assistant/WebApp/Configurators/AWS.hs b/Assistant/WebApp/Configurators/AWS.hs --- a/Assistant/WebApp/Configurators/AWS.hs +++ b/Assistant/WebApp/Configurators/AWS.hs @@ -10,7 +10,7 @@ module Assistant.WebApp.Configurators.AWS where import Assistant.WebApp.Common -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote #ifdef WITH_S3 import qualified Remote.S3 as S3 #endif @@ -24,7 +24,6 @@ import Creds import Assistant.Gpg import Git.Remote -import Assistant.WebApp.Utility import qualified Data.Text as T import qualified Data.Map as M diff --git a/Assistant/WebApp/Configurators/Delete.hs b/Assistant/WebApp/Configurators/Delete.hs --- a/Assistant/WebApp/Configurators/Delete.hs +++ b/Assistant/WebApp/Configurators/Delete.hs @@ -11,9 +11,9 @@ import Assistant.WebApp.Common import Assistant.DeleteRemote -import Assistant.WebApp.Utility import Assistant.DaemonStatus import Assistant.ScanRemotes +import Assistant.Sync import qualified Remote import qualified Git import Config.Files @@ -91,9 +91,10 @@ {- Disable syncing to this repository, and all - remotes. This stops all transfers, and all - file watching. -} - changeSyncable Nothing False - rs <- liftAssistant $ syncRemotes <$> getDaemonStatus - mapM_ (\r -> changeSyncable (Just r) False) rs + liftAssistant $ do + changeSyncable Nothing False + rs <- syncRemotes <$> getDaemonStatus + mapM_ (\r -> changeSyncable (Just r) False) rs {- Make all directories writable, so all annexed - content can be deleted. -} diff --git a/Assistant/WebApp/Configurators/Edit.hs b/Assistant/WebApp/Configurators/Edit.hs --- a/Assistant/WebApp/Configurators/Edit.hs +++ b/Assistant/WebApp/Configurators/Edit.hs @@ -10,12 +10,12 @@ module Assistant.WebApp.Configurators.Edit where import Assistant.WebApp.Common -import Assistant.WebApp.Utility import Assistant.WebApp.Gpg import Assistant.DaemonStatus -import Assistant.MakeRemote (uniqueRemoteName) +import Assistant.WebApp.MakeRemote (uniqueRemoteName) import Assistant.WebApp.Configurators.XMPP (xmppNeeded) import Assistant.ScanRemotes +import Assistant.Sync import qualified Assistant.WebApp.Configurators.AWS as AWS import qualified Assistant.WebApp.Configurators.IA as IA #ifdef WITH_S3 @@ -124,7 +124,7 @@ Nothing -> addScanRemotes True =<< syncDataRemotes <$> getDaemonStatus when syncableChanged $ - changeSyncable mremote (repoSyncable newc) + liftAssistant $ changeSyncable mremote (repoSyncable newc) where syncableChanged = repoSyncable oldc /= repoSyncable newc associatedDirectoryChanged = repoAssociatedDirectory oldc /= repoAssociatedDirectory newc diff --git a/Assistant/WebApp/Configurators/Fsck.hs b/Assistant/WebApp/Configurators/Fsck.hs --- a/Assistant/WebApp/Configurators/Fsck.hs +++ b/Assistant/WebApp/Configurators/Fsck.hs @@ -22,6 +22,10 @@ import qualified Remote import Assistant.DaemonStatus import qualified Annex.Branch +import Assistant.Fsck +import Config +import Git.Config +import qualified Annex {- This adds a form to the page. It does not handle posting of the form, - because unlike a typical yesod form that posts using the same url @@ -41,7 +45,7 @@ - some Annex action on it. -} withFsckForm :: (ScheduledActivity -> Annex ()) -> Handler () withFsckForm a = do - ((res, _form), _enctype) <- runFsckForm False defaultFsck + ((res, _form), _enctype) <- runFsckForm False $ defaultFsck Nothing case res of FormSuccess activity -> liftAnnex $ a activity _ -> noop @@ -109,8 +113,9 @@ liftAnnex $ zip <$> (map T.pack <$> Remote.prettyListUUIDs us) <*> pure us -defaultFsck :: ScheduledActivity -defaultFsck = ScheduledSelfFsck (Schedule Daily AnyTime) (Duration $ 60*60) +defaultFsck :: Maybe Remote -> ScheduledActivity +defaultFsck Nothing = ScheduledSelfFsck (Schedule Daily AnyTime) (Duration $ 60*60) +defaultFsck (Just r) = ScheduledRemoteFsck (Remote.uuid r) (Schedule Daily AnyTime) (Duration $ 60*60) showFsckStatus :: ScheduledActivity -> Widget showFsckStatus activity = do @@ -122,7 +127,12 @@ getConfigFsckR = postConfigFsckR postConfigFsckR :: Handler Html postConfigFsckR = page "Consistency checks" (Just Configuration) $ do - checks <- liftAnnex $ S.toList <$> (scheduleGet =<< getUUID) + scheduledchecks <- liftAnnex $ + S.toList <$> (scheduleGet =<< getUUID) + rs <- liftAssistant $ + filter fsckableRemote . syncRemotes <$> getDaemonStatus + recommendedchecks <- liftAnnex $ map defaultFsck + <$> filterM (not <$$> checkFscked) (Nothing : map Just rs) $(widgetFile "configurators/fsck") changeSchedule :: Handler () -> Handler Html @@ -147,3 +157,39 @@ postChangeActivityR u oldactivity = changeSchedule $ withFsckForm $ \newactivity -> scheduleChange u $ S.insert newactivity . S.delete oldactivity + +data FsckPreferences = FsckPreferences + { enableFsckNudge :: Bool + } + +getFsckPreferences :: Annex FsckPreferences +getFsckPreferences = FsckPreferences + <$> (annexFsckNudge <$> Annex.getGitConfig) + +fsckPreferencesAForm :: FsckPreferences -> MkAForm FsckPreferences +fsckPreferencesAForm def = FsckPreferences + <$> areq (checkBoxField `withNote` nudgenote) "Reminders" (Just $ enableFsckNudge def) + where + nudgenote = [whamlet|Remind me when using repositories that lack consistency checks.|] + +runFsckPreferencesForm :: Handler ((FormResult FsckPreferences, Widget), Enctype) +runFsckPreferencesForm = do + prefs <- liftAnnex getFsckPreferences + runFormPostNoToken $ renderBootstrap $ fsckPreferencesAForm prefs + +showFsckPreferencesForm :: Widget +showFsckPreferencesForm = do + ((res, form), enctype) <- liftH $ runFsckPreferencesForm + case res of + FormSuccess _ -> noop + _ -> $(widgetFile "configurators/fsck/preferencesform") + +postConfigFsckPreferencesR :: Handler Html +postConfigFsckPreferencesR = do + ((res, _form), _enctype) <- runFsckPreferencesForm + case res of + FormSuccess prefs -> + liftAnnex $ setConfig (annexConfig "fscknudge") + (boolConfig $ enableFsckNudge prefs) + _ -> noop + redirect ConfigFsckR diff --git a/Assistant/WebApp/Configurators/IA.hs b/Assistant/WebApp/Configurators/IA.hs --- a/Assistant/WebApp/Configurators/IA.hs +++ b/Assistant/WebApp/Configurators/IA.hs @@ -14,7 +14,7 @@ #ifdef WITH_S3 import qualified Remote.S3 as S3 import qualified Remote.Helper.AWS as AWS -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote #endif import qualified Remote import qualified Types.Remote as Remote diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs --- a/Assistant/WebApp/Configurators/Local.hs +++ b/Assistant/WebApp/Configurators/Local.hs @@ -12,7 +12,7 @@ import Assistant.WebApp.Common import Assistant.WebApp.OtherRepos import Assistant.WebApp.Gpg -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote import Assistant.Sync import Init import qualified Git diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs --- a/Assistant/WebApp/Configurators/Ssh.hs +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -13,7 +13,7 @@ import Assistant.WebApp.Common import Assistant.WebApp.Gpg import Assistant.Ssh -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote import Logs.Remote import Remote import Types.StandardGroups @@ -21,7 +21,6 @@ import Utility.Gpg import Types.Remote (RemoteConfig) import Git.Remote -import Assistant.WebApp.Utility import qualified Remote.GCrypt as GCrypt import Annex.UUID import Logs.UUID @@ -390,7 +389,7 @@ remoteCommand = shellWrap $ intercalate "&&" $ catMaybes [ Just $ "mkdir -p " ++ shellEscape remotedir , Just $ "cd " ++ shellEscape remotedir - , if rsynconly then Nothing else Just "if [ ! -d .git ]; then git init --bare --shared; fi" + , if rsynconly then Nothing else Just "if [ ! -d .git ]; then git init --bare --shared && git config receive.denyNonFastforwards false; fi" , if rsynconly || newgcrypt then Nothing else Just "git annex init" , if needsPubKey origsshdata then addAuthorizedKeysCommand (hasCapability origsshdata GitAnnexShellCapable) remotedir . sshPubKey <$> keypair diff --git a/Assistant/WebApp/Configurators/WebDAV.hs b/Assistant/WebApp/Configurators/WebDAV.hs --- a/Assistant/WebApp/Configurators/WebDAV.hs +++ b/Assistant/WebApp/Configurators/WebDAV.hs @@ -13,12 +13,11 @@ import Creds #ifdef WITH_WEBDAV import qualified Remote.WebDAV as WebDAV -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote import qualified Remote import Types.Remote (RemoteConfig) import Types.StandardGroups import Logs.Remote -import Assistant.WebApp.Utility import Git.Remote import qualified Data.Map as M diff --git a/Assistant/WebApp/Control.hs b/Assistant/WebApp/Control.hs --- a/Assistant/WebApp/Control.hs +++ b/Assistant/WebApp/Control.hs @@ -13,8 +13,8 @@ import Config.Files import Utility.LogFile import Assistant.DaemonStatus -import Assistant.WebApp.Utility import Assistant.Alert +import Assistant.TransferSlots import Control.Concurrent import System.Posix (getProcessID, signalProcess, sigTERM) @@ -26,16 +26,16 @@ getShutdownConfirmedR :: Handler Html getShutdownConfirmedR = do - {- Remove all alerts for currently running activities. -} liftAssistant $ do + {- Remove all alerts for currently running activities. -} updateAlertMap $ M.filter $ \a -> alertClass a /= Activity void $ addAlert shutdownAlert - {- Stop transfers the assistant is running, - - otherwise they would continue past shutdown. - - Pausing transfers prevents more being started up (and stops - - the transfer processes). -} - ts <- liftAssistant $ M.keys . currentTransfers <$> getDaemonStatus - mapM_ pauseTransfer ts + {- Stop transfers the assistant is running, + - otherwise they would continue past shutdown. + - Pausing transfers prevents more being started up (and stops + - the transfer processes). -} + ts <- M.keys . currentTransfers <$> getDaemonStatus + mapM_ pauseTransfer ts page "Shutdown" Nothing $ do {- Wait 2 seconds before shutting down, to give the web - page time to load in the browser. -} diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -10,10 +10,10 @@ module Assistant.WebApp.DashBoard where import Assistant.WebApp.Common -import Assistant.WebApp.Utility import Assistant.WebApp.RepoList import Assistant.WebApp.Notifications import Assistant.TransferQueue +import Assistant.TransferSlots import Assistant.DaemonStatus import Utility.NotificationBroadcaster import Logs.Transfer @@ -31,7 +31,7 @@ transfersDisplay :: Bool -> Widget transfersDisplay warnNoScript = do webapp <- liftH getYesod - current <- liftH $ M.toList <$> getCurrentTransfers + current <- liftAssistant $ M.toList <$> getCurrentTransfers queued <- take 10 <$> liftAssistant getTransferQueue autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int) let transfers = simplifyTransfers $ current ++ queued @@ -139,15 +139,15 @@ getPauseTransferR :: Transfer -> Handler () getPauseTransferR = noscript postPauseTransferR postPauseTransferR :: Transfer -> Handler () -postPauseTransferR = pauseTransfer +postPauseTransferR = liftAssistant . pauseTransfer getStartTransferR :: Transfer -> Handler () getStartTransferR = noscript postStartTransferR postStartTransferR :: Transfer -> Handler () -postStartTransferR = startTransfer +postStartTransferR = liftAssistant . startTransfer getCancelTransferR :: Transfer -> Handler () getCancelTransferR = noscript postCancelTransferR postCancelTransferR :: Transfer -> Handler () -postCancelTransferR = cancelTransfer False +postCancelTransferR = liftAssistant . cancelTransfer False noscript :: (Transfer -> Handler ()) -> Transfer -> Handler () noscript a t = a t >> redirectBack diff --git a/Assistant/WebApp/Gpg.hs b/Assistant/WebApp/Gpg.hs --- a/Assistant/WebApp/Gpg.hs +++ b/Assistant/WebApp/Gpg.hs @@ -18,7 +18,7 @@ import qualified Annex.Branch import qualified Git.GCrypt import qualified Remote.GCrypt as GCrypt -import Assistant.MakeRemote +import Assistant.WebApp.MakeRemote import Logs.Remote import qualified Data.Map as M diff --git a/Assistant/WebApp/MakeRemote.hs b/Assistant/WebApp/MakeRemote.hs new file mode 100644 --- /dev/null +++ b/Assistant/WebApp/MakeRemote.hs @@ -0,0 +1,36 @@ +{- git-annex assistant webapp making remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.MakeRemote ( + module Assistant.MakeRemote, + module Assistant.WebApp.MakeRemote +) where + +import Assistant.Common +import Assistant.WebApp.Types +import Assistant.Sync +import qualified Remote +import qualified Config +import Config.Cost +import Types.StandardGroups +import Git.Remote +import Logs.PreferredContent +import Assistant.MakeRemote + +import Utility.Yesod + +{- Runs an action that creates or enables a cloud remote, + - and finishes setting it up, then starts syncing with it, + - and finishes by displaying the page to edit it. -} +setupCloudRemote :: StandardGroup -> Maybe Cost -> Annex RemoteName -> Handler a +setupCloudRemote defaultgroup mcost maker = do + r <- liftAnnex $ addRemote maker + liftAnnex $ do + setStandardGroup (Remote.uuid r) defaultgroup + maybe noop (Config.setRemoteCost r) mcost + liftAssistant $ syncRemote r + redirect $ EditNewCloudRepositoryR $ Remote.uuid r diff --git a/Assistant/WebApp/Repair.hs b/Assistant/WebApp/Repair.hs --- a/Assistant/WebApp/Repair.hs +++ b/Assistant/WebApp/Repair.hs @@ -10,71 +10,26 @@ module Assistant.WebApp.Repair where import Assistant.WebApp.Common -import Assistant.WebApp.Utility import Assistant.WebApp.RepoList -import Remote (prettyUUID) -import Command.Repair (repairAnnexBranch) -import Git.Repair (runRepairOf) -import Logs.FsckResults -import Annex.UUID -import Utility.Batch -import Config.Files - -import Control.Concurrent.Async +import Remote (prettyUUID, remoteFromUUID) +import Annex.UUID (getUUID) +import Assistant.Repair getRepairRepositoryR :: UUID -> Handler Html getRepairRepositoryR = postRepairRepositoryR postRepairRepositoryR :: UUID -> Handler Html postRepairRepositoryR u = page "Repair repository" Nothing $ do repodesc <- liftAnnex $ prettyUUID u + repairingmainrepo <- (==) u <$> liftAnnex getUUID $(widgetFile "control/repairrepository") getRepairRepositoryRunR :: UUID -> Handler Html getRepairRepositoryRunR = postRepairRepositoryRunR postRepairRepositoryRunR :: UUID -> Handler Html postRepairRepositoryRunR u = do - -- Stop the watcher from running while running repairs. - changeSyncable Nothing False - - fsckthread <- liftAssistant $ runRepair u - - -- Start the watcher running again. This also triggers it to do a - -- startup scan, which is especially important if the git repo - -- repair removed files from the index file. Those files will be - -- seen as new, and re-added to the repository. - changeSyncable Nothing True - - liftAnnex $ writeFsckResults u Nothing - + r <- liftAnnex $ remoteFromUUID u + void $ liftAssistant $ runRepair u r True page "Repair repository" Nothing $ do let repolist = repoListDisplay $ mainRepoSelector { nudgeAddMore = True } $(widgetFile "control/repairrepository/done") - -runRepair :: UUID -> Assistant () -runRepair u = do - fsckresults <- liftAnnex (readFsckResults u) - myu <- liftAnnex getUUID - if u == myu - then localrepair fsckresults - else remoterepair fsckresults - where - localrepair fsckresults = do - -- This intentionally runs the repair inside the Annex - -- monad, which is not stricktly necessary, but keeps - -- other threads that might be trying to use the Annex - -- from running until it completes. - needfsck <- liftAnnex $ do - (ok, stillmissing, modifiedbranches) <- inRepo $ - runRepairOf fsckresults True - repairAnnexBranch stillmissing modifiedbranches - return (not ok) - when needfsck $ - backgroundfsck [ Param "--fast" ] - - remoterepair fsckresults = do - error "TODO: remote repair" - - backgroundfsck params = liftIO $ void $ async $ do - program <- readProgramFile - batchCommand program (Param "fsck" : params) diff --git a/Assistant/WebApp/RepoList.hs b/Assistant/WebApp/RepoList.hs --- a/Assistant/WebApp/RepoList.hs +++ b/Assistant/WebApp/RepoList.hs @@ -12,7 +12,6 @@ import Assistant.WebApp.Common import Assistant.DaemonStatus import Assistant.WebApp.Notifications -import Assistant.WebApp.Utility import Assistant.Ssh import qualified Annex import qualified Remote @@ -208,7 +207,7 @@ flipSync :: Bool -> UUID -> Handler () flipSync enable uuid = do mremote <- liftAnnex $ Remote.remoteFromUUID uuid - changeSyncable mremote enable + liftAssistant $ changeSyncable mremote enable redirectBack getRepositoriesReorderR :: Handler () diff --git a/Assistant/WebApp/Utility.hs b/Assistant/WebApp/Utility.hs deleted file mode 100644 --- a/Assistant/WebApp/Utility.hs +++ /dev/null @@ -1,138 +0,0 @@ -{- git-annex assistant webapp utilities - - - - Copyright 2012 Joey Hess - - - - Licensed under the GNU AGPL version 3 or higher. - -} - -module Assistant.WebApp.Utility where - -import Assistant.Common -import Assistant.WebApp.Types -import Assistant.DaemonStatus -import Assistant.TransferQueue -import Assistant.Types.TransferSlots -import Assistant.TransferSlots -import Assistant.Sync -import qualified Remote -import qualified Types.Remote as Remote -import qualified Remote.List as Remote -import qualified Assistant.Threads.Transferrer as Transferrer -import Logs.Transfer -import qualified Config -import Config.Cost -import Config.Files -import Git.Config -import Assistant.Threads.Watcher -import Assistant.NamedThread -import Types.StandardGroups -import Git.Remote -import Logs.PreferredContent -import Assistant.MakeRemote - -import qualified Data.Map as M -import Control.Concurrent -import System.Posix.Signals (signalProcessGroup, sigTERM, sigKILL) -import System.Posix.Process (getProcessGroupIDOf) -import Utility.Yesod - -{- Use Nothing to change autocommit setting; or a remote to change - - its sync setting. -} -changeSyncable :: Maybe Remote -> Bool -> Handler () -changeSyncable Nothing enable = do - liftAnnex $ Config.setConfig key (boolConfig enable) - liftIO . maybe noop (`throwTo` signal) - =<< liftAssistant (namedThreadId watchThread) - where - key = Config.annexConfig "autocommit" - signal - | enable = ResumeWatcher - | otherwise = PauseWatcher -changeSyncable (Just r) True = do - changeSyncFlag r True - liftAssistant $ syncRemote r -changeSyncable (Just r) False = do - changeSyncFlag r False - liftAssistant updateSyncRemotes - {- Stop all transfers to or from this remote. - - XXX Can't stop any ongoing scan, or git syncs. -} - void $ liftAssistant $ dequeueTransfers tofrom - mapM_ (cancelTransfer False) =<< - filter tofrom . M.keys <$> - liftAssistant (currentTransfers <$> getDaemonStatus) - where - tofrom t = transferUUID t == Remote.uuid r - -changeSyncFlag :: Remote -> Bool -> Handler () -changeSyncFlag r enabled = liftAnnex $ do - Config.setConfig key (boolConfig enabled) - void Remote.remoteListRefresh - where - key = Config.remoteConfig (Remote.repo r) "sync" - -pauseTransfer :: Transfer -> Handler () -pauseTransfer = cancelTransfer True - -cancelTransfer :: Bool -> Transfer -> Handler () -cancelTransfer pause t = do - m <- getCurrentTransfers - unless pause $ - {- remove queued transfer -} - void $ liftAssistant $ dequeueTransfers $ equivilantTransfer t - {- stop running transfer -} - maybe noop stop (M.lookup t m) - where - stop info = liftAssistant $ do - {- When there's a thread associated with the - - transfer, it's signaled first, to avoid it - - displaying any alert about the transfer having - - failed when the transfer process is killed. -} - liftIO $ maybe noop signalthread $ transferTid info - liftIO $ maybe noop killproc $ transferPid info - if pause - then void $ alterTransferInfo t $ - \i -> i { transferPaused = True } - else void $ removeTransfer t - signalthread tid - | pause = throwTo tid PauseTransfer - | otherwise = killThread tid - {- In order to stop helper processes like rsync, - - kill the whole process group of the process running the transfer. -} - killproc pid = void $ tryIO $ do - g <- getProcessGroupIDOf pid - void $ tryIO $ signalProcessGroup sigTERM g - threadDelay 50000 -- 0.05 second grace period - void $ tryIO $ signalProcessGroup sigKILL g - -startTransfer :: Transfer -> Handler () -startTransfer t = do - m <- getCurrentTransfers - maybe startqueued go (M.lookup t m) - where - go info = maybe (start info) resume $ transferTid info - startqueued = do - is <- liftAssistant $ map snd <$> getMatchingTransfers (== t) - maybe noop start $ headMaybe is - resume tid = do - liftAssistant $ alterTransferInfo t $ - \i -> i { transferPaused = False } - liftIO $ throwTo tid ResumeTransfer - start info = liftAssistant $ do - program <- liftIO readProgramFile - inImmediateTransferSlot program $ - Transferrer.genTransfer t info - -getCurrentTransfers :: Handler TransferMap -getCurrentTransfers = currentTransfers <$> liftAssistant getDaemonStatus - -{- Runs an action that creates or enables a cloud remote, - - and finishes setting it up, then starts syncing with it, - - and finishes by displaying the page to edit it. -} -setupCloudRemote :: StandardGroup -> Maybe Cost -> Annex RemoteName -> Handler a -setupCloudRemote defaultgroup mcost maker = do - r <- liftAnnex $ addRemote maker - liftAnnex $ do - setStandardGroup (Remote.uuid r) defaultgroup - maybe noop (Config.setRemoteCost r) mcost - liftAssistant $ syncRemote r - redirect $ EditNewCloudRepositoryR $ Remote.uuid r diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes --- a/Assistant/WebApp/routes +++ b/Assistant/WebApp/routes @@ -20,6 +20,7 @@ /config/xmpp/for/frield XMPPConfigForPairFriendR GET POST /config/xmpp/needcloudrepo/#UUID NeedCloudRepoR GET /config/fsck ConfigFsckR GET POST +/config/fsck/preferences ConfigFsckPreferencesR POST /config/addrepository AddRepositoryR GET /config/repository/new NewRepositoryR GET POST diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,35 @@ +git-annex (4.20131101) unstable; urgency=low + + * The "git annex content" command is renamed to "git annex wanted". + * New --want-get and --want-drop options which can be used to + test preferred content settings. + For example, "git annex find --in . --want-drop" + * assistant: When autostarted, wait 5 seconds before running the startup + scan, to avoid contending with the user's desktop login process. + * webapp: When setting up a bare shared repository, enable non-fast-forward + pushes. + * sync: Show a hint about receive.denyNonFastForwards when a push fails. + * directory, webdav: Fix bug introduced in version 4.20131002 that + caused the chunkcount file to not be written. Work around repositories + without such a file, so files can still be retreived from them. + * assistant: Automatically repair damanged git repository, if it can + be done without losing data. + * assistant: Support repairing git remotes that are locally accessible + (eg, on removable drives). + * add: Fix reversion in 4.20130827 when adding unlocked files that have + not yet been committed. + * unannex: New, much slower, but more safe behavior: Copies files out of + the annex. This avoids an unannex of one file breaking other files that + link to the same content. Also, it means that the content + remains in the annex using up space until cleaned up with + "git annex unused". + (The behavior of unannex --fast has not changed; it still hard links + to content in the annex. --fast was not made the default because it is + potentially unsafe; editing such a hard linked file can unexpectedly + change content stored in the annex.) + + -- Joey Hess Fri, 01 Nov 2013 11:34:27 -0400 + git-annex (4.20131024) unstable; urgency=low * webapp: Fix bug when adding a remote and git-remote-gcrypt diff --git a/Command/Assistant.hs b/Command/Assistant.hs --- a/Command/Assistant.hs +++ b/Command/Assistant.hs @@ -14,43 +14,55 @@ import Init import Config.Files import qualified Build.SysConfig +import Utility.HumanTime import System.Environment def :: [Command] -def = [noRepo checkAutoStart $ dontCheck repoExists $ - withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption, autoStartOption] $ +def = [noRepo checkAutoStart $ dontCheck repoExists $ withOptions options $ command "assistant" paramNothing seek SectionCommon "automatically handle changes"] +options :: [Option] +options = + [ Command.Watch.foregroundOption + , Command.Watch.stopOption + , autoStartOption + , startDelayOption + ] + autoStartOption :: Option autoStartOption = Option.flag [] "autostart" "start in known repositories" +startDelayOption :: Option +startDelayOption = Option.field [] "startdelay" paramNumber "delay before running startup scan" + seek :: [CommandSeek] seek = [withFlag Command.Watch.stopOption $ \stopdaemon -> withFlag Command.Watch.foregroundOption $ \foreground -> withFlag autoStartOption $ \autostart -> - withNothing $ start foreground stopdaemon autostart] + withField startDelayOption (pure . maybe Nothing parseDuration) $ \startdelay -> + withNothing $ start foreground stopdaemon autostart startdelay] -start :: Bool -> Bool -> Bool -> CommandStart -start foreground stopdaemon autostart +start :: Bool -> Bool -> Bool -> Maybe Duration -> CommandStart +start foreground stopdaemon autostart startdelay | autostart = do - liftIO autoStart + liftIO $ autoStart startdelay stop | otherwise = do ensureInitialized - Command.Watch.start True foreground stopdaemon + Command.Watch.start True foreground stopdaemon startdelay {- Run outside a git repository. Check to see if any parameter is - --autostart and enter autostart mode. -} checkAutoStart :: IO () checkAutoStart = ifM (elem "--autostart" <$> getArgs) - ( autoStart + ( autoStart Nothing , error "Not in a git repository." ) -autoStart :: IO () -autoStart = do +autoStart :: Maybe Duration -> IO () +autoStart startdelay = do dirs <- liftIO readAutoStartFile when (null dirs) $ do f <- autoStartFile @@ -67,5 +79,10 @@ go haveionice program dir = do setCurrentDirectory dir if haveionice - then boolSystem "ionice" [Param "-c3", Param program, Param "assistant"] - else boolSystem program [Param "assistant"] + then boolSystem "ionice" (Param "-c3" : Param program : baseparams) + else boolSystem program baseparams + where + baseparams = + [ Param "assistant" + , Param $ "--startdelay=" ++ fromDuration (fromMaybe (Duration 5) startdelay) + ] diff --git a/Command/Content.hs b/Command/Content.hs deleted file mode 100644 --- a/Command/Content.hs +++ /dev/null @@ -1,48 +0,0 @@ -{- git-annex command - - - - Copyright 2013 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Command.Content where - -import Common.Annex -import Command -import qualified Remote -import Logs.PreferredContent - -import qualified Data.Map as M - -def :: [Command] -def = [command "content" (paramPair paramRemote (paramOptional paramExpression)) seek - SectionSetup "get or set preferred content expression"] - -seek :: [CommandSeek] -seek = [withWords start] - -start :: [String] -> CommandStart -start = parse - where - parse (name:[]) = go name performGet - parse (name:expr:[]) = go name $ \uuid -> do - showStart "content" name - performSet expr uuid - parse _ = error "Specify a repository." - - go name a = do - u <- Remote.nameToUUID name - next $ a u - -performGet :: UUID -> CommandPerform -performGet uuid = do - m <- preferredContentMapRaw - liftIO $ putStrLn $ fromMaybe "" $ M.lookup uuid m - next $ return True - -performSet :: String -> UUID -> CommandPerform -performSet expr uuid = case checkPreferredContentExpression expr of - Just e -> error $ "Parse error: " ++ e - Nothing -> do - preferredContentSet uuid expr - next $ return True diff --git a/Command/Drop.hs b/Command/Drop.hs --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -139,7 +139,7 @@ unsafe = showNote "unsafe" hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" -{- In auto mode, only runs the action if there are enough copies +{- In auto mode, only runs the action if there are enough - copies on other semitrusted repositories. - - Passes any numcopies attribute of the file on to the action as an diff --git a/Command/Status.hs b/Command/Status.hs --- a/Command/Status.hs +++ b/Command/Status.hs @@ -126,7 +126,7 @@ , bad_data_size , local_annex_keys , local_annex_size - , known_annex_keys + , known_annex_files , known_annex_size , bloom_info , backend_usage @@ -136,7 +136,7 @@ [ local_dir , const local_annex_keys , const local_annex_size - , const known_annex_keys + , const known_annex_files , const known_annex_size ] local_slow_stats :: [FilePath -> Stat] @@ -183,22 +183,22 @@ local_dir :: FilePath -> Stat local_dir dir = stat "directory" $ json id $ return dir +local_annex_keys :: Stat +local_annex_keys = stat "local annex keys" $ json show $ + countKeys <$> cachedPresentData + local_annex_size :: Stat local_annex_size = stat "local annex size" $ json id $ showSizeKeys <$> cachedPresentData -local_annex_keys :: Stat -local_annex_keys = stat "local annex keys" $ json show $ - countKeys <$> cachedPresentData +known_annex_files :: Stat +known_annex_files = stat "annexed files in working tree" $ json show $ + countKeys <$> cachedReferencedData known_annex_size :: Stat -known_annex_size = stat "known annex size" $ json id $ +known_annex_size = stat "size of annexed files in working tree" $ json id $ showSizeKeys <$> cachedReferencedData -known_annex_keys :: Stat -known_annex_keys = stat "known annex keys" $ json show $ - countKeys <$> cachedReferencedData - tmp_size :: Stat tmp_size = staleSize "temporary directory size" gitAnnexTmpDir @@ -360,7 +360,7 @@ | unknownSizeKeys d == 0 = "" | otherwise = aside $ "+ " ++ show (unknownSizeKeys d) ++ - " keys of unknown size" + " unknown size" staleSize :: String -> (Git.Repo -> FilePath) -> Stat staleSize label dirspec = go =<< lift (dirKeys dirspec) diff --git a/Command/Sync.hs b/Command/Sync.hs --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -184,7 +184,11 @@ showStart "push" (Remote.name remote) next $ next $ do showOutput - inRepo $ pushBranch remote branch + ok <- inRepo $ pushBranch remote branch + unless ok $ do + warning $ unwords [ "Pushing to " ++ Remote.name remote ++ " failed." ] + showLongNote "(non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)" + return ok {- Pushes a regular branch like master to a remote. Also pushes the git-annex - branch. @@ -211,6 +215,9 @@ - But overwriting of data on synced/git-annex can happen, in a race. - The only difference caused by using a forced push in that case is that - the last repository to push wins the race, rather than the first to push. + - + - The sync push will fail to overwrite if receive.denyNonFastforwards is + - set on the remote. -} pushBranch :: Remote -> Git.Ref -> Git.Repo -> IO Bool pushBranch remote branch g = tryIO (directpush g) `after` syncpush g diff --git a/Command/Unannex.hs b/Command/Unannex.hs --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -13,11 +13,11 @@ import Command import Config import qualified Annex -import Logs.Location import Annex.Content import Annex.Content.Direct import qualified Git.Command import qualified Git.LsFiles as LsFiles +import Utility.CopyFile def :: [Command] def = [command "unannex" paramPaths seek SectionUtility @@ -60,28 +60,24 @@ cleanupIndirect :: FilePath -> Key -> CommandCleanup cleanupIndirect file key = do + src <- calcRepo $ gitAnnexLocation key ifM (Annex.getState Annex.fast) - ( goFast - , go + ( hardlinkfrom src + , copyfrom src ) - return True where -#ifdef mingw32_HOST_OS - goFast = go -#else - goFast = do - -- fast mode: hard link to content in annex - src <- calcRepo $ gitAnnexLocation key - -- creating a hard link could fall; fall back to non fast mode + copyfrom src = + thawContent file `after` liftIO (copyFileExternal src file) + hardlinkfrom src = +#ifndef mingw32_HOST_OS + -- creating a hard link could fall; fall back to copying ifM (liftIO $ catchBoolIO $ createLink src file >> return True) - ( thawContent file - , go + ( return True + , copyfrom src ) +#else + copyfrom src #endif - go = do - fromAnnex key file - logStatus key InfoMissing - performDirect :: FilePath -> Key -> CommandPerform performDirect file key = do diff --git a/Command/Uninit.hs b/Command/Uninit.hs --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -11,7 +11,6 @@ import Command import qualified Git import qualified Git.Command -import qualified Annex import qualified Command.Unannex import Init import qualified Annex.Branch @@ -38,7 +37,7 @@ seek :: [CommandSeek] seek = [ withFilesNotInGit $ whenAnnexed startCheckIncomplete - , withFilesInGit $ whenAnnexed startUnannex + , withFilesInGit $ whenAnnexed Command.Unannex.start , withNothing start ] @@ -50,14 +49,6 @@ , "Perhaps this was left behind by an interrupted git annex add?" , "Not continuing with uninit; either delete or git annex add the file and retry." ] - -startUnannex :: FilePath -> (Key, Backend) -> CommandStart -startUnannex file info = do - -- Force fast mode before running unannex. This way, if multiple - -- files link to a key, it will be left in the annex and hardlinked - -- to by each. - Annex.changeState $ \s -> s { Annex.fast = True } - Command.Unannex.start file info start :: CommandStart start = next $ next $ do diff --git a/Command/Wanted.hs b/Command/Wanted.hs new file mode 100644 --- /dev/null +++ b/Command/Wanted.hs @@ -0,0 +1,48 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Wanted where + +import Common.Annex +import Command +import qualified Remote +import Logs.PreferredContent + +import qualified Data.Map as M + +def :: [Command] +def = [command "wanted" (paramPair paramRemote (paramOptional paramExpression)) seek + SectionSetup "get or set preferred content expression"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start = parse + where + parse (name:[]) = go name performGet + parse (name:expr:[]) = go name $ \uuid -> do + showStart "wanted" name + performSet expr uuid + parse _ = error "Specify a repository." + + go name a = do + u <- Remote.nameToUUID name + next $ a u + +performGet :: UUID -> CommandPerform +performGet uuid = do + m <- preferredContentMapRaw + liftIO $ putStrLn $ fromMaybe "" $ M.lookup uuid m + next $ return True + +performSet :: String -> UUID -> CommandPerform +performSet expr uuid = case checkPreferredContentExpression expr of + Just e -> error $ "Parse error: " ++ e + Nothing -> do + preferredContentSet uuid expr + next $ return True diff --git a/Command/Watch.hs b/Command/Watch.hs --- a/Command/Watch.hs +++ b/Command/Watch.hs @@ -11,6 +11,7 @@ import Assistant import Command import Option +import Utility.HumanTime def :: [Command] def = [notBareRepo $ withOptions [foregroundOption, stopOption] $ @@ -19,7 +20,7 @@ seek :: [CommandSeek] seek = [withFlag stopOption $ \stopdaemon -> withFlag foregroundOption $ \foreground -> - withNothing $ start False foreground stopdaemon] + withNothing $ start False foreground stopdaemon Nothing] foregroundOption :: Option foregroundOption = Option.flag [] "foreground" "do not daemonize" @@ -27,9 +28,9 @@ stopOption :: Option stopOption = Option.flag [] "stop" "stop daemon" -start :: Bool -> Bool -> Bool -> CommandStart -start assistant foreground stopdaemon = do +start :: Bool -> Bool -> Bool -> Maybe Duration -> CommandStart +start assistant foreground stopdaemon startdelay = do if stopdaemon then stopDaemon - else startDaemon assistant foreground Nothing Nothing -- does not return + else startDaemon assistant foreground startdelay Nothing Nothing -- does not return stop diff --git a/Command/WebApp.hs b/Command/WebApp.hs --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -69,7 +69,7 @@ url <- liftIO . readFile =<< fromRepo gitAnnexUrlFile liftIO $ openBrowser browser f url Nothing Nothing - , startDaemon True True listenhost $ Just $ + , startDaemon True True Nothing listenhost $ Just $ \origout origerr url htmlshim -> if isJust listenhost then maybe noop (`hPutStrLn` url) origout @@ -155,7 +155,7 @@ _wait <- takeMVar v state <- Annex.new =<< Git.CurrentRepo.get Annex.eval state $ - startDaemon True True listenhost $ Just $ + startDaemon True True Nothing listenhost $ Just $ sendurlback v sendurlback v _origout _origerr url _htmlshim = do recordUrl url diff --git a/Git/Repair.hs b/Git/Repair.hs --- a/Git/Repair.hs +++ b/Git/Repair.hs @@ -124,9 +124,13 @@ {- Try to retrieve a set of missing objects, from the remotes of a - repository. Returns any that could not be retreived. + - + - If another clone of the repository exists locally, which might not be a + - remote of the repo being repaired, its path can be passed as a reference + - repository. -} -retrieveMissingObjects :: MissingObjects -> Repo -> IO MissingObjects -retrieveMissingObjects missing r +retrieveMissingObjects :: MissingObjects -> Maybe FilePath -> Repo -> IO MissingObjects +retrieveMissingObjects missing referencerepo r | S.null missing = return missing | otherwise = withTmpDir "tmprepo" $ \tmpdir -> do unlessM (boolSystem "git" [Params "init", File tmpdir]) $ @@ -137,12 +141,19 @@ then return stillmissing else pullremotes tmpr (remotes r) fetchallrefs stillmissing where - pullremotes _tmpr [] _ stillmissing = return stillmissing + pullremotes tmpr [] fetchrefs stillmissing = case referencerepo of + Nothing -> return stillmissing + Just p -> ifM (fetchfrom p fetchrefs tmpr) + ( do + void $ copyObjects tmpr r + findMissing (S.toList stillmissing) r + , return stillmissing + ) pullremotes tmpr (rmt:rmts) fetchrefs s | S.null s = return s | otherwise = do putStrLn $ "Trying to recover missing objects from remote " ++ repoDescribe rmt - ifM (fetchsome rmt fetchrefs tmpr) + ifM (fetchfrom (repoLocation rmt) fetchrefs tmpr) ( do void $ copyObjects tmpr r stillmissing <- findMissing (S.toList s) r @@ -155,9 +166,9 @@ ] pullremotes tmpr rmts fetchrefs s ) - fetchsome rmt ps = runBool $ + fetchfrom fetchurl ps = runBool $ [ Param "fetch" - , Param (repoLocation rmt) + , Param fetchurl , Params "--force --update-head-ok --quiet" ] ++ ps -- fetch refs and tags @@ -427,14 +438,15 @@ putStrLn "Running git fsck ..." fsckresult <- findBroken False g if foundBroken fsckresult - then runRepairOf fsckresult forced g + then runRepairOf fsckresult forced Nothing g else do putStrLn "No problems found." return (True, S.empty, []) -runRepairOf :: FsckResults -> Bool -> Repo -> IO (Bool, MissingObjects, [Branch]) -runRepairOf fsckresult forced g = do + +runRepairOf :: FsckResults -> Bool -> Maybe FilePath -> Repo -> IO (Bool, MissingObjects, [Branch]) +runRepairOf fsckresult forced referencerepo g = do missing <- cleanCorruptObjects fsckresult g - stillmissing <- retrieveMissingObjects missing g + stillmissing <- retrieveMissingObjects missing referencerepo g if S.null stillmissing then successfulfinish stillmissing [] else do diff --git a/GitAnnex.hs b/GitAnnex.hs --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -54,7 +54,7 @@ import qualified Command.Semitrust import qualified Command.Dead import qualified Command.Group -import qualified Command.Content +import qualified Command.Wanted import qualified Command.Schedule import qualified Command.Ungroup import qualified Command.Vicfg @@ -118,7 +118,7 @@ , Command.Semitrust.def , Command.Dead.def , Command.Group.def - , Command.Content.def + , Command.Wanted.def , Command.Schedule.def , Command.Ungroup.def , Command.Vicfg.def diff --git a/GitAnnex/Options.hs b/GitAnnex/Options.hs --- a/GitAnnex/Options.hs +++ b/GitAnnex/Options.hs @@ -16,6 +16,7 @@ import qualified Annex import qualified Remote import qualified Limit +import qualified Limit.Wanted import qualified Option options :: [Option] @@ -33,19 +34,23 @@ , Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob) "skip files matching the glob pattern" , Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob) - "don't skip files matching the glob pattern" + "limit to files matching the glob pattern" , Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote) - "skip files not present in a remote" + "match files present in a remote" , Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber) "skip files with fewer copies" , Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName) - "skip files not using a key-value backend" + "match files using a key-value backend" , Option [] ["inallgroup"] (ReqArg Limit.addInAllGroup paramGroup) - "skip files not present in all remotes in a group" + "match files present in all remotes in a group" , Option [] ["largerthan"] (ReqArg Limit.addLargerThan paramSize) - "skip files larger than a size" + "match files larger than a size" , Option [] ["smallerthan"] (ReqArg Limit.addSmallerThan paramSize) - "skip files smaller than a size" + "match files smaller than a size" + , Option [] ["want-get"] (NoArg Limit.Wanted.addWantGet) + "match files the repository wants to get" + , Option [] ["want-drop"] (NoArg Limit.Wanted.addWantDrop) + "match files the repository wants to drop" , Option ['T'] ["time-limit"] (ReqArg Limit.addTimeLimit paramTime) "stop after the specified amount of time" , Option [] ["user-agent"] (ReqArg setuseragent paramName) diff --git a/Limit.hs b/Limit.hs --- a/Limit.hs +++ b/Limit.hs @@ -27,6 +27,7 @@ import Types.Key import Types.Group import Types.FileMatcher +import Types.Limit import Logs.Group import Utility.HumanTime import Utility.DataUnits @@ -40,10 +41,6 @@ import Types.FileMatcher #endif #endif - -type MatchFiles = AssumeNotPresent -> FileInfo -> Annex Bool -type MkLimit = String -> Either String MatchFiles -type AssumeNotPresent = S.Set UUID {- Checks if there are user-specified limits. -} limited :: Annex Bool diff --git a/Limit/Wanted.hs b/Limit/Wanted.hs new file mode 100644 --- /dev/null +++ b/Limit/Wanted.hs @@ -0,0 +1,21 @@ +{- git-annex limits by wanted status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Limit.Wanted where + +import Common.Annex +import Annex.Wanted +import Limit +import Types.FileMatcher + +addWantGet :: Annex () +addWantGet = addLimit $ Right $ const $ + \fileinfo -> wantGet False (Just $ matchFile fileinfo) + +addWantDrop :: Annex () +addWantDrop = addLimit $ Right $ const $ + \fileinfo -> wantDrop False Nothing (Just $ matchFile fileinfo) diff --git a/Logs/PreferredContent.hs b/Logs/PreferredContent.hs --- a/Logs/PreferredContent.hs +++ b/Logs/PreferredContent.hs @@ -26,10 +26,10 @@ import qualified Annex import Logs import Logs.UUIDBased -import Limit import qualified Utility.Matcher import Annex.FileMatcher import Annex.UUID +import Types.Limit import Types.Group import Types.Remote (RemoteConfig) import Logs.Group diff --git a/Remote.hs b/Remote.hs --- a/Remote.hs +++ b/Remote.hs @@ -39,7 +39,8 @@ showTriedRemotes, showLocations, forceTrust, - logStatus + logStatus, + checkAvailable ) where import qualified Data.Map as M @@ -274,3 +275,7 @@ where costmap = M.fromListWith (++) . map costpair costpair r = (cost r, [r]) + +checkAvailable :: Bool -> Remote -> IO Bool +checkAvailable assumenetworkavailable = + maybe (return assumenetworkavailable) doesDirectoryExist . localpath diff --git a/Remote/Bup.hs b/Remote/Bup.hs --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -64,6 +64,7 @@ , hasKeyCheap = bupLocal buprepo , whereisKey = Nothing , remoteFsck = Nothing + , repairRepo = Nothing , config = c , repo = r , gitconfig = gc diff --git a/Remote/Directory.hs b/Remote/Directory.hs --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -55,6 +55,7 @@ hasKeyCheap = True, whereisKey = Nothing, remoteFsck = Nothing, + repairRepo = Nothing, config = M.empty, repo = r, gitconfig = gc, @@ -109,9 +110,13 @@ ifM (check chunkcount) ( do chunks <- listChunks f <$> readFile chunkcount - ifM (and <$> mapM check chunks) + ifM (allM check chunks) ( a chunks , return False ) - , go fs + , do + chunks <- probeChunks f check + if null chunks + then go fs + else a chunks ) withStoredFiles :: ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -108,6 +108,7 @@ , hasKeyCheap = repoCheap r , whereisKey = Nothing , remoteFsck = Nothing + , repairRepo = Nothing , config = M.empty , localpath = localpathCalc r , repo = r diff --git a/Remote/Git.hs b/Remote/Git.hs --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -117,6 +117,9 @@ , remoteFsck = if Git.repoIsUrl r then Nothing else Just $ fsckOnRemote r + , repairRepo = if Git.repoIsUrl r + then Nothing + else Just $ repairRemote r , config = M.empty , localpath = localpathCalc r , repo = r @@ -418,6 +421,10 @@ , ("GIT_DIR", Git.localGitDir r') ] ++ env batchCommandEnv program (Param "fsck" : params) (Just env') + +{- The passed repair action is run in the Annex monad of the remote. -} +repairRemote :: Git.Repo -> Annex Bool -> Annex (IO Bool) +repairRemote r a = return $ Remote.Git.onLocal r a {- Runs an action on a local repository inexpensively, by making an annex - monad using that repository. -} diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -60,6 +60,7 @@ hasKeyCheap = False, whereisKey = Nothing, remoteFsck = Nothing, + repairRepo = Nothing, config = c, repo = r, gitconfig = gc, diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs --- a/Remote/Helper/Chunked.hs +++ b/Remote/Helper/Chunked.hs @@ -43,6 +43,10 @@ chunkCount :: ChunkExt chunkCount = ".chunkcount" +{- An infinite stream of extensions to use for chunks. -} +chunkStream :: [ChunkExt] +chunkStream = map (\n -> ".chunk" ++ show n) [1 :: Integer ..] + {- Parses the String from the chunkCount file, and returns the files that - are used to store the chunks. -} listChunks :: FilePath -> String -> [FilePath] @@ -50,15 +54,28 @@ where count = fromMaybe 0 $ readish chunkcount -{- An infinite stream of extensions to use for chunks. -} -chunkStream :: [ChunkExt] -chunkStream = map (\n -> ".chunk" ++ show n) [1 :: Integer ..] +{- For use when there is no chunkCount file; uses the action to find + - chunks, and returns them, or Nothing if none found. Relies on + - storeChunks's finalizer atomically moving the chunks into place once all + - are written. + - + - This is only needed to work around a bug that caused the chunkCount file + - not to be written. + -} +probeChunks :: FilePath -> (FilePath -> IO Bool) -> IO [FilePath] +probeChunks basedest check = go [] $ map (basedest ++) chunkStream + where + go l [] = return (reverse l) + go l (c:cs) = ifM (check c) + ( go (c:l) cs + , go l [] + ) {- Given the base destination to use to store a value, - generates a stream of temporary destinations (just one when not chunking) - and passes it to an action, which should chunk and store the data, - and return the destinations it stored to, or [] on error. Then - - calls the storer to write the chunk count (if chunking). Finally, the + - calls the recorder to write the chunk count (if chunking). Finally, the - finalizer is called to rename the tmp into the dest - (and do any other cleanup). -} @@ -68,7 +85,7 @@ where go = do stored <- storer tmpdests - when (isNothing chunksize) $ do + when (isJust chunksize) $ do let chunkcount = basef ++ chunkCount recorder chunkcount (show $ length stored) finalizer tmp dest diff --git a/Remote/Hook.hs b/Remote/Hook.hs --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -53,6 +53,7 @@ hasKeyCheap = False, whereisKey = Nothing, remoteFsck = Nothing, + repairRepo = Nothing, config = M.empty, localpath = Nothing, repo = r, diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -80,6 +80,7 @@ , hasKeyCheap = False , whereisKey = Nothing , remoteFsck = Nothing + , repairRepo = Nothing , config = M.empty , repo = r , gitconfig = gc diff --git a/Remote/S3.hs b/Remote/S3.hs --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -63,6 +63,7 @@ hasKeyCheap = False, whereisKey = Nothing, remoteFsck = Nothing, + repairRepo = Nothing, config = c, repo = r, gitconfig = gc, diff --git a/Remote/Web.hs b/Remote/Web.hs --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -57,6 +57,7 @@ hasKeyCheap = False, whereisKey = Just getUrls, remoteFsck = Nothing, + repairRepo = Nothing, config = M.empty, gitconfig = gc, localpath = Nothing, diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -66,6 +66,7 @@ hasKeyCheap = False, whereisKey = Nothing, remoteFsck = Nothing, + repairRepo = Nothing, config = c, repo = r, gitconfig = gc, @@ -198,8 +199,14 @@ withStoredFiles r k baseurl user pass onerr a | isJust $ chunkSize $ config r = do let chunkcount = keyurl ++ chunkCount - maybe (onerr chunkcount) (a . listChunks keyurl . L8.toString) - =<< davGetUrlContent chunkcount user pass + v <- davGetUrlContent chunkcount user pass + case v of + Just s -> a $ listChunks keyurl $ L8.toString s + Nothing -> do + chunks <- probeChunks keyurl $ \u -> (== Right True) <$> davUrlExists u user pass + if null chunks + then onerr chunkcount + else a chunks | otherwise = a [keyurl] where keyurl = davLocation baseurl k ++ keyFile k diff --git a/Seek.hs b/Seek.hs --- a/Seek.hs +++ b/Seek.hs @@ -96,7 +96,8 @@ withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> (FilePath -> CommandStart) -> CommandSeek withFilesUnlocked' typechanged a params = prepFiltered a unlockedfiles where - check f = liftIO (notSymlink f) <&&> isJust <$> catKeyFileHEAD f + check f = liftIO (notSymlink f) <&&> + (isJust <$> catKeyFile f <||> isJust <$> catKeyFileHEAD f) unlockedfiles = filterM check =<< seekHelper typechanged params {- Finds files that may be modified. -} diff --git a/Test.hs b/Test.hs --- a/Test.hs +++ b/Test.hs @@ -370,13 +370,13 @@ git_annex env "get" ["--auto", annexedfile] @? "get --auto of file failed with default preferred content" annexed_notpresent annexedfile - git_annex env "content" [".", "standard"] @? "set expression to standard failed" + git_annex env "wanted" [".", "standard"] @? "set expression to standard failed" git_annex env "group" [".", "client"] @? "set group to standard failed" git_annex env "get" ["--auto", annexedfile] @? "get --auto of file failed for client" annexed_present annexedfile git_annex env "ungroup" [".", "client"] @? "ungroup failed" - git_annex env "content" [".", "standard"] @? "set expression to standard failed" + git_annex env "wanted" [".", "standard"] @? "set expression to standard failed" git_annex env "group" [".", "manual"] @? "set group to manual failed" -- drop --auto with manual leaves the file where it is git_annex env "drop" ["--auto", annexedfile] @? "drop --auto of file failed with manual preferred content" @@ -388,7 +388,7 @@ annexed_notpresent annexedfile git_annex env "ungroup" [".", "client"] @? "ungroup failed" - git_annex env "content" [".", "exclude=*"] @? "set expression to exclude=* failed" + git_annex env "wanted" [".", "exclude=*"] @? "set expression to exclude=* failed" git_annex env "get" [annexedfile] @? "get of file failed" annexed_present annexedfile git_annex env "drop" ["--auto", annexedfile] @? "drop --auto of file failed with exclude=*" @@ -806,7 +806,6 @@ _ <- git_annex env "uninit" [] -- exit status not checked; does abnormal exit checkregularfile annexedfile doesDirectoryExist ".git" @? ".git vanished in uninit" - not <$> doesDirectoryExist ".git/annex" @? ".git/annex still present after uninit" test_upgrade :: TestEnv -> Test test_upgrade env = "git-annex upgrade" ~: intmpclonerepo env $ do diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -41,6 +41,7 @@ , annexWebDownloadCommand :: Maybe String , annexCrippledFileSystem :: Bool , annexLargeFiles :: Maybe String + , annexFsckNudge :: Bool , coreSymlinks :: Bool , gcryptId :: Maybe String } @@ -68,6 +69,7 @@ , annexWebDownloadCommand = getmaybe (annex "web-download-command") , annexCrippledFileSystem = getbool (annex "crippledfilesystem") False , annexLargeFiles = getmaybe (annex "largefiles") + , annexFsckNudge = getbool (annex "fscknudge") True , coreSymlinks = getbool "core.symlinks" True , gcryptId = getmaybe "core.gcrypt-id" } diff --git a/Types/Limit.hs b/Types/Limit.hs new file mode 100644 --- /dev/null +++ b/Types/Limit.hs @@ -0,0 +1,20 @@ +{- types for limits + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.Limit where + +import Common.Annex +import Types.FileMatcher + +import qualified Data.Set as S + +type MkLimit = String -> Either String MatchFiles + +type AssumeNotPresent = S.Set UUID +type MatchFiles = AssumeNotPresent -> FileInfo -> Annex Bool diff --git a/Types/Remote.hs b/Types/Remote.hs --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -69,6 +69,8 @@ -- without transferring all the data to the local repo -- The parameters are passed to the fsck command on the remote. remoteFsck :: Maybe ([CommandParam] -> a (IO Bool)), + -- Runs an action to repair the remote's git repository. + repairRepo :: Maybe (a Bool -> a (IO Bool)), -- a Remote has a persistent configuration store config :: RemoteConfig, -- git repo for the Remote diff --git a/Utility/HumanTime.hs b/Utility/HumanTime.hs --- a/Utility/HumanTime.hs +++ b/Utility/HumanTime.hs @@ -35,9 +35,11 @@ go n [] = return n go n s = do num <- readish s :: Maybe Integer - let (c:rest) = dropWhile isDigit s - u <- M.lookup c unitmap - go (n + num * u) rest + case dropWhile isDigit s of + (c:rest) -> do + u <- M.lookup c unitmap + go (n + num * u) rest + _ -> return $ n + num fromDuration :: Duration -> String fromDuration Duration { durationSeconds = d } diff --git a/Utility/Scheduled.hs b/Utility/Scheduled.hs --- a/Utility/Scheduled.hs +++ b/Utility/Scheduled.hs @@ -275,13 +275,16 @@ toScheduledTime v = case words v of (s:ampm:[]) | map toUpper ampm == "AM" -> - go s (\h -> if h == 12 then 0 else h) + go s h0 | map toUpper ampm == "PM" -> - go s (+ 12) + go s (\h -> (h0 h) + 12) | otherwise -> Nothing (s:[]) -> go s id _ -> Nothing where + h0 h + | h == 12 = 0 + | otherwise = h go :: String -> (Int -> Int) -> Maybe ScheduledTime go s adjust = let (h, m) = separate (== ':') s @@ -318,8 +321,8 @@ arbitrary = oneof [ pure AnyTime , SpecificTime - <$> nonNegative arbitrary - <*> nonNegative arbitrary + <$> choose (0, 23) + <*> choose (1, 59) ] instance Arbitrary Recurrance where diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,35 @@ +git-annex (4.20131101) unstable; urgency=low + + * The "git annex content" command is renamed to "git annex wanted". + * New --want-get and --want-drop options which can be used to + test preferred content settings. + For example, "git annex find --in . --want-drop" + * assistant: When autostarted, wait 5 seconds before running the startup + scan, to avoid contending with the user's desktop login process. + * webapp: When setting up a bare shared repository, enable non-fast-forward + pushes. + * sync: Show a hint about receive.denyNonFastForwards when a push fails. + * directory, webdav: Fix bug introduced in version 4.20131002 that + caused the chunkcount file to not be written. Work around repositories + without such a file, so files can still be retreived from them. + * assistant: Automatically repair damanged git repository, if it can + be done without losing data. + * assistant: Support repairing git remotes that are locally accessible + (eg, on removable drives). + * add: Fix reversion in 4.20130827 when adding unlocked files that have + not yet been committed. + * unannex: New, much slower, but more safe behavior: Copies files out of + the annex. This avoids an unannex of one file breaking other files that + link to the same content. Also, it means that the content + remains in the annex using up space until cleaned up with + "git annex unused". + (The behavior of unannex --fast has not changed; it still hard links + to content in the annex. --fast was not made the default because it is + potentially unsafe; editing such a hard linked file can unexpectedly + change content stored in the annex.) + + -- Joey Hess Fri, 01 Nov 2013 11:34:27 -0400 + git-annex (4.20131024) unstable; urgency=low * webapp: Fix bug when adding a remote and git-remote-gcrypt diff --git a/doc/assistant.mdwn b/doc/assistant.mdwn --- a/doc/assistant.mdwn +++ b/doc/assistant.mdwn @@ -31,12 +31,13 @@ ## colophon -The git-annex assistant is being -[crowd funded on +The git-annex assistant was [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). -[[/assistant/Thanks]] to all my backers. This kickstarter is now closed, and there is a new home-made crowdfunding project to support the project for 2013-2014 [here](https://campaign.joeyh.name/). +[[/assistant/Thanks]] to all my backers. This kickstarter is now closed, +and there is a new home-made crowdfunding project to support the project +for 2013-2014 [here](https://campaign.joeyh.name/). -I blog about my work on the git-annex assistant on a daily basis -in [[this_blog|design/assistant/blog]]. Follow along! +I blog about my work on git-annex and the assistant on a daily basis +in [[this_blog|/devblog]]. Follow along! See also: The [[design|/design/assistant]] pages. diff --git a/doc/assistant/release_notes.mdwn b/doc/assistant/release_notes.mdwn --- a/doc/assistant/release_notes.mdwn +++ b/doc/assistant/release_notes.mdwn @@ -1,3 +1,16 @@ +## version 4.20131024 + +This version fixes several different bugs that could cause the webapp to +refuse to create a repository. Several other bugs are also fixed, including +a bug that caused it to not add files on Android. + +New in this release is the ability to use the webapp to set up scheduled +consistency checks of your repositories. Many problems with repositories +are now automatically corrected, and it can even repair damaged git +repositories. + +This is a recommended upgrade. + ## version 4.20131002 Now you can use the webapp to set up an encrypted git repository on a diff --git a/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment b/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment new file mode 100644 --- /dev/null +++ b/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmTho3mActvetF1iQdmui6gH1t6WE6c284" + nickname="Michael" + subject="SHA256e" + date="2013-10-30T02:00:45Z" + content=""" +I'd really like to have a SHA256e backend -- same as SHA256E but making sure that extensions of the files in .git/annex are converted to lower case. I normally try to convert filenames from cameras etc to lower case, but not all people that I share annex with do so consistently. +In my use case, I need to be able to find duplicates among files and .jpg vs .JPG throws git annex dedup off. Otherwise E backends are superior to non-E for me. Thanks, Michael. +"""]] diff --git a/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment b/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment new file mode 100644 --- /dev/null +++ b/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM" + nickname="Jarno" + subject="Non-E backend drawbacks?" + date="2013-10-30T21:25:00Z" + content=""" +The page states \"[non-E backends] can confuse some programs\". I like the ideal simplicity and recoverability of pure checksum backends but \"confusion\" sounds a bit worrying. Any practical examples of these problems to help me choose? +"""]] diff --git a/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment b/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment new file mode 100644 --- /dev/null +++ b/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 6" + date="2013-11-01T15:47:26Z" + content=""" +Some examples of problems with the raw SHA backends include, IIRC, calibre, and many programs on OSX. These programs look at the extension of the filename the symlink points at. +"""]] diff --git a/doc/bugs/Build_error_on_Linux.mdwn b/doc/bugs/Build_error_on_Linux.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Build_error_on_Linux.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. +Building on Linux, with a particular combination of flags, failed due to missing `async`. + +### What steps will reproduce the problem? +1. Configure with the following flag combination + + cryptohash -quvi -feed tdfa -testsuite -android production -dns -xmpp -pairing -webapp -assistant dbus inotify -webdav s3 + +2. Attempt to build and you'll get an error on line 16 of `Utility/Batch.hs` because `Control.Concurrent.Async` isn't available. + +### What version of git-annex are you using? On what operating system? +Version 4.20131024 on Linux + +### Please provide any additional information below. + +This is the patch I applied to `git-annex.cabal`: + + CPP-Options: -DWITH_KQUEUE + C-Sources: Utility/libkqueue.c + + + if os(linux) + + Build-Depends: async + + + if os(linux) && flag(Dbus) + Build-Depends: dbus (>= 0.10.3) + CPP-Options: -DWITH_DBUS + +> Feel async is core enough it should depend on it unconditionally. +> [[done]] --[[Joey]] diff --git a/doc/bugs/Finding_an_Unused_file.mdwn b/doc/bugs/Finding_an_Unused_file.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Finding_an_Unused_file.mdwn @@ -0,0 +1,152 @@ +### Please describe the problem. + +SHA256E makes it difficult or impossible to find the original filename + +### What steps will reproduce the problem? + +I have these unused files: + + $ git annex unused + unused . (checking for unused data...) (checking master...) (checking backupbook/HEAD...) (checking b + aster...) (checking lang/a-...) (checking lang/master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256-s9107031--2611c08c9822179c443f001f0bd7ecadf29adcd28edfa4cd1d8938d289cd3950 + 2 SHA256E-s31131336--58c48adad8e5f091981549dfbb2d9ec1003c8c46d1a660673fabefe722358f9b.flac + 3 SHA256-s9941549--db12950459ba039e6e6f6102a0811213c11a717d140d6e2f169049b958a5e047 + 4 SHA256-s11544438--174514684e03035cc741fa397a1b46f925899bd29189a98173f8f2a136d95ace + 5 SHA256E-s23445007--4659fae3eda6db7c528af6439fafab1496c740d02bdb67892450a8a2208fb29b.flac + 6 SHA256E-s47080709--7d331788ae7fee16bccce060c62cafa3cdafc9e0a2b387c0843cfe5871f51fa6.flac + 7 SHA256E-s33262563--3280607d6d397f84a02542c5ab2e5a9c44d60256b330e3d075078694f0c7f709.flac + 8 SHA256-s6522640--a1fa374afd62e8c85a115f18f78e679722f63980191c1e11ef84a49ae86f5b4f + 9 SHA256E-s29266138--27e792d64b6d4a4d44bdafe6867ca25ba79480d1b650cf385e67ff28a1fc5c31.flac + 10 SHA256-s14568326--023eb9fcefc063ee3ea495f4d382a8feac795d0e1a81c585781f5d369db2e00c + 11 SHA256-s11907175--73b66220bdbf0ca92209605b93d95d5f9e8247745f9c4367ff20cb53e11c24ac + 12 SHA256-s9193267--e159038b78c1b239d4cb8eeb892c7acf0e7b82feac7f5b5808dad477605e8478 + 13 SHA256E-s39329047--880ffab99c4b4c48224f409acb0cd797737e4ee70eacdba9bc7d7628ba3d05d7.flac + 14 SHA256E-s31400468--17ad7b13757d4e4d6e5d193d8295a30f566e00eea82bc19f50dc14b2bbf79ef3.flac + 15 SHA256E-s51514687--f83eb092ddcdf35e7f729bfa2cc0914b404de60f0b42f69f207cf01766061f16.flac + 16 SHA256E-s36235648--54e3f893b498b205b0b96bd87d5f14c71712e891c518070a0e1c730d92b3b0ac.flac + 17 SHA256-s10001177--35afb0b91c8b9f711b2d3b0fe7433ecc3bb13aece78c6170a47323c94233133d + 18 SHA256-s11830142--3dabe97ccbb68045d9ea82036ddf7211a3925a1bf682e05a32bcdf9b07bec676 + 19 SHA256E-s23102994--f848ed216e6ba17e6b539f31caa5af36266c367ed55dcff243445848b01fbeef.flac + 20 SHA256E-s38505547--967763ddd42daf782afa9299e67cb5c834153cb20242b50115dda566b24a68c4.flac + 21 SHA256-s11874975--14774a404526c4b68ede146f527202c59a4bee88376707c93df7da3bdb5345f2 + 22 SHA256-s10188700--18fd4ee62c2b3b1d8944eae528b59de2a45d493a291440edc9b30881ba10ece3 + 23 SHA256E-s53024109--eba924a26f7c602b60a83f208b2204ef3b570fd92ff39fbc067eeaec7c443ab6.flac + 24 SHA256-s11315225--0a8d2165166995e819e8c78302c45b1eeca9b79a5d77a3574885cbf8e18f265a + 25 SHA256-s12249573--79b9e551051232079f24902078e1fe5b7daed684e8acaa7cf29c191404a7c3c7 + 26 SHA256E-s47991289--79cfd8db5b3cbe7f50c335bfe0d148c38ff36dbc97b17ea3aff23d642bd5d167.flac + 27 SHA256E-s76961343--80e91f73e2f3ae6790d752d380118b3fadc223f9f1449354daa0095b5713986c.flac + 28 SHA256E-s44706648--ce782096aa5c0d58a12f7cbd6dbfe032fa7b0b4810219e23f906bd7fa0d96336.flac + 29 SHA256-s9393784--1e0e0190030352b3583ce515cd6dfb0db9f2ae39809a462099482947193bfd0f + 30 SHA256E-s11534608--bfcc0fdcb1ed112b1737d4fe7d2aa88511187d4c1c132bf1b7ab2cda9a7d90ad.flac + 31 SHA256E-s27916580--1f196e1b6312421f9d9bcd227a3885a5dec299fc47748cd34d5e59fe26374187.flac + 32 SHA256-s12099941--0a33af0293acb7cffa1727cc04efa4d7abb29a39e8f2165bfd565e2b18879430 + 33 SHA256E-s35578262--01d6f6022161bcaa825cb353012f85bacf3dcb930337ba506fc94b663b0d8043.flac + 34 SHA256E-s38880984--fa326bd0db5c9e400a4256e77acd488fede23ec714aeab7deb6952091a09e318.flac + 35 SHA256E-s27784704--a426f885d79d88d8b42b8a23ce267fb1c1855e8f7ff6833ec9ea61860b3c6819.flac + 36 SHA256-s11827390--17c1e84815805ea7e01555fdfa038a643a97621b84073679d5c649c020750652 + 37 SHA256E-s16632423--9735e37261624360f6a54fd1bc8a893cb5344afdc910e6b9e00c5791036759b0.flac + 38 SHA256-s10809006--7dd7ff078bdf4571bee6ab06213df970f427a8a2af476d4511a9c08ed0fb814d + 39 SHA256E-s33043364--3603942143fa4b1c0082ab7ba1db9b536621402eea3ac94e08ef0197062a2ef6.flac + 40 SHA256E-s33278052--62e1038e26894c6c014af4e77d2d9f79074ff4fc1ca9ca8c16fcc98eee5277c7.flac + 41 SHA256E-s47299134--0b604bff3ce12d077691d6e6428648da879c021537c094c72633209d5302fd97.flac + 42 SHA256E-s14882445--16c6aa4aaf8e60da617701281dfdfa372cb049636ae0f10f5aefcd8ce6c472af.flac + 43 SHA256E-s18168736--dfd7bfe9a433daef89f6f0015dfddaf7ac3611dd31813b0aed049171ac008323.flac + 44 SHA256-s10060170--9e3b638c4f397d8dbaa2fa9fc2bfacf6f9e93f00c80ee1fdd26971d25aff86ce + 45 SHA256E-s25630107--26fedc816dee58cb999cd2be8a58f3e8726ff056ba9c2abfbb9d149e7a92a230.flac + 46 SHA256-s9148609--3f29a2ae1ebd2e4857bb5c92aefb2f50f39a5a984e14645f658d15795a7c72a5 + 47 SHA256-s10576861--7d95469441ef205616ed795012d8a4c59acc00be00aca66bcacd8b041c2499c5 + 48 SHA256E-s23806802--8725d04f04925f4cd456bdb2108a48682cf51a498d97af9f71c58a412b8db9ab.flac + 49 SHA256E-s43187837--e83648296fa4553556596464efd4ab313529c0062071c9a40113fecede7a5de6.flac + 50 SHA256-s10384945--055a01b7e06f3165dddd10beb3e98ba4ca47b35abf9d5b2f26187bf07b9aa401 + 51 SHA256E-s54253300--b011eea8ec7ea51f22e0fe09645ffe83183e9589a9784ebc2d6dd0c559f07322.flac + 52 SHA256-s10335583--008c17c1a994884c6b9c52e10d17373b9160136ffab940992ef0f9d08eac45c1 + 53 SHA256-s9769970--b36fcb01d9df7627106e3a3c283944457f3e282ab62878c186162d465530cfbc + 54 SHA256-s10329964--dc4d82e085af3ade48d33cf828f8e7eecc27d4c33c448965e1b8bb59832ec473 + 55 SHA256-s10625243--2494e2ef2c64fc77dcb063f7b58079ad668cf46862b7f11eb28943d45f21b8dd + 56 SHA256E-s44585111--e35b0774729ccf547cd62e652425579781e3cbd76b33a17a0127bdbedb90606a.flac + 57 SHA256E-s42060728--9e0a533e640086fadefad7167d37d5b3c5de899e1ac5890bfd7a88524701ff14.flac + 58 SHA256E-s30013479--ae34a897e2f0e1124f04396e75dd41749511ce9930a2891ae3a066597ad518c5.flac + 59 SHA256E-s38896704--ca6211442c33d9c44b997bef8f1079f07cdd2eb0e9e3c1de1eda9ae8a705f137.flac + 60 SHA256-s13044938--8714a1db8781daa9f3993128dcda5a5ca904e075723cf38a11bc5e3695cf126f + 61 SHA256-s8814622--ba374e92c53ddb83605e2b500647117a00e4c6c463653cb0dd0311b76627c2ec + 62 SHA256E-s14900839--e94089b89629561ebf771543236f31439af1e6fcdce7ff56b9a183041e95e7ea.flac + 63 SHA256E-s27370218--a4717cf0615cda099ea5740194da8b4e349f2f38f6ebbf5f6111ced7f56d1736.flac + 64 SHA256E-s57686070--6d02689ccf91317bdfa4d8694ccbf7a9ccf00a7e00b92733769194908b4087d9.flac + 65 SHA256-s8707306--7f05e3e3dc4336eb7012d5bbb6d3d65047552901858ee4967e4a70d100fd1deb + 66 SHA256E-s42482194--bb7a968ec9bc0a8813974af9173c38cb39a43aa7d0c2aec203ddb358119e1f25.flac + 67 SHA256E-s31951334--b19789a9bbb98cb25df7b8a1b6d8856256bc2851cd892c91dddba7cf736542c5.flac + 68 SHA256E-s35901187--8eae2684aa3566b632a1797bd09d112dd5f438c98b649c000d71640983a549f2.flac + 69 SHA256E-s40659631--f6ca3227b14c7c050877ce4cc1218fea7582426649cd68fc82a74cae5d6962b4.flac + 70 SHA256E-s41415180--9c6f0000da119bd70422fe9529c41efb3103ca943697bdfb685f119f6b5ae6fd.flac + 71 SHA256E-s52313976--e4a59c65e05bd9450ef595b3c08365810205731952e45a68ee0c89bc76fbd9fc.flac + 72 SHA256E-s44925214--eabc5f7172d5c2e094ff84a5ecce784c172b175c8eeca861aa3996749668ea42.flac + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + ok + +And running: + + $ git log --stat -S'SHA256-s13044938--8714a1db8781daa9f3993128dcda5a5ca904e075723cf38a11bc5e3695cf126f' + commit 767a63a54784139f13d69d12fcfbee8f6ca3df41 + Author: Matthew Forrester + Date: Fri Aug 23 21:38:31 2013 +0100 + + Remove Roger Shah - Openminded! as there is another copy in Unsorted with correct? filenames + + Amazon/Roger Shah/Openminded!/1114 - Shine (Album Mix).mp3 | 1 - + 1 file changed, 1 deletion(-) + + commit 6b04002c03287fb8918bdcdeaae393e862bebd4e + Author: Matthew Forrester + Date: Thu Aug 8 21:30:14 2013 +0100 + + Initial checkin + + Amazon/Roger Shah/Openminded!/1114 - Shine (Album Mix).mp3 | 1 + + 1 file changed, 1 insertion(+) + +Works but I cannot find out how to get the `.flac` files working: + + +`$ git log --stat -S'SHA256E-s38896704--ca6211442c33d9c44b997bef8f1079f07cdd2eb0e9e3c1de1eda9ae8a705f137.flac'` + +`$ git log --stat -S'SHA256E-s38896704--ca6211442c33d9c44b997bef8f1079f07cdd2eb0e9e3c1de1eda9ae8a705f137'` + +`$ git log --stat -S'SHA256-s38896704--ca6211442c33d9c44b997bef8f1079f07cdd2eb0e9e3c1de1eda9ae8a705f137'` + + +None of which give answers. I don't know if I'm doing it wrong (though I __think__ I'm doing what the instructions say), but I can't make it work. + + +### What version of git-annex are you using? On what operating system? + +$ uname -a +Linux fozz-desktop 3.8.0-31-generic #46-Ubuntu SMP Tue Sep 10 19:56:49 UTC 2013 i686 i686 i686 GNU/Linux + +$ git annex version +git-annex version: 3.20121112ubuntu2 +local repository version: 3 +default repository version: 3 +supported repository versions: 3 +upgrade supported from repository versions: 0 1 2 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> If `git log -S` does not find the key, then it was not used for any +> commit currently in the git repository. Which is certianly possible; +> for example `git annex add file; git rm file`. +> +> This is a dup of [[todo/wishlist: option to print more info with 'unused']]; [[done]] --[[Joey]] diff --git a/doc/bugs/OSX_app_issues/comment_17_0e6ac5e0a54ce78bdc56c62e6fb92846._comment b/doc/bugs/OSX_app_issues/comment_17_0e6ac5e0a54ce78bdc56c62e6fb92846._comment new file mode 100644 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_17_0e6ac5e0a54ce78bdc56c62e6fb92846._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="calmyournerves" + ip="85.3.250.239" + subject="comment 17" + date="2013-10-24T21:43:37Z" + content=""" +For 10.9 Mavericks see http://git-annex.branchable.com/bugs/git_annex_doesn__39__t_work_in_Max_OS_X_10.9/#comments +"""]] diff --git a/doc/bugs/Too_much_system_load_on_startup.mdwn b/doc/bugs/Too_much_system_load_on_startup.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Too_much_system_load_on_startup.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. +When I log in, if git annex is monitoring a large repo, my desktop is very sluggish getting started. Git-annex causes moderate CPU load, but keeps the disk IO very busy -delaying the opening of desktop applications. + +### What steps will reproduce the problem? +On Linux, with git-annex set to autostart and monitoring a folder with more than a few hundred files (I have a pdf library of a few thousand journal articles). + +### What version of git-annex are you using? On what operating system? +4.20131002 Ubuntu, from Hess's PPA. + +### Please provide any additional information below. + +I solved this problem by changing the call to git-annex in /etc/xdg/autostart/git-annex.desktop from: + +Exec=/usr/bin/git-annex assistant --autostart + +to + +Exec=sleep 5 ionice -c 3 /usr/bin/git-annex assistant --autostart + +This delays the start of git-annex for 5 seconds, letting the desktop get started, and forces git-annex to yield IO to other programs -preventing it from slowing them down by forcing them to wait for disk access. Since this is a background daemon with potentially high IO usage, but no need for quick responsiveness, perhaps that would make a decent default? + +> Added 5 second delay to existing ionice. Provisionally [[done]], +> although it does occur to me that the startup scan could add some delays +> in between actions to run more as a batch job. --[[Joey]] diff --git a/doc/bugs/Windows_and_Linux_in_direct_mode_confuses_git.mdwn b/doc/bugs/Windows_and_Linux_in_direct_mode_confuses_git.mdwn --- a/doc/bugs/Windows_and_Linux_in_direct_mode_confuses_git.mdwn +++ b/doc/bugs/Windows_and_Linux_in_direct_mode_confuses_git.mdwn @@ -377,3 +377,8 @@ # End of transcript or log. """]] + +> Apparently `test.git` had `receive.denyNonFastForwards` +> set to true, which prevents the forced pushing `git annex sync` +> needs to do. I have made it print out a hint about this setting +> when a push failes. [[done]] --[[Joey]] diff --git a/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn --- a/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn +++ b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn @@ -49,3 +49,5 @@ # End of transcript or log. """]] + +[[!tag moreinfo]] diff --git a/doc/bugs/copy_to_webdav_sometimes_doesn__39__t_work.mdwn b/doc/bugs/copy_to_webdav_sometimes_doesn__39__t_work.mdwn --- a/doc/bugs/copy_to_webdav_sometimes_doesn__39__t_work.mdwn +++ b/doc/bugs/copy_to_webdav_sometimes_doesn__39__t_work.mdwn @@ -67,3 +67,8 @@ 10672 open("/media/1und1/git-annex/dcf/85a/GPGHMACSHA1--10ff9e1cc8191235670c2fd95375bccf62004f33/GPGHMACSHA1--10ff9e1cc8191235670c2fd95375bccf62004f33.chunk1", O_RDONLY|O_NOCTTY|O_NONBLOCK|O_LARGEFILE) = 20 # End of transcript or log. """]] + +> There was a bug that caused it not to write the chunkcount file. +> I have fixed it, and put in a workaround so fsck, etc, will +> see that the file is stored on the remote despite there being no +> chunkcount file present. [[done]] --[[Joey]] diff --git a/doc/bugs/microsd__47__thumbdrives_seem_to_die_when_using_the_ARM_build.mdwn b/doc/bugs/microsd__47__thumbdrives_seem_to_die_when_using_the_ARM_build.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/microsd__47__thumbdrives_seem_to_die_when_using_the_ARM_build.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +1 thumb drive became corrupted when using as a server on raspberry pi, and 2 microSD cards when using as a client in my phone. Both happened during syncing largish repository. (corrupted = permanent input/output error) + +* Put git annex on my android phone with a 64GB (FAT) micro SD, fired up git annex, it got reported as corrupted half on hour later, reformatting worked but got reported as corrupted again. + +* Put git annex assistant on my raspberry pi, one of the thumbdrives in my LVM (ext4) got corrupted shortly after I began using the assistant. I replaced them with a cheap real SSD drive, and had no problem since. + +* Put git annex back on my android phone. Kept it going for an extended sync session, but it never started syncing. I kept it going for an hour or so, and my new 32GB microSD (FAT) got corrupted. + +The pattern is nothing like proof, but it seems to be too regular to be completely coincidental. The pattern seems to be: ARM + (SDcard|USBstick) + Assistant = drive corruption. + +My guess is that the ARM build might have some kind of unlucky write pattern or loop that causes increased wear, but I know very little of the interna. + +### What steps will reproduce the problem? + +* Get a raspberry pi and a USB stick, or an android phone and a microSD card +* Get an [ARM build of the assistant](https://github.com/tradloff/git-annex-RPi) +* Sync a largish (12GB) repository + +### What version of git-annex are you using? On what operating system? + +4.20131002 on the pi, 20131024 for the 32GB SD, and 20131015 for the 64GB SD. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +Unfortunately, daemon.log was unrecoverable along with the other files on the SD card. + +I can try and autosync the daemon.log somewhere if I happen to come along a bunch of scrap flash storage (not impossible). + +# End of transcript or log. +"""]] diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn --- a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn @@ -12,3 +12,6 @@ % Ttbomk, the softlinks and objects are enough to un-annex the files; side-stepping git's index if necessary. + +> `git annex repair` can now repair broken index files and other +> git repository corruption. [[done]] --[[Joey]] diff --git a/doc/design/assistant/disaster_recovery.mdwn b/doc/design/assistant/disaster_recovery.mdwn --- a/doc/design/assistant/disaster_recovery.mdwn +++ b/doc/design/assistant/disaster_recovery.mdwn @@ -14,8 +14,9 @@ * What about local remotes, eg removable drives? git-annex does attempt to commit to the git-annex branch of those. It will use the automatic fix if any are dangling. It does not commit to the master branch; indeed - a removable drive typically has a bare repository. So I think nothing to - do here. + a removable drive typically has a bare repository. + However, it does a scan for broken locks anyway if there's a problem + syncing. **done** * What about git-annex-shell? If the ssh remote has the assistant running, it can take care of it, and if not, it's a server, and perhaps the user should be required to fix up if it crashes during a commit. This should @@ -59,18 +60,26 @@ Add git fsck to scheduled self fsck **done** -TODO: Add git fsck of local remotes to scheduled remote fscks. - TODO: git fsck on ssh remotes? Probably not worth the complexity.. TODO: If committing to the repository fails, after resolving any dangling -lock files (see above), it should git fsck. +lock files (see above), it should git fsck. This is difficult, because +git commit will also fail if the commit turns out to be empty, or due to +other transient problems.. So commit failures are currently ignored by the +assistant. If git fsck finds problems, launch git repository repair. **done** git annex fsck --fast at end of repository repair to ensure git-annex branch is accurate. **done** +If syncing with a local repository fails, try to repair it. **done** + +TODO: "Repair" gcrypt remotes, by removing all refs and objects, +and re-pushing. (Since the objects are encrypted data, there is no way +to pull missing ones from anywhere..) +Need to preserve gcrypt-id while doing this! + TODO: along with displaying alert when there is a problem detected by consistency check, send an email alert. (Using system MTA?) @@ -87,10 +96,10 @@ Or: Display a message whenever a removable drive is detected to have been connected. I like this, but what about nudging the main repo? Could do it -every webapp startup, perhaps? +every webapp startup, perhaps? **done** There should be a "No thanks" button that prevents it nudging again for a -repo. +repo. **done** ## git repository repair diff --git a/doc/devblog/day_43__bugfix_day.mdwn b/doc/devblog/day_43__bugfix_day.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_43__bugfix_day.mdwn @@ -0,0 +1,26 @@ +Got well caught up on bug fixes and traffic. Backlog is down to 40. + +Made the assistant wait for a few seconds before doing the startup +scan when it's autostarted, since the desktop is often busy starting +up at that same time. + +Fixed an ugly bug with chunked webdav and directory special remotes +that caused it to not write a "chunkcount" file when storing data, +so it didn't think the data was present later. I was able to make it +recover nicely from that mistake, by probing for what chunks are actually +present. + +Several people turn out to have had problems with `git annex sync` not +working because receive.denyNonFastForwards is enabled. I made the webapp +not enable it when setting up a ssh repository, and I made `git annex sync` +print out a hint about this when it's failed to push. (I don't think this +problem affects the assistant's own syncing.) + +Made the assistant try to repair a damaged git repository without +prompting. It will only prompt when it fails to fetch all the lost +objects from remotes. + +Glad to see that others have managed to +[get git-annex to build on Max OS X 10.9](http://git-annex.branchable.com/bugs/git_annex_doesn__39__t_work_in_Max_OS_X_10.9/#comment-8e8ee5e50506a6fde029d236f4809df8). +Now I just need someone to offer up a ssh account on that OS, and I could +set up an autobuilder for it. diff --git a/doc/devblog/day_44__automatic_removable_drive_repair.mdwn b/doc/devblog/day_44__automatic_removable_drive_repair.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_44__automatic_removable_drive_repair.mdwn @@ -0,0 +1,16 @@ +Finally got the assistant to repair git repositories on removable drives, +or other local repos. Mostly this happens entirely automatically, whatever +data in the git repo on the drive has been corrupted can just be copied +to it from `~/annex/.git`. + +And, the assistant will launch a git fsck of such a repo whenever it fails +to sync with it, so the user does not even need to schedule periodic fscks. +Although it's still a good idea, since some git repository problems don't +prevent syncing from happening. + +Watching git annex heal problems like this is quite cool! + +One thing I had to defer till later is repairing corrupted gcrypt +repositories. I don't see a way to do it without deleting all the objects +in the gcrypt repository, and re-pushing everything. And even doing that +is tricky, since the `gcrypt-id` needs to stay the same. diff --git a/doc/devblog/day_45__command_line.mdwn b/doc/devblog/day_45__command_line.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_45__command_line.mdwn @@ -0,0 +1,9 @@ +All command line stuff today.. + +Added --want-get and --want-drop, which can be used to test preferred content settings +of a repository. For example `git annex find --in . --want-drop` will list the same +files that `git annex drop --auto` would try to drop. (Also renamed `git annex content` +to `git annex wanted`.) + +Finally laid to rest problems with `git annex unannex` when multiple files point to the +same key. It's a lot slower, but I'll stop getting bug reports about that. diff --git a/doc/devblog/day_46__wrapping_up_the_month.mdwn b/doc/devblog/day_46__wrapping_up_the_month.mdwn new file mode 100644 --- /dev/null +++ b/doc/devblog/day_46__wrapping_up_the_month.mdwn @@ -0,0 +1,18 @@ +Spent today reviewing my [[plans_for_the_month|assistant/disaster_recovery]] +and filling in a couple of missing peices. + +Noticed that I had forgotten to make repository repair clean up any stale +git locks, despite writing that code at the beginning of the month, and +added that in. + +Made the webapp notice when a repository that is being used does not have +any consistency checks configured, and encourage the user to set up checks. +This happens when the assistant is started (for the local repository), +and when removable drives containing repositories are plugged in. If the +reminders are annoying, they can be disabled with a couple clicks. + +And I think that just about wraps up the month. (If I get a chance, I would +still like to add recovery of git-remote-gcrypt encrypted git repositories.) + +My [[design/roadmap]] has next month dedicated to user-driven features +and polishing and bugfixing. diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn --- a/doc/encryption.mdwn +++ b/doc/encryption.mdwn @@ -1,3 +1,5 @@ +[[!toc]] + git-annex mostly does not use encryption. Anyone with access to a git repository can see all the filenames in it, its history, and can access any annexed file contents. diff --git a/doc/forum/Change_remote_server_address.mdwn b/doc/forum/Change_remote_server_address.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Change_remote_server_address.mdwn @@ -0,0 +1,6 @@ +Hi again, + +I have a SSH remote server that I registered to my git-annex repository via git-annex assistant. When I go to edit the settings for the repository from within git-annex assistant I noticed I can't edit the server address. If the server IP changes, how should I go about letting git-annex know of this? Can I just (1) shutdown git-annex assistant, (2) edit the 'url' line of the remote entry inside the repository's ``.git/config`` file, and then (3) start up git-annex assistant again? Is this a safe method for doing this? + +Regards, +Blake diff --git a/doc/forum/Deleting_Unused_Files_by_Age.mdwn b/doc/forum/Deleting_Unused_Files_by_Age.mdwn deleted file mode 100644 --- a/doc/forum/Deleting_Unused_Files_by_Age.mdwn +++ /dev/null @@ -1,1 +0,0 @@ -I periodically move unused files to one of my servers. What I would like to do is drop any unused file that has been unused for say more than 6 months? I would like to not drop all unused files. diff --git a/doc/forum/GPG_passphrase_handling.txt b/doc/forum/GPG_passphrase_handling.txt new file mode 100644 --- /dev/null +++ b/doc/forum/GPG_passphrase_handling.txt @@ -0,0 +1,74 @@ +Hello! +I'm using OSX 10.9 and have installed gpg (and gpg2, if it matters) through +homebrew and git-annex through cabal. I also installed +https://github.com/joeyh/git-remote-gcrypt like the UI told me. + +Whenever I'm trying to add an encrypted remote through the web UI I get a +lot of "You need a passphrase to unlock the secret key for user:" on stdout +and, obviously, I can't enter my passphrase (If I could I wouldn't make this +post to begin with :)) +Is this behavior normal? What should I do to work around it? +I did also try to not use the web UI by using this command: +git annex initremote rsync.net type=gcrypt gitrepo=user@host:directory encryption=pubkey keyid=X + +Because of this I can't copy files to my remotes. All I get is: +----- +$ git annex copy --to rsync.net +copy MySecretFile (gpg) +You need a passphrase to unlock the secret key for +user: "user" +4096-bit RSA key, ID X, created 2013-10-01 (main key ID Y) + +(checking rsync.net...) (to rsync.net...) gpg: no valid addressees +gpg: [stdin]: encryption failed: No user ID +failed +----- + +Yes, I am using gpg-agent. When other applications ask for my passphrase I get +the pinentry dialog from GPGTools, just like I've configured it in +~/.gnupg/gpg-agent.conf, but this isn't the case with git-annex. + +If I remove GPGTools from /usr/local/bin with: ``brew link --overwrite gnupg && +brew link --overwrite gnupg2'' it works *slightly* better. +I get that pinentry dialog I want but when I do a copy I get: +----- +$ git annex copy --to rsync.net +copy MySecretFile (gpg) (checking rsync.net...) (to rsync.net...) gpg: no valid addressees +gpg: [stdin]: encryption failed: no such user id +failed +----- + +--debug shows me it is executing gpg llke so: +----- +gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--encrypt","--no-encrypt-to","--no-default-recipient","--force-mdc","--no-textmode"] +----- + +$ git annex version +git-annex version: 4.20131024 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav glacier hook + +$ gpg --version +gpg (GnuPG) 2.0.22 +libgcrypt 1.5.3 +Copyright (C) 2013 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Home: ~/.gnupg +Supported algorithms: +Pubkey: RSA, ELG, DSA, ?, ? +Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH, + CAMELLIA128, CAMELLIA192, CAMELLIA256 +Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 +Compression: Uncompressed, ZIP, ZLIB, BZIP2 + + $ gpg-agent --version +gpg-agent (GnuPG/MacGPG2) 2.0.22 +libgcrypt 1.5.3 +Copyright (C) 2013 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. diff --git a/doc/forum/known_and_local_annex_keys.mdwn b/doc/forum/known_and_local_annex_keys.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/known_and_local_annex_keys.mdwn @@ -0,0 +1,14 @@ +I have a direct repository with the assistant running (v4.20131002). The repo is in the backup group and should be grabbing every known annex file. Yet `git annex status` says: + + local annex keys: 1386 + local annex size: 94.53 gigabytes + known annex keys: 1387 + known annex size: 94.53 gigabytes + +As you can seem there is one more known annex key than there are local ones. Neither `git annex get` nor `git annex sync` changes the numbers. According to `tree` the repo dir contains exactly 1387 files, which means that the missing file must exist on disk. + +1) How do I find out which is the file in question? How do I get it synchronized? + +Yesterday there were 1376 known annex keys. Today there are 1387. For some reason the number of keys went up by 11, and I would like to know how and why. The log suggests that a couple of files have been added from inside the repo directory, but I am positive that these files already existed in the annex directory yesterday, and that I ran `git annex sync` manually without the number of local and/or known keys going up. + +2) Why weren't the files synced before? Is there a way to find out whether they were or were not part of the annex yesterday - maybe list all files sorted by the date they were added? diff --git a/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn b/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn @@ -0,0 +1,254 @@ +I've been playing with a mixed setup, and I frequentely end up with conflicts which can be ascribed to mixing direct windows repos with indirect linux one(s), and making renames on the indirect ones. +Possibly someone can address what I miss of the git-annex/git interaction. + +#### versions involved #### + +linux: git-annex version: 4.20131024 +win: git-annex version: 4.20131024-gca7b71e + +### Steps to reproduce behaviour ### + +###### 1. On linux, i setup bare origin "casa" and client repo "local": + + [michele@home ~]$ git init --bare casa + Initialized empty Git repository in /home/sambahome/michele/casa/ + [michele@home ~]$ cd casa + [michele@home casa]$ git annex init casa + init casa ok + (Recording state in git...) + [michele@home ~]$ cd ..; git clone casa local + Cloning into 'local'... + done. + warning: remote HEAD refers to nonexistent ref, unable to checkout. + [michele@home ~]$ cd local; git annex init local + init local ok + (Recording state in git...) + [michele@home local]$ echo lintest > lintest + [michele@home local]$ git annex add lintest + add lintest (checksum...) ok + (Recording state in git...) + [michele@home local]$ git annex sync + (merging origin/git-annex into git-annex...) + (Recording state in git...) + commit + ok + pull origin + ok + push origin + Counting objects: 18, done. + Delta compression using up to 4 threads. + Compressing objects: 100% (12/12), done. + Writing objects: 100% (16/16), 1.48 KiB | 0 bytes/s, done. + Total 16 (delta 1), reused 0 (delta 0) + To /home/sambahome/michele/casa + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master + ok + ``` + +###### 2. On windows I clone origin, and I sync empty + + ```cmd + M:\>git clone ssh://michele@home/home/michele/casa win + Cloning into 'win'... + remote: Counting objects: 20, done. + remote: Compressing objects: 100% (15/15), done. + remote: Total 20 (delta 3), reused 0 (delta 0) + Receiving objects: 100% (20/20), done. + Resolving deltas: 100% (3/3), done. + M:\>cd win + M:\win>git annex status + Detected a crippled filesystem. + Enabling direct mode. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + repository mode: direct + trusted repositories: (merging origin/git-annex origin/synced/git-annex into git-annex...) + (Recording state in git...) + 0 + semitrusted repositories: 4 + 00000000-0000-0000-0000-000000000001 -- web + 598ecfac-087d-49a3-b48d-beafd0d71805 -- origin (casa) + b2699c17-d0bc-40a0-b447-a64ad109b2a2 -- here (ALICUDI:M:\win) + bd4166eb-296b-4f0f-a3be-6c25e4c7cbb0 -- local + untrusted repositories: 0 + transfers in progress: none + available local disk space: unknown + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 1 + known annex size: 8 bytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + SHA256E: 1 + ``` + +###### 3. I copy content from local to win + + ```bash + [michele@home local]$ git annex copy --to origin lintest + copy lintest (to origin...) ok + (Recording state in git...) + [michele@home local]$ git annex sync + ...runs ok... + ``` + +###### 4. and + + ```cmd + M:\win>git annex sync + ...works + M:\win>git annex get . + ...works + M:\win>cat lintest + lintest + ``` + +so far so good. +###### 5. Now the renaming part (performed on linux indirect repo) + + ```bash + [michele@home local]$ git mv lintest renamed + [michele@home local]$ git annex list + here + |origin + ||web + ||| + XX_ renamed + [michele@home local]$ git annex sync + ...works + ``` + +###### 6. now, by issuing sync on windows I start getting a "push issue": + + ``` + M:\win>git annex sync + commit + ok + pull origin + remote: Counting objects: 3, done. + remote: Total 2 (delta 0), reused 0 (delta 0) + Unpacking objects: 100% (2/2), done. + From ssh://home/home/michele/casa + c3b7a63..a0854bf master -> origin/master + c3b7a63..a0854bf synced/master -> origin/synced/master + ok + push origin + Counting objects: 9, done. + Delta compression using up to 2 threads. + Compressing objects: 100% (4/4), done. + Writing objects: 100% (5/5), 484 bytes, done. + Total 5 (delta 1), reused 0 (delta 0) + To ssh://michele@home/home/michele/casa + 6c18669..8cc74a0 git-annex -> synced/git-annex + ! [rejected] master -> synced/master (non-fast-forward) + error: failed to push some refs to 'ssh://michele@home/home/michele/casa' + hint: Updates were rejected because a pushed branch tip is behind its remote + hint: counterpart. Check out this branch and merge the remote changes + hint: (e.g. 'git pull') before pushing again. + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + failed + git-annex: sync: 1 failed + + M:\win> + ``` + +at this stage I tried to issue a git annex merge, git annex sync, leading to same result. + +somewhere in the forum I read i could try issuing a git pull origin master (this could be the problem). +and the result is as such: + + ``` + M:\win>git pull master + fatal: 'master' does not appear to be a git repository + fatal: The remote end hung up unexpectedly + + M:\win>git pull origin master + From ssh://home/home/michele/casa + * branch master -> FETCH_HEAD + Updating c3b7a63..a0854bf + error: Your local changes to the following files would be overwritten by merge: + lintest + Please, commit your changes or stash them before you can merge. + Aborting + + M:\win>git status + # On branch master + # Your branch is behind 'origin/master' by 1 commit, and can be fast-forwarded. + # + # Changes not staged for commit: + # (use "git add ..." to update what will be committed) + # (use "git checkout -- ..." to discard changes in working directory) + # + # modified: lintest + # + no changes added to commit (use "git add" and/or "git commit -a") + + M:\win>cat lintest + lintest + ``` + +well, ok it appears modified for some weirdness (crlf?), we can live with it. + + ``` + M:\win>git checkout -->> this replaces files contents with simlink (due to git pull above?) + ``` + +at this stage content is lost, and annex has no knowledge about it. + + ``` + M:\win>git annex fsck + fsck lintest ok + + M:\win>cat lintest + .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e90 + M:\win>git annex list + here + |origin + ||web + ||| + XX_ lintest + ``` + +still I cannot sync, but now i can pull origin master: + + ```cmd + M:\win>git pull origin master From ssh://home/home/michele/casa * branch master -> FETCH_HEAD Updating c3b7a63..a0854bf Fast-forward lintest => renamed | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lintest => renamed (100%) + ``` + +this doesnt restore content (annex thinks its already there: + + ``` + M:\win>cat renamed + .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715 + ``` + +I think my mistake is the use of the ```git checkout``` in direct mode. But why is the file detected as modified in the first place ? + +note: as long as i didn't drop on origin, i still can recover contents by 'forcing' a content refresh: + + ``` + M:\win>git annex get renamed --from origin + get renamed (from origin...) + SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715 + 8 100% 7.81kB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 30 bytes received 153 bytes 8.13 bytes/sec + total size is 8 speedup is 0.04 + ok + (Recording state in git...) + + M:\win>cat renamed + .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e909d + M:\win>git annex fsck + fsck renamed (fixing direct mode) (checksum...) ok + + M:\win>cat renamed + lintest +``` + + + + + + diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -398,14 +398,14 @@ Removes a repository from a group. -* `content repository [expression]` +* `wanted repository [expression]` When run with an expression, configures the content that is preferred to be held in the archive. See PREFERRED CONTENT below. For example: - git annex content . "include=*.mp3 or include=*.ogg" + git annex wanted . "include=*.mp3 or include=*.ogg" Without an expression, displays the current preferred content setting of the repository. @@ -673,16 +673,20 @@ * `unannex [path ...]` - Use this to undo an accidental `git annex add` command. You can use - `git annex unannex` to move content out of the annex at any point, - even if you've already committed it. + Use this to undo an accidental `git annex add` command. It puts the + file back how it was before the add. + Note that for safety, the content of the file remains in the annex (as a + hard link), until you use `git annex unused` and `git annex dropunused`. + This is not the command you should use if you intentionally annexed a file and don't want its contents any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. - In `--fast` mode, this command leaves content in the annex, simply making - a hard link to it. + Normally this does a slow copy of the file. In `--fast` mode, it + instead makes a hard link from the file to the content in the annex. + But use --fast mode with caution, because editing the file will + change the content in the annex. * `uninit` @@ -947,6 +951,18 @@ The size can be specified with any commonly used units, for example, "0.5 gb" or "100 KiloBytes" +* `--want-get` + + Matches files that the preferred content settings for the repository + make it want to get. Note that this will match even files that are + already present, unless limited with eg, `--not --in .` + +* `--want-drop` + + Matches files that the preferred content settings for the repository + make it want to drop. Note that this will match even files that have + already been dropped, unless limited with eg, `--in .` + * `--not` Inverts the next file matching option. For example, to only act on @@ -973,7 +989,7 @@ Each repository has a preferred content setting, which specifies content that the repository wants to have present. These settings can be configured -using `git annex vicfg` or `git annex content`. +using `git annex vicfg` or `git annex wanted`. They are used by the `--auto` option, and by the git-annex assistant. The preferred content settings are similar, but not identical to @@ -1101,6 +1117,11 @@ is not needed, because they already wait for all writers of the file to close it. On Mac OSX, when not using direct mode this defaults to 1 second, to work around a bad interaction with software there. + +* `annex.fscknudge` + + When set to false, prevents the webapp from reminding you when using + repositories that lack consistency checks. * `annex.autocommit` diff --git a/doc/install/OSX/comment_34_c9362141d15a2f68a75df9f8bfe29da0._comment b/doc/install/OSX/comment_34_c9362141d15a2f68a75df9f8bfe29da0._comment new file mode 100644 --- /dev/null +++ b/doc/install/OSX/comment_34_c9362141d15a2f68a75df9f8bfe29da0._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl-xMSPoRHcT5d2nAc1K8pWVi-AexKkYik" + nickname="Ralf" + subject="Mac OS X Maverick - symbol not found" + date="2013-10-27T21:02:45Z" + content=""" +Just to mention that the beta dated 24 Oct 2013 and Joey's autobuild of 27 Oct both don't start with the following error message for git-annex, git-annex-webapp under Mac OS X 10.9 Maverick with latest XCode installed: + + dyld: Symbol not found: _objc_debug_taggedpointer_mask + Referenced from: /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation + Expected in: /Applications/git-annex.app/Contents/MacOS/bundle/I + in /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation + + Trace/BPT trap: 5 + +Many thanks. Can I help? +"""]] diff --git a/doc/install/OSX/comment_35_8106196c3fef70652cb2106e2d5857db._comment b/doc/install/OSX/comment_35_8106196c3fef70652cb2106e2d5857db._comment new file mode 100644 --- /dev/null +++ b/doc/install/OSX/comment_35_8106196c3fef70652cb2106e2d5857db._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 35" + date="2013-10-27T21:06:57Z" + content=""" +We do not yet have an autobuild for 10.9. You can build from source: +"""]] diff --git a/doc/install/cabal/comment_25_8a7664e6f9271718dc607a0782366c5b._comment b/doc/install/cabal/comment_25_8a7664e6f9271718dc607a0782366c5b._comment new file mode 100644 --- /dev/null +++ b/doc/install/cabal/comment_25_8a7664e6f9271718dc607a0782366c5b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="RaspberryPie" + ip="141.138.141.208" + subject="Bad version on Hackage" + date="2013-10-30T18:56:25Z" + content=""" +Quick note: The latest version in the Hackage repository (git-annex-4.20131024) fails to build, due to a faulty version of Assistant/Threads/Cronner.hs. +"""]] diff --git a/doc/install/fromscratch.mdwn b/doc/install/fromscratch.mdwn --- a/doc/install/fromscratch.mdwn +++ b/doc/install/fromscratch.mdwn @@ -24,6 +24,7 @@ * [regex-tdfa](http://hackage.haskell.org/package/regex-tdfa) * [extensible-exceptions](http://hackage.haskell.org/package/extensible-exceptions) * [feed](http://hackage.haskell.org/package/feed) + * [async](http://hackage.haskell.org/package/async) * Optional haskell stuff, used by the [[assistant]] and its webapp * [stm](http://hackage.haskell.org/package/stm) (version 2.3 or newer) @@ -48,7 +49,6 @@ * [network-protocol-xmpp](http://hackage.haskell.org/package/network-protocol-xmpp) * [dns](http://hackage.haskell.org/package/dns) * [xml-types](http://hackage.haskell.org/package/xml-types) - * [async](http://hackage.haskell.org/package/async) * [HTTP](http://hackage.haskell.org/package/HTTP) * [unix-compat](http://hackage.haskell.org/package/unix-compat) * [MonadCatchIO-transformers](http://hackage.haskell.org/package/MonadCatchIO-transformers) diff --git a/doc/news/version_4.20131101.mdwn b/doc/news/version_4.20131101.mdwn new file mode 100644 --- /dev/null +++ b/doc/news/version_4.20131101.mdwn @@ -0,0 +1,29 @@ +git-annex 4.20131101 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * The "git annex content" command is renamed to "git annex wanted". + * New --want-get and --want-drop options which can be used to + test preferred content settings. + For example, "git annex find --in . --want-drop" + * assistant: When autostarted, wait 5 seconds before running the startup + scan, to avoid contending with the user's desktop login process. + * webapp: When setting up a bare shared repository, enable non-fast-forward + pushes. + * sync: Show a hint about receive.denyNonFastForwards when a push fails. + * directory, webdav: Fix bug introduced in version 4.20131002 that + caused the chunkcount file to not be written. Work around repositories + without such a file, so files can still be retreived from them. + * assistant: Automatically repair damanged git repository, if it can + be done without losing data. + * assistant: Support repairing git remotes that are locally accessible + (eg, on removable drives). + * add: Fix reversion in 4.20130827 when adding unlocked files that have + not yet been committed. + * unannex: New, much slower, but more safe behavior: Copies files out of + the annex. This avoids an unannex of one file breaking other files that + link to the same content. Also, it means that the content + remains in the annex using up space until cleaned up with + "git annex unused". + (The behavior of unannex --fast has not changed; it still hard links + to content in the annex. --fast was not made the default because it is + potentially unsafe; editing such a hard linked file can unexpectedly + change content stored in the annex.)"""]] diff --git a/doc/preferred_content.mdwn b/doc/preferred_content.mdwn --- a/doc/preferred_content.mdwn +++ b/doc/preferred_content.mdwn @@ -7,7 +7,7 @@ smarter things. Preferred content settings can be edited using `git -annex vicfg`, or viewed and set at the command line with `git annex content`. +annex vicfg`, or viewed and set at the command line with `git annex wanted`. Each repository can have its own settings, and other repositories may also try to honor those settings. So there's no local `.git/config` setting it. @@ -83,6 +83,16 @@ `git annex enableremote $remote preferreddir=$dirname` (If no directory name is configured, it uses "public" by default.) + +## testing preferred content settings + +To check at the command line which files are matched by preferred content +settings, you can use the --want-get and --want-drop options. + +For example, "git annex find --want-get --not --in ." will find all the +files that "git annex get --auto" will want to get, and "git annex find +--want-drop --in ." will find all the files that "git annex drop --auto" +will want to drop. ## standard expressions diff --git a/doc/tips/Git_annex_and_Calibre.mdwn b/doc/tips/Git_annex_and_Calibre.mdwn --- a/doc/tips/Git_annex_and_Calibre.mdwn +++ b/doc/tips/Git_annex_and_Calibre.mdwn @@ -111,8 +111,10 @@ =================== You could also use direct mode in place of the auto unlock feature - git annex indirect + git annex direct The remove the `post-commit` git hook (or do not add it). Its a simpler solution, but remember that interaction between git annex direct -repositories and plain git are complex +repositories and plain git are complex and sometimes downright dangerous. See [[direct mode]] for details. + +In particular, do *not* called `git add *` in the above steps, as that will commit all books into git. diff --git a/doc/tips/flickrannex.mdwn b/doc/tips/flickrannex.mdwn --- a/doc/tips/flickrannex.mdwn +++ b/doc/tips/flickrannex.mdwn @@ -39,7 +39,7 @@ The photo name on flickr is currently the GPGHMACSHA1 version. Run the following command in your annex directory - git annex content flickr uuid include=*.jpg or include=*.jpeg or include=*.gif or include=*.png + git annex wanted flickr uuid include=*.jpg or include=*.jpeg or include=*.gif or include=*.png ## Encrypted mode The current version base64 encodes all the data, which results in ~35% larger filesize. @@ -47,7 +47,7 @@ I might look into yyenc instead. I'm not sure if it will work in the tEXt field. Run the following command in your annex directory - git annex content flickr exclude=largerthan=30mb + git annex wanted flickr exclude=largerthan=30mb ## Including directories as tags Get get each of the directories below the top level git directory added as tags to uploads: diff --git a/doc/tips/imapannex.mdwn b/doc/tips/imapannex.mdwn --- a/doc/tips/imapannex.mdwn +++ b/doc/tips/imapannex.mdwn @@ -24,4 +24,4 @@ git config annex.imap-hook '/usr/bin/python2 ~/imapannex/imapannex.py' git annex initremote imap type=hook hooktype=imap encryption=shared git annex describe imap "the imap library" - git annex content imap exclude=largerthan=30mb + git annex wanted imap exclude=largerthan=30mb diff --git a/doc/tips/offline_archive_drives.mdwn b/doc/tips/offline_archive_drives.mdwn --- a/doc/tips/offline_archive_drives.mdwn +++ b/doc/tips/offline_archive_drives.mdwn @@ -24,7 +24,7 @@ its label, so you can find it later: git annex group archivedrive archive - git annex content archivedrive standard + git annex wanted archivedrive standard git annex describe archivedrive "my first archive drive (SATA)" Or you can use the assistant to set up the drive for you. diff --git a/doc/todo/Deleting_Unused_Files_by_Age.mdwn b/doc/todo/Deleting_Unused_Files_by_Age.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/Deleting_Unused_Files_by_Age.mdwn @@ -0,0 +1,13 @@ +I periodically move unused files to one of my servers. What I would like to +do is drop any unused file that has been unused for say more than 6 months? +I would like to not drop all unused files. + +> It strikes me that this is quite similar to how git handles deleting +> stale refs with the reflog. So, if `git annex unused` were changed to +> also look at the reflog, it would keep all files referred to by all refs +> in the reflog, until the reflog expires. You could then set reflog expiry +> to 6 months, and be done. +> +> However, I think that many users expect git annex unused to be able to +> immediately find and remove a file after it's been deleted. So this +> probably needs to be a configurable behavior. --[[Joey]] diff --git a/doc/todo/whishlist:_git_annex_drop_--dry-run.mdwn b/doc/todo/whishlist:_git_annex_drop_--dry-run.mdwn new file mode 100644 --- /dev/null +++ b/doc/todo/whishlist:_git_annex_drop_--dry-run.mdwn @@ -0,0 +1,5 @@ +It'd be useful to be able to see what `git annex drop` would do *before* asking it to drop any files. + +For example, I just set up my preferred contents expressions, and I don't know if I got them right. Before dropping anything from this repo, it'd be nice to check what would happen. I know git annex drop will only drop files that are above their minimum numcopies, but I'd still like to avoid heavyweight copying in case I got my preferred contents expressions wrong. + +> [[done]]; added --want-get and --want-drop. --[[Joey]] diff --git a/git-annex.1 b/git-annex.1 --- a/git-annex.1 +++ b/git-annex.1 @@ -366,13 +366,13 @@ .IP "\fBungroup repository groupname\fP" Removes a repository from a group. .IP -.IP "\fBcontent repository [expression]\fP" +.IP "\fBwanted repository [expression]\fP" When run with an expression, configures the content that is preferred to be held in the archive. See PREFERRED CONTENT below. .IP For example: .IP - git annex content . "include=*.mp3 or include=*.ogg" + git annex wanted . "include=*.mp3 or include=*.ogg" .IP Without an expression, displays the current preferred content setting of the repository. @@ -618,16 +618,20 @@ git annex reinject /tmp/foo.iso foo.iso .IP .IP "\fBunannex [path ...]\fP" -Use this to undo an accidental \fBgit annex add\fP command. You can use -\fBgit annex unannex\fP to move content out of the annex at any point, -even if you've already committed it. +Use this to undo an accidental \fBgit annex add\fP command. It puts the +file back how it was before the add. .IP +Note that for safety, the content of the file remains in the annex (as a +hard link), until you use \fBgit annex unused\fP and \fBgit annex dropunused\fP. +.IP This is not the command you should use if you intentionally annexed a file and don't want its contents any more. In that case you should use \fBgit annex drop\fP instead, and you can also \fBgit rm\fP the file. .IP -In \fB\-\-fast\fP mode, this command leaves content in the annex, simply making -a hard link to it. +Normally this does a slow copy of the file. In \fB\-\-fast\fP mode, it +instead makes a hard link from the file to the content in the annex. +But use \-\-fast mode with caution, because editing the file will +change the content in the annex. .IP .IP "\fBuninit\fP" Use this to stop using git annex. It will unannex every file in the @@ -854,6 +858,16 @@ The size can be specified with any commonly used units, for example, "0.5 gb" or "100 KiloBytes" .IP +.IP "\fB\-\-want\-get\fP" +Matches files that the preferred content settings for the repository +make it want to get. Note that this will match even files that are +already present, unless limited with eg, \fB\-\-not \-\-in .\fP +.IP +.IP "\fB\-\-want\-drop\fP" +Matches files that the preferred content settings for the repository +make it want to drop. Note that this will match even files that have +already been dropped, unless limited with eg, \fB\-\-in .\fP +.IP .IP "\fB\-\-not\fP" Inverts the next file matching option. For example, to only act on files with less than 3 copies, use \fB\-\-not \-\-copies=3\fP @@ -874,7 +888,7 @@ .SH PREFERRED CONTENT Each repository has a preferred content setting, which specifies content that the repository wants to have present. These settings can be configured -using \fBgit annex vicfg\fP or \fBgit annex content\fP. +using \fBgit annex vicfg\fP or \fBgit annex wanted\fP. They are used by the \fB\-\-auto\fP option, and by the git\-annex assistant. .PP The preferred content settings are similar, but not identical to @@ -989,6 +1003,10 @@ is not needed, because they already wait for all writers of the file to close it. On Mac OSX, when not using direct mode this defaults to 1 second, to work around a bad interaction with software there. +.IP +.IP "\fBannex.fscknudge\fP" +When set to false, prevents the webapp from reminding you when using +repositories that lack consistency checks. .IP .IP "\fBannex.autocommit\fP" Set to false to prevent the git\-annex assistant from automatically diff --git a/git-annex.cabal b/git-annex.cabal --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 4.20131024 +Version: 4.20131101 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess @@ -82,7 +82,7 @@ extensible-exceptions, dataenc, SHA, process, json, base (>= 4.5 && < 4.9), monad-control, MonadCatchIO-transformers, IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process, - SafeSemaphore, uuid, random, dlist, unix-compat + SafeSemaphore, uuid, random, dlist, unix-compat, async -- Need to list these because they're generated from .hsc files. Other-Modules: Utility.Touch Utility.Mounts Include-Dirs: Utility @@ -121,7 +121,7 @@ CPP-Options: -DWITH_WEBDAV if flag(Assistant) && ! os(windows) && ! os(solaris) - Build-Depends: async, stm (>= 2.3) + Build-Depends: stm (>= 2.3) CPP-Options: -DWITH_ASSISTANT if flag(Android) diff --git a/templates/configurators/fsck.hamlet b/templates/configurators/fsck.hamlet --- a/templates/configurators/fsck.hamlet +++ b/templates/configurators/fsck.hamlet @@ -9,13 +9,23 @@ Running the consistency check involves reading all the files in the # repository, which can take a long time if it's large. Running just a # little at a time will eventually check the whole repository. - $if (not (null checks)) + $if (not (null scheduledchecks))

Currently scheduled checks: - $forall c <- checks + $forall c <- scheduledchecks ^{showFsckForm False c}

^{showFsckStatus c}

- Add a check: - ^{showFsckForm True defaultFsck} + $if null (recommendedchecks) + Add a check: + ^{showFsckForm True (defaultFsck Nothing)} + $else + # + Some repositories are not yet checked. # + Please consider adding these checks: + $forall c <- recommendedchecks + ^{showFsckForm True c} +

+ Configuration + ^{showFsckPreferencesForm} diff --git a/templates/configurators/fsck/preferencesform.hamlet b/templates/configurators/fsck/preferencesform.hamlet new file mode 100644 --- /dev/null +++ b/templates/configurators/fsck/preferencesform.hamlet @@ -0,0 +1,6 @@ +
+
+ ^{form} +
+