diff --git a/Annex/UUID.hs b/Annex/UUID.hs --- a/Annex/UUID.hs +++ b/Annex/UUID.hs @@ -36,12 +36,7 @@ where gen [] = error $ "no output from " ++ command gen (l:_) = toUUID l - command = SysConfig.uuid - params - -- request a random uuid be generated - | command == "uuid" = ["-m"] - -- uuidgen generates random uuid by default - | otherwise = [] + (command:params) = words SysConfig.uuid {- Get current repository's UUID. -} getUUID :: Annex UUID diff --git a/Assistant/Install.hs b/Assistant/Install.hs new file mode 100644 --- /dev/null +++ b/Assistant/Install.hs @@ -0,0 +1,71 @@ +{- Assistant installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Install where + +import Assistant.Common +import Assistant.Install.AutoStart +import Assistant.Ssh +import Locations.UserConfig +import Utility.FileMode + +#ifdef darwin_HOST_OS +import Utility.OSX +#else +import Utility.FreeDesktop +#endif + +import System.Posix.Env + +standaloneAppBase :: IO (Maybe FilePath) +standaloneAppBase = getEnv "GIT_ANNEX_APP_BASE" + +{- The standalone app does not have an installation process. + - So when it's run, it needs to set up autostarting of the assistant + - daemon, as well as writing the programFile, and putting a + - git-annex-shell wrapper into ~/.ssh + - + - Note that this is done every time it's started, so if the user moves + - it around, the paths this sets up won't break. + -} +ensureInstalled :: IO () +ensureInstalled = go =<< standaloneAppBase + where + go Nothing = noop + go (Just base) = do + let program = base ++ "runshell git-annex" + programfile <- programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile program + +#ifdef darwin_HOST_OS + autostartfile <- userAutoStart osxAutoStartLabel +#else + autostartfile <- autoStartPath "git-annex" + <$> userConfigDir +#endif + installAutoStart program autostartfile + + {- This shim is only updated if it doesn't + - already exist with the right content. This + - ensures that there's no race where it would have + - worked, but is unavailable due to being updated. -} + sshdir <- sshDir + let shim = sshdir "git-annex-shell" + let content = unlines + [ "#!/bin/sh" + , "set -e" + , "exec", base "runshell" ++ + " git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"" + ] + curr <- catchDefaultIO "" $ readFileStrict shim + when (curr /= content) $ do + createDirectoryIfMissing True (parentDir shim) + writeFile shim content + modifyFileMode shim $ addModes [ownerExecuteMode] diff --git a/Assistant/Install/AutoStart.hs b/Assistant/Install/AutoStart.hs new file mode 100644 --- /dev/null +++ b/Assistant/Install/AutoStart.hs @@ -0,0 +1,38 @@ +{- Assistant autostart file installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Install.AutoStart where + +import Utility.FreeDesktop +#ifdef darwin_HOST_OS +import Utility.OSX +import Utility.Path +import System.Directory +#endif + +installAutoStart :: FilePath -> FilePath -> IO () +installAutoStart command file = do +#ifdef darwin_HOST_OS + createDirectoryIfMissing True (parentDir file) + writeFile file $ genOSXAutoStartFile osxAutoStartLabel command + ["assistant", "--autostart"] +#else + writeDesktopMenuFile (fdoAutostart command) file +#endif + +osxAutoStartLabel :: String +osxAutoStartLabel = "com.branchable.git-annex.assistant" + +fdoAutostart :: FilePath -> DesktopEntry +fdoAutostart command = genDesktopEntry + "Git Annex Assistant" + "Autostart" + False + (command ++ " assistant --autostart") + [] diff --git a/Assistant/MakeRemote.hs b/Assistant/MakeRemote.hs --- a/Assistant/MakeRemote.hs +++ b/Assistant/MakeRemote.hs @@ -25,6 +25,7 @@ import qualified Data.Text as T import qualified Data.Map as M +import Data.Char {- Sets up and begins syncing with a new ssh or rsync remote. -} makeSshRemote :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> Bool -> SshData -> IO () @@ -55,13 +56,10 @@ void remoteListRefresh maybe (error "failed to add remote") return =<< Remote.byName (Just name) -{- Inits a rsync special remote, and returns the name of the remote. -} +{- Inits a rsync special remote, and returns its name. -} makeRsyncRemote :: String -> String -> Annex String -makeRsyncRemote name location = makeRemote name location $ const $ do - (u, c) <- Command.InitRemote.findByName name - c' <- R.setup Rsync.remote u $ M.union config c - describeUUID u name - configSet u c' +makeRsyncRemote name location = makeRemote name location $ + const $ makeSpecialRemote name Rsync.remote config where config = M.fromList [ ("encryption", "shared") @@ -69,6 +67,14 @@ , ("type", "rsync") ] +{- Inits a special remote. -} +makeSpecialRemote :: String -> RemoteType -> R.RemoteConfig -> Annex () +makeSpecialRemote name remotetype config = do + (u, c) <- Command.InitRemote.findByName name + c' <- R.setup remotetype u $ M.union config c + describeUUID u name + configSet u c' + {- Returns the name of the git remote it created. If there's already a - remote at the location, returns its name. -} makeGitRemote :: String -> String -> Annex String @@ -86,7 +92,7 @@ r <- fromRepo id if not (any samelocation $ Git.remotes r) then do - let name = uniqueRemoteName r basename 0 + let name = uniqueRemoteName basename 0 r a name return name else return basename @@ -94,14 +100,19 @@ samelocation x = Git.repoLocation x == location {- Generate an unused name for a remote, adding a number if - - necessary. -} -uniqueRemoteName :: Git.Repo -> String -> Int -> String -uniqueRemoteName r basename n + - necessary. + - + - Ensures that the returned name is a legal git remote name. -} +uniqueRemoteName :: String -> Int -> Git.Repo -> String +uniqueRemoteName basename n r | null namecollision = name - | otherwise = uniqueRemoteName r basename (succ n) + | otherwise = uniqueRemoteName legalbasename (succ n) r where namecollision = filter samename (Git.remotes r) samename x = Git.remoteName x == Just name name - | n == 0 = basename - | otherwise = basename ++ show n + | n == 0 = legalbasename + | otherwise = legalbasename ++ show n + legalbasename = filter legal basename + legal '_' = True + legal c = isAlphaNum c diff --git a/Assistant/Ssh.hs b/Assistant/Ssh.hs --- a/Assistant/Ssh.hs +++ b/Assistant/Ssh.hs @@ -119,10 +119,20 @@ writeFile keyfile $ unlines $ filter (/= keyline) ls {- Implemented as a shell command, so it can be run on remote servers over - - ssh. -} + - ssh. + - + - The ~/.ssh/git-annex-shell wrapper script is created if not already + - present. + -} addAuthorizedKeysCommand :: Bool -> SshPubKey -> String addAuthorizedKeysCommand rsynconly pubkey = join "&&" [ "mkdir -p ~/.ssh" + , join "; " + [ "if [ ! -e " ++ wrapper ++ " ]" + , "then (" ++ join ";" (map echoval script) ++ ") > " ++ wrapper + , "fi" + ] + , "chmod 700 " ++ wrapper , "touch ~/.ssh/authorized_keys" , "chmod 600 ~/.ssh/authorized_keys" , unwords @@ -131,15 +141,23 @@ , ">>~/.ssh/authorized_keys" ] ] + where + echoval v = "echo " ++ shellEscape v + wrapper = "~/.ssh/git-annex-shell" + script = + [ "#!/bin/sh" + , "set -e" + , "exec git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"" + ] authorizedKeysLine :: Bool -> SshPubKey -> String authorizedKeysLine rsynconly pubkey {- TODO: Locking down rsync is difficult, requiring a rather - long perl script. -} | rsynconly = pubkey - | otherwise = limitcommand "git-annex-shell -c" ++ pubkey + | otherwise = limitcommand ++ pubkey where - limitcommand c = "command=\"perl -e 'exec qw(" ++ c ++ "), $ENV{SSH_ORIGINAL_COMMAND}'\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding " + limitcommand = "command=\"~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding " {- Generates a ssh key pair. -} genSshKeyPair :: IO SshKeyPair @@ -205,6 +223,10 @@ knownHost hostname = do sshdir <- sshDir ifM (doesFileExist $ sshdir "known_hosts") - ( not . null <$> readProcess "ssh-keygen" ["-F", T.unpack hostname] + ( not . null <$> checkhost , return False ) + where + {- ssh-keygen -F can crash on some old known_hosts file -} + checkhost = catchDefaultIO "" $ + readProcess "ssh-keygen" ["-F", T.unpack hostname] diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -205,15 +205,6 @@ openfiles <- S.fromList . map fst3 . filter openwrite <$> liftIO (Lsof.queryDir tmpdir) - -- TODO this is here for debugging a problem on - -- OSX, and is pretty expensive, so remove later - liftIO $ debug thisThread - [ "checking changes:" - , show changes - , "vs open files:" - , show openfiles - ] - let checked = map (check openfiles) changes {- If new events are received when files are closed, diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -22,8 +22,6 @@ import Remote.List import qualified Types.Remote as Remote -import Control.Concurrent -import qualified Control.Exception as E import qualified Data.Set as S #if WITH_DBUS @@ -31,6 +29,8 @@ import DBus.Client import DBus import Data.Word (Word32) +import Control.Concurrent +import qualified Control.Exception as E #else #warning Building without dbus support; will use mtab polling #endif diff --git a/Assistant/Threads/NetWatcher.hs b/Assistant/Threads/NetWatcher.hs --- a/Assistant/Threads/NetWatcher.hs +++ b/Assistant/Threads/NetWatcher.hs @@ -19,13 +19,12 @@ import Remote.List import qualified Types.Remote as Remote -import qualified Control.Exception as E - #if WITH_DBUS import Utility.DBus import DBus.Client import DBus import Data.Word (Word32) +import qualified Control.Exception as E #else #warning Building without dbus support; will poll for network connection changes #endif @@ -34,11 +33,11 @@ thisThread = "NetWatcher" netWatcherThread :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> NamedThread -netWatcherThread st dstatus scanremotes = thread $ #if WITH_DBUS +netWatcherThread st dstatus scanremotes = thread $ dbusThread st dstatus scanremotes #else - noop +netWatcherThread _ _ _ = thread noop #endif where thread = NamedThread thisThread diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs --- a/Assistant/Threads/WebApp.hs +++ b/Assistant/Threads/WebApp.hs @@ -20,6 +20,7 @@ import Assistant.WebApp.Configurators.Local import Assistant.WebApp.Configurators.Ssh import Assistant.WebApp.Configurators.Pairing +import Assistant.WebApp.Configurators.S3 import Assistant.WebApp.Documentation import Assistant.WebApp.OtherRepos import Assistant.ThreadedMonad diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs --- a/Assistant/WebApp.hs +++ b/Assistant/WebApp.hs @@ -114,13 +114,13 @@ webapp <- lift getYesod [whamlet||] -{- A button with an icon, and maybe label, that can be clicked to perform - - some action. +{- A button with an icon, and maybe label or tooltip, that can be + - clicked to perform some action. - With javascript, clicking it POSTs the Route, and remains on the same - page. - With noscript, clicking it GETs the Route. -} -actionButton :: Route WebApp -> (Maybe String) -> String -> String -> Widget -actionButton route label buttonclass iconclass = $(widgetFile "actionbutton") +actionButton :: Route WebApp -> (Maybe String) -> (Maybe String) -> String -> String -> Widget +actionButton route label tooltip buttonclass iconclass = $(widgetFile "actionbutton") type UrlRenderFunc = Route WebApp -> [(Text, Text)] -> Text type UrlRenderer = MVar (UrlRenderFunc) diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs --- a/Assistant/WebApp/Configurators.hs +++ b/Assistant/WebApp/Configurators.hs @@ -65,6 +65,7 @@ Just c -> case M.lookup "type" c of Just "rsync" -> u `enableswith` EnableRsyncR Just "directory" -> u `enableswith` EnableDirectoryR + Just "S3" -> u `enableswith` EnableS3R _ -> Nothing u `enableswith` r = Just (u, Just $ r u) list l = runAnnex [] $ do diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs --- a/Assistant/WebApp/Configurators/Local.hs +++ b/Assistant/WebApp/Configurators/Local.hs @@ -90,19 +90,22 @@ {- On first run, if run in the home directory, default to putting it in - ~/Desktop/annex, when a Desktop directory exists, and ~/annex otherwise. - - - If run in another directory, the user probably wants to put it there. -} + - If run in another directory, that the user can write to, + - the user probably wants to put it there. -} defaultRepositoryPath :: Bool -> IO FilePath defaultRepositoryPath firstrun = do cwd <- liftIO $ getCurrentDirectory home <- myHomeDir if home == cwd && firstrun - then do + then inhome + else ifM (canWrite cwd) ( return cwd, inhome ) + where + inhome = do desktop <- userDesktopDir ifM (doesDirectoryExist desktop) ( relHome $ desktop gitAnnexAssistantDefaultDir , return $ "~" gitAnnexAssistantDefaultDir ) - else return cwd newRepositoryForm :: FilePath -> Form RepositoryPath newRepositoryForm defpath msg = do diff --git a/Assistant/WebApp/Configurators/S3.hs b/Assistant/WebApp/Configurators/S3.hs new file mode 100644 --- /dev/null +++ b/Assistant/WebApp/Configurators/S3.hs @@ -0,0 +1,124 @@ +{- git-annex assistant webapp configurator for Amazon S3 + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Configurators.S3 where + +import Assistant.Common +import Assistant.MakeRemote +import Assistant.Sync +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.SideBar +import Assistant.ThreadedMonad +import Utility.Yesod +import qualified Remote.S3 as S3 +import Logs.Remote +import Remote (prettyListUUIDs) +import Types.Remote (RemoteConfig) + +import Yesod +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Map as M + +s3Configurator :: Widget -> Handler RepHtml +s3Configurator a = bootstrap (Just Config) $ do + sideBarDisplay + setTitle "Add an Amazon S3 repository" + a + +data StorageClass = StandardRedundancy | ReducedRedundancy + deriving (Eq, Enum, Bounded) + +instance Show StorageClass where + show StandardRedundancy = "STANDARD" + show ReducedRedundancy = "REDUCED_REDUNDANCY" + +data S3Input = S3Input + { accessKeyID :: Text + , secretAccessKey :: Text + -- Free form text for datacenter because Amazon adds new ones. + , datacenter :: Text + , storageClass :: StorageClass + , repoName :: Text + } + +data S3Creds = S3Creds Text Text + +extractCreds :: S3Input -> S3Creds +extractCreds i = S3Creds (accessKeyID i) (secretAccessKey i) + +s3InputAForm :: AForm WebApp WebApp S3Input +s3InputAForm = S3Input + <$> areq textField "Access Key ID" Nothing + <*> areq passwordField "Secret Access Key" Nothing + <*> areq textField "Datacenter" (Just "US") + <*> areq (selectFieldList storageclasses) "Storage class" (Just StandardRedundancy) + <*> areq textField "Repository name" (Just "S3") + where + storageclasses :: [(Text, StorageClass)] + storageclasses = + [ ("Standard redundancy", StandardRedundancy) + , ("Reduced redundancy (costs less)", ReducedRedundancy) + ] + +s3CredsAForm :: AForm WebApp WebApp S3Creds +s3CredsAForm = S3Creds + <$> areq textField "Access Key ID" Nothing + <*> areq passwordField "Secret Access Key" Nothing + +getAddS3R :: Handler RepHtml +getAddS3R = s3Configurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap s3InputAForm + case result of + FormSuccess s3input -> lift $ do + let name = T.unpack $ repoName s3input + makeS3Remote (extractCreds s3input) name $ M.fromList + [ ("encryption", "shared") + , ("type", "S3") + , ("datacenter", T.unpack $ datacenter s3input) + , ("storageclass", show $ storageClass s3input) + ] + _ -> showform form enctype + where + showform form enctype = do + let authtoken = webAppFormAuthToken + $(widgetFile "configurators/adds3") + +getEnableS3R :: UUID -> Handler RepHtml +getEnableS3R uuid = s3Configurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap s3CredsAForm + case result of + FormSuccess s3creds -> lift $ do + m <- runAnnex M.empty readRemoteLog + let name = fromJust $ M.lookup "name" $ + fromJust $ M.lookup uuid m + makeS3Remote s3creds name M.empty + _ -> showform form enctype + where + showform form enctype = do + let authtoken = webAppFormAuthToken + description <- lift $ runAnnex "" $ + T.pack . concat <$> prettyListUUIDs [uuid] + $(widgetFile "configurators/enables3") + +makeS3Remote :: S3Creds -> String -> RemoteConfig -> Handler () +makeS3Remote (S3Creds ak sk) name config = do + webapp <- getYesod + let st = fromJust $ threadState webapp + remotename <- runAnnex name $ fromRepo $ uniqueRemoteName name 0 + liftIO $ do + S3.s3SetCredsEnv ( T.unpack ak, T.unpack sk) + r <- runThreadState st $ addRemote $ do + makeSpecialRemote name S3.remote config + return remotename + syncNewRemote st (daemonStatus webapp) (scanRemotes webapp) r + redirect RepositoriesR diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs --- a/Assistant/WebApp/Configurators/Ssh.hs +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -24,7 +24,7 @@ import Data.Text (Text) import qualified Data.Text as T import qualified Data.Map as M -import Network.BSD +import Network.Socket import System.Posix.User sshConfigurator :: Widget -> Handler RepHtml @@ -63,13 +63,15 @@ check_hostname = checkM (liftIO . checkdns) textField checkdns t = do let h = T.unpack t - r <- catchMaybeIO $ getHostByName h - return $ case r of + r <- catchMaybeIO $ getAddrInfo canonname (Just h) Nothing + return $ case catMaybes . map addrCanonName <$> r of -- canonicalize input hostname if it had no dot - Just hostentry + Just (fullname:_) | '.' `elem` h -> Right t - | otherwise -> Right $ T.pack $ hostName hostentry + | otherwise -> Right $ T.pack fullname + Just [] -> Right t Nothing -> Left bad_hostname + canonname = Just $ defaultHints { addrFlags = [AI_CANONNAME] } check_username = checkBool (all (`notElem` "/:@ \t") . T.unpack) bad_username textField @@ -175,7 +177,8 @@ - a special ssh key will need to be generated just for this server. - - Once logged into the server, probe to see if git-annex-shell is - - available, or rsync. + - available, or rsync. Note that on OSX, ~/.ssh/git-annex-shell may be + - present, while git-annex-shell is not in PATH. -} testServer :: SshInput -> IO (Either ServerStatus SshData) testServer (SshInput { hostname = Nothing }) = return $ @@ -200,6 +203,7 @@ [ report "loggedin" , checkcommand "git-annex-shell" , checkcommand "rsync" + , checkcommand osx_shim ] knownhost <- knownHost hn let sshopts = filter (not . null) $ extraopts ++ @@ -214,6 +218,7 @@ parsetranscript . fst <$> sshTranscript sshopts "" parsetranscript s | reported "git-annex-shell" = UsableSshInput + | reported osx_shim = UsableSshInput | reported "rsync" = UsableRsyncServer | reported "loggedin" = UnusableServer "Neither rsync nor git-annex are installed on the server. Perhaps you should go install them?" @@ -224,6 +229,7 @@ checkcommand c = "if which " ++ c ++ "; then " ++ report c ++ "; fi" token r = "git-annex-probe " ++ r report r = "echo " ++ token r + osx_shim = "~/.ssh/git-annex-shell" {- Runs a ssh command; if it fails shows the user the transcript, - and if it succeeds, runs an action. -} diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -44,7 +44,6 @@ webapp <- lift getYesod current <- lift $ M.toList <$> getCurrentTransfers queued <- liftIO $ getTransferQueue $ transferQueue webapp - let ident = "transfers" autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int) let transfers = simplifyTransfers $ current ++ queued if null transfers @@ -54,6 +53,7 @@ ) else $(widgetFile "dashboard/transfers") where + ident = "transfers" isrunning info = not $ transferPaused info || isNothing (startedTime info) @@ -142,7 +142,7 @@ return False ) where -#if OSX +#ifdef darwin_HOST_OS cmd = "open" #else cmd = "xdg-open" diff --git a/Assistant/WebApp/Documentation.hs b/Assistant/WebApp/Documentation.hs --- a/Assistant/WebApp/Documentation.hs +++ b/Assistant/WebApp/Documentation.hs @@ -9,15 +9,37 @@ module Assistant.WebApp.Documentation where +import Assistant.Common import Assistant.WebApp import Assistant.WebApp.Types import Assistant.WebApp.SideBar +import Assistant.Install (standaloneAppBase) import Utility.Yesod +import Build.SysConfig (packageversion) import Yesod +{- The full license info may be included in a file on disk that can + - be read in and displayed. -} +licenseFile :: IO (Maybe FilePath) +licenseFile = do + base <- standaloneAppBase + return $ ( "LICENSE") <$> base + getAboutR :: Handler RepHtml getAboutR = bootstrap (Just About) $ do sideBarDisplay setTitle "About git-annex" + builtinlicense <- isJust <$> liftIO licenseFile $(widgetFile "documentation/about") + +getLicenseR :: Handler RepHtml +getLicenseR = do + v <- liftIO licenseFile + case v of + Nothing -> redirect AboutR + Just f -> bootstrap (Just About) $ do + -- no sidebar, just pages of legalese.. + setTitle "License" + license <- liftIO $ readFile f + $(widgetFile "documentation/license") diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs --- a/Assistant/WebApp/Types.hs +++ b/Assistant/WebApp/Types.hs @@ -28,7 +28,7 @@ import Data.Text (Text, pack, unpack) import Control.Concurrent.STM -staticFiles "static" +publicFiles "static" mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes --- a/Assistant/WebApp/routes +++ b/Assistant/WebApp/routes @@ -2,6 +2,7 @@ /noscript NoScriptR GET /noscript/auto NoScriptAutoR GET /about AboutR GET +/about/license LicenseR GET /config ConfigR GET /config/repository RepositoriesR GET @@ -15,7 +16,8 @@ /config/repository/add/ssh/confirm/#SshData ConfirmSshR GET /config/repository/add/ssh/make/git/#SshData MakeSshGitR GET /config/repository/add/ssh/make/rsync/#SshData MakeSshRsyncR GET -/config/repository/add/rsync.net AddRsyncNetR GET +/config/repository/add/cloud/rsync.net AddRsyncNetR GET +/config/repository/add/cloud/S3 AddS3R GET /config/repository/pair/start StartPairR GET /config/repository/pair/inprogress/#SecretReminder InprogressPairR GET @@ -23,6 +25,7 @@ /config/repository/enable/rsync/#UUID EnableRsyncR GET /config/repository/enable/directory/#UUID EnableDirectoryR GET +/config/repository/enable/S3/#UUID EnableS3R GET /transfers/#NotificationId TransfersR GET /sidebar/#NotificationId SideBarR GET diff --git a/Build/Configure.hs b/Build/Configure.hs --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -19,7 +19,7 @@ , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] "" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid -m", "uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs --- a/Build/InstallDesktopFile.hs +++ b/Build/InstallDesktopFile.hs @@ -15,15 +15,16 @@ import Utility.Path import Utility.Monad import Locations.UserConfig +import Utility.OSX +import Assistant.Install.AutoStart import Control.Applicative -import Control.Monad import System.Directory import System.Environment import System.Posix.User -import System.Posix.Types import System.Posix.Files import System.FilePath +import Data.Maybe {- The command can be either just "git-annex", or the full path to use - to run it. -} @@ -43,49 +44,42 @@ (command ++ " assistant --autostart") [] -isRoot :: IO Bool -isRoot = do - uid <- fromIntegral <$> getRealUserID - return $ uid == 0 +systemwideInstall :: IO Bool +systemwideInstall = isroot <||> destdirset + where + isroot = do + uid <- fromIntegral <$> getRealUserID + return $ uid == (0 :: Int) + destdirset = isJust <$> catchMaybeIO (getEnv "DESTDIR") inDestDir :: FilePath -> IO FilePath inDestDir f = do destdir <- catchDefaultIO "" (getEnv "DESTDIR") - return $ destdir f + return $ destdir ++ "/" ++ f writeFDODesktop :: FilePath -> IO () writeFDODesktop command = do - datadir <- ifM isRoot ( return systemDataDir, userDataDir ) + datadir <- ifM systemwideInstall ( return systemDataDir, userDataDir ) writeDesktopMenuFile (desktop command) =<< inDestDir (desktopMenuFilePath "git-annex" datadir) - configdir <- ifM isRoot ( return systemConfigDir, userConfigDir ) - writeDesktopMenuFile (autostart command) + configdir <- ifM systemwideInstall ( return systemConfigDir, userConfigDir ) + installAutoStart command =<< inDestDir (autoStartPath "git-annex" configdir) - ifM isRoot - ( return () - , do - programfile <- inDestDir =<< programFile - createDirectoryIfMissing True (parentDir programfile) - writeFile programfile command - ) - writeOSXDesktop :: FilePath -> IO () writeOSXDesktop command = do - home <- myHomeDir - - let base = "Library" "LaunchAgents" label ++ ".plist" - autostart <- ifM isRoot ( inDestDir $ "/" base , inDestDir $ home base) - createDirectoryIfMissing True (parentDir autostart) - writeFile autostart $ genOSXAutoStartFile label command + installAutoStart command =<< inDestDir =<< ifM systemwideInstall + ( return $ systemAutoStart osxAutoStartLabel + , userAutoStart osxAutoStartLabel + ) + {- Install the OSX app in non-self-contained mode. -} let appdir = "git-annex.app" installOSXAppFile appdir "Contents/Info.plist" Nothing installOSXAppFile appdir "Contents/Resources/git-annex.icns" Nothing - installOSXAppFile appdir "Contents/MacOS/git-annex" (Just webappscript) + installOSXAppFile appdir "Contents/MacOS/git-annex-webapp" (Just webappscript) where - label = "com.branchable.git-annex.assistant" webappscript = unlines [ "#!/bin/sh" , command ++ " webapp" @@ -93,9 +87,9 @@ installOSXAppFile :: FilePath -> FilePath -> Maybe String -> IO () installOSXAppFile appdir appfile mcontent = do - let src = "ui-macos" appdir appfile + let src = "standalone" "macos" appdir appfile home <- myHomeDir - dest <- ifM isRoot + dest <- ifM systemwideInstall ( return $ "/Applications" appdir appfile , return $ home "Desktop" appdir appfile ) @@ -106,33 +100,23 @@ mode <- fileMode <$> getFileStatus src setFileMode dest mode -genOSXAutoStartFile :: String -> String -> String -genOSXAutoStartFile label command = unlines - [ "" - , "" - , "" - , "" - , "Label" - , "" ++ label ++ "" - , "ProgramArguments" - , "" - , "" ++ command ++ "" - , "assistant" - , "--autostart" - , "" - , "RunAtLoad" - , "" - , "" - ] - -writeDesktop :: FilePath -> IO () +install :: FilePath -> IO () +install command = do #ifdef darwin_HOST_OS -writeDesktop = writeOSXDesktop + writeOSXDesktop command #else -writeDesktop = writeFDODesktop + writeFDODesktop command #endif + ifM systemwideInstall + ( return () + , do + programfile <- inDestDir =<< programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile command + ) +main :: IO () main = getArgs >>= go where go [] = error "specify git-annex command" - go (command:_) = writeDesktop command + go (command:_) = install command diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,33 @@ +git-annex (3.20121001) unstable; urgency=low + + * fsck: Now has an incremental mode. Start a new incremental fsck pass + with git annex fsck --incremental. Now the fsck can be interrupted + as desired, and resumed with git annex fsck --more. + Thanks, Justin Azoff + * New --time-limit option, makes long git-annex commands stop after + a specified amount of time. + * fsck: New --incremental-schedule option which is nice for scheduling + eg, monthly incremental fsck runs in cron jobs. + * Fix fallback to ~/Desktop when xdg-user-dir is not available. + Closes: #688833 + * S3: When using a shared cipher, S3 credentials are not stored encrypted + in the git repository, as that would allow anyone with access to + the repository access to the S3 account. Instead, they're stored + in a 600 mode file in the local git repo. + * webapp: Avoid crashing when ssh-keygen -F chokes on an invalid known_hosts + file. + * Always do a system wide installation when DESTDIR is set. Closes: #689052 + * The Makefile now builds with the new yesod by default. + Systems like Debian that have the old yesod 1.0.1 should set + GIT_ANNEX_LOCAL_FEATURES=-DWITH_OLD_YESOD + * copy: Avoid updating the location log when no copy is performed. + * configure: Test that uuid -m works, falling back to plain uuid if not. + * Avoid building the webapp on Debian architectures that do not yet + have template haskell and thus yesod. (Should be available for arm soonish + I hope). + + -- Joey Hess Mon, 01 Oct 2012 13:56:55 -0400 + git-annex (3.20120924) unstable; urgency=low * assistant: New command, a daemon which does everything watch does, @@ -23,7 +53,7 @@ files and reading from checksum commands. * sync: Pushes the git-annex branch to remote/synced/git-annex, rather than directly to remote/git-annex. - * Now supports matchig files that are present on a number of remotes + * Now supports matching files that are present on a number of remotes with a specified trust level. Example: --copies=trusted:2 Thanks, Nicolas Pouillard diff --git a/COPYRIGHT b/COPYRIGHT --- a/COPYRIGHT +++ b/COPYRIGHT @@ -9,7 +9,7 @@ Copyright: © 2012 Joey Hess License: AGPL-3+ -Files: doc/logo* */favicon.ico ui-macos/git-annex.app/Contents/Resources/git-annex.icns +Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns Copyright: 2007 Henrik Nyh 2010 Joey Hess License: other diff --git a/Command/Fsck.hs b/Command/Fsck.hs --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,8 +7,6 @@ module Command.Fsck where -import System.Posix.Process (getProcessID) - import Common.Annex import Command import qualified Annex @@ -27,7 +25,14 @@ import Config import qualified Option import Types.Key +import Utility.HumanTime +import System.Posix.Process (getProcessID) +import Data.Time.Clock.POSIX +import Data.Time +import System.Posix.Types (EpochTime) +import System.Locale + def :: [Command] def = [withOptions options $ command "fsck" paramPaths seek "check for problems"] @@ -35,25 +40,71 @@ fromOption :: Option fromOption = Option.field ['f'] "from" paramRemote "check remote" +startIncrementalOption :: Option +startIncrementalOption = Option.flag ['S'] "incremental" "start an incremental fsck" + +moreIncrementalOption :: Option +moreIncrementalOption = Option.flag ['m'] "more" "continue an incremental fsck" + +incrementalScheduleOption :: Option +incrementalScheduleOption = Option.field [] "incremental-schedule" paramTime + "schedule incremental fscking" + options :: [Option] -options = [fromOption] +options = + [ fromOption + , startIncrementalOption + , moreIncrementalOption + , incrementalScheduleOption + ] seek :: [CommandSeek] seek = [ withField fromOption Remote.byName $ \from -> - withFilesInGit $ whenAnnexed $ start from - , withBarePresentKeys startBare + withIncremental $ \i -> withFilesInGit $ whenAnnexed $ start from i + , withIncremental $ \i -> withBarePresentKeys $ startBare i ] -start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart -start from file (key, backend) = do +withIncremental :: (Incremental -> CommandSeek) -> CommandSeek +withIncremental = withValue $ do + i <- maybe (return False) (checkschedule . parseDuration) + =<< Annex.getField (Option.name incrementalScheduleOption) + starti <- Annex.getFlag (Option.name startIncrementalOption) + morei <- Annex.getFlag (Option.name moreIncrementalOption) + case (i, starti, morei) of + (False, False, False) -> return NonIncremental + (False, True, _) -> startIncremental + (False ,False, True) -> ContIncremental <$> getStartTime + (True, _, _) -> + maybe startIncremental (return . ContIncremental . Just) + =<< getStartTime + where + startIncremental = do + recordStartTime + return StartIncremental + + checkschedule Nothing = error "bad --incremental-schedule value" + checkschedule (Just delta) = do + Annex.addCleanup "" $ do + v <- getStartTime + case v of + Nothing -> noop + Just started -> do + now <- liftIO getPOSIXTime + when (now - realToFrac started >= delta) $ + resetStartTime + return True + +start :: Maybe Remote -> Incremental -> FilePath -> (Key, Backend) -> CommandStart +start from inc file (key, backend) = do numcopies <- numCopies file - showStart "fsck" file case from of - Nothing -> next $ perform key file backend numcopies - Just r -> next $ performRemote key file backend numcopies r + Nothing -> go $ perform key file backend numcopies + Just r -> go $ performRemote key file backend numcopies r + where + go = runFsck inc file key -perform :: Key -> FilePath -> Backend -> Maybe Int -> CommandPerform +perform :: Key -> FilePath -> Backend -> Maybe Int -> Annex Bool perform key file backend numcopies = check -- order matters [ fixLink key file @@ -65,13 +116,13 @@ {- To fsck a remote, the content is retrieved to a tmp file, - and checked locally. -} -performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> CommandPerform +performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> Annex Bool performRemote key file backend numcopies remote = dispatch =<< Remote.hasKey remote key where dispatch (Left err) = do showNote err - stop + return False dispatch (Right True) = withtmp $ \tmpfile -> ifM (getfile tmpfile) ( go True (Just tmpfile) @@ -111,30 +162,23 @@ error "fsck should be run without parameters in a bare repository" map a <$> loggedKeys -startBare :: Key -> CommandStart -startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of +startBare :: Incremental -> Key -> CommandStart +startBare inc key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of Nothing -> stop - Just backend -> do - showStart "fsck" (key2file key) - next $ performBare key backend + Just backend -> runFsck inc (key2file key) key $ performBare key backend {- Note that numcopies cannot be checked in a bare repository, because - getting the numcopies value requires a working copy with .gitattributes - files. -} -performBare :: Key -> Backend -> CommandPerform +performBare :: Key -> Backend -> Annex Bool performBare key backend = check [ verifyLocationLog key (key2file key) , checkKeySize key , checkBackend backend key ] -check :: [Annex Bool] -> CommandPerform -check = sequence >=> dispatch - where - dispatch vs - | all (== True) vs = next $ return True - | otherwise = stop - +check :: [Annex Bool] -> Annex Bool +check cs = all id <$> sequence cs {- Checks that the file's symlink points correctly to the content. -} fixLink :: Key -> FilePath -> Annex Bool @@ -303,3 +347,91 @@ Remote.logStatus remote key InfoMissing return $ (if ok then "dropped from " else "failed to drop from ") ++ Remote.name remote + +data Incremental = StartIncremental | ContIncremental (Maybe EpochTime) | NonIncremental + deriving (Eq) + +runFsck :: Incremental -> FilePath -> Key -> Annex Bool -> CommandStart +runFsck inc file key a = ifM (needFsck inc key) + ( do + showStart "fsck" file + next $ do + ok <- a + when ok $ + recordFsckTime key + next $ return ok + , stop + ) + +{- Check if a key needs to be fscked, with support for incremental fscks. -} +needFsck :: Incremental -> Key -> Annex Bool +needFsck (ContIncremental Nothing) _ = return True +needFsck (ContIncremental starttime) key = do + fscktime <- getFsckTime key + return $ fscktime < starttime +needFsck _ _ = return True + +{- To record the time that a key was last fscked, without + - modifying its mtime, we set the timestamp of its parent directory. + - Each annexed file is the only thing in its directory, so this is fine. + - + - To record that the file was fscked, the directory's sticky bit is set. + - (None of the normal unix behaviors of the sticky bit should matter, so + - we can reuse this permission bit.) + - + - Note that this relies on the parent directory being deleted when a file + - is dropped. That way, if it's later added back, the fsck record + - won't still be present. + -} +recordFsckTime :: Key -> Annex () +recordFsckTime key = do + parent <- parentDir <$> inRepo (gitAnnexLocation key) + liftIO $ void $ tryIO $ do + touchFile parent + setSticky parent + +getFsckTime :: Key -> Annex (Maybe EpochTime) +getFsckTime key = do + parent <- parentDir <$> inRepo (gitAnnexLocation key) + liftIO $ catchDefaultIO Nothing $ do + s <- getFileStatus parent + return $ if isSticky $ fileMode s + then Just $ modificationTime s + else Nothing + +{- Records the start time of an interactive fsck. + - + - To guard against time stamp damange (for example, if an annex directory + - is copied without -a), the fsckstate file contains a time that should + - be identical to its modification time. -} +recordStartTime :: Annex () +recordStartTime = do + f <- fromRepo gitAnnexFsckState + createAnnexDirectory $ parentDir f + liftIO $ do + nukeFile f + h <- openFile f WriteMode + t <- modificationTime <$> getFileStatus f + hPutStr h $ showTime $ realToFrac t + hClose h + where + showTime :: POSIXTime -> String + showTime = show + +resetStartTime :: Annex () +resetStartTime = liftIO . nukeFile =<< fromRepo gitAnnexFsckState + +{- Gets the incremental fsck start time. -} +getStartTime :: Annex (Maybe EpochTime) +getStartTime = do + f <- fromRepo gitAnnexFsckState + liftIO $ catchDefaultIO Nothing $ do + timestamp <- modificationTime <$> getFileStatus f + t <- readishTime <$> readFile f + return $ if Just (realToFrac timestamp) == t + then Just timestamp + else Nothing + where + readishTime :: String -> Maybe POSIXTime + readishTime s = utcTimeToPOSIXSeconds <$> + parseTime defaultTimeLocale "%s%Qs" s diff --git a/Command/Move.hs b/Command/Move.hs --- a/Command/Move.hs +++ b/Command/Move.hs @@ -92,15 +92,16 @@ ok <- upload (Remote.uuid dest) key (Just file) noRetry $ Remote.storeKey dest key (Just file) if ok - then finish + then finish True else do when fastcheck $ warning "This could have failed because --fast is enabled." stop - Right True -> finish + Right True -> finish False where - finish = do - Remote.logStatus dest key InfoPresent + finish remotechanged = do + when remotechanged $ + Remote.logStatus dest key InfoPresent if move then do whenM (inAnnex key) $ removeAnnex key diff --git a/Command/WebApp.hs b/Command/WebApp.hs --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -17,6 +17,7 @@ import Assistant.TransferSlots import Assistant.Threads.WebApp import Assistant.WebApp +import Assistant.Install import Utility.WebApp import Utility.Daemon (checkDaemon, lockPidFile) import Init @@ -39,6 +40,7 @@ start :: CommandStart start = notBareRepo $ do + liftIO $ ensureInstalled ifM isInitialized ( go , liftIO startNoRepo ) stop where diff --git a/GitAnnex.hs b/GitAnnex.hs --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -141,9 +141,11 @@ "skip files with fewer copies" , Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName) "skip files not using a key-value backend" + , Option ['T'] ["time-limit"] (ReqArg Limit.addTimeLimit paramTime) + "stop after the specified amount of time" ] ++ Option.matcher where - setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readish v } + setnumcopies v = Annex.changeState $ \s -> s { Annex.forcenumcopies = readish v } setgitconfig :: String -> Annex () setgitconfig v = do newg <- inRepo $ Git.Config.store v diff --git a/INSTALL b/INSTALL --- a/INSTALL +++ b/INSTALL @@ -1,85 +1,34 @@ -## OS-specific instructions +## Pick your OS -* [[OSX]] -* [[Debian]] -* [[Ubuntu]] -* [[Fedora]] -* [[FreeBSD]] -* [[openSUSE]] -* [[ArchLinux]] -* [[NixOS]] -* [[Gentoo]] -* [[ScientificLinux5]] - This should cover RHEL5 clones such as CentOS5 and so on -* Windows: [[sorry, not possible yet|todo/windows_support]] +Quick commands given where possible, but see the individual pages for +details. +[[!table format=dsv header=no data=""" +[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2) **beta** +[[Linux|linux_standalone]] | [download prebuilt linux tarball](http://downloads.kitenet.net/git-annex/linux/) +[[Debian]] | `apt-get install git-annex` +[[Ubuntu]] | `apt-get install git-annex` +[[FreeBSD]] | `pkg_add -r hs-git-annex` +[[ArchLinux]] | `yaourt -Sy git-annex` +[[NixOS]] | `nix-env -i git-annex` +[[Gentoo]] | `emerge git-annex` +[[ScientificLinux5]] | (and other RHEL5 clones like CentOS5) +[[Fedora]] | +[[openSUSE]] | +Windows | [[sorry, Windows not supported yet|todo/windows_support]] +"""]] + ## Using cabal -As a haskell package, git-annex can be installed using cabal. For example: +As a haskell package, git-annex can be installed using cabal. +Start by installing the [Haskell Platform](http://hackage.haskell.org/platform/), +and then: cabal install git-annex --bindir=$HOME/bin -The above downloads the latest release. Alternatively, you can [[download]] -it yourself and [[manually_build_with_cabal|install/cabal]]. - -## Installation by hand - -This is not recommended, it's easier to let cabal pull in the many haskell -libraries. To build and use git-annex by hand, you will need: +That installs the latest release. Alternatively, you can [[download]] +git-annex yourself and [[manually_build_with_cabal|install/cabal]]. -* Haskell stuff - * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer) - * [MissingH](http://github.com/jgoerzen/missingh/wiki) - * [pcre-light](http://hackage.haskell.org/package/pcre-light) - * [utf8-string](http://hackage.haskell.org/package/utf8-string) - * [SHA](http://hackage.haskell.org/package/SHA) - * [dataenc](http://hackage.haskell.org/package/dataenc) - * [monad-control](http://hackage.haskell.org/package/monad-control) - * [lifted-base](http://hackage.haskell.org/package/lifted-base) - * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack) - * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck) - * [HTTP](http://hackage.haskell.org/package/HTTP) - * [json](http://hackage.haskell.org/package/json) - * [IfElse](http://hackage.haskell.org/package/IfElse) - * [bloomfilter](http://hackage.haskell.org/package/bloomfilter) - * [edit-distance](http://hackage.haskell.org/package/edit-distance) - * [hS3](http://hackage.haskell.org/package/hS3) (optional) -* Optional haskell stuff, used by the [[assistant]] and its webapp (edit Makefile to disable) - * [stm](http://hackage.haskell.org/package/stm) - (version 2.3 or newer) - * [hinotify](http://hackage.haskell.org/package/hinotify) - (Linux only) - * [dbus](http://hackage.haskell.org/package/dbus) - * [yesod](http://hackage.haskell.org/package/yesod) - * [yesod-static](http://hackage.haskell.org/package/yesod-static) - * [yesod-default](http://hackage.haskell.org/package/yesod-default) - * [data-default](http://hackage.haskell.org/package/data-default) - * [case-insensitive](http://hackage.haskell.org/package/case-insensitive) - * [http-types](http://hackage.haskell.org/package/http-types) - * [transformers](http://hackage.haskell.org/package/transformers) - * [wai](http://hackage.haskell.org/package/wai) - * [wai-logger](http://hackage.haskell.org/package/wai-logger) - * [warp](http://hackage.haskell.org/package/warp) - * [blaze-builder](http://hackage.haskell.org/package/blaze-builder) - * [blaze-html](http://hackage.haskell.org/package/blaze-html) - * [crypto-api](http://hackage.haskell.org/package/crypto-api) - * [hamlet](http://hackage.haskell.org/package/hamlet) - * [clientsession](http://hackage.haskell.org/package/clientsession) - * [network-multicast](http://hackage.haskell.org/package/network-multicast) - * [network-info](http://hackage.haskell.org/package/network-info) -* Shell commands - * [git](http://git-scm.com/) - * [uuid](http://www.ossp.org/pkg/lib/uuid/) - (or `uuidgen` from util-linux) - * [xargs](http://savannah.gnu.org/projects/findutils/) - * [rsync](http://rsync.samba.org/) - * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended) - * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended; - a sha1 command will also do) - * [gpg](http://gnupg.org/) (optional; needed for encryption) - * [lsof](ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/) - (optional; recommended for watch mode) - * multicast DNS support, provided on linux by [nss-mdns](http://www.0pointer.de/lennart/projects/nss-mdns/) - (optional; recommended for the assistant to support pairing well) - * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs) +## Installation from scratch -Then just [[download]] git-annex and run: `make; make install` +This is not recommended, but if you really want to, see [[fromscratch]]. diff --git a/Limit.hs b/Limit.hs --- a/Limit.hs +++ b/Limit.hs @@ -9,6 +9,7 @@ import Text.Regex.PCRE.Light.Char8 import System.Path.WildMatch +import Data.Time.Clock.POSIX import Common.Annex import qualified Annex @@ -17,6 +18,7 @@ import qualified Backend import Annex.Content import Logs.Trust +import Utility.HumanTime type Limit = Utility.Matcher.Token (FilePath -> Annex Bool) @@ -106,3 +108,17 @@ where wanted = Backend.lookupBackendName name check = return . maybe False ((==) wanted . snd) + +addTimeLimit :: String -> Annex () +addTimeLimit s = do + let seconds = fromMaybe (error "bad time-limit") $ parseDuration s + start <- liftIO getPOSIXTime + let cutoff = start + seconds + addLimit $ const $ do + now <- liftIO getPOSIXTime + if now > cutoff + then do + warning $ "Time limit (" ++ s ++ ") reached!" + liftIO $ exitWith $ ExitFailure 101 + else return True + diff --git a/Locations.hs b/Locations.hs --- a/Locations.hs +++ b/Locations.hs @@ -18,7 +18,9 @@ gitAnnexBadDir, gitAnnexBadLocation, gitAnnexUnusedLog, + gitAnnexFsckState, gitAnnexTransferDir, + gitAnnexCredsDir, gitAnnexJournalDir, gitAnnexJournalLock, gitAnnexIndex, @@ -130,7 +132,16 @@ gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") -{- .git/annex/transfer/ is used is used to record keys currently +{- .git/annex/fsckstate is used to store information about incremental fscks. -} +gitAnnexFsckState :: Git.Repo -> FilePath +gitAnnexFsckState r = gitAnnexDir r "fsckstate" + +{- .git/annex/creds/ is used to store credentials to access some special + - remotes. -} +gitAnnexCredsDir :: Git.Repo -> FilePath +gitAnnexCredsDir r = addTrailingPathSeparator $ gitAnnexDir r "creds" + +{- .git/annex/transfer/ is used to record keys currently - being transferred, and other transfer bookkeeping info. -} gitAnnexTransferDir :: Git.Repo -> FilePath gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r "transfer" diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -5,7 +5,9 @@ # If you get build failures due to missing haskell libraries, # you can turn off some of these features. -FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_OLD_YESOD -DWITH_PAIRING +# +# If you're using an old version of yesod, enable -DWITH_OLD_YESOD +FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING bins=git-annex mans=git-annex.1 git-annex-shell.1 @@ -21,7 +23,6 @@ OPTFLAGS=-DWITH_KQUEUE clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o ifeq ($(OS),Darwin) -OPTFLAGS=-DWITH_KQUEUE -DOSX # Ensure OSX compiler builds for 32 bit when using 32 bit ghc GHCARCH:=$(shell ghc -e 'print System.Info.arch') ifeq ($(GHCARCH),i386) @@ -132,5 +133,79 @@ # Upload to hackage. hackage: sdist @cabal upload dist/*.tar.gz + +THIRDPARTY_BINS=git curl lsof xargs rsync uuid wget gpg \ + sha1sum sha224sum sha256sum sha384sum sha512sum + +LINUXSTANDALONE_DEST=$(GIT_ANNEX_TMP_BUILD_DIR)/git-annex.linux +linuxstandalone: $(bins) + rm -rf "$(LINUXSTANDALONE_DEST)" + + cp -R standalone/linux "$(LINUXSTANDALONE_DEST)" + + install -d "$(LINUXSTANDALONE_DEST)/bin" + cp git-annex "$(LINUXSTANDALONE_DEST)/bin/" + strip "$(LINUXSTANDALONE_DEST)/bin/git-annex" + ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell" + zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE + + for bin in $(THIRDPARTY_BINS); do \ + cp "$$(which "$$bin")" "$(LINUXSTANDALONE_DEST)/bin/" || echo "$$bin not available; skipping"; \ + done + + install -d "$(LINUXSTANDALONE_DEST)/git-core" + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x) + + touch "$(LINUXSTANDALONE_DEST)/libdirs.tmp" + for lib in $$(ldd "$(LINUXSTANDALONE_DEST)"/bin/* $$(find "$(LINUXSTANDALONE_DEST)"/git-core/ -type f) | grep -v -f standalone/linux/glibc-libs | grep -v "not a dynamic executable" | egrep '^ ' | sed 's/^\t//' | sed 's/.*=> //' | cut -d ' ' -f 1 | sort | uniq); do \ + dir=$$(dirname "$$lib"); \ + install -d "$(LINUXSTANDALONE_DEST)/$$dir"; \ + echo "$$dir" >> "$(LINUXSTANDALONE_DEST)/libdirs.tmp"; \ + cp "$$lib" "$(LINUXSTANDALONE_DEST)/$$dir"; \ + if [ -L "$lib" ]; then \ + link=$$(readlink -f "$$lib"); \ + cp "$$link" "$(LINUXSTANDALONE_DEST)/$$(dirname "$$link")"; \ + fi; \ + done + sort "$(LINUXSTANDALONE_DEST)/libdirs.tmp" | uniq > "$(LINUXSTANDALONE_DEST)/libdirs" + rm -f "$(LINUXSTANDALONE_DEST)/libdirs.tmp" + + cd $(GIT_ANNEX_TMP_BUILD_DIR) && tar czf git-annex-standalone-$(shell dpkg --print-architecture).tar.gz git-annex.linux + +OSXAPP_DEST=$(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg/git-annex.app +OSXAPP_BASE=$(OSXAPP_DEST)/Contents/MacOS +osxapp: $(bins) + rm -rf "$(OSXAPP_DEST)" + install -d $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg + cp -R standalone/osx/git-annex.app "$(OSXAPP_DEST)" + + install -d "$(OSXAPP_BASE)/bin" + cp git-annex "$(OSXAPP_BASE)/bin/" + strip "$(OSXAPP_BASE)/bin/git-annex" + ln -sf git-annex "$(OSXAPP_BASE)/bin/git-annex-shell" + gzcat standalone/licences.gz > $(OSXAPP_BASE)/LICENSE + cp $(OSXAPP_BASE)/LICENSE $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg/LICENSE.txt + + for bin in $(THIRDPARTY_BINS); do \ + cp "$$(which "$$bin")" "$(OSXAPP_BASE)/bin/" || echo "$$bin not available; skipping"; \ + done + + install -d "$(OSXAPP_BASE)/git-core" + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)"/git-core && tar x) + + touch "$(OSXAPP_BASE)/libdirs.tmp" + for lib in $$(otool -L "$(OSXAPP_BASE)"/bin/* "$(OSXAPP_BASE)"/git-core/* | egrep '^ ' | cut -d ' ' -f 1 | sed 's/^ //' | sort | uniq); do \ + dir=$$(dirname "$$lib"); \ + install -d "$(OSXAPP_BASE)/$$dir"; \ + echo "$$dir" >> "$(OSXAPP_BASE)/libdirs.tmp"; \ + cp "$$lib" "$(OSXAPP_BASE)/$$dir"; \ + done + sort "$(OSXAPP_BASE)/libdirs.tmp" | uniq > "$(OSXAPP_BASE)/libdirs" + rm -f "$(OSXAPP_BASE)/libdirs.tmp" + rm -f tmp/git-annex.dmg + hdiutil create -size 640m -format UDRW -srcfolder $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg \ + -volname git-annex -o tmp/git-annex.dmg + rm -f tmp/git-annex.dmg.bz2 + bzip2 tmp/git-annex.dmg .PHONY: $(bins) test install diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs --- a/Remote/Helper/Encryptable.hs +++ b/Remote/Helper/Encryptable.hs @@ -88,6 +88,11 @@ Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache }) return $ Just cipher +{- Checks if there is a trusted (non-shared) cipher. -} +isTrustedCipher :: RemoteConfig -> Bool +isTrustedCipher c = + isJust (M.lookup "cipherkeys" c) && isJust (M.lookup "cipher" c) + {- Gets encryption Cipher, and encrypted version of Key. -} cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) cipherKey Nothing _ = return Nothing diff --git a/Remote/S3.hs b/Remote/S3.hs --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Remote.S3 (remote) where +module Remote.S3 (remote, s3SetCredsEnv) where import Network.AWS.AWSConnection import Network.AWS.S3Object @@ -27,6 +27,8 @@ import Crypto import Annex.Content import Utility.Base64 +import Annex.Perms +import Utility.FileMode remote :: RemoteType remote = RemoteType { @@ -85,12 +87,12 @@ use fullconfig = do gitConfigSpecialRemote u fullconfig "s3" "true" - s3SetCreds fullconfig + s3SetCreds fullconfig u defaulthost = do c' <- encryptionSetup c let fullconfig = c' `M.union` defaults - genBucket fullconfig + genBucket fullconfig u use fullconfig archiveorg = do @@ -206,7 +208,7 @@ when (isNothing $ config r) $ error $ "Missing configuration for special remote " ++ name r let bucket = M.lookup "bucket" $ fromJust $ config r - conn <- s3Connection $ fromJust $ config r + conn <- s3Connection (fromJust $ config r) (uuid r) case (bucket, conn) of (Just b, Just c) -> action (c, b) _ -> return noconn @@ -235,9 +237,9 @@ | isSpace c = [] | otherwise = "&" ++ show (ord c) ++ ";" -genBucket :: RemoteConfig -> Annex () -genBucket c = do - conn <- s3ConnectionRequired c +genBucket :: RemoteConfig -> UUID -> Annex () +genBucket c u = do + conn <- s3ConnectionRequired c u showAction "checking bucket" loc <- liftIO $ getBucketLocation conn bucket case loc of @@ -253,13 +255,13 @@ bucket = fromJust $ M.lookup "bucket" c datacenter = fromJust $ M.lookup "datacenter" c -s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection -s3ConnectionRequired c = - maybe (error "Cannot connect to S3") return =<< s3Connection c +s3ConnectionRequired :: RemoteConfig -> UUID -> Annex AWSConnection +s3ConnectionRequired c u = + maybe (error "Cannot connect to S3") return =<< s3Connection c u -s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) -s3Connection c = do - creds <- s3GetCreds c +s3Connection :: RemoteConfig -> UUID -> Annex (Maybe AWSConnection) +s3Connection c u = do + creds <- s3GetCreds c u case creds of Just (ak, sk) -> return $ Just $ AWSConnection host port ak sk _ -> do @@ -272,51 +274,77 @@ [(p, _)] -> p _ -> error $ "bad S3 port value: " ++ s -{- S3 creds come from the environment if set. - - Otherwise, might be stored encrypted in the remote's config. -} -s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String)) -s3GetCreds c = maybe fromconfig (return . Just) =<< liftIO getenv +{- S3 creds come from the environment if set, otherwise from the cache + - in gitAnnexCredsDir, or failing that, might be stored encrypted in + - the remote's config. -} +s3GetCreds :: RemoteConfig -> UUID -> Annex (Maybe (String, String)) +s3GetCreds c u = maybe fromcache (return . Just) =<< liftIO getenv where getenv = liftM2 (,) <$> get s3AccessKey <*> get s3SecretKey where get = catchMaybeIO . getEnv - setenv (ak, sk) = do - setEnv s3AccessKey ak True - setEnv s3SecretKey sk True + fromcache = do + d <- fromRepo gitAnnexCredsDir + let f = d fromUUID u + v <- liftIO $ catchMaybeIO $ readFile f + case lines <$> v of + Just (ak:sk:[]) -> return $ Just (ak, sk) + _ -> fromconfig fromconfig = do mcipher <- remoteCipher c case (M.lookup "s3creds" c, mcipher) of - (Just s3creds, Just cipher) -> - liftIO $ decrypt s3creds cipher + (Just s3creds, Just cipher) -> do + creds <- liftIO $ decrypt s3creds cipher + case creds of + [ak, sk] -> do + s3CacheCreds (ak, sk) u + return $ Just (ak, sk) + _ -> do error "bad s3creds" _ -> return Nothing - decrypt s3creds cipher = do - creds <- lines <$> - withDecryptedContent cipher - (return $ L.pack $ fromB64 s3creds) - (return . L.unpack) - case creds of - [ak, sk] -> do - setenv (ak, sk) - return $ Just (ak, sk) - _ -> do error "bad s3creds" + decrypt s3creds cipher = lines <$> + withDecryptedContent cipher + (return $ L.pack $ fromB64 s3creds) + (return . L.unpack) -{- Stores S3 creds encrypted in the remote's config if possible. -} -s3SetCreds :: RemoteConfig -> Annex RemoteConfig -s3SetCreds c = do - creds <- s3GetCreds c +{- Stores S3 creds encrypted in the remote's config if possible to do so + - securely, and otherwise locally in gitAnnexCredsDir. -} +s3SetCreds :: RemoteConfig -> UUID -> Annex RemoteConfig +s3SetCreds c u = do + creds <- s3GetCreds c u case creds of Just (ak, sk) -> do mcipher <- remoteCipher c case mcipher of - Just cipher -> do + Just cipher | isTrustedCipher c -> do s <- liftIO $ withEncryptedContent cipher (return $ L.pack $ unlines [ak, sk]) (return . L.unpack) return $ M.insert "s3creds" (toB64 s) c - Nothing -> return c + _ -> do + s3CacheCreds (ak, sk) u + return c _ -> return c + +{- The S3 creds are cached in gitAnnexCredsDir. -} +s3CacheCreds :: (String, String) -> UUID -> Annex () +s3CacheCreds (ak, sk) u = do + d <- fromRepo gitAnnexCredsDir + createAnnexDirectory d + liftIO $ do + let f = d fromUUID u + h <- openFile f WriteMode + modifyFileMode f $ removeModes + [groupReadMode, otherReadMode] + hPutStr h $ unlines [ak, sk] + hClose h + +{- Sets the S3 creds in the environment. -} +s3SetCredsEnv :: (String, String) -> IO () +s3SetCredsEnv (ak, sk) = do + setEnv s3AccessKey ak True + setEnv s3SecretKey sk True s3AccessKey :: String s3AccessKey = "AWS_ACCESS_KEY_ID" diff --git a/Setup.hs b/Setup.hs --- a/Setup.hs +++ b/Setup.hs @@ -58,6 +58,6 @@ installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () installDesktopFile copyDest verbosity pkg lbi = - InstallDesktopFile.writeDesktop $ dstBinDir "git-annex" + InstallDesktopFile.install $ dstBinDir "git-annex" where dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest diff --git a/Usage.hs b/Usage.hs --- a/Usage.hs +++ b/Usage.hs @@ -77,6 +77,8 @@ paramType = "TYPE" paramDate :: String paramDate = "DATE" +paramTime :: String +paramTime = "TIME" paramFormat :: String paramFormat = "FORMAT" paramFile :: String diff --git a/Utility/Applicative.o b/Utility/Applicative.o deleted file mode 100644 Binary files a/Utility/Applicative.o and /dev/null differ diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -102,8 +102,8 @@ #if WITH_KQUEUE type DirWatcherHandle = ThreadId watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle -watchDir dir ignored hooks runstartup = do - kq <- runstartup $ Kqueue.initKqueue dir ignored +watchDir dir prune hooks runstartup = do + kq <- runstartup $ Kqueue.initKqueue dir prune forkIO $ Kqueue.runHooks kq hooks #else type DirWatcherHandle = () diff --git a/Utility/Directory.o b/Utility/Directory.o deleted file mode 100644 Binary files a/Utility/Directory.o and /dev/null differ diff --git a/Utility/Exception.o b/Utility/Exception.o deleted file mode 100644 Binary files a/Utility/Exception.o and /dev/null differ diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs --- a/Utility/FileMode.hs +++ b/Utility/FileMode.hs @@ -63,9 +63,12 @@ , ownerReadMode, groupReadMode ] +checkMode :: FileMode -> FileMode -> Bool +checkMode checkfor mode = checkfor `intersectFileModes` mode == checkfor + {- Checks if a file mode indicates it's a symlink. -} isSymLink :: FileMode -> Bool -isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode +isSymLink = checkMode symbolicLinkMode {- Checks if a file has any executable bits set. -} isExecutable :: FileMode -> Bool @@ -88,3 +91,12 @@ combineModes [] = undefined combineModes [m] = m combineModes (m:ms) = foldl unionFileModes m ms + +stickyMode :: FileMode +stickyMode = 512 + +isSticky :: FileMode -> Bool +isSticky = checkMode stickyMode + +setSticky :: FilePath -> IO () +setSticky f = modifyFileMode f $ addModes [stickyMode] diff --git a/Utility/FileSystemEncoding.o b/Utility/FileSystemEncoding.o deleted file mode 100644 Binary files a/Utility/FileSystemEncoding.o and /dev/null differ diff --git a/Utility/HumanTime.hs b/Utility/HumanTime.hs new file mode 100644 --- /dev/null +++ b/Utility/HumanTime.hs @@ -0,0 +1,26 @@ +{- Time for humans. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.HumanTime where + +import Utility.PartialPrelude + +import Data.Time.Clock.POSIX (POSIXTime) + +{- Parses a human-input time duration, of the form "5h" or "1m". -} +parseDuration :: String -> Maybe POSIXTime +parseDuration s = do + num <- readish s :: Maybe Integer + units <- findUnits =<< lastMaybe s + return $ fromIntegral num * units + where + findUnits 's' = Just 1 + findUnits 'm' = Just 60 + findUnits 'h' = Just $ 60 * 60 + findUnits 'd' = Just $ 60 * 60 * 24 + findUnits 'y' = Just $ 60 * 60 * 24 * 365 + findUnits _ = Nothing diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -198,7 +198,8 @@ go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo) where go (Just newdirinfo) = do - let changes = olddirinfo // newdirinfo + let changes = filter (not . pruner . changedFile) $ + olddirinfo // newdirinfo let (added, deleted) = partition isAdd changes -- Scan newly added directories to add to the map. diff --git a/Utility/Misc.o b/Utility/Misc.o deleted file mode 100644 Binary files a/Utility/Misc.o and /dev/null differ diff --git a/Utility/Monad.o b/Utility/Monad.o deleted file mode 100644 Binary files a/Utility/Monad.o and /dev/null differ diff --git a/Utility/OSX.hs b/Utility/OSX.hs new file mode 100644 --- /dev/null +++ b/Utility/OSX.hs @@ -0,0 +1,43 @@ +{- OSX stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.OSX where + +import Utility.Path + +import System.FilePath + +autoStartBase :: String -> FilePath +autoStartBase label = "Library" "LaunchAgents" label ++ ".plist" + +systemAutoStart :: String -> FilePath +systemAutoStart label = "/" autoStartBase label + +userAutoStart :: String -> IO FilePath +userAutoStart label = do + home <- myHomeDir + return $ home autoStartBase label + +{- Generates an OSX autostart plist file with a given label, command, and + - params to run at boot or login. -} +genOSXAutoStartFile :: String -> String -> [String] -> String +genOSXAutoStartFile label command params = unlines + [ "" + , "" + , "" + , "" + , "Label" + , "" ++ label ++ "" + , "ProgramArguments" + , "" + , unlines $ map (\v -> "" ++ v ++ "") (command:params) + , "" + , "RunAtLoad" + , "" + , "" + ] + diff --git a/Utility/PartialPrelude.o b/Utility/PartialPrelude.o deleted file mode 100644 Binary files a/Utility/PartialPrelude.o and /dev/null differ diff --git a/Utility/Path.o b/Utility/Path.o deleted file mode 100644 Binary files a/Utility/Path.o and /dev/null differ diff --git a/Utility/Process.hs b/Utility/Process.hs --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -101,14 +101,14 @@ , env = environ } -{- Waits for a ProcessHandle, and throws an exception if the process +{- Waits for a ProcessHandle, and throws an IOError if the process - did not exit successfully. -} forceSuccessProcess :: CreateProcess -> ProcessHandle -> IO () forceSuccessProcess p pid = do code <- waitForProcess pid case code of ExitSuccess -> return () - ExitFailure n -> error $ showCmd p ++ " exited " ++ show n + ExitFailure n -> fail $ showCmd p ++ " exited " ++ show n {- Waits for a ProcessHandle and returns True if it exited successfully. -} checkSuccessProcess :: ProcessHandle -> IO Bool diff --git a/Utility/Process.o b/Utility/Process.o deleted file mode 100644 Binary files a/Utility/Process.o and /dev/null differ diff --git a/Utility/SafeCommand.o b/Utility/SafeCommand.o deleted file mode 100644 Binary files a/Utility/SafeCommand.o and /dev/null differ diff --git a/Utility/TempFile.o b/Utility/TempFile.o deleted file mode 100644 Binary files a/Utility/TempFile.o and /dev/null differ diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -44,7 +44,7 @@ runBrowser :: String -> IO Bool runBrowser url = boolSystem cmd [Param url] where -#if OSX +#ifdef darwin_HOST_OS cmd = "open" #else cmd = "xdg-open" @@ -80,7 +80,14 @@ { addrFlags = [AI_ADDRCONFIG] , addrSocketType = Stream } - go addr = bracketOnError (open addr) close (use addr) + {- Repeated attempts because bind sometimes fails for an + - unknown reason on OSX. -} + go addr = go' 100 addr + go' :: Int -> AddrInfo -> IO Socket + go' 0 _ = error "unable to bind to local socket" + go' n addr = do + r <- tryIO $ bracketOnError (open addr) close (use addr) + either (const $ go' (pred n) addr) return r open addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr) close = sClose use addr sock = do diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,33 @@ +git-annex (3.20121001) unstable; urgency=low + + * fsck: Now has an incremental mode. Start a new incremental fsck pass + with git annex fsck --incremental. Now the fsck can be interrupted + as desired, and resumed with git annex fsck --more. + Thanks, Justin Azoff + * New --time-limit option, makes long git-annex commands stop after + a specified amount of time. + * fsck: New --incremental-schedule option which is nice for scheduling + eg, monthly incremental fsck runs in cron jobs. + * Fix fallback to ~/Desktop when xdg-user-dir is not available. + Closes: #688833 + * S3: When using a shared cipher, S3 credentials are not stored encrypted + in the git repository, as that would allow anyone with access to + the repository access to the S3 account. Instead, they're stored + in a 600 mode file in the local git repo. + * webapp: Avoid crashing when ssh-keygen -F chokes on an invalid known_hosts + file. + * Always do a system wide installation when DESTDIR is set. Closes: #689052 + * The Makefile now builds with the new yesod by default. + Systems like Debian that have the old yesod 1.0.1 should set + GIT_ANNEX_LOCAL_FEATURES=-DWITH_OLD_YESOD + * copy: Avoid updating the location log when no copy is performed. + * configure: Test that uuid -m works, falling back to plain uuid if not. + * Avoid building the webapp on Debian architectures that do not yet + have template haskell and thus yesod. (Should be available for arm soonish + I hope). + + -- Joey Hess Mon, 01 Oct 2012 13:56:55 -0400 + git-annex (3.20120924) unstable; urgency=low * assistant: New command, a daemon which does everything watch does, @@ -23,7 +53,7 @@ files and reading from checksum commands. * sync: Pushes the git-annex branch to remote/synced/git-annex, rather than directly to remote/git-annex. - * Now supports matchig files that are present on a number of remotes + * Now supports matching files that are present on a number of remotes with a specified trust level. Example: --copies=trusted:2 Thanks, Nicolas Pouillard diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -22,10 +22,11 @@ libghc-edit-distance-dev, libghc-hinotify-dev [linux-any], libghc-stm-dev (>= 2.3), - libghc-dbus-dev, - libghc-yesod-dev, - libghc-yesod-static-dev, - libghc-yesod-default-dev, + libghc-dbus-dev [linux-any], + libghc-yesod-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-yesod-static-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-yesod-default-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-hamlet-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], libghc-case-insensitive-dev, libghc-http-types-dev, libghc-transformers-dev, @@ -35,7 +36,6 @@ libghc-blaze-builder-dev, libghc-blaze-html-dev, libghc-crypto-api-dev, - libghc-hamlet-dev, libghc-clientsession-dev, libghc-network-multicast-dev, libghc-network-info-dev, @@ -59,8 +59,8 @@ rsync, wget | curl, openssh-client (>= 1:5.6p1) -Recommends: lsof, libnss-mdns, gnupg -Suggests: graphviz, bup +Recommends: lsof, gnupg +Suggests: graphviz, bup, libnss-mdns Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/debian/copyright b/debian/copyright --- a/debian/copyright +++ b/debian/copyright @@ -9,7 +9,7 @@ Copyright: © 2012 Joey Hess License: AGPL-3+ -Files: doc/logo* */favicon.ico ui-macos/git-annex.app/Contents/Resources/git-annex.icns +Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns Copyright: 2007 Henrik Nyh 2010 Joey Hess License: other diff --git a/debian/rules b/debian/rules --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,12 @@ #!/usr/bin/make -f + +ARCH = $(shell dpkg-architecture -qDEB_BUILD_ARCH) +ifeq (install ok installed,$(shell dpkg-query -W -f '$${Status}' libghc-yesod-dev 2>/dev/null)) +export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_OLD_YESOD -DWITH_WEBAPP -DWITH_PAIRING +else +export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_PAIRING +endif + %: dh $@ diff --git a/doc/assistant.mdwn b/doc/assistant.mdwn --- a/doc/assistant.mdwn +++ b/doc/assistant.mdwn @@ -4,7 +4,7 @@ It's very easy to use, and has all the power of git and git-annex. Note that the git-annex assistant is still beta quality code. See -[[the_errata]] for known infelicities. +[[the_errata|errata]] for known infelicities. ## installation diff --git a/doc/assistant/thanks.mdwn b/doc/assistant/thanks.mdwn --- a/doc/assistant/thanks.mdwn +++ b/doc/assistant/thanks.mdwn @@ -48,7 +48,8 @@ Andrew, Aaron De Vries, Monti Knazze, Jorge Canseco, Hamish, Mark Eichin And special thanks to Kevin McKenzie, who also gave me a login to a Mac OSX -machine, which has proven invaluable. +machine, which has proven invaluable, and Jimmy Tang who has helped +with Mac OSX autobuilding and packaging. ## Other Backers @@ -171,7 +172,7 @@ Christian Simonsen, Wouter Beugelsdijk, Adam Gibson, Gal Buki, James Marble, Alan Chambers, Bernd Wiehle, Simon Korzun, Daniel Glassey, Eero af Heurlin, Mikael, Timo Engelhardt, Wesley Faulkner, Jay Wo, Mike Belle, -David Fowlkes Jr., Jimmy Tang, Karl-Heinz Strass, Ed Mullins, Sam Flint, +David Fowlkes Jr., Karl-Heinz Strass, Ed Mullins, Sam Flint, Hendrica, Mark Emer Anderson, Joshua Cole, Jan Gondol, Henrik Lindhe, Albert Delgado, Patrick, Alexa Avellar, Chris, sebsn1349, Maxim Kachur, Andrew Marshall, Navjot Narula, Alwin Mathew, Christian Mangelsdorf, Avi diff --git a/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn --- a/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn +++ b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn @@ -48,3 +48,6 @@ > > Please test and see if it works, and also if the "looping" problem > still happens. --[[Joey]] + +>> Closing; I'm pretty sure the looping is just transfer retrying, to be +>> expected if they fail. [[done]] --[[Joey]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn b/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn @@ -0,0 +1,31 @@ +What steps will reproduce the problem? + +Use assistant and create repository the a folder in home dir. +Use textedit and save a new txt to the repository folder. + +What is the expected output? What do you see instead? + +The alias solution is broken. It should work more like Dropbox. +Textedit saves the file initially, but it is immediately locked. +Since it autosaves, it asks to unlock or duplicate. +Then gives the error: +"The file “Untitled 16.txt” cannot be unlocked." + +If the file exists: +The document “Untitled 14” could not be saved as “Untitled 14.txt”. You don’t have permission. + +If you open a file from the repository (now replaced by a symlink) with textedit, there are other problems: +- The filename will not be correct (will show the sha hash). +- It will ask to unlock, then give the error "You don’t have permission to write to the folder that the file “SHA256E-s8--8985d9832de2e28b5e1af64258c391a34d7528709ef916bac496e698c139020c.txt” is in." + +What version of git-annex are you using? On what operating system? + +OSX Lion +git-annex version: 3.20120924 + +Please provide any additional information below. + +Even if you fix these problems, automatic versioning in lion will probably don't work, and the symlinks seem a hackish solution and don't seem intuitive or easy to the end user. +The sync should be transparent but it's not, and it's error prone. It would even be best to keep file copies in the git repo and sync them with the original folder than make symlinks. + +Dropbox even allows to put a symlink in the dropbox directory, and it will sync the file. diff --git a/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn b/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn --- a/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn +++ b/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn @@ -23,3 +23,5 @@ git-annex-3.20120825 Max OS X 10.8.1 + +> [[done]]; see comments --[[Joey]] diff --git a/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn b/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn @@ -0,0 +1,15 @@ +What steps will reproduce the problem? + +From the webapp, go to Configuration > Manage repositories > Remote server. Enter a hostname that only has an IPv6 hostname (e.g. ipv6.google.com). Click Check this server. + +What is the expected output? What do you see instead? + +Expect the application to attempt to check the server via SSH. Instead, it results in error "cannot resolve host name". + +What version of git-annex are you using? On what operating system? + +git-annex 3.20120924 on Debian testing (amd64). + +Please provide any additional information below. + +> Thanks, [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn b/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn @@ -0,0 +1,13 @@ +the build is currently borked with + +
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_KQUEUE -DOSX --make git-annex -o tmp/git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+Assistant/Install.hs:19:8:
+Could not find module `Utility'
+Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1
+
+ +This was first introduced at commit e88e3ba + +> oops, -DOSX is not a good idea. [[done]] --[[Joey]] diff --git a/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn b/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn @@ -0,0 +1,14 @@ +Seems the last few commits have borked 'cabal configure' on OSX with the following error message + +
+[jtang@laplace git-annex (master)]$ cabal configure 
+Resolving dependencies...
+
+Build/InstallDesktopFile.hs:19:8:
+    Could not find module `Assistant.OSX'
+    Use -v to see a list of the files searched for.
+
+ +Looks like a missing module. + +> Was broken everywhere really, so I fixed it. [[done]] --[[Joey]] diff --git a/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn --- a/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn +++ b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn @@ -30,3 +30,5 @@ I have done this on my copy for testing it. For testing, I am also using two repositories on the same computer. I set this up from the command line, as the web app does not seem to support syncing to two different git folders on the same computer. + +> [[done]]; see comments --[[Joey]] diff --git a/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn b/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn --- a/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn +++ b/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn @@ -49,3 +49,5 @@ I ran " git annex watch -d --foreground" to watch what was going one, and just created a .gitignore file and the the commiter/lsof thread just loops over and over.... I only noticed as my laptop battery had drained at somepoint when git-annex was running in the background. [[!meta title="assistant: lsof/committer thread loops occassionally"]] + +> Closing this since it doesn't seem reproducible. [[done]] --[[Joey]] diff --git a/doc/bugs/pasting_into_annex_on_OSX.mdwn b/doc/bugs/pasting_into_annex_on_OSX.mdwn --- a/doc/bugs/pasting_into_annex_on_OSX.mdwn +++ b/doc/bugs/pasting_into_annex_on_OSX.mdwn @@ -15,4 +15,7 @@ Please provide any additional information below. > Ok, I've put in the one second delay to adding by default on OSX. -> I consider this bug [[done]], at least for now.. +> I consider this bug done, at least for now.. + +>> Reopening since I've heard from someone else that it can still happen. +>> --[[Joey]] diff --git a/doc/bugs/removable_device_configurator_chokes_on_spaces.mdwn b/doc/bugs/removable_device_configurator_chokes_on_spaces.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/removable_device_configurator_chokes_on_spaces.mdwn @@ -0,0 +1,18 @@ +# What steps will reproduce the problem? + +1. Get a removable device and create a filesystem with a label containing a space, e.g.: + # mkfs.ext4 -L "Backup Home" /dev/sdb1 +2. Open the webapp and add a new repository on a removabledevice +3. Select the devices mountpoint, e.g. /media/Backup\ Home + +# What is the expected output? What do you see instead? + +The configurator should remove spaces for branch names, but it actually seems to call git remote add with "Backup Home" as argument which is invalid. + +The assistant produces an internal server error and subsequently crashes completely. + +# What version of git-annex are you using? On what operating system? + +git-annex 3.20120924 from the Debian package in sid on Debian wheezy, amd64. + +> Thanks for reporting this, I've fixed it in git. [[done]] --[[Joey]] diff --git a/doc/bugs/xdg-user-dir_error.mdwn b/doc/bugs/xdg-user-dir_error.mdwn new file mode 100644 --- /dev/null +++ b/doc/bugs/xdg-user-dir_error.mdwn @@ -0,0 +1,8 @@ +Run *git annex webapp* in Debian Sid with KDE. + +It opens the web browser with this error: **Internal Server Error** *xdg-user-dir ["DESKTOP"] exited 127*. + +I've tried to changing the xdg-user-dir config with no success. + +> This is fixed in my master branch. Workaround: Install the xdg-user-dirs +> package. [[done]] --[[Joey]] diff --git a/doc/contact.mdwn b/doc/contact.mdwn --- a/doc/contact.mdwn +++ b/doc/contact.mdwn @@ -8,3 +8,4 @@ of managing their large personal files. For realtime chat, use the `#vcs-home` channel on irc.oftc.net. +You can also watch incoming commits there. diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn --- a/doc/design/assistant.mdwn +++ b/doc/design/assistant.mdwn @@ -25,7 +25,7 @@ * [[desymlink]] * [[deltas]] -* [[!traillink leftovers]] +* [[leftovers]] ## blog diff --git a/doc/design/assistant/OSX.mdwn b/doc/design/assistant/OSX.mdwn --- a/doc/design/assistant/OSX.mdwn +++ b/doc/design/assistant/OSX.mdwn @@ -4,6 +4,9 @@ * icon to start webapp **done** * Use OSX's "network reachability functionality" to detect when on a network +* daily build seems to have a bad yesod-static, resulting in the url + for `/static/jquery-full.js` redirecting to `/jquery-full.js?etag=foo', + which is a 404. Works ok when I build it on oberon. Gripes: diff --git a/doc/design/assistant/blog/day_91__break.mdwn b/doc/design/assistant/blog/day_91__break.mdwn new file mode 100644 --- /dev/null +++ b/doc/design/assistant/blog/day_91__break.mdwn @@ -0,0 +1,7 @@ +Mostly took a break from working on the assistant today. Instead worked +on adding incremental fsck to git-annex. Well, that will be something +that assistant will use, eventually, probably. + +Jimmy and I have been working on a self-contained OSX app for using the +assistant, that doesn't depend on installing git, etc. More on that +once we have something that works. diff --git a/doc/design/assistant/blog/day_92__S3.mdwn b/doc/design/assistant/blog/day_92__S3.mdwn new file mode 100644 --- /dev/null +++ b/doc/design/assistant/blog/day_92__S3.mdwn @@ -0,0 +1,23 @@ +Amazon S3 was the second most popular choice in the +[[polls/prioritizing_special_remotes]] poll, and since I'm not sure how +I want to support phone/mp3 players, I did it first. + +So I added a configurator today to easily set up an Amazon S3 repository. +That was straightforward and didn't take long since git-annex already +supported S3. + +The hard part, of course, is key distribution. Since the webapp so far +can only configure the shared encryption method, and not fullblown gpg keys, +I didn't feel it would be secure to store the S3 keys in the git repository. +Anyone with access to that git repo would have full access to S3 ... just not +acceptable. Instead, the webapp stores the keys in a 600 mode file locally, +and they're not distributed at all. + +When the same S3 repository is enabled on another computer, it prompts for +keys then too. I did add a hint about using the IAM Management Console in +this case -- it should be possible to set up users in IAM who can only +access a single bucket, although I have not tried to set that up. + +--- + +Also, more work on the standalone OSX app. diff --git a/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn b/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn new file mode 100644 --- /dev/null +++ b/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn @@ -0,0 +1,23 @@ +Various bug fixes, and work on the OSX app today: + +* Avoid crashing when ssh-keygen fails due to not being able to parse + `authorized_keys`.. seems a lot of people have crufty unparsable + `authorized_keys` files. +* On OSX, for some reason the webapp was failing to start sometimes due + to bind failing with EINVAL. I don't understand why, as that should + only happen if the socket is already bound, which it should not as + it's just been created. I was able to work around this by retrying + with a new socket when bind fails. +* When setting up `authorized_keys` to let `git-annex-shell` be run, + it had been inserting a perl oneliner into it. I changed that + to instead call a `~/.ssh/git-annex-shell` wrapper script that it sets + up. The benefits are it no longer needs perl, and it's less ugly, + and the standalone OSX app can modify the wrapper script to point to + wherever it's installed today (people like to move these things around I + guess). +* Made the standalone OSX app set up autostarting when it's first run. +* Spent rather a long time collecting the licenses of all the software that + will be bundled with the standalone OSX app. Ended up with a file + containing 3954 lines of legalese. Happily, all the software appears + redistributable, and free software; even the couple of OSX system libraries + we're bundling are licensed under the APSL. diff --git a/doc/design/assistant/blog/day_93__easy_install.mdwn b/doc/design/assistant/blog/day_93__easy_install.mdwn new file mode 100644 --- /dev/null +++ b/doc/design/assistant/blog/day_93__easy_install.mdwn @@ -0,0 +1,34 @@ +I hear that people want the git-annex assistant to be easy to install +without messing about building it from source.. + +## on OSX + +So Jimmy and I have been working all week on making an easily installed OSX +app of the assistant. This is a .dmz file that bundles all the dependencies +(git, etc) in, so it can be installed with one click. + +It seems to basically work. You can get it [[here|install/OSX]]. + +Unfortunatly, the [[bugs/pasting_into_annex_on_OSX]] bug resurfaced while +testing this.. So I can't really recommend using it on real data yet. + +Still, any testing you can do is gonna be really helpful. I'm squashing OSX +bugs right and left. + +## on Linux + +First of all, the git-annex assistant is now available in Debian unstable, +and in Arch Linux's AUR. Proper packages. + +For all the other Linux distributions, I have a workaround. It's +a big hack, but it seems to work.. at least on Debian stable. + +I've just put up a [[install/linux_standalone]] tarball, which has **no +library dependencies** apart from glibc, and doesn't even need git to be +installed on your system. + +## on FreeBSD + +The FreeBSD port has been updated to include the git-annex assistant too.. + +[[!meta title="day 94 easy install"]] diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn --- a/doc/design/assistant/cloud.mdwn +++ b/doc/design/assistant/cloud.mdwn @@ -28,6 +28,20 @@ or some other repo in the cloud she pushed to. Once both steps are done, the assistant will transfer the file from the cloud to Bob. +* dvcs-autosync uses jabber; all repos need to have the same jabber account + configured, and send self-messages. An alternative would be to have + different accounts that join a channel or message each other. Still needs + account configuration. +* irc could be used. With a default irc network, and an agreed-upon channel, + no configuration should be needed. IRC might be harder to get through + some firewalls, and is prone to netsplits, etc. IRC networks have reasons + to be wary of bots using them. Only basic notifications could be done over + irc, as it has little security. +* When there's a ssh server involved, code could be run on it to notify + logged-in clients. But this is not a general solution to this problem. +* pubsubhubbub does not seem like an option; its hubs want to pull down + a feed over http. + ## storing git repos in the cloud Of course, one option is to just use github etc to store the git repo. diff --git a/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment b/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment new file mode 100644 --- /dev/null +++ b/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://launchpad.net/~gdr-go2" + nickname="gdr-go2" + subject="Maybe a DEB?2" + date="2012-09-27T09:44:14Z" + content=""" +Month 3 was all about easy setup, so I kind of expected to download a deb package and just install it, not to download a whole bunch of haskell libraries. Is there a chance that you will release some packages? +"""]] diff --git a/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment b/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment new file mode 100644 --- /dev/null +++ b/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.14.141" + subject="comment 17" + date="2012-09-27T18:44:11Z" + content=""" +@gdr A package with the assistant is available in Debian unstable. +"""]] diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 14 "Amazon S3" 8 "Amazon Glacier" 6 "Box.com" 44 "My phone (or MP3 player)" 4 "Tahoe-LAFS" 3 "OpenStack SWIFT" 12 "Google Drive"]] +[[!poll open=yes 14 "Amazon S3 (done)" 9 "Amazon Glacier" 6 "Box.com" 49 "My phone (or MP3 player)" 7 "Tahoe-LAFS" 3 "OpenStack SWIFT" 14 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn --- a/doc/design/assistant/webapp.mdwn +++ b/doc/design/assistant/webapp.mdwn @@ -25,6 +25,8 @@ over http by the web app * Display the `inotify max_user_watches` exceeded message. **done** * Display something sane when kqueue runs out of file descriptors. +* allow renaming git remotes and/or setting git-annex repo descriptions +* allow removing git remotes ## first start **done** diff --git a/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn b/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn @@ -0,0 +1,15 @@ +I'm having trouble install from cabal on Debian-testing since the new beta released. + + % sudo aptitude remove --purge cabal-install + % rm -rf $HOME/.cabal + % sudo aptitude install cabal-install + % cabal update + % cabal install git-annex + +The output all follows this general syntax: + + depends on which failed to install + +On the flip side, I upgrade my Debian to sid and it installed just fine through aptitude. + +(Apologies for my English, and if this is simply a user error) diff --git a/doc/forum/linux_standalone_tarballs.mdwn b/doc/forum/linux_standalone_tarballs.mdwn new file mode 100644 --- /dev/null +++ b/doc/forum/linux_standalone_tarballs.mdwn @@ -0,0 +1,1 @@ +Just saw that there is now a linux standalone tarball, I've a SL5 (RHEL5) based machine which I can churn out git-annex binaries if there is interest. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -258,10 +258,26 @@ With parameters, only the specified files are checked. To check a remote to fsck, specify --from. - + To avoid expensive checksum calculations (and expensive transfers when - fscking a remote), specify --fast + fscking a remote), specify --fast. + To start a new incremental fsck, specify --incremental. Then + the next time you fsck, you can specify --more to skip over + files that have already been checked, and continue where it left off. + + The --incremental-schedule option makes a new incremental fsck be + started a configurable time after the last incremental fsck was started. + Once the current incremental fsck has completely finished, it causes + a new one to start. + + Maybe you'd like to run a fsck for 5 hours at night, picking up each + night where it left off. You'd like this to continue until all files + have been fscked. And once it's done, you'd like a new fsck pass to start, + but no more often than once a month. Then put this in a nightly cron job: + + git annex fsck --incremental-schedule 30d --time-limit 5h + * unused Checks the annex for data that does not correspond to any files present @@ -510,6 +526,17 @@ Overrides the `annex.numcopies` setting, forcing git-annex to ensure the specified number of copies exist. +* --time-limit=time + + Limits how long a git-annex command runs. The time can be something + like "5h", or "30m" or even "45s" or "10d". + + Note that git-annex may continue running a little past the specified + time limit, in order to finish processing a file. + + Also, note that if the time limit prevents git-annex from doing all it + was asked to, it will exit with a special code, 101. + * --trust=repository * --semitrust=repository * --untrust=repository @@ -583,6 +610,12 @@ of copies, or more. Note that it does not check remotes to verify that the copies still exist. +* --copies=trustlevel:number + + Matches only files that git-annex believes have the specified number of + copies, on remotes with the specified trust level. For example, + "--copies=trusted:2" + * --inbackend=name Matches only files whose content is stored using the specified key-value @@ -755,7 +788,7 @@ * `remote..annex-bup-split-options` Options to pass to bup split when storing content in this remote. - For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k" + For example, to limit the bandwidth to 100Kbyte/s, set it to "--bwlimit 100k" (There is no corresponding option for bup join.) * `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options` diff --git a/doc/index.mdwn b/doc/index.mdwn --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -5,8 +5,7 @@ [[!sidebar content=""" [[!img logo_small.png link=no]] -* **[[download]]** -* [[install]] +* **[[install]]** * [[assistant]] * [[tips]] * [[bugs]] diff --git a/doc/install.mdwn b/doc/install.mdwn --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -1,85 +1,34 @@ -## OS-specific instructions +## Pick your OS -* [[OSX]] -* [[Debian]] -* [[Ubuntu]] -* [[Fedora]] -* [[FreeBSD]] -* [[openSUSE]] -* [[ArchLinux]] -* [[NixOS]] -* [[Gentoo]] -* [[ScientificLinux5]] - This should cover RHEL5 clones such as CentOS5 and so on -* Windows: [[sorry, not possible yet|todo/windows_support]] +Quick commands given where possible, but see the individual pages for +details. +[[!table format=dsv header=no data=""" +[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2) **beta** +[[Linux|linux_standalone]] | [download prebuilt linux tarball](http://downloads.kitenet.net/git-annex/linux/) +[[Debian]] | `apt-get install git-annex` +[[Ubuntu]] | `apt-get install git-annex` +[[FreeBSD]] | `pkg_add -r hs-git-annex` +[[ArchLinux]] | `yaourt -Sy git-annex` +[[NixOS]] | `nix-env -i git-annex` +[[Gentoo]] | `emerge git-annex` +[[ScientificLinux5]] | (and other RHEL5 clones like CentOS5) +[[Fedora]] | +[[openSUSE]] | +Windows | [[sorry, Windows not supported yet|todo/windows_support]] +"""]] + ## Using cabal -As a haskell package, git-annex can be installed using cabal. For example: +As a haskell package, git-annex can be installed using cabal. +Start by installing the [Haskell Platform](http://hackage.haskell.org/platform/), +and then: cabal install git-annex --bindir=$HOME/bin -The above downloads the latest release. Alternatively, you can [[download]] -it yourself and [[manually_build_with_cabal|install/cabal]]. - -## Installation by hand - -This is not recommended, it's easier to let cabal pull in the many haskell -libraries. To build and use git-annex by hand, you will need: +That installs the latest release. Alternatively, you can [[download]] +git-annex yourself and [[manually_build_with_cabal|install/cabal]]. -* Haskell stuff - * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer) - * [MissingH](http://github.com/jgoerzen/missingh/wiki) - * [pcre-light](http://hackage.haskell.org/package/pcre-light) - * [utf8-string](http://hackage.haskell.org/package/utf8-string) - * [SHA](http://hackage.haskell.org/package/SHA) - * [dataenc](http://hackage.haskell.org/package/dataenc) - * [monad-control](http://hackage.haskell.org/package/monad-control) - * [lifted-base](http://hackage.haskell.org/package/lifted-base) - * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack) - * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck) - * [HTTP](http://hackage.haskell.org/package/HTTP) - * [json](http://hackage.haskell.org/package/json) - * [IfElse](http://hackage.haskell.org/package/IfElse) - * [bloomfilter](http://hackage.haskell.org/package/bloomfilter) - * [edit-distance](http://hackage.haskell.org/package/edit-distance) - * [hS3](http://hackage.haskell.org/package/hS3) (optional) -* Optional haskell stuff, used by the [[assistant]] and its webapp (edit Makefile to disable) - * [stm](http://hackage.haskell.org/package/stm) - (version 2.3 or newer) - * [hinotify](http://hackage.haskell.org/package/hinotify) - (Linux only) - * [dbus](http://hackage.haskell.org/package/dbus) - * [yesod](http://hackage.haskell.org/package/yesod) - * [yesod-static](http://hackage.haskell.org/package/yesod-static) - * [yesod-default](http://hackage.haskell.org/package/yesod-default) - * [data-default](http://hackage.haskell.org/package/data-default) - * [case-insensitive](http://hackage.haskell.org/package/case-insensitive) - * [http-types](http://hackage.haskell.org/package/http-types) - * [transformers](http://hackage.haskell.org/package/transformers) - * [wai](http://hackage.haskell.org/package/wai) - * [wai-logger](http://hackage.haskell.org/package/wai-logger) - * [warp](http://hackage.haskell.org/package/warp) - * [blaze-builder](http://hackage.haskell.org/package/blaze-builder) - * [blaze-html](http://hackage.haskell.org/package/blaze-html) - * [crypto-api](http://hackage.haskell.org/package/crypto-api) - * [hamlet](http://hackage.haskell.org/package/hamlet) - * [clientsession](http://hackage.haskell.org/package/clientsession) - * [network-multicast](http://hackage.haskell.org/package/network-multicast) - * [network-info](http://hackage.haskell.org/package/network-info) -* Shell commands - * [git](http://git-scm.com/) - * [uuid](http://www.ossp.org/pkg/lib/uuid/) - (or `uuidgen` from util-linux) - * [xargs](http://savannah.gnu.org/projects/findutils/) - * [rsync](http://rsync.samba.org/) - * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended) - * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended; - a sha1 command will also do) - * [gpg](http://gnupg.org/) (optional; needed for encryption) - * [lsof](ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/) - (optional; recommended for watch mode) - * multicast DNS support, provided on linux by [nss-mdns](http://www.0pointer.de/lennart/projects/nss-mdns/) - (optional; recommended for the assistant to support pairing well) - * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs) +## Installation from scratch -Then just [[download]] git-annex and run: `make; make install` +This is not recommended, but if you really want to, see [[fromscratch]]. diff --git a/doc/install/Linux_standalone.mdwn b/doc/install/Linux_standalone.mdwn new file mode 100644 --- /dev/null +++ b/doc/install/Linux_standalone.mdwn @@ -0,0 +1,11 @@ +If your Linux distribution does not have git-annex packaged up for you, +you can either build it [[fromscratch]], or you can use a handy +prebuilt tarball. + +This tarball should work on most Linux systems. It does not depend +on anything except for glibc. + +[download tarball](http://downloads.kitenet.net/git-annex/linux/) + +Warning: This is a last resort. Most Linux users should instead +[[install]] git-annex from their distribution of choice. diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -1,3 +1,23 @@ +## git-annex.app + +For easy installation, [Jimmy Tang](http://www.sgenomics.org/~jtang/) +builds a standalone git-annex.app of the git-annex assistant. + +* [beta release of git-annex.app](http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2) +* [daily build of git-annex.app](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/ref/master/git-annex.dmg.bz2) ([build logs](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/)) + * [past builds](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/sha1/) -- directories are named from the commitid's + +## using Brew + +
+sudo brew update
+sudo brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
+cabal update
+cabal install git-annex --bindir=$HOME/bin
+
+ +## using MacPorts + Install the Haskell Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute @@ -10,6 +30,8 @@ sudo cabal update cabal install git-annex --bindir=$HOME/bin + +## PATH setup Do not forget to add to your PATH variable your ~/bin folder. In your .bashrc, for example: 
diff --git a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
deleted file mode 100644
--- a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
+++ /dev/null
@@ -1,13 +0,0 @@
-[[!comment format=mdwn
- username="http://www.schleptet.net/~cfm/"
- ip="64.30.148.100"
- subject="comment 1"
- date="2011-08-30T14:31:36Z"
- content="""
-You can also use Homebrew instead of MacPorts.  Homebrew's `haskell-platform` is up-to-date, too:
-
-    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
-    ln -s /usr/local/include/pcre.h /usr/include/pcre.h
-
-As of this writing, however, Homebrew's `md5sha1sum` has a broken mirror.  I wound up getting that from MacPorts anyway.
-"""]]
diff --git a/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment
--- a/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment
+++ b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment
@@ -4,4 +4,5 @@
  date="2012-07-24T15:09:29Z"
  content="""
 I've moved some outdated comments about installing on OSX to [[old_comments]].
+And also moved away some comments that helped build the instructions above.
 """]]
diff --git a/doc/install/OSX/comment_3_733147cebe501c60f2141b711f1d7f24._comment b/doc/install/OSX/comment_3_733147cebe501c60f2141b711f1d7f24._comment
deleted file mode 100644
--- a/doc/install/OSX/comment_3_733147cebe501c60f2141b711f1d7f24._comment
+++ /dev/null
@@ -1,16 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawnnIQkoUQo4RYzjUNyiB3v6yJ5aR41WG8k"
- nickname="Markus"
- subject="Updated install instructions with homebrew"
- date="2012-08-07T06:46:47Z"
- content="""
-To install git annex with homebrew simply do:
-
-    brew update
-    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
-    cabal install git-annex
-
-Then link the binary to your `PATH` e.g. with
-
-    ln -s ~/.cabal/bin/git-annex* /usr/local/bin/
-"""]]
diff --git a/doc/install/OSX/comment_4_d513e21512a9b207983d38abf348d00f._comment b/doc/install/OSX/comment_4_d513e21512a9b207983d38abf348d00f._comment
deleted file mode 100644
--- a/doc/install/OSX/comment_4_d513e21512a9b207983d38abf348d00f._comment
+++ /dev/null
@@ -1,16 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawm_-2XlXNyd6cCLI4n_jaBNqVUOWwJquko"
- nickname="David"
- subject="installing via homebrew"
- date="2012-09-05T11:11:55Z"
- content="""
-I had to:
-
-    cabal update
-
-before:
-
-    cabal install git-annex
-
-
-"""]]
diff --git a/doc/install/fromscratch.mdwn b/doc/install/fromscratch.mdwn
new file mode 100644
--- /dev/null
+++ b/doc/install/fromscratch.mdwn
@@ -0,0 +1,60 @@
+To install git-annex from scratch, you need a lot of stuff. Really
+quite a lot.
+
+* Haskell stuff
+  * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer)
+  * [MissingH](http://github.com/jgoerzen/missingh/wiki)
+  * [pcre-light](http://hackage.haskell.org/package/pcre-light)
+  * [utf8-string](http://hackage.haskell.org/package/utf8-string)
+  * [SHA](http://hackage.haskell.org/package/SHA)
+  * [dataenc](http://hackage.haskell.org/package/dataenc)
+  * [monad-control](http://hackage.haskell.org/package/monad-control)
+  * [lifted-base](http://hackage.haskell.org/package/lifted-base)
+  * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
+  * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
+  * [HTTP](http://hackage.haskell.org/package/HTTP)
+  * [json](http://hackage.haskell.org/package/json)
+  * [IfElse](http://hackage.haskell.org/package/IfElse)
+  * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
+  * [edit-distance](http://hackage.haskell.org/package/edit-distance)
+  * [hS3](http://hackage.haskell.org/package/hS3) (optional)
+* Optional haskell stuff, used by the [[assistant]] and its webapp (edit Makefile to disable)
+  * [stm](http://hackage.haskell.org/package/stm)
+    (version 2.3 or newer)
+  * [hinotify](http://hackage.haskell.org/package/hinotify)
+    (Linux only)
+  * [dbus](http://hackage.haskell.org/package/dbus)
+  * [yesod](http://hackage.haskell.org/package/yesod)
+  * [yesod-static](http://hackage.haskell.org/package/yesod-static)
+  * [yesod-default](http://hackage.haskell.org/package/yesod-default)
+  * [data-default](http://hackage.haskell.org/package/data-default)
+  * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)
+  * [http-types](http://hackage.haskell.org/package/http-types)
+  * [transformers](http://hackage.haskell.org/package/transformers)
+  * [wai](http://hackage.haskell.org/package/wai)
+  * [wai-logger](http://hackage.haskell.org/package/wai-logger)
+  * [warp](http://hackage.haskell.org/package/warp)
+  * [blaze-builder](http://hackage.haskell.org/package/blaze-builder)
+  * [blaze-html](http://hackage.haskell.org/package/blaze-html)
+  * [crypto-api](http://hackage.haskell.org/package/crypto-api)
+  * [hamlet](http://hackage.haskell.org/package/hamlet)
+  * [clientsession](http://hackage.haskell.org/package/clientsession)
+  * [network-multicast](http://hackage.haskell.org/package/network-multicast)
+  * [network-info](http://hackage.haskell.org/package/network-info)
+* Shell commands
+  * [git](http://git-scm.com/)
+  * [uuid](http://www.ossp.org/pkg/lib/uuid/)
+    (or `uuidgen` from util-linux)
+  * [xargs](http://savannah.gnu.org/projects/findutils/)
+  * [rsync](http://rsync.samba.org/)
+  * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
+    a sha1 command will also do)
+  * [gpg](http://gnupg.org/) (optional; needed for encryption)
+  * [lsof](ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/)
+    (optional; recommended for watch mode)
+  * multicast DNS support, provided on linux by [nss-mdns](http://www.0pointer.de/lennart/projects/nss-mdns/)
+    (optional; recommended for the assistant to support pairing well)
+  * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
+
+Then just [[download]] git-annex and run: `make; make install`
diff --git a/doc/license.mdwn b/doc/license.mdwn
--- a/doc/license.mdwn
+++ b/doc/license.mdwn
@@ -10,5 +10,5 @@
 
 git-annex contains a variety of other code, artwork, etc copyright by
 others, under a variety of licences, including the [[LGPL]], BSD,
-MIT, and Apache 2.0 license. For a detailed overview, and pointers to the
-full licenses of these components, see the COPYRIGHT file in the source.
+MIT, and Apache 2.0 licenses. For details, see
+[this file](http://source.git-annex.branchable.com/?p=source.git;a=blob_plain;f=debian/copyright;hb=HEAD).
diff --git a/doc/news/.version_3.20120924.mdwn.swp b/doc/news/.version_3.20120924.mdwn.swp
deleted file mode 100644
Binary files a/doc/news/.version_3.20120924.mdwn.swp and /dev/null differ
diff --git a/doc/news/version_3.20120629.mdwn b/doc/news/version_3.20120629.mdwn
deleted file mode 100644
--- a/doc/news/version_3.20120629.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-git-annex 3.20120629 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * cabal: Only try to use inotify on Linux.
-   * Version build dependency on STM, and allow building without it,
-     which disables the watch command.
-   * Avoid ugly failure mode when moving content from a local repository
-     that is not available.
-   * Got rid of the last place that did utf8 decoding.
-   * Accept arbitrarily encoded repository filepaths etc when reading
-     git config output. This fixes support for remotes with unusual characters
-     in their names.
-   * sync: Automatically resolves merge conflicts."""]]
diff --git a/doc/news/version_3.20120924.mdwn b/doc/news/version_3.20120924.mdwn
--- a/doc/news/version_3.20120924.mdwn
+++ b/doc/news/version_3.20120924.mdwn
@@ -2,6 +2,8 @@
 [[!toggleable text="""
    * assistant: New command, a daemon which does everything watch does,
      as well as automatically syncing file contents between repositories.
+     See [[assistant]] for details and [[assistant/errata]] for known
+     issues in this beta feature.
    * webapp: An interface for managing and configuring the assistant.
    * The default backend used when adding files to the annex is changed
      from SHA256 to SHA256E, to simplify interoperability with OSX, media
diff --git a/doc/news/version_3.20121001.mdwn b/doc/news/version_3.20121001.mdwn
new file mode 100644
--- /dev/null
+++ b/doc/news/version_3.20121001.mdwn
@@ -0,0 +1,27 @@
+git-annex 3.20121001 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * fsck: Now has an incremental mode. Start a new incremental fsck pass
+     with git annex fsck --incremental. Now the fsck can be interrupted
+     as desired, and resumed with git annex fsck --more.
+     Thanks, Justin Azoff
+   * New --time-limit option, makes long git-annex commands stop after
+     a specified amount of time.
+   * fsck: New --incremental-schedule option which is nice for scheduling
+     eg, monthly incremental fsck runs in cron jobs.
+   * Fix fallback to ~/Desktop when xdg-user-dir is not available.
+     Closes: #[688833](http://bugs.debian.org/688833)
+   * S3: When using a shared cipher, S3 credentials are not stored encrypted
+     in the git repository, as that would allow anyone with access to
+     the repository access to the S3 account. Instead, they're stored
+     in a 600 mode file in the local git repo.
+   * webapp: Avoid crashing when ssh-keygen -F chokes on an invalid known\_hosts
+     file.
+   * Always do a system wide installation when DESTDIR is set. Closes: #[689052](http://bugs.debian.org/689052)
+   * The Makefile now builds with the new yesod by default.
+     Systems like Debian that have the old yesod 1.0.1 should set
+     GIT\_ANNEX\_LOCAL\_FEATURES=-DWITH\_OLD\_YESOD
+   * copy: Avoid updating the location log when no copy is performed.
+   * configure: Test that uuid -m works, falling back to plain uuid if not.
+   * Avoid building the webapp on Debian architectures that do not yet
+     have template haskell and thus yesod. (Should be available for arm soonish
+     I hope)."""]]
diff --git a/doc/not/comment_6_547fc59b19ad66d7280c53a7f923ea08._comment b/doc/not/comment_6_547fc59b19ad66d7280c53a7f923ea08._comment
new file mode 100644
--- /dev/null
+++ b/doc/not/comment_6_547fc59b19ad66d7280c53a7f923ea08._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlatTbI0K-qydpeYHl37iseqPNvERcdIMk"
+ nickname="Tiago"
+ subject="comment 6"
+ date="2012-09-27T10:17:18Z"
+ content="""
+Stefan: \"annex is for managing big binary files that not get modified most of the time and only added/synced or deleted.\"
+
+While this is true,  the kickstarter title for assistant was \"Like dropbox\", and dropbox makes it transparent to edit files and they work with the filesystem.
+So with assistant, lock/unlock should be automated and transparent to the user. Otherwise it's confusing and not simple at all to use, and at least OSX keeps giving errors because of the way it handles aliases.
+So something like sharebox is essential to be included in assistant in my opinion.
+
+"""]]
diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -9,8 +9,10 @@
 The standard environment variables `AWS_ACCESS_KEY_ID` and 
 `AWS_SECRET_ACCESS_KEY` are used to supply login credentials
 for Amazon. When encryption is enabled, they are stored in encrypted form
-by `git annex initremote`, so you do not need to keep the environment
-variables set after the initial initalization of the remote.
+by `git annex initremote`. Without encryption, they are stored in a
+file only you can read inside the local git repository. So you do not
+need to keep the environment variables set after the initial
+initalization of the remote.
 
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.
diff --git a/doc/tips/setup_a_public_repository_on_a_web_site.mdwn b/doc/tips/setup_a_public_repository_on_a_web_site.mdwn
new file mode 100644
--- /dev/null
+++ b/doc/tips/setup_a_public_repository_on_a_web_site.mdwn
@@ -0,0 +1,27 @@
+Let's say you want to distribute some big files to the whole world.
+You can of course, just drop them onto a website. But perhaps you'd like to
+use git-annex to manage those files. And as an added bonus, why not let
+anyone in the world clone your site and use `git-annex get`!
+
+My site like this is [downloads.kitenet.net](http://downloads.kitenet.net).
+Here's how I set it up. --[[Joey]]
+
+1. Set up a web site. I used Apache, and configured it to follow symlinks.
+   `Options FollowSymLinks`
+2. Put some files on the website. Make sure it works.
+4. `git init; git annex init`
+3. We want users to be able to clone the git repository over http, because
+   git-annex can download files from it over http as well. For this to
+   work, `git update-server-info` needs to get run after commits. So
+   put it in the git `post-update` hook.
+5. `git annex add; git commit -m added`
+6. Make sure users can still download files from the site directly.
+7. Instruct advanced users to clone a http url that ends with the "/.git/"
+   directory. For example, for downloads.kitenet.net, the clone url
+   is `http://downloads.kitenet.net/.git/`
+
+When users clone over http, and run git-annex, it will
+automatically learn all about your repository and be able to download files
+right out of it, also using http. 
+
+Enjoy!
diff --git a/doc/todo/incremental_fsck.mdwn b/doc/todo/incremental_fsck.mdwn
--- a/doc/todo/incremental_fsck.mdwn
+++ b/doc/todo/incremental_fsck.mdwn
@@ -9,3 +9,11 @@
 indicate the content has been fsked, and the mtime indicate the time
 of last fsck. Anything that dropped or put in content would need to
 clear the sticky bit. --[[Joey]] 
+
+> Basic incremental fsck is done now.
+> 
+> Some enhancements would include:
+> 
+> * --max-age=30d  Once the incremental fsck completes and was started 30 days ago,
+>   start a new one.
+> * --time-limit --size-limit --file-limit: Limit how long the fsck runs.
diff --git a/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn b/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn
new file mode 100644
--- /dev/null
+++ b/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn
@@ -0,0 +1,10 @@
+Similar to how
+
+    git commit -m 'foo'
+
+works, if I run 
+
+    git annex sync -m 'my hovercraft is full of eels'
+
+git annex should use that commit message instead of the default ones. That way, I could use [[sync]] directly and not be forced to commit prior to syncing just to make sure I have a useful commit message.
+
diff --git a/git-annex.1 b/git-annex.1
--- a/git-annex.1
+++ b/git-annex.1
@@ -234,8 +234,24 @@
 To check a remote to fsck, specify \-\-from.
 .IP
 To avoid expensive checksum calculations (and expensive transfers when
-fscking a remote), specify \-\-fast
+fscking a remote), specify \-\-fast.
 .IP
+To start a new incremental fsck, specify \-\-incremental. Then
+the next time you fsck, you can specify \-\-more to skip over
+files that have already been checked, and continue where it left off.
+.IP
+The \-\-incremental\-schedule option makes a new incremental fsck be
+started a configurable time after the last incremental fsck was started.
+Once the current incremental fsck has completely finished, it causes
+a new one to start.
+.IP
+Maybe you'd like to run a fsck for 5 hours at night, picking up each
+night where it left off. You'd like this to continue until all files
+have been fscked. And once it's done, you'd like a new fsck pass to start,
+but no more often than once a month. Then put this in a nightly cron job:
+.IP
+ git annex fsck \-\-incremental\-schedule 30d \-\-time\-limit 5h
+.IP
 .IP "unused"
 Checks the annex for data that does not correspond to any files present
 in any tag or branch, and prints a numbered list of the data.
@@ -453,6 +469,16 @@
 Overrides the annex.numcopies setting, forcing git\-annex to ensure the
 specified number of copies exist.
 .IP
+.IP "\-\-time\-limit=time"
+Limits how long a git\-annex command runs. The time can be something
+like "5h", or "30m" or even "45s" or "10d".
+.IP
+Note that git\-annex may continue running a little past the specified
+time limit, in order to finish processing a file.
+.IP
+Also, note that if the time limit prevents git\-annex from doing all it 
+was asked to, it will exit with a special code, 101.
+.IP
 .IP "\-\-trust=repository"
 .IP "\-\-semitrust=repository"
 .IP "\-\-untrust=repository"
@@ -517,6 +543,11 @@
 of copies, or more. Note that it does not check remotes to verify that
 the copies still exist.
 .IP
+.IP "\-\-copies=trustlevel:number"
+Matches only files that git\-annex believes have the specified number of
+copies, on remotes with the specified trust level. For example,
+"\-\-copies=trusted:2"
+.IP
 .IP "\-\-inbackend=name"
 Matches only files whose content is stored using the specified key\-value
 backend.
@@ -660,7 +691,7 @@
 .IP
 .IP "remote..annex\-bup\-split\-options"
 Options to pass to bup split when storing content in this remote.
-For example, to limit the bandwidth to 100Kbye/s, set it to "\-\-bwlimit 100k"
+For example, to limit the bandwidth to 100Kbyte/s, set it to "\-\-bwlimit 100k"
 (There is no corresponding option for bup join.)
 .IP
 .IP "annex.ssh\-options, annex.rsync\-options, annex.bup\-split\-options"
diff --git a/git-annex.cabal b/git-annex.cabal
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120924
+Version: 3.20121001
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -88,9 +88,6 @@
   if flag(Pairing) && flag(Webapp)
     Build-Depends: network-multicast, network-info
     CPP-Options: -DWITH_PAIRING
-
-  if os(darwin)
-    CPP-Options: -DOSX
 
 Test-Suite test
   Type: exitcode-stdio-1.0
diff --git a/standalone/licences.gz b/standalone/licences.gz
new file mode 100644
Binary files /dev/null and b/standalone/licences.gz differ
diff --git a/standalone/linux/README b/standalone/linux/README
new file mode 100644
--- /dev/null
+++ b/standalone/linux/README
@@ -0,0 +1,23 @@
+To start the git-annex webapp, run the git-annex-webapp script in this
+directory.
+
+To enter an environment with git-annex in PATH, use runshell
+
+This should work on any Linux system of the appropriate architecture.
+More or less. There are no external dependencies, except for glibc.
+Any recent-ish version of glibc should work (2.13 is ok; so is 2.11).
+
+
+How it works: This directory contains a lot of libraries and programs
+that git-annex needs. But it's not a chroot. Instead, runshell sets
+PATH and LD_LIBRARY_PATH to point to the stuff in this directory.
+
+The glibc libs are not included. Instead, it runs with the host system's
+glibc. We trust that glibc's excellent backwards and forward compatability
+is good enough to run binaries that were linked for a newer or older
+version. Of course, this could fail. Particularly if the binaries try to
+use some new glibc feature. But hopefully not.
+
+Why not bundle glibc too? I've not gotten it to work! The host system's 
+ld-linux.so will be used for sure, as that's hardcoded into the binaries.
+When I tried including libraries from glibc in here, everything segfaulted.
diff --git a/standalone/linux/git-annex-webapp b/standalone/linux/git-annex-webapp
new file mode 100644
--- /dev/null
+++ b/standalone/linux/git-annex-webapp
@@ -0,0 +1,25 @@
+#!/bin/sh
+base="$(dirname $0)"
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+# If this is a standalone app, set a variable that git-annex can use to
+# install itself.
+if [ -e "$base/bin/git-annex" ]; then
+	GIT_ANNEX_APP_BASE="$base"
+	export GIT_ANNEX_APP_BASE
+fi
+
+"$base/runshell" git-annex webapp "$@"
diff --git a/standalone/linux/glibc-libs b/standalone/linux/glibc-libs
new file mode 100644
--- /dev/null
+++ b/standalone/linux/glibc-libs
@@ -0,0 +1,43 @@
+libanl-.*.so
+libutil-.*.so
+libnss_hesiod-.*.so
+libcrypt-.*.so
+libnss_compat-.*.so
+libm-.*.so
+libr.so
+libpcprofile.so
+libnss_nis-.*.so
+libSegFault.so
+libpthread-.*.so
+librt-.*.so
+libnss_dns-.*.so
+libdl-.*.so
+libBrokenLocale-.*.so
+libnss_nisplus-.*.so
+libthread_db-1.0.so
+libmemusage.so
+libcidn-.*.so
+libnss_files-.*.so
+libnsl-.*.so
+libc-.*.so
+ld-.*.so
+libnss_nis.so
+libthread_db.so
+libanl.so
+libr.so
+libnss_compat.so
+libm.so
+libnss_dns.so
+libpthread.so
+libc.so
+librt.so
+libcidn.so
+libnss_nisplus.so
+libnsl.so
+libutil.so
+libBrokenLocale.so
+ld-linux.so
+libnss_files.so
+libdl.so
+libnss_hesiod.so
+libcrypt.so
diff --git a/standalone/linux/runshell b/standalone/linux/runshell
new file mode 100644
--- /dev/null
+++ b/standalone/linux/runshell
@@ -0,0 +1,48 @@
+#!/bin/sh
+# Runs a shell command (or interactive shell) using the binaries and
+# libraries bundled with this app.
+
+set -e
+
+base="$(dirname $0)"
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+
+if [ ! -e "$base/bin/git-annex" ]; then
+	echo "** base directory $base does not contain bin/git-annex" >&2
+	exit 1
+fi
+if [ ! -e "$base/bin/git" ]; then
+	echo "** base directory $base does not contain bin/git" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+# Put our binaries first, to avoid issues with out of date or incompatable
+# system binaries.
+PATH=$base/bin:$PATH
+export PATH
+
+for lib in $(cat $base/libdirs); do
+	LD_LIBRARY_PATH="$base/$lib:$LD_LIBRARY_PATH"
+done
+export LD_LIBRARY_PATH
+
+GIT_EXEC_PATH=$base/git-core
+export GIT_EXEC_PATH
+
+if [ "$1" ]; then
+	cmd="$1"
+	shift 1
+	exec "$cmd" "$@"
+else
+	$SHELL
+fi
diff --git a/standalone/osx/git-annex.app/Contents/Info.plist b/standalone/osx/git-annex.app/Contents/Info.plist
new file mode 100644
--- /dev/null
+++ b/standalone/osx/git-annex.app/Contents/Info.plist
@@ -0,0 +1,45 @@
+
+
+
+
+	CFBundleDevelopmentRegion
+	English
+	CFBundleExecutable
+	git-annex-webapp
+	NSHumanReadableCopyright
+	GPL 3
+	CFBundleGetInfoString
+	0.0.1
+	CFBundleIconFile
+	git-annex
+	CFBundleIdentifier
+	com.branchable.git-annex
+	CFBundleInfoDictionaryVersion
+	6.0
+	CFBundleName
+	GIT-ANNEX
+	CFBundlePackageType
+	APPL
+	CFBundleShortVersionString
+	0.0.1
+	CFBundleSignature
+	git-annex
+	CFBundleVersion
+	0.0.1
+	NSAppleScriptEnabled
+	
+	CGDisableCoalescedUpdates
+	
+	LSMinimumSystemVersion
+	10.5
+	CFBundleDisplayName
+	Start git-annex webapp
+	LSMinimumSystemVersionByArchitecture
+	
+		i386
+		10.5.0
+		x86_64
+		10.6.0
+	
+
+
diff --git a/standalone/osx/git-annex.app/Contents/MacOS/git-annex-webapp b/standalone/osx/git-annex.app/Contents/MacOS/git-annex-webapp
new file mode 100644
--- /dev/null
+++ b/standalone/osx/git-annex.app/Contents/MacOS/git-annex-webapp
@@ -0,0 +1,25 @@
+#!/bin/sh
+base="$(dirname $0)"
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+# If this is a standalone app, set a variable that git-annex can use to
+# install itself.
+if [ -e "$base/bin/git-annex" ]; then
+	GIT_ANNEX_APP_BASE="$base"
+	export GIT_ANNEX_APP_BASE
+fi
+
+"$base/runshell" git-annex webapp "$@"
diff --git a/standalone/osx/git-annex.app/Contents/MacOS/runshell b/standalone/osx/git-annex.app/Contents/MacOS/runshell
new file mode 100644
--- /dev/null
+++ b/standalone/osx/git-annex.app/Contents/MacOS/runshell
@@ -0,0 +1,53 @@
+#!/bin/sh
+# Runs a shell command (or interactive shell) using the binaries and
+# libraries bundled with this app.
+
+set -e
+
+base="$(dirname $0)"
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+
+if [ ! -e "$base/bin/git-annex" ]; then
+	echo "** base directory $base does not contain bin/git-annex" >&2
+	exit 1
+fi
+if [ ! -e "$base/bin/git" ]; then
+	echo "** base directory $base does not contain bin/git" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+# Put our binaries first, to avoid issues with out of date or incompatable
+# system binaries.
+PATH=$base/bin:$PATH
+export PATH
+
+# Using DYLD_FALLBACK_LIBRARY_PATH rather than DYLD_LIBRARY_PATH, so as
+# not to force binaries to link against possibly the wrong versions of
+# libraries found in the path. With DYLD_FALLBACK_LIBRARY_PATH, the
+# system's versions of libraries will be used when possible, and otherwise
+# it will fall back to using the libraries bundled with this app.
+for lib in $(cat $base/libdirs); do
+	DYLD_FALLBACK_LIBRARY_PATH="$base/$lib:$DYLD_FALLBACK_LIBRARY_PATH"
+done
+export DYLD_FALLBACK_LIBRARY_PATH
+
+GIT_EXEC_PATH=$base/git-core
+export GIT_EXEC_PATH
+
+if [ "$1" ]; then
+	cmd="$1"
+	shift 1
+	exec "$cmd" "$@"
+else
+	$SHELL
+fi
diff --git a/standalone/osx/git-annex.app/Contents/Resources/git-annex.icns b/standalone/osx/git-annex.app/Contents/Resources/git-annex.icns
new file mode 100644
Binary files /dev/null and b/standalone/osx/git-annex.app/Contents/Resources/git-annex.icns differ
diff --git a/templates/actionbutton.hamlet b/templates/actionbutton.hamlet
--- a/templates/actionbutton.hamlet
+++ b/templates/actionbutton.hamlet
@@ -1,2 +1,2 @@
-
+
    #{fromMaybe "" label}
diff --git a/templates/configurators/adds3.hamlet b/templates/configurators/adds3.hamlet
new file mode 100644
--- /dev/null
+++ b/templates/configurators/adds3.hamlet
@@ -0,0 +1,38 @@
+

+  

+    Adding an Amazon S3 repository
+  

+    Amazon S3 is a cloud storage #
+    provider. If you need a professional level of storage for your data, #
+    this is a good choice. #
+    
+      Pricing details, including one year Free Usage Tier promotion
+  

+     Do keep in mind that all your data #
+    will be synced to Amazon S3. You will be charged by Amazon for data #
+    uploaded to S3, as well as data downloaded from S3, and a monthly fee #
+    for data storage. #
+  

+    All data will be encrypted before being sent to Amazon S3.
+  

+    When you sign up to Amazon S3, they provide you with an Access #
+    Key ID, and a Secret Access Key. You will need to enter both below. #
+    These access keys will be stored in a file that only you can #
+    access. #
+    
+      Look up your access keys
+  

+    

+      

+        ^{form}
+        ^{authtoken}
+        

+