diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,6 +7,15 @@ ## Unreleased +## [0.1.3] - 2018-11-26 + +### Changed + +- #77 retry transient HTTP exceptions @domenkozar +- #151 prevent mingled store paths output @domenkozar +- #141 prevent unncessary warning about /etc/nix missing @domenkozar +- #142 ditch hpack @domenkozar + ## [0.1.2] - 2018-09-27 ### Changed diff --git a/cachix.cabal b/cachix.cabal --- a/cachix.cabal +++ b/cachix.cabal @@ -1,13 +1,6 @@ -cabal-version: 1.12 - --- This file has been generated from package.yaml by hpack version 0.31.0. --- --- see: https://github.com/sol/hpack --- --- hash: 098d91889f12eca7fc2fd5f1197c1ec823f43b19fd5be38a630ea965ce99dae8 - +cabal-version: 2.2 name: cachix -version: 0.1.2 +version: 0.1.3 synopsis: Command line client for Nix binary cache hosting https://cachix.org homepage: https://github.com/cachix/cachix#readme bug-reports: https://github.com/cachix/cachix/issues @@ -17,15 +10,22 @@ license: Apache-2.0 license-file: LICENSE build-type: Simple + extra-source-files: - README.md ChangeLog.md + README.md +common defaults + default-extensions: OverloadedStrings NoImplicitPrelude RecordWildCards DeriveGeneric + ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fwarn-tabs -fwarn-unused-imports -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns + default-language: Haskell2010 + source-repository head type: git location: https://github.com/cachix/cachix library + import: defaults exposed-modules: Cachix.Client Cachix.Client.Commands @@ -39,10 +39,10 @@ Cachix.Client.URI other-modules: Paths_cachix + autogen-modules: + Paths_cachix hs-source-dirs: src - default-extensions: OverloadedStrings NoImplicitPrelude RecordWildCards DeriveGeneric - ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fwarn-tabs -fwarn-unused-imports -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns build-depends: async , base >=4.7 && <5 @@ -55,6 +55,7 @@ , conduit-extra , cookie , cryptonite + , data-default , dhall , directory , ed25519 @@ -73,6 +74,7 @@ , process , protolude , resourcet + , retry , safe-exceptions , servant >=0.14.1 , servant-auth @@ -85,63 +87,25 @@ , unix , uri-bytestring , versions - default-language: Haskell2010 executable cachix + import: defaults main-is: Main.hs other-modules: Paths_cachix + autogen-modules: + Paths_cachix hs-source-dirs: cachix - default-extensions: OverloadedStrings NoImplicitPrelude RecordWildCards DeriveGeneric - ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fwarn-tabs -fwarn-unused-imports -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns -threaded -rtsopts -with-rtsopts=-N + ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: - async - , base >=4.7 && <5 - , base16-bytestring - , base64-bytestring - , bifunctors - , bytestring + base >=4.7 && <5 , cachix , cachix-api - , conduit >=1.3.0 - , conduit-extra - , cookie - , cryptonite - , dhall - , directory - , ed25519 - , filepath - , fsnotify - , here - , http-client - , http-client-tls - , http-conduit - , http-types - , lzma-conduit - , megaparsec - , memory - , mmorph - , optparse-applicative - , process - , protolude - , resourcet - , safe-exceptions - , servant >=0.14.1 - , servant-auth - , servant-auth-client >=0.3.3.0 - , servant-client >=0.14 - , servant-client-core >=0.14.1 - , servant-streaming-client >=0.3.0.0 - , streaming - , text - , unix - , uri-bytestring - , versions - default-language: Haskell2010 build-tool-depends: hspec-discover:hspec-discover test-suite cachix-test + import: defaults type: exitcode-stdio-1.0 main-is: Main.hs other-modules: @@ -152,50 +116,11 @@ Paths_cachix hs-source-dirs: test - default-extensions: OverloadedStrings NoImplicitPrelude RecordWildCards DeriveGeneric - ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fwarn-tabs -fwarn-unused-imports -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns -threaded -rtsopts -with-rtsopts=-N + ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: - async - , base >=4.7 && <5 - , base16-bytestring - , base64-bytestring - , bifunctors - , bytestring + base >=4.7 && <5 , cachix , cachix-api - , conduit >=1.3.0 - , conduit-extra - , cookie - , cryptonite - , dhall - , directory - , ed25519 - , filepath - , fsnotify - , here , hspec - , http-client - , http-client-tls - , http-conduit - , http-types - , lzma-conduit - , megaparsec - , memory - , mmorph - , optparse-applicative - , process , protolude - , resourcet - , safe-exceptions - , servant >=0.14.1 - , servant-auth - , servant-auth-client >=0.3.3.0 - , servant-client >=0.14 - , servant-client-core >=0.14.1 - , servant-streaming-client >=0.3.0.0 - , streaming - , text - , unix - , uri-bytestring - , versions - default-language: Haskell2010 + , here diff --git a/src/Cachix/Client/Commands.hs b/src/Cachix/Client/Commands.hs --- a/src/Cachix/Client/Commands.hs +++ b/src/Cachix/Client/Commands.hs @@ -16,8 +16,10 @@ import Control.Monad (forever) import Control.Exception.Safe (MonadThrow, throwM) import Control.Monad.Trans.Resource (ResourceT) +import Control.Retry (recoverAll, RetryStatus(rsIterNumber)) import qualified Data.ByteString.Base64 as B64 import Data.Conduit +import Data.Default (def) import Data.Conduit.Process import qualified Data.Conduit.List as CL import Data.Conduit.Lzma ( compress ) @@ -189,8 +191,9 @@ dropEnd index xs = take (length xs - index) xs pushStorePath :: Env -> Text -> Text -> IO () -pushStorePath env name storePath = do +pushStorePath env name storePath = retryPath $ \retrystatus -> do -- use secret key from config or env + -- TODO: this shouldn't happen for each store path maybeEnvSK <- lookupEnv "CACHIX_SIGNING_KEY" let matches Config{..} = filter (\bc -> Config.name bc == name) binaryCaches maybeBCSK = case config env of @@ -205,13 +208,19 @@ (cachixBCClient name) (Api.NarInfoC storeHash) case res of - Right NoContent -> return () - Left err | isErr err status404 -> go sk - | otherwise -> panic $ show err -- TODO: retry + Right NoContent -> pass -- we're done as store path is already in the cache + Left err | isErr err status404 -> go sk retrystatus + | otherwise -> escalate $ void res where - go sk = do + retryText :: RetryStatus -> Text + retryText retrystatus = + if rsIterNumber retrystatus == 0 + then "" + else "(retry #" <> show (rsIterNumber retrystatus) <> ") " + go sk retrystatus = do let (storeHash, storeSuffix) = splitStorePath $ toS storePath - putStrLn $ "pushing " <> storePath + -- we append newline instead of putStrLn due to https://github.com/haskell/text/issues/242 + putStr $ "pushing " <> retryText retrystatus <> storePath <> "\n" narSizeRef <- liftIO $ newIORef 0 fileSizeRef <- liftIO $ newIORef 0 @@ -235,15 +244,13 @@ let newClientEnv = (clientenv env) { baseUrl = (baseUrl (clientenv env)) { baseUrlHost = toS name <> "." <> baseUrlHost (baseUrl (clientenv env))} } - -- TODO: http retry: retry package? NoContent <- escalate <=< (`runClientM` newClientEnv) $ Api.createNar (cachixBCClient name) (contentType (Proxy :: Proxy Api.XNixNar), conduitToStreaming) closeStdout exitcode <- waitForStreamingProcess cph -- TODO: print process stderr? - when (exitcode /= ExitSuccess) $ - panic $ "Failed with " <> show exitcode <> ": " <> show cmd + when (exitcode /= ExitSuccess) $ throwM $ NarStreamingError exitcode $ show cmd narSize <- readIORef narSizeRef narHashB16 <- readIORef narHashRef @@ -287,7 +294,11 @@ -- Utils --- TODO: should one error abort the whole pushing process? Or do a retry? Or keep going and then fail? +-- Retry up to 5 times for each store path. +-- Catches all exceptions except skipAsyncExceptions +retryPath :: (RetryStatus -> IO a) -> IO a +retryPath = recoverAll def + escalate :: (Exception exc, MonadThrow m) => Either exc a -> m a escalate = escalateAs identity diff --git a/src/Cachix/Client/InstallationMode.hs b/src/Cachix/Client/InstallationMode.hs --- a/src/Cachix/Client/InstallationMode.hs +++ b/src/Cachix/Client/InstallationMode.hs @@ -26,6 +26,7 @@ | AmbiguousInput Text | NoInput Text | NoConfig Text + | NarStreamingError ExitCode Text deriving (Show, Typeable) instance Exception CachixException diff --git a/src/Cachix/Client/NixConf.hs b/src/Cachix/Client/NixConf.hs --- a/src/Cachix/Client/NixConf.hs +++ b/src/Cachix/Client/NixConf.hs @@ -31,7 +31,6 @@ , trustedPublicKeysKey ) where -import Control.Exception (catch) import Data.Char (isSpace) import Data.Text (unwords, unlines) import Data.List (nub) @@ -41,6 +40,7 @@ import System.Directory ( doesFileExist, createDirectoryIfMissing , getXdgDirectory, XdgDirectory(..) ) +import System.FilePath.Posix ( takeDirectory ) import Cachix.Api (BinaryCache(..)) import Cachix.Client.NixVersion (NixVersion(..)) @@ -119,6 +119,7 @@ write :: NixVersion -> NixConfLoc -> NixConf -> IO () write nixversion ncl nc = do filename <- getFilename ncl + createDirectoryIfMissing True (takeDirectory filename) writeFile filename $ render nixversion nc read :: NixConfLoc -> IO (Maybe NixConf) @@ -148,8 +149,6 @@ dir <- case ncl of Global -> return "/etc/nix" Local -> getXdgDirectory XdgConfig "nix" - _ <- catch (createDirectoryIfMissing True dir) $ \e -> - hPutStr stderr ("Warning: Couldn't create " <> dir <> " :" <> show (e :: IOException)) return $ dir <> "/nix.conf" -- nix.conf Parser @@ -169,11 +168,12 @@ return $ constr (fmap toS values) parseOther :: Parser NixConfLine -parseOther = Other . toS <$> Mega.manyTill anyChar eol +parseOther = Mega.try $ Other . toS <$> Mega.someTill anyChar (void eol <|> Mega.eof) parseAltLine :: Parser NixConfLine parseAltLine = - parseLine Substituters "substituters" + (const (Other "") <$> eol) + <|> parseLine Substituters "substituters" <|> parseLine TrustedPublicKeys "trusted-public-keys" <|> parseLine TrustedUsers "trusted-users" <|> parseLine TrustedPublicKeys "binary-cache-public-keys" diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -2,7 +2,6 @@ import Protolude import Test.Hspec.Runner -import Test.Hspec.Formatters import qualified Spec main :: IO () diff --git a/test/NixConfSpec.hs b/test/NixConfSpec.hs --- a/test/NixConfSpec.hs +++ b/test/NixConfSpec.hs @@ -16,10 +16,12 @@ propertyNix1 :: Text -> Expectation propertyNix1 x = NixConf.render Nix1XX <$> parse x `shouldBe` Right x +bc :: BinaryCache bc = BinaryCache { name = "name" , uri = "https://name.cachix.org" , publicSigningKeys = ["pub"] + , githubUsername = "foobar" } spec :: Spec @@ -115,6 +117,8 @@ parse "substituters = a b c\n" `shouldBe` (Right $ NixConf [Substituters ["a", "b", "c"]]) it "parses equal sign after the first key as literal" $ parse "substituters = a b c= d\n" `shouldBe` (Right $ NixConf [Substituters ["a", "b", "c=", "d"]]) + it "parses with missing endline" $ + parse "allowed-users = *" `shouldBe` (Right $ NixConf [Other "allowed-users = *"]) it "parses a complex example" $ parse realExample `shouldBe` (Right $ NixConf [ Other "" , Substituters ["a","b","c"] @@ -125,6 +129,7 @@ , Other "" , Other ""]) +realExample :: Text realExample = [hereLit| substituters = a b c trusted-users = him me