diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,36 @@ ## Unreleased +## [0.3.7] - 2020-03-12 + +### Added + +- Allow specifying number of parallel jobs when pushing: cachix push mycache -j8 + +### Changed + +- Print stderr during streaming of nars + +### Fixed + +- Use max up to 8 cores to prevent performance issues + on machines with lots of cores + +- Get deriver and references from C++ structures rather + then shelling out (slight performance improvement) + +- #251: Assert nar hash before creating narinfo + + Fixes rare but annoying bug of "bad nar archive" from Nix. + Never managed to reproduce. One theory is that the path disappears as + its deleted by GC or nix-store --delete. + +- Print correct path when passing --nixos-folder and encountering an error + + ## [0.3.6] - 2020-02-22 + +### Fixed - #275: support LTS-13.x and GHC 8.2.2 diff --git a/cachix.cabal b/cachix.cabal --- a/cachix.cabal +++ b/cachix.cabal @@ -1,6 +1,6 @@ cabal-version: 2.2 name: cachix -version: 0.3.6 +version: 0.3.7 license: Apache-2.0 license-file: LICENSE copyright: 2018 Domen Kožar @@ -126,7 +126,7 @@ import: defaults main-is: Main.hs build-tool-depends: hspec-discover:hspec-discover -any - ghc-options: -threaded -rtsopts -with-rtsopts=-N + ghc-options: -threaded -rtsopts -with-rtsopts=-maxN8 hs-source-dirs: cachix other-modules: Paths_cachix autogen-modules: Paths_cachix diff --git a/src/Cachix/Client/Commands.hs b/src/Cachix/Client/Commands.hs --- a/src/Cachix/Client/Commands.hs +++ b/src/Cachix/Client/Commands.hs @@ -168,7 +168,7 @@ store <- wait (storeAsync env) void $ pushClosure - (mapConcurrentlyBounded 4) + (mapConcurrentlyBounded (numJobs opts)) (clientenv env) store PushCache diff --git a/src/Cachix/Client/Exception.hs b/src/Cachix/Client/Exception.hs --- a/src/Cachix/Client/Exception.hs +++ b/src/Cachix/Client/Exception.hs @@ -12,6 +12,7 @@ | NoConfig Text | NetRcParseError Text | NarStreamingError ExitCode Text + | NarHashMismatch Text | DeprecatedCommand Text | AccessDeniedBinaryCache Text | BinaryCacheNotFound Text diff --git a/src/Cachix/Client/InstallationMode.hs b/src/Cachix/Client/InstallationMode.hs --- a/src/Cachix/Client/InstallationMode.hs +++ b/src/Cachix/Client/InstallationMode.hs @@ -100,10 +100,10 @@ nixosBinaryCache maybeConfig bc UseOptions {useNixOSFolder = baseDirectory} = do eitherPermissions <- try $ getPermissions (toS baseDirectory) :: IO (Either SomeException Permissions) case eitherPermissions of - Left _ -> throwIO $ NixOSInstructions noEtcPermissionInstructions + Left _ -> throwIO $ NixOSInstructions $ noEtcPermissionInstructions $ toS baseDirectory Right permissions | writable permissions -> installFiles - | otherwise -> throwIO $ NixOSInstructions noEtcPermissionInstructions + | otherwise -> throwIO $ NixOSInstructions $ noEtcPermissionInstructions $ toS baseDirectory where installFiles = do createDirectoryIfMissing True $ toS toplevel @@ -121,10 +121,10 @@ glueModuleFile = toplevel <> ".nix" cacheModuleFile :: Text cacheModuleFile = toplevel <> "/" <> toS (name bc) <> ".nix" - noEtcPermissionInstructions :: Text - noEtcPermissionInstructions = + noEtcPermissionInstructions :: Text -> Text + noEtcPermissionInstructions dir = [iTrim| -Could not install NixOS configuration to /etc/nixos/ due to lack of write permissions. +Could not install NixOS configuration to ${dir} due to lack of write permissions. Pass `--nixos-folder /etc/mynixos/` as an alternative location with write permissions. |] diff --git a/src/Cachix/Client/OptionsParser.hs b/src/Cachix/Client/OptionsParser.hs --- a/src/Cachix/Client/OptionsParser.hs +++ b/src/Cachix/Client/OptionsParser.hs @@ -77,7 +77,8 @@ data PushOptions = PushOptions - { compressionLevel :: Int + { compressionLevel :: Int, + numJobs :: Int } deriving (Show) @@ -102,12 +103,21 @@ <$> option (auto >>= validatedLevel) ( long "compression-level" + <> short 'c' <> metavar "[0..9]" <> help "The compression level for XZ compression.\ \ Take compressor *and* decompressor memory usage into account before using [7..9]!" <> showDefault <> value 2 + ) + <*> option + auto + ( long "jobs" + <> short 'j' + <> help "Number of threads used for pushing store paths." + <> showDefault + <> value 4 ) push = (\opts cache f -> Push $ f opts cache) <$> pushOptions <*> nameArg <*> (pushPaths <|> pushWatchStore) pushPaths = diff --git a/src/Cachix/Client/Push.hs b/src/Cachix/Client/Push.hs --- a/src/Cachix/Client/Push.hs +++ b/src/Cachix/Client/Push.hs @@ -29,7 +29,6 @@ import Control.Concurrent.Async (mapConcurrently) import qualified Control.Concurrent.QSem as QSem import Control.Exception.Safe (MonadMask, throwM) -import Control.Monad ((>=>)) import Control.Monad.Trans.Resource (ResourceT) import Control.Retry (RetryPolicy, RetryStatus, constantDelay, limitRetries, recoverAll) import Crypto.Sign.Ed25519 @@ -48,7 +47,6 @@ import Servant.Client.Streaming hiding (ClientError) import Servant.Conduit () import qualified System.Nix.Base32 -import System.Process (readProcess) data PushCache = PushCache @@ -133,8 +131,7 @@ storePathSize :: Int64 storePathSize = Store.validPathInfoNarSize pathinfo onAttempt cb retrystatus storePathSize - -- TODO: print process stderr? - (ClosedStream, (stdoutStream, closeStdout), ClosedStream, cph) <- liftIO $ streamingProcess cmd + (ClosedStream, stdoutStream, Inherited, cph) <- liftIO $ streamingProcess cmd withXzipCompressor cb $ \xzCompressor -> do let stream' = stdoutStream @@ -158,43 +155,43 @@ (Api.createNar (cachixBCStreamingClient name) stream') $ escalate >=> \NoContent -> do - closeStdout exitcode <- waitForStreamingProcess cph when (exitcode /= ExitSuccess) $ throwM $ NarStreamingError exitcode $ show cmd - -- TODO: we're done, so we can leave withClientM. Doing so - -- will allow more concurrent requests when the number - -- of XZ compressors is limited - narSize <- readIORef narSizeRef - narHash <- ("sha256:" <>) . System.Nix.Base32.encode <$> readIORef narHashRef - fileHash <- readIORef fileHashRef - fileSize <- readIORef fileSizeRef - deriverRaw <- T.strip . toS <$> readProcess "nix-store" ["-q", "--deriver", toS storePath] mempty - let deriver = - if deriverRaw == "unknown-deriver" - then deriverRaw - else T.drop 11 deriverRaw - references <- sort . T.lines . T.strip . toS <$> readProcess "nix-store" ["-q", "--references", toS storePath] mempty - let fp = fingerprint storePath narHash narSize references - sig = dsign (signingSecretKey $ pushCacheSigningKey cache) fp - nic = - Api.NarInfoCreate - { Api.cStoreHash = storeHash, - Api.cStoreSuffix = storeSuffix, - Api.cNarHash = narHash, - Api.cNarSize = narSize, - Api.cFileSize = fileSize, - Api.cFileHash = toS fileHash, - Api.cReferences = fmap (T.drop 11) references, - Api.cDeriver = deriver, - Api.cSig = toS $ B64.encode $ unSignature sig - } - escalate $ Api.isNarInfoCreateValid nic - -- Upload narinfo with signature - escalate <=< (`runClientM` clientEnv) $ - Api.createNarinfo - (cachixBCClient name) - (Api.NarInfoC storeHash) - nic + return NoContent + (_ :: NoContent) <- liftIO $ do + narSize <- readIORef narSizeRef + narHash <- ("sha256:" <>) . System.Nix.Base32.encode <$> readIORef narHashRef + narHashNix <- Store.validPathInfoNarHash pathinfo + when (narHash /= toS narHashNix) $ throwM $ NarHashMismatch "Nar hash mismatch between nix-store --dump and nix db" + fileHash <- readIORef fileHashRef + fileSize <- readIORef fileSizeRef + deriver <- toS <$> Store.validPathInfoDeriver pathinfo + referencesPathSet <- Store.validPathInfoReferences pathinfo + references <- sort <$> Store.traversePathSet (pure . toS) referencesPathSet + let fp = fingerprint storePath narHash narSize references + sig = dsign (signingSecretKey $ pushCacheSigningKey cache) fp + nic = + Api.NarInfoCreate + { Api.cStoreHash = storeHash, + Api.cStoreSuffix = storeSuffix, + Api.cNarHash = narHash, + Api.cNarSize = narSize, + Api.cFileSize = fileSize, + Api.cFileHash = toS fileHash, + Api.cReferences = fmap (T.drop 11) references, + Api.cDeriver = + if deriver == "unknown-deriver" + then deriver + else T.drop 11 deriver, + Api.cSig = toS $ B64.encode $ unSignature sig + } + escalate $ Api.isNarInfoCreateValid nic + -- Upload narinfo with signature + escalate <=< (`runClientM` clientEnv) $ + Api.createNarinfo + (cachixBCClient name) + (Api.NarInfoC storeHash) + nic onDone cb -- Catches all exceptions except skipAsyncExceptions diff --git a/src/Cachix/Client/Store.hs b/src/Cachix/Client/Store.hs --- a/src/Cachix/Client/Store.hs +++ b/src/Cachix/Client/Store.hs @@ -13,6 +13,9 @@ followLinksToStorePath, queryPathInfo, validPathInfoNarSize, + validPathInfoNarHash, + validPathInfoDeriver, + validPathInfoReferences, -- * Get closures computeFSClosure, @@ -121,6 +124,35 @@ [C.pure| long { (*$fptr-ptr:(refValidPathInfo* vpi))->narSize } |] + +-- | Copy the narHash field of a ValidPathInfo struct. Source: store-api.hh +validPathInfoNarHash :: ForeignPtr (Ref ValidPathInfo) -> IO ByteString +validPathInfoNarHash vpi = + unsafePackMallocCString + =<< [C.exp| const char + *{ strdup((*$fptr-ptr:(refValidPathInfo* vpi))->narHash.to_string().c_str()) } + |] + +-- | Deriver field of a ValidPathInfo struct. Source: store-api.hh +validPathInfoDeriver :: ForeignPtr (Ref ValidPathInfo) -> IO ByteString +validPathInfoDeriver vpi = + unsafePackMallocCString + =<< [C.throwBlock| const char* + { + std::optional deriver = (*$fptr-ptr:(refValidPathInfo* vpi))->deriver; + return strdup((deriver == "" ? "unknown-deriver" : deriver->c_str())); + } + |] + +-- | References field of a ValidPathInfo struct. Source: store-api.hh +validPathInfoReferences :: ForeignPtr (Ref ValidPathInfo) -> IO PathSet +validPathInfoReferences vpi = do + ptr <- + [C.exp| const PathSet* + { new PathSet((*$fptr-ptr:(refValidPathInfo* vpi))->references) } + |] + fptr <- newForeignPtr finalizePathSet ptr + pure $ PathSet fptr ----- PathSet ----- newtype PathSet = PathSet (ForeignPtr (C.Set C.CxxString))