cachix 0.3.6 → 0.3.7

raw patch · 8 files changed

+117/−48 lines, 8 files

Files

• CHANGELOG.md +29/−0
• cachix.cabal +2/−2
• src/Cachix/Client/Commands.hs +1/−1
• src/Cachix/Client/Exception.hs +1/−0
• src/Cachix/Client/InstallationMode.hs +5/−5
• src/Cachix/Client/OptionsParser.hs +11/−1
• src/Cachix/Client/Push.hs +36/−39
• src/Cachix/Client/Store.hs +32/−0

@@ -7,7 +7,36 @@  ## Unreleased +## [0.3.7] - 2020-03-12++### Added++- Allow specifying number of parallel jobs when pushing: cachix push mycache -j8++### Changed++- Print stderr during streaming of nars    ++### Fixed++- Use max up to 8 cores to prevent performance issues+  on machines with lots of cores++- Get deriver and references from C++ structures rather+  then shelling out (slight performance improvement)++- #251: Assert nar hash before creating narinfo+    +  Fixes rare but annoying bug of "bad nar archive" from Nix.+  Never managed to reproduce. One theory is that the path disappears as+  its deleted by GC or nix-store --delete.++- Print correct path when passing --nixos-folder and encountering an error++ ## [0.3.6] - 2020-02-22++### Fixed  - #275: support LTS-13.x and GHC 8.2.2 

@@ -1,6 +1,6 @@ cabal-version:      2.2 name:               cachix-version: 0.3.6+version: 0.3.7 license:            Apache-2.0 license-file:       LICENSE copyright:          2018 Domen Kožar@@ -126,7 +126,7 @@   import:             defaults   main-is:            Main.hs   build-tool-depends: hspec-discover:hspec-discover -any-  ghc-options:        -threaded -rtsopts -with-rtsopts=-N+  ghc-options:        -threaded -rtsopts -with-rtsopts=-maxN8   hs-source-dirs:     cachix   other-modules:      Paths_cachix   autogen-modules:    Paths_cachix

@@ -168,7 +168,7 @@   store <- wait (storeAsync env)   void $     pushClosure-      (mapConcurrentlyBounded 4)+      (mapConcurrentlyBounded (numJobs opts))       (clientenv env)       store       PushCache

@@ -12,6 +12,7 @@   | NoConfig Text   | NetRcParseError Text   | NarStreamingError ExitCode Text+  | NarHashMismatch Text   | DeprecatedCommand Text   | AccessDeniedBinaryCache Text   | BinaryCacheNotFound Text

@@ -100,10 +100,10 @@ nixosBinaryCache maybeConfig bc UseOptions {useNixOSFolder = baseDirectory} = do   eitherPermissions <- try $ getPermissions (toS baseDirectory) :: IO (Either SomeException Permissions)   case eitherPermissions of-    Left _ -> throwIO $ NixOSInstructions noEtcPermissionInstructions+    Left _ -> throwIO $ NixOSInstructions $ noEtcPermissionInstructions $ toS baseDirectory     Right permissions       | writable permissions -> installFiles-      | otherwise -> throwIO $ NixOSInstructions noEtcPermissionInstructions+      | otherwise -> throwIO $ NixOSInstructions $ noEtcPermissionInstructions $ toS baseDirectory   where     installFiles = do       createDirectoryIfMissing True $ toS toplevel@@ -121,10 +121,10 @@     glueModuleFile = toplevel <> ".nix"     cacheModuleFile :: Text     cacheModuleFile = toplevel <> "/" <> toS (name bc) <> ".nix"-    noEtcPermissionInstructions :: Text-    noEtcPermissionInstructions =+    noEtcPermissionInstructions :: Text -> Text+    noEtcPermissionInstructions dir =       [iTrim|-Could not install NixOS configuration to /etc/nixos/ due to lack of write permissions.+Could not install NixOS configuration to ${dir} due to lack of write permissions.  Pass `--nixos-folder /etc/mynixos/` as an alternative location with write permissions.       |]

@@ -77,7 +77,8 @@  data PushOptions   = PushOptions-      { compressionLevel :: Int+      { compressionLevel :: Int,+        numJobs :: Int       }   deriving (Show) @@ -102,12 +103,21 @@         <$> option           (auto >>= validatedLevel)           ( long "compression-level"+              <> short 'c'               <> metavar "[0..9]"               <> help                 "The compression level for XZ compression.\                 \ Take compressor *and* decompressor memory usage into account before using [7..9]!"               <> showDefault               <> value 2+          )+        <*> option+          auto+          ( long "jobs"+              <> short 'j'+              <> help "Number of threads used for pushing store paths."+              <> showDefault+              <> value 4           )     push = (\opts cache f -> Push $ f opts cache) <$> pushOptions <*> nameArg <*> (pushPaths <|> pushWatchStore)     pushPaths =

@@ -29,7 +29,6 @@ import Control.Concurrent.Async (mapConcurrently) import qualified Control.Concurrent.QSem as QSem import Control.Exception.Safe (MonadMask, throwM)-import Control.Monad ((>=>)) import Control.Monad.Trans.Resource (ResourceT) import Control.Retry (RetryPolicy, RetryStatus, constantDelay, limitRetries, recoverAll) import Crypto.Sign.Ed25519@@ -48,7 +47,6 @@ import Servant.Client.Streaming hiding (ClientError) import Servant.Conduit () import qualified System.Nix.Base32-import System.Process (readProcess)  data PushCache   = PushCache@@ -133,8 +131,7 @@       storePathSize :: Int64       storePathSize = Store.validPathInfoNarSize pathinfo   onAttempt cb retrystatus storePathSize-  -- TODO: print process stderr?-  (ClosedStream, (stdoutStream, closeStdout), ClosedStream, cph) <- liftIO $ streamingProcess cmd+  (ClosedStream, stdoutStream, Inherited, cph) <- liftIO $ streamingProcess cmd   withXzipCompressor cb $ \xzCompressor -> do     let stream' =           stdoutStream@@ -158,43 +155,43 @@           (Api.createNar (cachixBCStreamingClient name) stream')         $ escalate           >=> \NoContent -> do-            closeStdout             exitcode <- waitForStreamingProcess cph             when (exitcode /= ExitSuccess) $ throwM $ NarStreamingError exitcode $ show cmd-            -- TODO: we're done, so we can leave withClientM. Doing so-            --       will allow more concurrent requests when the number-            --       of XZ compressors is limited-            narSize <- readIORef narSizeRef-            narHash <- ("sha256:" <>) . System.Nix.Base32.encode <$> readIORef narHashRef-            fileHash <- readIORef fileHashRef-            fileSize <- readIORef fileSizeRef-            deriverRaw <- T.strip . toS <$> readProcess "nix-store" ["-q", "--deriver", toS storePath] mempty-            let deriver =-                  if deriverRaw == "unknown-deriver"-                    then deriverRaw-                    else T.drop 11 deriverRaw-            references <- sort . T.lines . T.strip . toS <$> readProcess "nix-store" ["-q", "--references", toS storePath] mempty-            let fp = fingerprint storePath narHash narSize references-                sig = dsign (signingSecretKey $ pushCacheSigningKey cache) fp-                nic =-                  Api.NarInfoCreate-                    { Api.cStoreHash = storeHash,-                      Api.cStoreSuffix = storeSuffix,-                      Api.cNarHash = narHash,-                      Api.cNarSize = narSize,-                      Api.cFileSize = fileSize,-                      Api.cFileHash = toS fileHash,-                      Api.cReferences = fmap (T.drop 11) references,-                      Api.cDeriver = deriver,-                      Api.cSig = toS $ B64.encode $ unSignature sig-                    }-            escalate $ Api.isNarInfoCreateValid nic-            -- Upload narinfo with signature-            escalate <=< (`runClientM` clientEnv) $-              Api.createNarinfo-                (cachixBCClient name)-                (Api.NarInfoC storeHash)-                nic+            return NoContent+    (_ :: NoContent) <- liftIO $ do+      narSize <- readIORef narSizeRef+      narHash <- ("sha256:" <>) . System.Nix.Base32.encode <$> readIORef narHashRef+      narHashNix <- Store.validPathInfoNarHash pathinfo+      when (narHash /= toS narHashNix) $ throwM $ NarHashMismatch "Nar hash mismatch between nix-store --dump and nix db"+      fileHash <- readIORef fileHashRef+      fileSize <- readIORef fileSizeRef+      deriver <- toS <$> Store.validPathInfoDeriver pathinfo+      referencesPathSet <- Store.validPathInfoReferences pathinfo+      references <- sort <$> Store.traversePathSet (pure . toS) referencesPathSet+      let fp = fingerprint storePath narHash narSize references+          sig = dsign (signingSecretKey $ pushCacheSigningKey cache) fp+          nic =+            Api.NarInfoCreate+              { Api.cStoreHash = storeHash,+                Api.cStoreSuffix = storeSuffix,+                Api.cNarHash = narHash,+                Api.cNarSize = narSize,+                Api.cFileSize = fileSize,+                Api.cFileHash = toS fileHash,+                Api.cReferences = fmap (T.drop 11) references,+                Api.cDeriver =+                  if deriver == "unknown-deriver"+                    then deriver+                    else T.drop 11 deriver,+                Api.cSig = toS $ B64.encode $ unSignature sig+              }+      escalate $ Api.isNarInfoCreateValid nic+      -- Upload narinfo with signature+      escalate <=< (`runClientM` clientEnv) $+        Api.createNarinfo+          (cachixBCClient name)+          (Api.NarInfoC storeHash)+          nic     onDone cb  -- Catches all exceptions except skipAsyncExceptions

@@ -13,6 +13,9 @@     followLinksToStorePath,     queryPathInfo,     validPathInfoNarSize,+    validPathInfoNarHash,+    validPathInfoDeriver,+    validPathInfoReferences,      -- * Get closures     computeFSClosure,@@ -121,6 +124,35 @@       [C.pure| long         { (*$fptr-ptr:(refValidPathInfo* vpi))->narSize }       |]++-- | Copy the narHash field of a ValidPathInfo struct. Source: store-api.hh+validPathInfoNarHash :: ForeignPtr (Ref ValidPathInfo) -> IO ByteString+validPathInfoNarHash vpi =+  unsafePackMallocCString+    =<< [C.exp| const char+        *{ strdup((*$fptr-ptr:(refValidPathInfo* vpi))->narHash.to_string().c_str()) }+      |]++-- | Deriver field of a ValidPathInfo struct. Source: store-api.hh+validPathInfoDeriver :: ForeignPtr (Ref ValidPathInfo) -> IO ByteString+validPathInfoDeriver vpi =+  unsafePackMallocCString+    =<< [C.throwBlock| const char*+        {+          std::optional<Path> deriver = (*$fptr-ptr:(refValidPathInfo* vpi))->deriver;+          return strdup((deriver == "" ? "unknown-deriver" : deriver->c_str()));+        }+      |]++-- | References field of a ValidPathInfo struct. Source: store-api.hh+validPathInfoReferences :: ForeignPtr (Ref ValidPathInfo) -> IO PathSet+validPathInfoReferences vpi = do+  ptr <-+    [C.exp| const PathSet*+            { new PathSet((*$fptr-ptr:(refValidPathInfo* vpi))->references) }+        |]+  fptr <- newForeignPtr finalizePathSet ptr+  pure $ PathSet fptr  ----- PathSet ----- newtype PathSet = PathSet (ForeignPtr (C.Set C.CxxString))