hercules-ci-agent 0.10.5 → 0.10.6

raw patch · 16 files changed

+142/−46 lines, 16 files

Files

• CHANGELOG.md +10/−0
• cbits/hercules-aliases.h +5/−1
• cbits/hercules-error.hh +7/−0
• cbits/hercules-logger.hh +9/−0
• cbits/nix-2.4/hercules-store.cxx +14/−0
• cbits/nix-2.4/hercules-store.hh +13/−3
• hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +4/−1
• hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs +17/−2
• hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +15/−12
• hercules-ci-agent-worker/Hercules/Agent/Worker/Error.hs +9/−0
• hercules-ci-agent-worker/Hercules/Agent/Worker/Evaluate.hs +8/−11
• hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +6/−0
• hercules-ci-agent.cabal +1/−1
• hercules-ci-agent/Hercules/Agent/Evaluate.hs +4/−5
• hercules-ci-nix-daemon/daemon.cc +19/−4
• src/Hercules/Agent/WorkerProtocol/Command/Eval.hs +1/−6

@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.6] - 2024-05-05++### Added++- Nix 2.28 support++### Fixed++- Always fetch dependency closure when max-jobs == 0+ ## [0.10.5] - 2024-11-15  ### Added

@@ -2,9 +2,13 @@  #include "hercules-store.hh" #include "hercules-logger.hh"-#include "derivations.hh" #include <hercules-ci-cnix/store.hxx> #include <hercules-ci-cnix/expr.hxx>+#if NIX_IS_AT_LEAST(2,28,0)+#include <nix/store/derivations.hh>+#else+#include "derivations.hh"+#endif  // inline-c-cpp doesn't seem to handle namespace operator or template // syntax so we help it a bit for now. This definition can be inlined

@@ -1,5 +1,12 @@+#if NIX_IS_AT_LEAST(2, 28, 0)++#include <nix/store/store-api.hh>++#else #include <nix/config.h> #include <nix/store-api.hh>+#endif+ #include <string>  namespace hercules {

@@ -1,10 +1,19 @@ #pragma once +#if NIX_IS_AT_LEAST(2, 28, 0)+#include <nix/util/error.hh>+#include <nix/util/logging.hh>+#include <nix/util/sync.hh>+#include <nix/main/shared.hh>+#else #include <nix/config.h> #include <nix/error.hh> #include <nix/shared.hh> #include <nix/sync.hh> #include <nix/logging.hh>+#endif++ #include <queue> #include <string> 

@@ -2,6 +2,19 @@ #include <cstring> #include <memory> #include <math.h>++#if NIX_IS_AT_LEAST(2,28,0)++#include <nix/store/build-result.hh>+#include <nix/store/derivations.hh>+#include <nix/store/globals.hh>+#include <nix/store/path-with-outputs.hh>+#include <nix/store/store-api.hh>+#include <nix/util/callback.hh>+#include <nix/expr/get-drvs.hh>+#include <nix/main/shared.hh>++#else #include <nix/config.h> #include <nix/shared.hh> #include <nix/store-api.hh>@@ -15,6 +28,7 @@ #include <nix/gc-store.hh> #endif #include <nix/path-with-outputs.hh>+#endif  #include "hercules-store.hh" 

@@ -1,6 +1,15 @@ #include <cstdio> #include <cstring> #include <math.h>++#if NIX_IS_AT_LEAST(2,28,0)+#include <nix/util/util.hh>+#include <nix/store/derivations.hh>+#include <nix/store/path-with-outputs.hh>+#include <nix/store/store-api.hh>+#include <nix/main/shared.hh>++#else #include <nix/config.h> #include <nix/shared.hh> #include <nix/store-api.hh>@@ -8,10 +17,11 @@ #include <nix/get-drvs.hh> #include <nix/derivations.hh> #include <nix/globals.hh>-#include <nix/globals.hh>-#if NIX_IS_AT_LEAST(2,13,0)-#include <nix/path-with-outputs.hh>+#  if NIX_IS_AT_LEAST(2,13,0)+#    include <nix/path-with-outputs.hh>+#  endif #endif+ #include "HsFFI.h"  #if NIX_IS_AT_LEAST(2,19,0)

@@ -16,6 +16,7 @@   ( getDerivationOutputs,   ) import Hercules.CNix qualified as CNix+import Hercules.CNix.Settings qualified as CNix.Settings import Hercules.CNix.Store (Store) import Katip import Protolude hiding (yield)@@ -44,8 +45,10 @@     Just drv -> pure drv     Nothing -> panic $ "Could not retrieve derivation " <> show drvStorePath <> " from local store or binary caches."   drvPlatform <- liftIO $ CNix.getDerivationPlatform derivation+  maxJobs <- liftIO $ CNix.Settings.getMaxBuildJobs   let mayNeedRemoteBuild = drvPlatform `elem` Command.Build.materializePlatforms build-      materialize = materialize0 || mayNeedRemoteBuild+      willNeedRemoteBuild = maxJobs == 0+      materialize = materialize0 || mayNeedRemoteBuild || willNeedRemoteBuild   nixBuildResult <- liftIO $ buildDerivation store drvStorePath derivation (extraPaths <$ guard (not materialize))   katipAddContext (sl "result" (show nixBuildResult :: Text)) $     logLocM DebugS "Build result"

@@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-}+{-# LANGUAGE CPP #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-}@@ -34,11 +35,15 @@  C.include "<cstring>" -C.include "<nix/config.h>"+#if NIX_IS_AT_LEAST(2, 28, 0) -C.include "<nix/shared.hh>"+-- C.include  +#else+C.include "<nix/config.h>"+C.include "<nix/shared.hh>" C.include "<nix/globals.hh>"+#endif  C.include "<hercules-ci-cnix/string.hxx>" @@ -52,10 +57,20 @@  initLogger :: IO () initLogger =+#if NIX_IS_AT_LEAST(2, 28, 0)   [C.throwBlock| void {+    nix::logger = std::make_unique<HerculesLogger>();+    herculesLogger = dynamic_cast<HerculesLogger *>(nix::logger.get());+    if (herculesLogger == nullptr) {+      throw std::runtime_error("Failed to cast logger to HerculesLogger");+    }+  }|]+#else+  [C.throwBlock| void {     herculesLogger = new HerculesLogger();     nix::logger = herculesLogger;   }|]+#endif  popMany :: IO (Vector LogEntry) popMany =

@@ -31,30 +31,33 @@  C.include "<cstring>" -C.include "<nix/config.h>"+#if NIX_IS_AT_LEAST(2, 28, 0) -C.include "<nix/shared.hh>"+C.include "<nix/store/build-result.hh>"+C.include "<nix/store/derivations.hh>"+C.include "<nix/store/path-with-outputs.hh>"+C.include "<nix/store/store-api.hh>"+C.include "<nix/util/signals.hh>" +#else+C.include "<nix/config.h>"+C.include "<nix/shared.hh>" C.include "<nix/store-api.hh>"- C.include "<nix/get-drvs.hh>"- C.include "<nix/derivations.hh>"- C.include "<nix/globals.hh>"--#if NIX_IS_AT_LEAST(2,19,0)+#  if NIX_IS_AT_LEAST(2,19,0) C.include "<nix/signals.hh>"-#else+#  else -- redundant? C.include "<nix/fs-accessor.hh>"-#endif--#if NIX_IS_AT_LEAST(2,7,0)+#  endif+#  if NIX_IS_AT_LEAST(2,7,0) C.include "<nix/build-result.hh>"-#endif+#  endif  C.include "<nix/path-with-outputs.hh>"+#endif  C.include "<hercules-ci-cnix/store.hxx>" 

@@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-}+{-# LANGUAGE CPP #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-} @@ -17,8 +18,16 @@ C.context (C.cppCtx <> C.fptrCtx <> C.bsCtx <> Hercules.CNix.Store.Context.context)  C.include "<hercules-error.hh>"++#if NIX_IS_AT_LEAST(2, 28, 0)++C.include "<nix/util/error.hh>"++#else C.include "<nix/error.hh>" C.include "<nix/util.hh>"+#endif+ C.include "<iostream>" C.include "<sstream>" 

@@ -52,7 +52,7 @@ import Hercules.Agent.WorkerProtocol.Event.AttributeIFD qualified as Event.AttributeIFD import Hercules.Agent.WorkerProtocol.ViaJSON qualified as ViaJSON import Hercules.CNix as CNix-import Hercules.CNix.Expr (Match (IsAttrs, IsString), NixAttrs, RawValue, addAllowedPath, addInternalAllowedPaths, autoCallFunction, evalArgs, getAttrBool, getAttrList, getAttrs, getDrvFile, getFlakeFromArchiveUrl, getFlakeFromGit, getRecurseForDerivations, getStringIgnoreContext, isDerivation, isFunctor, match, rawValueType, rtValue, toRawValue, toValue, withEvalStateConduit)+import Hercules.CNix.Expr (Match (IsAttrs, IsString), NixAttrs, RawValue, addAllowedPath, addInternalAllowedPaths, autoCallFunction, getAttrBool, getAttrList, getAttrs, getDrvFile, getFlakeFromArchiveUrl, getFlakeFromGit, getRecurseForDerivations, getStringIgnoreContext, isDerivation, isFunctor, match, rawValueType, rtValue, toRawValue, toValue, valueFromExpressionString, withEvalStateConduit) import Hercules.CNix.Expr.Context (EvalState) import Hercules.CNix.Expr.Raw qualified import Hercules.CNix.Expr.Schema (MonadEval, PSObject, dictionaryToMap, fromPSObject, provenance, requireDict, traverseArray, (#.), (#?), (#?!), (#??), ($?), (|!), type (->?), type (.))@@ -92,14 +92,6 @@ evalEnvStore :: EvalEnv -> Store evalEnvStore = nixStore . herculesStore . evalEnvHerculesState --- TODO: test-autoArgArgs :: Map Text Eval.Arg -> [ByteString]-autoArgArgs kvs = do-  (k, v) <- M.toList kvs-  case v of-    Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s]-    Eval.ExprArg s -> ["--arg", encodeUtf8 k, s]- -- Ensure that a nested build invocation does not happen by a mistake in wiring. -- -- (It does not happen, but this de-escalates a potential recursion bug to just@@ -286,8 +278,13 @@     katipAddContext (sl "storeURI" (decode s)) $       logLocM DebugS "EvalState loaded."     args <--      liftIO $-        evalArgs evalState (autoArgArgs (Eval.autoArguments eval))+      liftIO do+        argMap <-+          Eval.autoArguments eval+            & M.traverseWithKey \k arg -> do+              valueFromExpressionString evalState arg ("Argument " <> encodeUtf8 k)+        toValue evalState (M.mapKeys encodeUtf8 argMap)+     Data.Conduit.handleC (yieldAttributeError store []) $       do         homeExpr <- getHomeExpr evalState eval

@@ -36,6 +36,11 @@  C.include "<cstring>" +#if NIX_IS_AT_LEAST(2, 28, 0)++C.include "<nix/main/shared.hh>"++#else C.include "<nix/config.h>"  C.include "<nix/shared.hh>"@@ -47,6 +52,7 @@ C.include "<nix/derivations.hh>"  C.include "<nix/globals.hh>"+#endif  C.include "hercules-aliases.h" 

@@ -1,7 +1,7 @@ cabal-version: 2.4  name:           hercules-ci-agent-version:        0.10.5+version:        0.10.6 synopsis:       Runs Continuous Integration tasks on your machines category:       Nix, CI, Testing, DevOps homepage:       https://docs.hercules-ci.com

@@ -448,11 +448,10 @@                in case do                     inputId <- EvaluateTask.autoArguments task & M.lookup k                     EvaluateTask.inputMetadata task & M.lookup (EvaluateTask.path inputId) of-                    Nothing -> Eval.ExprArg argPath+                    Nothing -> argPath                     Just attrs ->-                      Eval.ExprArg $-                        -- TODO pass directly to avoid having to escape (or just escape properly)-                        "builtins.fromJSON ''" <> BL.toStrict (A.encode attrs) <> "'' // { outPath = " <> argPath <> "; }"+                      -- TODO pass directly to avoid having to escape (or just escape properly)+                      "builtins.fromJSON ''" <> BL.toStrict (A.encode attrs) <> "'' // { outPath = " <> argPath <> "; }"     adHocSystem <-       readFileMaybe (projectDir </> "ci-default-system.txt") @@ -583,7 +582,7 @@   CNix.Store ->   FilePath ->   FilePath ->-  Map Text Eval.Arg ->+  Map Text ByteString ->   [ EvaluateTask.NixPathElement       (EvaluateTask.SubPathOf FilePath)   ] ->

@@ -5,6 +5,17 @@  #include <string_view> +#if NIX_IS_AT_LEAST(2,28,0)++#include <nix/util/error.hh>+#include <nix/util/fmt.hh>+#include <nix/util/config-global.hh>+#include <nix/util/serialise.hh>+#include <nix/util/signals.hh>+#include <nix/store/daemon.hh>+#include <nix/main/shared.hh>++#else #include <nix/config.h> #include <nix/shared.hh> #include <nix/error.hh>@@ -13,14 +24,18 @@ #include <nix/store-api.hh> #include <nix/daemon.hh> -using namespace nix;+#  if NIX_IS_AT_LEAST(2,19,0)+#    include <nix/signals.hh>+#  endif -#if NIX_IS_AT_LEAST(2,19,0)-#include <nix/signals.hh>+#  if NIX_IS_AT_LEAST(2,24,0)+#    include <nix/config-global.hh>+#  endif #endif +using namespace nix;+ #if NIX_IS_AT_LEAST(2,24,0)-#include <nix/config-global.hh> using nix::unix::closeOnExec; #endif 

@@ -13,7 +13,7 @@ data Eval = Eval   { cwd :: FilePath,     file :: Text,-    autoArguments :: Map Text Arg,+    autoArguments :: Map Text ByteString,     -- TODO: Also set at worker start, so remove this here to avoid ambiguity?     extraNixOptions :: [(Text, Text)],     gitSource :: ViaJSON GitSource,@@ -25,9 +25,4 @@     allowInsecureBuiltinFetchers :: Bool,     allowedPaths :: [ByteString]   }-  deriving (Generic, Binary, Show, Eq)--data Arg-  = LiteralArg ByteString-  | ExprArg ByteString   deriving (Generic, Binary, Show, Eq)