diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.6] - 2024-05-05 + +### Added + +- Nix 2.28 support + +### Fixed + +- Always fetch dependency closure when max-jobs == 0 + ## [0.10.5] - 2024-11-15 ### Added diff --git a/cbits/hercules-aliases.h b/cbits/hercules-aliases.h --- a/cbits/hercules-aliases.h +++ b/cbits/hercules-aliases.h @@ -2,9 +2,13 @@ #include "hercules-store.hh" #include "hercules-logger.hh" -#include "derivations.hh" #include #include +#if NIX_IS_AT_LEAST(2,28,0) +#include +#else +#include "derivations.hh" +#endif // inline-c-cpp doesn't seem to handle namespace operator or template // syntax so we help it a bit for now. This definition can be inlined diff --git a/cbits/hercules-error.hh b/cbits/hercules-error.hh --- a/cbits/hercules-error.hh +++ b/cbits/hercules-error.hh @@ -1,5 +1,12 @@ +#if NIX_IS_AT_LEAST(2, 28, 0) + +#include + +#else #include #include +#endif + #include namespace hercules { diff --git a/cbits/hercules-logger.hh b/cbits/hercules-logger.hh --- a/cbits/hercules-logger.hh +++ b/cbits/hercules-logger.hh @@ -1,10 +1,19 @@ #pragma once +#if NIX_IS_AT_LEAST(2, 28, 0) +#include +#include +#include +#include +#else #include #include #include #include #include +#endif + + #include #include diff --git a/cbits/nix-2.4/hercules-store.cxx b/cbits/nix-2.4/hercules-store.cxx --- a/cbits/nix-2.4/hercules-store.cxx +++ b/cbits/nix-2.4/hercules-store.cxx @@ -2,6 +2,19 @@ #include #include #include + +#if NIX_IS_AT_LEAST(2,28,0) + +#include +#include +#include +#include +#include +#include +#include +#include + +#else #include #include #include @@ -15,6 +28,7 @@ #include #endif #include +#endif #include "hercules-store.hh" diff --git a/cbits/nix-2.4/hercules-store.hh b/cbits/nix-2.4/hercules-store.hh --- a/cbits/nix-2.4/hercules-store.hh +++ b/cbits/nix-2.4/hercules-store.hh @@ -1,6 +1,15 @@ #include #include #include + +#if NIX_IS_AT_LEAST(2,28,0) +#include +#include +#include +#include +#include + +#else #include #include #include @@ -8,10 +17,11 @@ #include #include #include -#include -#if NIX_IS_AT_LEAST(2,13,0) -#include +# if NIX_IS_AT_LEAST(2,13,0) +# include +# endif #endif + #include "HsFFI.h" #if NIX_IS_AT_LEAST(2,19,0) diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs @@ -16,6 +16,7 @@ ( getDerivationOutputs, ) import Hercules.CNix qualified as CNix +import Hercules.CNix.Settings qualified as CNix.Settings import Hercules.CNix.Store (Store) import Katip import Protolude hiding (yield) @@ -44,8 +45,10 @@ Just drv -> pure drv Nothing -> panic $ "Could not retrieve derivation " <> show drvStorePath <> " from local store or binary caches." drvPlatform <- liftIO $ CNix.getDerivationPlatform derivation + maxJobs <- liftIO $ CNix.Settings.getMaxBuildJobs let mayNeedRemoteBuild = drvPlatform `elem` Command.Build.materializePlatforms build - materialize = materialize0 || mayNeedRemoteBuild + willNeedRemoteBuild = maxJobs == 0 + materialize = materialize0 || mayNeedRemoteBuild || willNeedRemoteBuild nixBuildResult <- liftIO $ buildDerivation store drvStorePath derivation (extraPaths <$ guard (not materialize)) katipAddContext (sl "result" (show nixBuildResult :: Text)) $ logLocM DebugS "Build result" diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs @@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-} +{-# LANGUAGE CPP #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-} @@ -34,11 +35,15 @@ C.include "" -C.include "" +#if NIX_IS_AT_LEAST(2, 28, 0) -C.include "" +-- C.include +#else +C.include "" +C.include "" C.include "" +#endif C.include "" @@ -52,10 +57,20 @@ initLogger :: IO () initLogger = +#if NIX_IS_AT_LEAST(2, 28, 0) [C.throwBlock| void { + nix::logger = std::make_unique(); + herculesLogger = dynamic_cast(nix::logger.get()); + if (herculesLogger == nullptr) { + throw std::runtime_error("Failed to cast logger to HerculesLogger"); + } + }|] +#else + [C.throwBlock| void { herculesLogger = new HerculesLogger(); nix::logger = herculesLogger; }|] +#endif popMany :: IO (Vector LogEntry) popMany = diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -31,30 +31,33 @@ C.include "" -C.include "" +#if NIX_IS_AT_LEAST(2, 28, 0) -C.include "" +C.include "" +C.include "" +C.include "" +C.include "" +C.include "" +#else +C.include "" +C.include "" C.include "" - C.include "" - C.include "" - C.include "" - -#if NIX_IS_AT_LEAST(2,19,0) +# if NIX_IS_AT_LEAST(2,19,0) C.include "" -#else +# else -- redundant? C.include "" -#endif - -#if NIX_IS_AT_LEAST(2,7,0) +# endif +# if NIX_IS_AT_LEAST(2,7,0) C.include "" -#endif +# endif C.include "" +#endif C.include "" diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Error.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Error.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Error.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Error.hs @@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-} +{-# LANGUAGE CPP #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-} @@ -17,8 +18,16 @@ C.context (C.cppCtx <> C.fptrCtx <> C.bsCtx <> Hercules.CNix.Store.Context.context) C.include "" + +#if NIX_IS_AT_LEAST(2, 28, 0) + +C.include "" + +#else C.include "" C.include "" +#endif + C.include "" C.include "" diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Evaluate.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Evaluate.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Evaluate.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Evaluate.hs @@ -52,7 +52,7 @@ import Hercules.Agent.WorkerProtocol.Event.AttributeIFD qualified as Event.AttributeIFD import Hercules.Agent.WorkerProtocol.ViaJSON qualified as ViaJSON import Hercules.CNix as CNix -import Hercules.CNix.Expr (Match (IsAttrs, IsString), NixAttrs, RawValue, addAllowedPath, addInternalAllowedPaths, autoCallFunction, evalArgs, getAttrBool, getAttrList, getAttrs, getDrvFile, getFlakeFromArchiveUrl, getFlakeFromGit, getRecurseForDerivations, getStringIgnoreContext, isDerivation, isFunctor, match, rawValueType, rtValue, toRawValue, toValue, withEvalStateConduit) +import Hercules.CNix.Expr (Match (IsAttrs, IsString), NixAttrs, RawValue, addAllowedPath, addInternalAllowedPaths, autoCallFunction, getAttrBool, getAttrList, getAttrs, getDrvFile, getFlakeFromArchiveUrl, getFlakeFromGit, getRecurseForDerivations, getStringIgnoreContext, isDerivation, isFunctor, match, rawValueType, rtValue, toRawValue, toValue, valueFromExpressionString, withEvalStateConduit) import Hercules.CNix.Expr.Context (EvalState) import Hercules.CNix.Expr.Raw qualified import Hercules.CNix.Expr.Schema (MonadEval, PSObject, dictionaryToMap, fromPSObject, provenance, requireDict, traverseArray, (#.), (#?), (#?!), (#??), ($?), (|!), type (->?), type (.)) @@ -92,14 +92,6 @@ evalEnvStore :: EvalEnv -> Store evalEnvStore = nixStore . herculesStore . evalEnvHerculesState --- TODO: test -autoArgArgs :: Map Text Eval.Arg -> [ByteString] -autoArgArgs kvs = do - (k, v) <- M.toList kvs - case v of - Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s] - Eval.ExprArg s -> ["--arg", encodeUtf8 k, s] - -- Ensure that a nested build invocation does not happen by a mistake in wiring. -- -- (It does not happen, but this de-escalates a potential recursion bug to just @@ -286,8 +278,13 @@ katipAddContext (sl "storeURI" (decode s)) $ logLocM DebugS "EvalState loaded." args <- - liftIO $ - evalArgs evalState (autoArgArgs (Eval.autoArguments eval)) + liftIO do + argMap <- + Eval.autoArguments eval + & M.traverseWithKey \k arg -> do + valueFromExpressionString evalState arg ("Argument " <> encodeUtf8 k) + toValue evalState (M.mapKeys encodeUtf8 argMap) + Data.Conduit.handleC (yieldAttributeError store []) $ do homeExpr <- getHomeExpr evalState eval diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs @@ -36,6 +36,11 @@ C.include "" +#if NIX_IS_AT_LEAST(2, 28, 0) + +C.include "" + +#else C.include "" C.include "" @@ -47,6 +52,7 @@ C.include "" C.include "" +#endif C.include "hercules-aliases.h" diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.10.5 +version: 0.10.6 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com diff --git a/hercules-ci-agent/Hercules/Agent/Evaluate.hs b/hercules-ci-agent/Hercules/Agent/Evaluate.hs --- a/hercules-ci-agent/Hercules/Agent/Evaluate.hs +++ b/hercules-ci-agent/Hercules/Agent/Evaluate.hs @@ -448,11 +448,10 @@ in case do inputId <- EvaluateTask.autoArguments task & M.lookup k EvaluateTask.inputMetadata task & M.lookup (EvaluateTask.path inputId) of - Nothing -> Eval.ExprArg argPath + Nothing -> argPath Just attrs -> - Eval.ExprArg $ - -- TODO pass directly to avoid having to escape (or just escape properly) - "builtins.fromJSON ''" <> BL.toStrict (A.encode attrs) <> "'' // { outPath = " <> argPath <> "; }" + -- TODO pass directly to avoid having to escape (or just escape properly) + "builtins.fromJSON ''" <> BL.toStrict (A.encode attrs) <> "'' // { outPath = " <> argPath <> "; }" adHocSystem <- readFileMaybe (projectDir "ci-default-system.txt") @@ -583,7 +582,7 @@ CNix.Store -> FilePath -> FilePath -> - Map Text Eval.Arg -> + Map Text ByteString -> [ EvaluateTask.NixPathElement (EvaluateTask.SubPathOf FilePath) ] -> diff --git a/hercules-ci-nix-daemon/daemon.cc b/hercules-ci-nix-daemon/daemon.cc --- a/hercules-ci-nix-daemon/daemon.cc +++ b/hercules-ci-nix-daemon/daemon.cc @@ -5,6 +5,17 @@ #include +#if NIX_IS_AT_LEAST(2,28,0) + +#include +#include +#include +#include +#include +#include +#include + +#else #include #include #include @@ -13,14 +24,18 @@ #include #include -using namespace nix; +# if NIX_IS_AT_LEAST(2,19,0) +# include +# endif -#if NIX_IS_AT_LEAST(2,19,0) -#include +# if NIX_IS_AT_LEAST(2,24,0) +# include +# endif #endif +using namespace nix; + #if NIX_IS_AT_LEAST(2,24,0) -#include using nix::unix::closeOnExec; #endif diff --git a/src/Hercules/Agent/WorkerProtocol/Command/Eval.hs b/src/Hercules/Agent/WorkerProtocol/Command/Eval.hs --- a/src/Hercules/Agent/WorkerProtocol/Command/Eval.hs +++ b/src/Hercules/Agent/WorkerProtocol/Command/Eval.hs @@ -13,7 +13,7 @@ data Eval = Eval { cwd :: FilePath, file :: Text, - autoArguments :: Map Text Arg, + autoArguments :: Map Text ByteString, -- TODO: Also set at worker start, so remove this here to avoid ambiguity? extraNixOptions :: [(Text, Text)], gitSource :: ViaJSON GitSource, @@ -25,9 +25,4 @@ allowInsecureBuiltinFetchers :: Bool, allowedPaths :: [ByteString] } - deriving (Generic, Binary, Show, Eq) - -data Arg - = LiteralArg ByteString - | ExprArg ByteString deriving (Generic, Binary, Show, Eq)