diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.8] - 2026-01-06 + +### Added + +- Nix 2.29, 2.30, 2.31, 2.32, 2.33 support + ## [0.10.7] - 2025-07-18 ### Fixed @@ -804,6 +810,11 @@ - Initial release +[0.10.8]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.7...hercules-ci-agent-0.10.8 +[0.10.7]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.6...hercules-ci-agent-0.10.7 +[0.10.6]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.5...hercules-ci-agent-0.10.6 +[0.10.5]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.4...hercules-ci-agent-0.10.5 +[0.10.4]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.3...hercules-ci-agent-0.10.4 [0.10.3]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.2...hercules-ci-agent-0.10.3 [0.10.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.10.1...hercules-ci-agent-0.10.2 [0.10.1]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.12...hercules-ci-agent-0.10.1 diff --git a/cbits/hercules-error.hh b/cbits/hercules-error.hh --- a/cbits/hercules-error.hh +++ b/cbits/hercules-error.hh @@ -1,4 +1,7 @@ #include +#if NIX_IS_AT_LEAST(2, 32, 0) +#include +#endif #include namespace hercules { @@ -6,7 +9,12 @@ class HerculesBuildError : public nix::BuildError { public: nix::StorePath drv; +#if NIX_IS_AT_LEAST(2, 32, 0) + HerculesBuildError(const std::string msg, nix::StorePath drv) + : BuildError(nix::BuildResult::Failure::MiscFailure, msg), drv(drv) {}; +#else HerculesBuildError(const std::string msg, nix::StorePath drv) : BuildError(msg), drv(drv) {}; +#endif }; void copyErrorStrings(const nix::Error &err, const char **msg, const char **trace) noexcept; diff --git a/cbits/hercules-logger.hh b/cbits/hercules-logger.hh --- a/cbits/hercules-logger.hh +++ b/cbits/hercules-logger.hh @@ -3,11 +3,10 @@ #include #include #include -#include - #include #include +#include class HerculesLogger final : public nix::Logger { diff --git a/cbits/nix-2.4/hercules-store.cxx b/cbits/nix-2.4/hercules-store.cxx --- a/cbits/nix-2.4/hercules-store.cxx +++ b/cbits/nix-2.4/hercules-store.cxx @@ -10,20 +10,26 @@ #include #include #include -#include #include "hercules-store.hh" using namespace nix; -WrappingStore::WrappingStore(const Params& params, ref storeToWrap) - : Store(params), wrappedStore(storeToWrap) {} +#if NIX_IS_AT_LEAST(2, 29, 0) +WrappingStore::WrappingStore(ref storeToWrap) + : Store(storeToWrap->config), wrappedStore(storeToWrap) {} +#else +WrappingStore::WrappingStore(ref storeToWrap) + : Store({}), wrappedStore(storeToWrap) {} +#endif WrappingStore::~WrappingStore() {} +#if !NIX_IS_AT_LEAST(2, 31, 0) std::string WrappingStore::getUri() { return "wrapped:" + wrappedStore->getUri(); -}; +} +#endif bool WrappingStore::isValidPathUncached(const StorePath & path) { return wrappedStore->isValidPath(path); // caches again. Not much we can do. @@ -154,21 +160,29 @@ includeDerivers); } +#if NIX_IS_AT_LEAST(2, 30, 0) +MissingPaths WrappingStore::queryMissing(const std::vector & targets) { + return wrappedStore->queryMissing(targets); +} +#else void WrappingStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); } +#endif void WrappingStore::connect() { wrappedStore->connect(); }; +#if !NIX_IS_AT_LEAST(2, 33, 0) Path WrappingStore::toRealPath(const Path& storePath) { return wrappedStore->toRealPath(storePath); -}; +} +#endif; unsigned int WrappingStore::getProtocol() { return wrappedStore->getProtocol(); @@ -179,20 +193,44 @@ return wrappedStore->isTrustedClient(); } +#if NIX_IS_AT_LEAST(2, 32, 0) +void WrappingStore::registerDrvOutput(const Realisation & output) { + wrappedStore->registerDrvOutput(output); +} + +std::shared_ptr WrappingStore::getFSAccessor(const StorePath & path, bool requireValidPath) { + return wrappedStore->getFSAccessor(path, requireValidPath); +} +#endif + ///// -HerculesStore::HerculesStore(const Params& params, ref storeToWrap) - : StoreConfig(params) - , WrappingStore(params, storeToWrap) {} +#if NIX_IS_AT_LEAST(2, 29, 0) +HerculesStore::HerculesStore(ref storeToWrap) + : WrappingStore(storeToWrap) {} +#else +HerculesStore::HerculesStore(ref storeToWrap) + : StoreConfig(nix::StringMap {}) + , WrappingStore(storeToWrap) {} +#endif +#if !NIX_IS_AT_LEAST(2, 29, 0) const std::string HerculesStore::name() { return "wrapped " + wrappedStore->name(); } +#endif +#if NIX_IS_AT_LEAST(2, 33, 0) void HerculesStore::queryRealisationUncached(const DrvOutput &drvOutput, + Callback> callback) noexcept { + wrappedStore->queryRealisation(drvOutput, std::move(callback)); +} +#else +void HerculesStore::queryRealisationUncached(const DrvOutput &drvOutput, Callback> callback) noexcept { wrappedStore->queryRealisation(drvOutput, std::move(callback)); } +#endif void HerculesStore::ensurePath(const StorePath& path) { /* We avoid asking substituters for paths, since @@ -216,10 +254,17 @@ }; // Avoid substituting in evaluator, see `ensurePath` for more details +#if NIX_IS_AT_LEAST(2, 30, 0) +MissingPaths HerculesStore::queryMissing(const std::vector & targets) { + throw nix::Error("HerculesStore::queryMissing is not implemented"); + return MissingPaths{}; +} +#else void HerculesStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { -}; +} +#endif void HerculesStore::buildPaths(const std::vector & derivedPaths, BuildMode buildMode, std::shared_ptr evalStore) { std::exception_ptr exceptionToThrow(nullptr); diff --git a/cbits/nix-2.4/hercules-store.hh b/cbits/nix-2.4/hercules-store.hh --- a/cbits/nix-2.4/hercules-store.hh +++ b/cbits/nix-2.4/hercules-store.hh @@ -6,24 +6,26 @@ #include #include #include -#include #include "HsFFI.h" using FSAccessor = nix::SourceAccessor; + using namespace nix; class WrappingStore : public Store { public: ref wrappedStore; - WrappingStore(const Params & params, ref storeToWrap); + WrappingStore(ref storeToWrap); virtual ~WrappingStore(); +#if !NIX_IS_AT_LEAST(2, 31, 0) virtual std::string getUri() override; +#endif protected: @@ -98,23 +100,33 @@ virtual ref getFSAccessor(bool requireValidPath) override; +#if NIX_IS_AT_LEAST(2, 32, 0) + virtual void registerDrvOutput(const Realisation & output) override; + virtual std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override; +#endif + virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) override; virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override; +#if NIX_IS_AT_LEAST(2, 30, 0) + virtual MissingPaths queryMissing(const std::vector & targets) override; +#else virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; +#endif virtual unsigned int getProtocol() override; virtual void connect() override; +#if !NIX_IS_AT_LEAST(2, 33, 0) virtual Path toRealPath(const Path & storePath) override; - +#endif virtual std::optional isTrustedClient() override; @@ -125,14 +137,21 @@ StorePathSet ensuredPaths; void (* builderCallback)(std::vector*, std::exception_ptr *exceptionToThrow); - HerculesStore(const Params & params, ref storeToWrap); + HerculesStore(ref storeToWrap); +#if !NIX_IS_AT_LEAST(2, 29, 0) virtual const std::string name() override; +#endif // Overrides +#if NIX_IS_AT_LEAST(2, 33, 0) virtual void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override; +#else + virtual void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override; +#endif virtual void ensurePath(const StorePath & path) override; @@ -144,9 +163,13 @@ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override; +#if NIX_IS_AT_LEAST(2, 30, 0) + virtual MissingPaths queryMissing(const std::vector & targets) override; +#else virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; +#endif // Additions diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs @@ -35,7 +35,6 @@ C.include "" - C.include "" C.include "hercules-aliases.h" @@ -240,7 +239,8 @@ >>= \case 0 -> LogEntry.Int <$> peek uintPtr 1 -> - LogEntry.String . decode + LogEntry.String + . decode <$> (unsafePackMallocCString =<< peek stringPtr) _ -> panic "convertAndDeleteFields invalid internal type" diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -37,6 +37,10 @@ C.include "" C.include "" +#if NIX_IS_AT_LEAST(2, 29, 0) +C.include "" +#endif + C.include "" C.include "" @@ -90,6 +94,7 @@ } deriving (Show) +{- ORMOLU_DISABLE -} getDerivation :: Store -> StorePath -> IO (Maybe Derivation) getDerivation (Store store) derivationPath = nullableMoveToForeignPtrWrapper @@ -108,18 +113,32 @@ } catch (nix::Interrupted &e) { throw e; } catch (nix::Error &e) { +#if NIX_IS_AT_LEAST(2, 31, 0) + printTalkative("ignoring exception during drv lookup in %s: %s", currentStore->config.getHumanReadableURI().c_str(), e.what()); +#else printTalkative("ignoring exception during drv lookup in %s: %s", currentStore->getUri(), e.what()); +#endif } catch (std::exception &e) { +#if NIX_IS_AT_LEAST(2, 31, 0) + printTalkative("ignoring exception during drv lookup in %s: %s", currentStore->config.getHumanReadableURI().c_str(), e.what()); +#else printTalkative("ignoring exception during drv lookup in %s: %s", currentStore->getUri(), e.what()); +#endif } catch (...) { // FIXME: remove this and make the "specific" catches above work on darwin +#if NIX_IS_AT_LEAST(2, 31, 0) + printTalkative("ignoring unknown exception during drv lookup in %s: %s", currentStore->config.getHumanReadableURI().c_str()); +#else printTalkative("ignoring unknown exception during drv lookup in %s: %s", currentStore->getUri()); +#endif } } return derivation; }|] +{- ORMOLU_ENABLE -} -- | @buildDerivation derivationPath derivationText@ +{- ORMOLU_DISABLE -} buildDerivation :: Store -> StorePath -> Derivation -> Maybe [ByteString] -> IO BuildResult buildDerivation (Store store) derivationPath derivation extraInputs = let extraInputsMerged = C8.intercalate "\n" (fromMaybe [] extraInputs) @@ -175,6 +194,71 @@ nix::BuildResult result = store.buildDerivation(derivationPath, *derivation); +#if NIX_IS_AT_LEAST(2, 32, 0) + if (auto *success = result.tryGetSuccess()) { + switch (success->status) { + case nix::BuildResult::Success::Built: + status = 0; + break; + case nix::BuildResult::Success::Substituted: + status = 1; + break; + case nix::BuildResult::Success::AlreadyValid: + status = 2; + break; + case nix::BuildResult::Success::ResolvesToAlreadyValid: + status = 13; + break; + default: + status = -1; + break; + } + } else if (auto *failure = result.tryGetFailure()) { + switch (failure->status) { + case nix::BuildResult::Failure::PermanentFailure: + status = 3; + break; + case nix::BuildResult::Failure::InputRejected: + status = 4; + break; + case nix::BuildResult::Failure::OutputRejected: + status = 5; + break; + case nix::BuildResult::Failure::TransientFailure: + status = 6; + break; + case nix::BuildResult::Failure::CachedFailure: + status = 7; + break; + case nix::BuildResult::Failure::TimedOut: + status = 8; + break; + case nix::BuildResult::Failure::MiscFailure: + status = 9; + break; + case nix::BuildResult::Failure::DependencyFailed: + status = 10; + break; + case nix::BuildResult::Failure::LogLimitExceeded: + status = 11; + break; + case nix::BuildResult::Failure::NotDeterministic: + status = 12; + break; + case nix::BuildResult::Failure::NoSubstituters: + status = 14; + break; + case nix::BuildResult::Failure::HashMismatch: + status = 15; + break; + default: + status = -2; + break; + } + } else { + status = -3; + } +#else switch (result.status) { case nix::BuildResult::Built: status = 0; @@ -219,9 +303,25 @@ status = result.success() ? -1 : -2; break; } +#endif +#if NIX_IS_AT_LEAST(2, 32, 0) + auto *successPtr = result.tryGetSuccess(); + auto *failurePtr = result.tryGetFailure(); + assert(!(successPtr && failurePtr)); // Both can't be true + assert(successPtr || failurePtr); // One must be true + + success = successPtr != nullptr; + if (failurePtr) { + printError(failurePtr->errorMsg); + errorMessage = stringdup(failurePtr->errorMsg); + } else { + errorMessage = stringdup(""); + } +#else printError(result.errorMsg); success = result.success(); errorMessage = stringdup(result.errorMsg); +#endif startTime = result.startTime; stopTime = result.stopTime; } @@ -241,3 +341,4 @@ stopTime = stopTimeValue, errorMessage = toS errorMessageValue } +{- ORMOLU_ENABLE -} diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs @@ -36,8 +36,6 @@ C.include "" -C.include "" - C.include "hercules-aliases.h" C.using "namespace nix" @@ -51,7 +49,7 @@ ( liftIO [C.block| refHerculesStore* { refStore &s = *$(refStore *wrappedStore); - refHerculesStore hs(new HerculesStore({}, s)); + refHerculesStore hs(new HerculesStore(s)); return new refHerculesStore(hs); } |] ) diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.10.7 +version: 0.10.8 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -17,14 +17,21 @@ cbits/hercules-error.hh cbits/hercules-logger.hh cbits/nix-2.4/hercules-store.hh + cbits/nix-2.4/hercules-store.cxx data/default-herculesCI-for-flake.nix testdata/vm-test-run-agent-test.drv + hercules-ci-nix-daemon/daemon.cxx flag ide description: Whether to enable IDE workarounds. You shouldn't need this. default: False +flag nix-2_31 + description: nix-store >= 2.31 + default: False + manual: False + source-repository head type: git location: https://github.com/hercules-ci/hercules-ci-agent @@ -56,8 +63,12 @@ -- match the C++ language standard Nix is using common cxx-opts - cxx-options: - -std=c++2a + if flag(nix-2_31) + cxx-options: + -std=c++23 + else + cxx-options: + -std=c++2a cxx-options: -Wall @@ -70,12 +81,22 @@ if impl(ghc >= 8.10) ghc-options: -optcxx-Wall - -optcxx-std=c++2a + if flag(nix-2_31) + ghc-options: + -optcxx-std=c++23 + else + ghc-options: + -optcxx-std=c++2a else -- Remove soon ghc-options: - -optc-std=c++2a -optc-Wall + if flag(nix-2_31) + ghc-options: + -optc-std=c++23 + else + ghc-options: + -optc-std=c++2a if os(darwin) ghc-options: -pgmc=clang++ @@ -377,10 +398,11 @@ , uuid , vector pkgconfig-depends: - nix-store >= 2.28 - , nix-expr >= 2.28 + nix-store >= 2.28 && < 2.34 + , nix-expr >= 2.28 && < 2.34 + , nix-flake >= 2.28 && < 2.34 -- TODO: replace stack overflow detection and remove nix-main - , nix-main >= 2.28 + , nix-main >= 2.28 && < 2.34 default-language: Haskell2010 test-suite hercules-ci-agent-unit-tests @@ -468,10 +490,8 @@ executable hercules-ci-nix-daemon import: cxx-opts - main-is: main.cc - cxx-sources: hercules-ci-nix-daemon/daemon.cc + main-is: hercules-ci-nix-daemon/daemon.cxx default-language: Haskell2010 - hs-source-dirs: hercules-ci-nix-daemon pkgconfig-depends: - nix-store >= 2.28 - , nix-main >= 2.28 + nix-store >= 2.28 && < 2.34 + , nix-main >= 2.28 && < 2.34 diff --git a/hercules-ci-agent/Hercules/Agent/Build.hs b/hercules-ci-agent/Hercules/Agent/Build.hs --- a/hercules-ci-agent/Hercules/Agent/Build.hs +++ b/hercules-ci-agent/Hercules/Agent/Build.hs @@ -9,15 +9,15 @@ import Data.Vector (Vector) import Hercules.API.Agent.Build qualified as API.Build import Hercules.API.Agent.Build.BuildEvent qualified as BuildEvent -import Hercules.API.Agent.OutputInfo - ( OutputInfo, - ) -import Hercules.API.Agent.OutputInfo qualified as OutputInfo import Hercules.API.Agent.Build.BuildEvent.Pushed qualified as Pushed import Hercules.API.Agent.Build.BuildTask ( BuildTask, ) import Hercules.API.Agent.Build.BuildTask qualified as BuildTask +import Hercules.API.Agent.OutputInfo + ( OutputInfo, + ) +import Hercules.API.Agent.OutputInfo qualified as OutputInfo import Hercules.API.Logs.LogEntry (LogEntry) import Hercules.API.Servant (noContent) import Hercules.API.TaskStatus (TaskStatus) @@ -38,12 +38,12 @@ import Hercules.Agent.WorkerProtocol.Command.Build qualified as Command.Build import Hercules.Agent.WorkerProtocol.Event qualified as Event import Hercules.Agent.WorkerProtocol.Event.BuildResult qualified as BuildResult +import Hercules.Agent.WorkerProtocol.OutputInfo qualified as Proto import Hercules.Agent.WorkerProtocol.ViaJSON (ViaJSON (ViaJSON)) import Hercules.CNix.Store qualified as CNix import Hercules.Error (defaultRetry) import Protolude import System.Process -import qualified Hercules.Agent.WorkerProtocol.OutputInfo as Proto performBuild :: (Vector LogEntry -> IO ()) -> BuildTask.BuildTask -> App TaskStatus performBuild sendLogEntries buildTask = katipAddContext (sl "taskDerivationPath" buildTask.derivationPath) $ do @@ -106,11 +106,7 @@ Just BuildResult.BuildSuccess {outputs = outs'} -> do let outs = convertOutputs (BuildTask.derivationPath buildTask) outs' reportOutputInfos buildTask outs -#if MIN_VERSION_cachix(1, 4, 0) && ! MIN_VERSION_cachix(1, 5, 0) - CNix.withStore $ \store -> push store buildTask outs -#else - asks (Cachix.Env.store . Env.cachixEnv) >>= \store -> push store buildTask outs -#endif + pushToStore buildTask outs reportSuccess buildTask pure $ TaskStatus.Successful () Just BuildResult.BuildFailure {errorMessage = errorMessage} -> @@ -119,6 +115,15 @@ pure $ TaskStatus.Terminated () Nothing -> pure $ TaskStatus.Exceptional "Build did not complete" +-- | Push outputs to the store using the appropriate method based on cachix version +pushToStore :: BuildTask -> Map Text OutputInfo -> App () +pushToStore buildTask outs = do +#if MIN_VERSION_cachix(1, 4, 0) && ! MIN_VERSION_cachix(1, 5, 0) + CNix.withStore $ \store -> push store buildTask outs +#else + asks (Cachix.Env.store . Env.cachixEnv) >>= \store -> push store buildTask outs +#endif + convertOutputs :: Text -> [Proto.OutputInfo] -> Map Text OutputInfo convertOutputs deriver = foldMap $ \oi -> M.singleton (decodeUtf8With lenientDecode oi.name) $ @@ -127,13 +132,13 @@ convertOutputInfo :: Text -> Proto.OutputInfo -> OutputInfo convertOutputInfo deriver oi = OutputInfo.OutputInfo - { OutputInfo.deriver = deriver, - name = decodeUtf8With lenientDecode oi.name, - path = decodeUtf8With lenientDecode oi.path, - size = fromIntegral oi.size, - hash = decodeUtf8With lenientDecode oi.hash, - references = Just (decodeUtf8With lenientDecode <$> oi.references) - } + { OutputInfo.deriver = deriver, + name = decodeUtf8With lenientDecode oi.name, + path = decodeUtf8With lenientDecode oi.path, + size = fromIntegral oi.size, + hash = decodeUtf8With lenientDecode oi.hash, + references = Just (decodeUtf8With lenientDecode <$> oi.references) + } push :: CNix.Store -> BuildTask -> Map Text OutputInfo -> App () push store buildTask outs = do diff --git a/hercules-ci-agent/Hercules/Agent/Cache.hs b/hercules-ci-agent/Hercules/Agent/Cache.hs --- a/hercules-ci-agent/Hercules/Agent/Cache.hs +++ b/hercules-ci-agent/Hercules/Agent/Cache.hs @@ -89,4 +89,4 @@ ( \path -> do CNix.signPath store key path ) - <&> foldMap (\case True -> (1, 1); False -> (1, 0)) + <&> foldMap (\case True -> (1, 1); False -> (1, 0)) diff --git a/hercules-ci-agent/Hercules/Agent/Cachix.hs b/hercules-ci-agent/Hercules/Agent/Cachix.hs --- a/hercules-ci-agent/Hercules/Agent/Cachix.hs +++ b/hercules-ci-agent/Hercules/Agent/Cachix.hs @@ -6,25 +6,25 @@ ) where -import qualified Cachix.Client.Push as Cachix.Push -import Cachix.Types.BinaryCache (CompressionMethod(XZ)) +import Cachix.Client.Push qualified as Cachix.Push +import Cachix.Types.BinaryCache (CompressionMethod (XZ)) #if MIN_VERSION_cachix(1,4,0) && ! MIN_VERSION_cachix(1,5,0) import qualified Cachix.Client.Store import qualified Cachix.Client.Store as Cachix #endif -import qualified Data.Conduit as Conduit import Control.Monad.IO.Unlift -import qualified Data.Map as M -import qualified Hercules.Agent.Cachix.Env as Agent.Cachix +import Data.Conduit qualified as Conduit +import Data.Map qualified as M +import Hercules.Agent.Cachix.Env qualified as Agent.Cachix import Hercules.Agent.Cachix.Info (activePushCaches) import Hercules.Agent.Env as Agent.Env hiding (activePushCaches) -import qualified Hercules.Agent.EnvironmentInfo as EnvInfo +import Hercules.Agent.EnvironmentInfo qualified as EnvInfo import Hercules.Agent.Log +import Hercules.CNix qualified as CNix import Hercules.CNix.Store (StorePath) import Hercules.Error -import qualified Hercules.Formats.CachixCache as CachixCache +import Hercules.Formats.CachixCache qualified as CachixCache import Protolude -import qualified Hercules.CNix as CNix #if MIN_VERSION_cachix(1,7,2) import qualified Cachix.Client.OptionsParser as Opts #endif @@ -42,62 +42,127 @@ maybeToEither (FatalError $ "Cache not found " <> cache) $ M.lookup cache pushCaches ul <- askUnliftIO - let pushParams = - Cachix.Push.PushParams - { pushParamsName = Agent.Cachix.pushCacheName pushCache, - pushParamsSecret = Agent.Cachix.pushCacheSecret pushCache, - pushParamsStore = cachixStore, + let pushParams = makePushParams pushCache cachixStore clientEnv + paths' <- convertPaths nixStore paths + void $ + Cachix.Push.pushClosure + (\f l -> liftIO $ Cachix.Push.mapConcurrentlyBounded workers (fmap (unliftIO ul) f) l) + pushParams + paths' + where #if MIN_VERSION_cachix(1,6,0) - pushOnClosureAttempt = \_ missing -> return missing, -#endif - pushParamsClientEnv = clientEnv, - pushParamsStrategy = \storePath -> -#if MIN_VERSION_cachix(1,4,0) && ! MIN_VERSION_cachix(1,5,0) - let ctx = withNamedContext "path" (Cachix.getPath storePath) + makePushParams pushCache cachixStore clientEnv = + Cachix.Push.PushParams + { pushParamsName = Agent.Cachix.pushCacheName pushCache, + pushParamsSecret = Agent.Cachix.pushCacheSecret pushCache, + pushParamsStore = cachixStore, + pushOnClosureAttempt = \_ missing -> return missing, + pushParamsClientEnv = clientEnv, + pushParamsStrategy = makePushStrategy + } #else - let ctx = withNamedContext "path" (show storePath :: Text) + makePushParams pushCache cachixStore clientEnv = + Cachix.Push.PushParams + { pushParamsName = Agent.Cachix.pushCacheName pushCache, + pushParamsSecret = Agent.Cachix.pushCacheSecret pushCache, + pushParamsStore = cachixStore, + pushParamsClientEnv = clientEnv, + pushParamsStrategy = makePushStrategy + } #endif - in Cachix.Push.PushStrategy - { onAlreadyPresent = pass, - onAttempt = \retryStatus size -> - ctx $ - withNamedContext "size" size $ - withNamedContext "retry" (show retryStatus :: Text) $ - logLocM DebugS "pushing", -#if MIN_VERSION_cachix(1,3,0) - on401 = \err -> throwIO $ FatalError $ "Cachix push is unauthorized: " <> show err, + +#if MIN_VERSION_cachix(1,7,2) + makePushStrategy storePath = + let ctx = makeContextForPath storePath + in Cachix.Push.PushStrategy + { onAlreadyPresent = pass, + onAttempt = \retryStatus size -> + ctx $ + withNamedContext "size" size $ + withNamedContext "retry" (show retryStatus :: Text) $ + logLocM DebugS "pushing", + on401 = makeOn401Handler, + onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, + onDone = ctx $ logLocM DebugS "push done", + compressionMethod = XZ, + compressionLevel = 2, + onUncompressedNARStream = \_ _ -> Conduit.awaitForever Conduit.yield, + chunkSize = Opts.defaultChunkSize, + numConcurrentChunks = Opts.defaultNumConcurrentChunks, + omitDeriver = False + } +#elif MIN_VERSION_cachix(1,6,0) + makePushStrategy storePath = + let ctx = makeContextForPath storePath + in Cachix.Push.PushStrategy + { onAlreadyPresent = pass, + onAttempt = \retryStatus size -> + ctx $ + withNamedContext "size" size $ + withNamedContext "retry" (show retryStatus :: Text) $ + logLocM DebugS "pushing", + on401 = makeOn401Handler, + onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, + onDone = ctx $ logLocM DebugS "push done", + compressionMethod = XZ, + compressionLevel = 2, + onUncompressedNARStream = \_ _ -> Conduit.awaitForever Conduit.yield, + omitDeriver = False + } +#elif MIN_VERSION_cachix(1,1,0) + makePushStrategy storePath = + let ctx = makeContextForPath storePath + in Cachix.Push.PushStrategy + { onAlreadyPresent = pass, + onAttempt = \retryStatus size -> + ctx $ + withNamedContext "size" size $ + withNamedContext "retry" (show retryStatus :: Text) $ + logLocM DebugS "pushing", + on401 = makeOn401Handler, + onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, + onDone = ctx $ logLocM DebugS "push done", + compressionMethod = XZ, + compressionLevel = 2, + omitDeriver = False + } #else - on401 = throwIO $ FatalError "Cachix push is unauthorized", + makePushStrategy storePath = + let ctx = makeContextForPath storePath + in Cachix.Push.PushStrategy + { onAlreadyPresent = pass, + onAttempt = \retryStatus size -> + ctx $ + withNamedContext "size" size $ + withNamedContext "retry" (show retryStatus :: Text) $ + logLocM DebugS "pushing", + on401 = makeOn401Handler, + onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, + onDone = ctx $ logLocM DebugS "push done", + withXzipCompressor = Cachix.Push.defaultWithXzipCompressor, + omitDeriver = False + } #endif - onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, - onDone = ctx $ logLocM DebugS "push done", -#if MIN_VERSION_cachix(1,1,0) - compressionMethod = XZ, - compressionLevel = 2, + +#if MIN_VERSION_cachix(1,4,0) && ! MIN_VERSION_cachix(1,5,0) + makeContextForPath storePath = withNamedContext "path" (Cachix.getPath storePath) #else - withXzipCompressor = Cachix.Push.defaultWithXzipCompressor, -#endif -#if MIN_VERSION_cachix(1,6,0) - onUncompressedNARStream = \_ _ -> Conduit.awaitForever Conduit.yield, + makeContextForPath storePath = withNamedContext "path" (show storePath :: Text) #endif -#if MIN_VERSION_cachix(1,7,2) - chunkSize = Opts.defaultChunkSize, - numConcurrentChunks = Opts.defaultNumConcurrentChunks, + +#if MIN_VERSION_cachix(1,3,0) + makeOn401Handler = \err -> throwIO $ FatalError $ "Cachix push is unauthorized: " <> show err +#else + makeOn401Handler = throwIO $ FatalError "Cachix push is unauthorized" #endif - omitDeriver = False - } - } + #if MIN_VERSION_cachix(1,4,0) && ! MIN_VERSION_cachix(1,5,0) - paths' <- paths & traverse (convertPath nixStore) + convertPaths nixStore paths = paths & traverse (convertPath nixStore) #else - let paths' = paths - _ = nixStore -- silence unused warning + convertPaths nixStore paths = do + let _ = nixStore -- silence unused warning + pure paths #endif - void $ - Cachix.Push.pushClosure - (\f l -> liftIO $ Cachix.Push.mapConcurrentlyBounded workers (fmap (unliftIO ul) f) l) - pushParams - paths' #if MIN_VERSION_cachix(1,4,0) && ! MIN_VERSION_cachix(1,5,0) diff --git a/hercules-ci-agent/Hercules/Agent/Compat.hs b/hercules-ci-agent/Hercules/Agent/Compat.hs --- a/hercules-ci-agent/Hercules/Agent/Compat.hs +++ b/hercules-ci-agent/Hercules/Agent/Compat.hs @@ -2,7 +2,7 @@ module Hercules.Agent.Compat (katipLevel) where -import qualified Katip as K +import Katip qualified as K import Protolude katipLevel :: K.Severity -> K.PermitFunc diff --git a/hercules-ci-agent/Hercules/Agent/Config.hs b/hercules-ci-agent/Hercules/Agent/Config.hs --- a/hercules-ci-agent/Hercules/Agent/Config.hs +++ b/hercules-ci-agent/Hercules/Agent/Config.hs @@ -83,7 +83,8 @@ combiCodec :: Combi' (Config 'Input) combiCodec = Config - <$> opt (textAtKey "apiBaseUrl") .=. herculesApiBaseURL + <$> opt (textAtKey "apiBaseUrl") + .=. herculesApiBaseURL <*> opt (boolAtKey "nixUserIsTrusted") .=. nixUserIsTrusted <*> opt ( combi @@ -129,7 +130,8 @@ mountableCodec :: Combi' Mountable mountableCodec = Mountable - <$> textAtKey "source" .=. Mountable.source + <$> textAtKey "source" + .=. Mountable.source <*> boolAtKey "readOnly" .=. Mountable.readOnly <*> combi ((throwImmediately . A.parseJSON) <$> Toml.embedJson "condition") diff --git a/hercules-ci-nix-daemon/daemon.cc b/hercules-ci-nix-daemon/daemon.cc deleted file mode 100644 --- a/hercules-ci-nix-daemon/daemon.cc +++ /dev/null @@ -1,113 +0,0 @@ -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -using namespace nix; - -using nix::unix::closeOnExec; - - -static void sigChldHandler(int sigNo) -{ - // Ensure we don't modify errno of whatever we've interrupted - auto saved_errno = errno; - // Reap all dead children. - while (waitpid(-1, 0, WNOHANG) > 0) ; - errno = saved_errno; -} - -static void setSigChldAction() -{ - struct sigaction act, oact; - act.sa_handler = sigChldHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, &oact)) - throw SysError("setting SIGCHLD handler"); -} - -extern "C" int main(int argc, char **argv) { - nix::initNix(); - - for (int i = 1; i < argc && argv[i]; i++) { - std::string arg(argv[i]); - if (arg == "--option") { - if (i + 2 < argc) { - nix::globalConfig.set(argv[i+1], argv[i+2]); - i += 2; - } - else { - std::cerr << "Not enough arguments to --option" << std::endl; - return 1; - } - } - } - - // See withNixDaemonProxy for stdin (ab)use - AutoCloseFD socketFD = dup(0); - closeOnExec(socketFD.get()); - - // Put /dev/null on stdin - int devnull = open("/dev/null", O_RDONLY); - dup2(devnull, 0); - close(devnull); - - setSigChldAction(); - while (1) { - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - try { - AutoCloseFD remote = accept(socketFD.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); - if (!remote) { - if (errno == EINTR) continue; - else throw SysError("hercules-ci-nix-daemon: accepting connection"); - } - closeOnExec(remote.get()); - ProcessOptions options; - options.errorPrefix = "hercules-ci-nix-daemon: unexpected error: "; - options.dieWithParent = false; - options.runExitHandlers = true; - options.allowVfork = false; - startProcess([&]() { - socketFD = -1; - - // Regular nix daemon creates a new session for connections and - // does not kill connection child processes. Those are not - // appropriate behaviors for a temporary proxy. - - FdSource from(remote.get()); - FdSink to(remote.get()); - // TODO: disable caching without interfering with user parameters - ref store = openStore(); - TrustedFlag trusted = NotTrusted; - daemon::RecursiveFlag recursive = daemon::NotRecursive; - daemon::processConnection( - store - , std::move(from) - , std::move(to) - , trusted - , recursive - ); - }); - } catch (Interrupted & e) { - return 0; - } catch (Error & error) { - ErrorInfo ei = error.info(); - ei.msg = HintFmt("hercules-ci-nix-daemon: error processing connection: %1%", ei.msg.str()); - logError(ei); - } - } - return 0; -} diff --git a/hercules-ci-nix-daemon/daemon.cxx b/hercules-ci-nix-daemon/daemon.cxx new file mode 100644 --- /dev/null +++ b/hercules-ci-nix-daemon/daemon.cxx @@ -0,0 +1,117 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if NIX_IS_AT_LEAST(2, 29, 0) +#include +#endif + +using namespace nix; + +using nix::unix::closeOnExec; + + +static void sigChldHandler(int sigNo) +{ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + // Reap all dead children. + while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; +} + +static void setSigChldAction() +{ + struct sigaction act, oact; + act.sa_handler = sigChldHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + +extern "C" int main(int argc, char **argv) { + nix::initNix(); + + for (int i = 1; i < argc && argv[i]; i++) { + std::string arg(argv[i]); + if (arg == "--option") { + if (i + 2 < argc) { + nix::globalConfig.set(argv[i+1], argv[i+2]); + i += 2; + } + else { + std::cerr << "Not enough arguments to --option" << std::endl; + return 1; + } + } + } + + // See withNixDaemonProxy for stdin (ab)use + AutoCloseFD socketFD = dup(0); + closeOnExec(socketFD.get()); + + // Put /dev/null on stdin + int devnull = open("/dev/null", O_RDONLY); + dup2(devnull, 0); + close(devnull); + + setSigChldAction(); + while (1) { + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + try { + AutoCloseFD remote = accept(socketFD.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + else throw SysError("hercules-ci-nix-daemon: accepting connection"); + } + closeOnExec(remote.get()); + ProcessOptions options; + options.errorPrefix = "hercules-ci-nix-daemon: unexpected error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess([&]() { + socketFD = -1; + + // Regular nix daemon creates a new session for connections and + // does not kill connection child processes. Those are not + // appropriate behaviors for a temporary proxy. + + FdSource from(remote.get()); + FdSink to(remote.get()); + // TODO: disable caching without interfering with user parameters + ref store = openStore(); + TrustedFlag trusted = NotTrusted; + daemon::RecursiveFlag recursive = daemon::NotRecursive; + daemon::processConnection( + store + , std::move(from) + , std::move(to) + , trusted + , recursive + ); + }); + } catch (Interrupted & e) { + return 0; + } catch (Error & error) { + ErrorInfo ei = error.info(); + ei.msg = HintFmt("hercules-ci-nix-daemon: error processing connection: %1%", ei.msg.str()); + logError(ei); + } + } + return 0; +} diff --git a/hercules-ci-nix-daemon/main.cc b/hercules-ci-nix-daemon/main.cc deleted file mode 100644 --- a/hercules-ci-nix-daemon/main.cc +++ /dev/null @@ -1,4 +0,0 @@ -/* - main() is in daemon.cc, because cabal doesn't pass - the correct flags to the main-is file. -*/ diff --git a/src/Hercules/Agent/NixFile.hs b/src/Hercules/Agent/NixFile.hs --- a/src/Hercules/Agent/NixFile.hs +++ b/src/Hercules/Agent/NixFile.hs @@ -29,6 +29,7 @@ -- * @onPush@ getVirtualValueByPath, parseExtraInputs, + resolveAndInvokeOutputs, ) where diff --git a/src/Hercules/Effect.hs b/src/Hercules/Effect.hs --- a/src/Hercules/Effect.hs +++ b/src/Hercules/Effect.hs @@ -283,13 +283,13 @@ BindMount {pathInContainer = "/nix/var/nix/daemon-socket/socket", pathInHost = toS forwardedSocketPath, readOnly = True} ] ++ [ BindMount {pathInContainer = "/etc", pathInHost = etcDir, readOnly = False} - | not (isExtraBind "/etc") + | not (isExtraBind "/etc") ] ++ [ - -- TODO: does this apply to crun? - -- we cannot bind mount this read-only because of https://github.com/opencontainers/runc/issues/1523 - BindMount {pathInContainer = "/etc/resolv.conf", pathInHost = "/etc/resolv.conf", readOnly = False} - | not (isExtraBind "/etc") && not (isExtraBind "/etc/resolv.conf") + -- TODO: does this apply to crun? + -- we cannot bind mount this read-only because of https://github.com/opencontainers/runc/issues/1523 + BindMount {pathInContainer = "/etc/resolv.conf", pathInHost = "/etc/resolv.conf", readOnly = False} + | not (isExtraBind "/etc") && not (isExtraBind "/etc/resolv.conf") ] ++ extraBindMounts_, executable = decodeUtf8With lenientDecode drvBuilder, diff --git a/src/Hercules/Effect/Container.hs b/src/Hercules/Effect/Container.hs --- a/src/Hercules/Effect/Container.hs +++ b/src/Hercules/Effect/Container.hs @@ -8,6 +8,7 @@ import Data.Aeson (Value (String), eitherDecode, encode, object, toJSON) import Data.Aeson.Lens import Data.ByteString qualified as BS +import Data.ByteString.Char8 qualified as BS.Char8 import Data.ByteString.Lazy qualified as BL import Data.Map qualified as M import Data.UUID.V4 qualified as UUID @@ -148,7 +149,7 @@ ) ( do let shovel = - handleEOF (BS.hGetLine master) >>= \case + handleEOF (BS.Char8.hGetLine master) >>= \case "" -> pass someBytes | "@nix" `BS.isPrefixOf` someBytes -> do -- TODO use it (example @nix { "action": "setPhase", "phase": "effectPhase" })