diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,54 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.11] - 2022-03-06 + +### BREAKING + + - The `nix-darwin` module uses new user id and group id numbers, to match the upstream `nix-darwin` module. + The main benefit of the upstream change is that the agent user will not appear as a normal (human) user on the system. + + To migrate, run: + + ```shell + # if you have deployed an older version of the nix-darwin PR before, + # the following delete commands will prevents this error: + # creating group _hercules-ci-agent... + #
attribute status: eDSRecordAlreadyExists + # DS Error: -14135 (eDSRecordAlreadyExists) + sudo dscl . -delete '/Groups/_hercules-ci-agent' + sudo dscl . -delete '/Users/_hercules-ci-agent' + + # update the flake inputs / expressions / channels + # and deploy the new version, e.g: + sudo darwin-rebuild switch + + sudo chown -R _hercules-ci-agent:_hercules-ci-agent /var/lib/hercules-ci-agent + ``` + +### Fixed + + - Non-builder build errors such as output cycles are now reported + in the build log. + + - `ciSystems` is now taken into account by the default `onPush.default` + job when the `herculesCI` attribute of a flake is a function. + + - `darwinConfigurations` is now filtered in accordance with `ciSystems`. + + +### Added + + - Nix verbosity can now be specified in the config file under the + new attribute `nixVerbosity`. + + - Cachix 1.3 support + + - Nix 2.14 support + + - Nix 2.13 support + + ## [0.9.10] - 2022-12-29 ### Fixed @@ -656,6 +704,7 @@ - Initial release +[0.9.11]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.10...hercules-ci-agent-0.9.11 [0.9.10]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.9...hercules-ci-agent-0.9.10 [0.9.9]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.8...hercules-ci-agent-0.9.9 [0.9.8]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.7...hercules-ci-agent-0.9.8 diff --git a/cbits/nix-2.4/hercules-store.cxx b/cbits/nix-2.4/hercules-store.cxx --- a/cbits/nix-2.4/hercules-store.cxx +++ b/cbits/nix-2.4/hercules-store.cxx @@ -239,9 +239,11 @@ return wrappedStore->getProtocol(); } +#if ! NIX_IS_AT_LEAST(2,14,0) void WrappingStore::createUser(const std::string & userName, uid_t userId) { wrappedStore->createUser(userName, userId); } +#endif ///// @@ -300,7 +302,20 @@ // TODO: don't ignore the Opaques for (auto & derivedPath : derivedPaths) { +#if NIX_IS_AT_LEAST(2,13,0) + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(derivedPath); + std::set outputs; std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + // paths.push_back(StorePathWithOutputs { built.drvPath, built.outputs }); + paths.push_back(s); + }, + [&](const StorePath & drvPath) { + // should this be substituted? + }, + }, sOrDrvPath); +#else + std::visit(overloaded { [&](DerivedPath::Built built) { paths.push_back(StorePathWithOutputs { built.drvPath, built.outputs }); }, @@ -308,6 +323,7 @@ // TODO: pass this to the callback as well }, }, derivedPath.raw()); +#endif } builderCallback(pathsPtr, &exceptionToThrow); diff --git a/cbits/nix-2.4/hercules-store.hh b/cbits/nix-2.4/hercules-store.hh --- a/cbits/nix-2.4/hercules-store.hh +++ b/cbits/nix-2.4/hercules-store.hh @@ -8,6 +8,10 @@ #include #include #include +#include +#if NIX_IS_AT_LEAST(2,13,0) +#include +#endif #include "HsFFI.h" using namespace nix; @@ -150,7 +154,9 @@ virtual Path toRealPath(const Path & storePath) override; +#if ! NIX_IS_AT_LEAST(2,14,0) virtual void createUser(const std::string & userName, uid_t userId) override; +#endif }; diff --git a/data/default-herculesCI-for-flake.nix b/data/default-herculesCI-for-flake.nix --- a/data/default-herculesCI-for-flake.nix +++ b/data/default-herculesCI-for-flake.nix @@ -23,19 +23,21 @@ (attrNames set) ); optionalAttrs = b: if b then a: a else _: { }; - optionalCall = f: a: if builtins.isFunction f then f a else f; # end lib + # Not quite the same as lib.toFunction, as it will not call a __functor. + optionalCall = f: a: if builtins.isFunction f then f a else f; + # flake -> evalArgs -> { flake | herculesCI } addDefaults = flake: evalArgs: let + herculesCI = optionalCall (flake.herculesCI or { }) evalArgs; args = evalArgs // { ciSystems = - if flake?herculesCI.ciSystems - then listToAttrs (map (sys: nameValuePair sys { }) flake.herculesCI.ciSystems) + if herculesCI?ciSystems + then listToAttrs (map (sys: nameValuePair sys { }) herculesCI.ciSystems) else args.herculesCI.ciSystems; }; - herculesCI = optionalCall (flake.herculesCI or { }) evalArgs; in herculesCI // optionalAttrs (!herculesCI?onPush) { onPush.default = { @@ -50,18 +52,24 @@ then attrs: attrs else intersectAttrs ciSystems; - isEnabledSystemConfig = ciSystems: + getSystemNixDarwin = sys: sys.config.nixpkgs.system; + + getSystemNixOS = sys: + if sys?options.nixpkgs.hostPlatform && sys.options.nixpkgs.hostPlatform.isDefined + then sys.config.nixpkgs.buildPlatform.system + else sys.config.nixpkgs.localSystem.system or sys.config.nixpkgs.system; + + isEnabledSystemConfig = getter: ciSystems: if ciSystems == null then sys: true else sys: - hasAttr sys.config.nixpkgs.localSystem.system ciSystems - || hasAttr sys.config.nixpkgs.system ciSystems; + hasAttr (getter sys) ciSystems; - filterSystemConfigs = ciSystems: filterAttrs (k: v: - if isEnabledSystemConfig ciSystems v + filterSystemConfigs = getter: ciSystems: filterAttrs (k: v: + if isEnabledSystemConfig getter ciSystems v then true - else builtins.trace "Ignoring flake attribute ${k} (system not in ciSystems)" false); + else builtins.trace "Ignoring flake attribute ${k} (system ${getter v} not in ciSystems)" false); translateFlakeAttr = args: k: v: if translations?${k} @@ -164,8 +172,8 @@ devShell = forSystems (sys: translateShell); devShells = forSystems (sys: mapAttrs (k: translateShell)); apps = forSystems (sys: mapAttrs (k: checkApp)); - nixosConfigurations = args: attrs: mapAttrs (k: sys: { config.system.build.toplevel = sys.config.system.build.toplevel; }) (filterSystemConfigs args.ciSystems attrs); - darwinConfigurations = args: attrs: mapAttrs (k: sys: { config.system.build.toplevel = sys.config.system.build.toplevel; }) (filterSystemConfigs args.ciSystems attrs); + nixosConfigurations = args: attrs: mapAttrs (k: sys: { config.system.build.toplevel = sys.config.system.build.toplevel; }) (filterSystemConfigs getSystemNixOS args.ciSystems attrs); + darwinConfigurations = args: attrs: mapAttrs (k: sys: { config.system.build.toplevel = sys.config.system.build.toplevel; }) (filterSystemConfigs getSystemNixDarwin args.ciSystems attrs); formatter = forSystems (sys: formatter: formatter); templates = args: attrs: mapAttrs checkTemplate attrs; diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -163,6 +163,7 @@ stopTime = 0; } catch (nix::Error &e) { + printError(e.msg()); status = -2; success = false; errorMessage = strdup(e.msg().c_str()); @@ -227,6 +228,7 @@ status = result.success() ? -1 : -2; break; } + printError(result.errorMsg); success = result.success(); errorMessage = strdup(result.errorMsg.c_str()); startTime = result.startTime; diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.9.10 +version: 0.9.11 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -66,8 +66,8 @@ custom-setup setup-depends: - base - , Cabal >= 2.2.0.0 + base < 5 + , Cabal >= 2.2.0.0 && < 4 , cabal-pkg-config-version-hook @@ -221,7 +221,7 @@ , hercules-ci-agent , hercules-ci-api , hercules-ci-api-core == 0.1.5.0 - , hercules-ci-api-agent == 0.4.6.1 + , hercules-ci-api-agent == 0.5.0.0 , hostname , http-client , http-client-tls diff --git a/hercules-ci-agent/Hercules/Agent.hs b/hercules-ci-agent/Hercules/Agent.hs --- a/hercules-ci-agent/Hercules/Agent.hs +++ b/hercules-ci-agent/Hercules/Agent.hs @@ -81,10 +81,10 @@ main :: IO () main = do - Init.initCNix opts <- Options.parse let cfgPath = Options.configFile opts cfg <- Config.finalizeConfig cfgPath =<< Config.readConfig cfgPath + Init.initCNix cfg Init.setupLogging cfg $ \logEnv -> do env <- Init.newEnv cfg logEnv case Options.mode opts of diff --git a/hercules-ci-agent/Hercules/Agent/Cachix.hs b/hercules-ci-agent/Hercules/Agent/Cachix.hs --- a/hercules-ci-agent/Hercules/Agent/Cachix.hs +++ b/hercules-ci-agent/Hercules/Agent/Cachix.hs @@ -47,7 +47,11 @@ withNamedContext "size" size $ withNamedContext "retry" (show retryStatus :: Text) $ logLocM DebugS "pushing", +#if MIN_VERSION_cachix(1,3,0) + on401 = \err -> throwIO $ FatalError $ "Cachix push is unauthorized: " <> show err, +#else on401 = throwIO $ FatalError "Cachix push is unauthorized", +#endif onError = \err -> throwIO $ FatalError $ "Error pushing to cachix: " <> show err, onDone = ctx $ logLocM DebugS "push done", #if MIN_VERSION_cachix(1,1,0) diff --git a/hercules-ci-agent/Hercules/Agent/Config.hs b/hercules-ci-agent/Hercules/Agent/Config.hs --- a/hercules-ci-agent/Hercules/Agent/Config.hs +++ b/hercules-ci-agent/Hercules/Agent/Config.hs @@ -19,6 +19,7 @@ import Data.Scientific (floatingOrInteger, fromFloatDigits) import qualified Data.Vector as V import GHC.Conc (getNumProcessors) +import Hercules.CNix.Verbosity (Verbosity (..)) import Katip (Severity (..)) import Protolude hiding (to) import qualified System.Environment @@ -56,6 +57,7 @@ binaryCachesPath :: Item purpose 'Required FilePath, secretsJsonPath :: Item purpose 'Required FilePath, logLevel :: Item purpose 'Required Severity, + nixVerbosity :: Item purpose 'Required Verbosity, labels :: Item purpose 'Required (Map Text A.Value), allowInsecureBuiltinFetchers :: Item purpose 'Required Bool } @@ -89,6 +91,8 @@ .= secretsJsonPath <*> dioptional (Toml.enumBounded "logLevel") .= logLevel + <*> dioptional (Toml.enumBounded "nixVerbosity") + .= nixVerbosity <*> dioptional (Toml.tableMap _KeyText embedJson "labels") .= labels <*> dioptional (Toml.bool "allowInsecureBuiltinFetchers") @@ -214,6 +218,7 @@ secretsJsonPath = secretsJsonP, workDirectory = workDir, logLevel = logLevel input & fromMaybe InfoS, + nixVerbosity = nixVerbosity input & fromMaybe Talkative, labels = fromMaybe mempty $ labels input, allowInsecureBuiltinFetchers = fromMaybe False $ allowInsecureBuiltinFetchers input } diff --git a/hercules-ci-agent/Hercules/Agent/Init.hs b/hercules-ci-agent/Hercules/Agent/Init.hs --- a/hercules-ci-agent/Hercules/Agent/Init.hs +++ b/hercules-ci-agent/Hercules/Agent/Init.hs @@ -13,6 +13,7 @@ import qualified Hercules.Agent.Token as Token import qualified Hercules.CNix import qualified Hercules.CNix.Util +import qualified Hercules.CNix.Verbosity import qualified Katip as K import qualified Network.HTTP.Client.TLS import Protolude @@ -65,8 +66,8 @@ emptyNamespace :: K.Namespace emptyNamespace = K.Namespace [] -initCNix :: IO () -initCNix = do +initCNix :: Config.FinalConfig -> IO () +initCNix cfg = do Hercules.CNix.init - Hercules.CNix.setTalkative + Hercules.CNix.Verbosity.setVerbosity $ Config.nixVerbosity cfg Hercules.CNix.Util.installDefaultSigINTHandler diff --git a/hercules-ci-agent/Hercules/Agent/Options.hs b/hercules-ci-agent/Hercules/Agent/Options.hs --- a/hercules-ci-agent/Hercules/Agent/Options.hs +++ b/hercules-ci-agent/Hercules/Agent/Options.hs @@ -8,7 +8,7 @@ import Hercules.Agent.CabalInfo (herculesAgentVersion) import Hercules.Agent.Config (ConfigPath (..)) import Options.Applicative -import Protolude hiding (option) +import Protolude data Options = Options { configFile :: ConfigPath, diff --git a/hercules-ci-nix-daemon/daemon.cc b/hercules-ci-nix-daemon/daemon.cc --- a/hercules-ci-nix-daemon/daemon.cc +++ b/hercules-ci-nix-daemon/daemon.cc @@ -90,14 +90,18 @@ ref store = openStore(); daemon::TrustedFlag trusted = daemon::NotTrusted; daemon::RecursiveFlag recursive = daemon::NotRecursive; +#if ! NIX_IS_AT_LEAST(2,14,0) std::function authHook = [](Store &){}; +#endif daemon::processConnection( store , from , to , trusted , recursive +#if ! NIX_IS_AT_LEAST(2,14,0) , authHook +#endif ); }); } catch (Interrupted & e) { diff --git a/src/Hercules/Agent/Producer.hs b/src/Hercules/Agent/Producer.hs --- a/src/Hercules/Agent/Producer.hs +++ b/src/Hercules/Agent/Producer.hs @@ -14,7 +14,6 @@ import Control.Monad.State import Data.Foldable import Data.Traversable -import Katip import UnliftIO.Exception import Prelude @@ -63,7 +62,7 @@ -- -- @withProducer (\write -> write "a" >> write "b") $ \producer -> consume producer@ withProducer :: - (MonadIO m, MonadUnliftIO m) => + (MonadUnliftIO m) => ((p -> m ()) -> m r) -> (Producer p r -> m a) -> m a @@ -122,7 +121,7 @@ -- - Alternatively, maybe use stm-delay (which uses GHC.Event for efficiency) -- https://hackage.haskell.org/package/stm-delay-0.1.1.1/docs/Control-Concurrent-STM-Delay.html withBoundedDelayBatchProducer :: - (MonadIO m, MonadUnliftIO m, KatipContext m) => + (MonadIO m, MonadUnliftIO m) => -- | Max time before flushing in microseconds Int -> -- | Max number of items in batch