diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.3] + +### Added + + - `hci` can now run in the effects sandbox + +### Fixed + + - Interrupt handling has been improved + ## [0.8.2] ### Added @@ -422,6 +432,8 @@ - Initial release +[0.8.3]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.2...hercules-ci-agent-0.8.3 +[0.8.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.1...hercules-ci-agent-0.8.2 [0.8.1]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.0...hercules-ci-agent-0.8.1 [0.8.0]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.5...hercules-ci-agent-0.8.0 [0.7.5]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.4...hercules-ci-agent-0.7.5 diff --git a/cbits/nix-2.4/hercules-store.cxx b/cbits/nix-2.4/hercules-store.cxx --- a/cbits/nix-2.4/hercules-store.cxx +++ b/cbits/nix-2.4/hercules-store.cxx @@ -101,7 +101,8 @@ } void WrappingStore::buildPaths( - const std::vector & paths, BuildMode buildMode) { + const std::vector & paths, BuildMode buildMode, + std::shared_ptr evalStore) { wrappedStore->buildPaths(paths, buildMode); } @@ -231,7 +232,7 @@ uint64_t & downloadSize, uint64_t & narSize) { }; -void HerculesStore::buildPaths(const std::vector & derivedPaths, BuildMode buildMode) { +void HerculesStore::buildPaths(const std::vector & derivedPaths, BuildMode buildMode, std::shared_ptr evalStore) { std::exception_ptr exceptionToThrow(nullptr); // responsibility for delete is transferred to builderCallback diff --git a/cbits/nix-2.4/hercules-store.hh b/cbits/nix-2.4/hercules-store.hh --- a/cbits/nix-2.4/hercules-store.hh +++ b/cbits/nix-2.4/hercules-store.hh @@ -72,7 +72,8 @@ virtual void buildPaths( const std::vector & paths, - BuildMode buildMode = bmNormal) override; + BuildMode buildMode = bmNormal, + std::shared_ptr evalStore = nullptr) override; virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override; @@ -134,7 +135,8 @@ virtual void buildPaths( const std::vector & paths, - BuildMode buildMode = bmNormal) override; + BuildMode buildMode = bmNormal, + std::shared_ptr evalStore = nullptr) override; virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override; diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs @@ -66,7 +66,7 @@ import Hercules.CNix.Std.Vector (StdVector) import qualified Hercules.CNix.Std.Vector as Std.Vector import Hercules.CNix.Store.Context (NixStorePathWithOutputs) -import Hercules.CNix.Util (triggerInterrupt) +import Hercules.CNix.Util (installDefaultSigINTHandler) import Hercules.Error import Katip import qualified Language.C.Inline.Cpp.Exceptions as C @@ -74,7 +74,6 @@ import Protolude hiding (bracket, catch, evalState, wait, withAsync, yield) import qualified System.Environment as Environment import System.IO (BufferMode (LineBuffering), hSetBuffering) -import System.Mem.Weak (deRefWeak) import System.Posix.IO (dup, fdToHandle, stdError) import System.Posix.Signals (Handler (Catch), installHandler, raiseSignal, sigINT, sigTERM) import System.Timeout (timeout) @@ -99,30 +98,12 @@ instance Exception BuildException -extendSigINT :: IO () -extendSigINT = do - mainThread <- myThreadId - weakId <- mkWeakThreadId mainThread - _oldHandler <- - installHandler - sigINT - ( Catch do - -- GHC RTS default behavior - mt <- deRefWeak weakId - for_ mt \t -> do - throwTo t (toException UserInterrupt) - -- Nix - triggerInterrupt - ) - Nothing - pass - main :: IO () main = do hSetBuffering stderr LineBuffering Hercules.CNix.Expr.init _ <- installHandler sigTERM (Catch $ raiseSignal sigINT) Nothing - extendSigINT + installDefaultSigINTHandler Logger.initLogger [options] <- Environment.getArgs let allOptions = diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -97,6 +97,7 @@ getDerivation (Store store) derivationPath = nullableMoveToForeignPtrWrapper =<< [C.throwBlock| Derivation *{ + ReceiveInterrupts _; StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath); std::list> stores = getDefaultSubstituters(); stores.push_front(*$(refStore* store)); @@ -132,6 +133,7 @@ alloca $ \stopTimePtr -> alloca $ \errorMessagePtr -> do [C.throwBlock| void { + ReceiveInterrupts _; Store &store = **$(refStore* store); bool &success = *$(bool *successPtr); int &status = *$(int *statusPtr); diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs @@ -10,6 +10,7 @@ import Hercules.CNix (Store) import qualified Hercules.CNix as CNix import Hercules.CNix.Store (Derivation) +import Hercules.Effect (RunEffectParams (..)) import qualified Hercules.Effect as Effect import Katip (KatipContext) import Protolude @@ -19,7 +20,16 @@ runEffect store command = do derivation <- prepareDerivation store command dir <- getCurrentDirectory - Effect.runEffect derivation (Command.Effect.token command) (Command.Effect.secretsPath command) (Command.Effect.apiBaseURL command) dir + Effect.runEffect + RunEffectParams + { runEffectDerivation = derivation, + runEffectToken = Just $ Command.Effect.token command, + runEffectSecretsConfigPath = Just $ Command.Effect.secretsPath command, + runEffectApiBaseURL = Command.Effect.apiBaseURL command, + runEffectDir = dir, + runEffectProjectId = Just $ Command.Effect.projectId command, + runEffectProjectPath = Just $ Command.Effect.projectPath command + } prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m Derivation prepareDerivation store command = do diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.8.2 +version: 0.8.3 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -69,6 +69,7 @@ Hercules.Agent.WorkerProtocol.Event.AttributeError Hercules.Agent.WorkerProtocol.Event.BuildResult Hercules.Agent.WorkerProtocol.LogSettings + Hercules.Agent.WorkerProtocol.Orphans Hercules.Effect Hercules.Effect.Container Data.Conduit.Extras @@ -189,7 +190,7 @@ , filepath , hercules-ci-agent , hercules-ci-api-core == 0.1.3.0 - , hercules-ci-api-agent == 0.4.0.0 + , hercules-ci-api-agent == 0.4.1.0 , hostname , http-client , http-client-tls diff --git a/hercules-ci-agent/Hercules/Agent/Effect.hs b/hercules-ci-agent/Hercules/Agent/Effect.hs --- a/hercules-ci-agent/Hercules/Agent/Effect.hs +++ b/hercules-ci-agent/Hercules/Agent/Effect.hs @@ -64,7 +64,9 @@ materializeDerivation = materialize, secretsPath = toS $ Config.staticSecretsDirectory config "secrets.json", token = Sensitive (EffectTask.token effectTask), - apiBaseURL = Config.herculesApiBaseURL config + apiBaseURL = Config.herculesApiBaseURL config, + projectId = EffectTask.projectId effectTask, + projectPath = EffectTask.projectPath effectTask } exitCode <- runWorker procSpec (stderrLineHandler "Effect worker") commandChan writeEvent logLocM DebugS $ "Worker exit: " <> logStr (show exitCode :: Text) diff --git a/hercules-ci-agent/Hercules/Agent/Init.hs b/hercules-ci-agent/Hercules/Agent/Init.hs --- a/hercules-ci-agent/Hercules/Agent/Init.hs +++ b/hercules-ci-agent/Hercules/Agent/Init.hs @@ -11,6 +11,7 @@ import qualified Hercules.Agent.ServiceInfo as ServiceInfo import qualified Hercules.Agent.Token as Token import qualified Hercules.CNix +import qualified Hercules.CNix.Util import qualified Katip as K import qualified Network.HTTP.Client.TLS import Protolude @@ -66,3 +67,4 @@ initCNix = do Hercules.CNix.init Hercules.CNix.setTalkative + Hercules.CNix.Util.installDefaultSigINTHandler diff --git a/src/Hercules/Agent/WorkerProtocol/Command/Effect.hs b/src/Hercules/Agent/WorkerProtocol/Command/Effect.hs --- a/src/Hercules/Agent/WorkerProtocol/Command/Effect.hs +++ b/src/Hercules/Agent/WorkerProtocol/Command/Effect.hs @@ -1,10 +1,13 @@ +{-# LANGUAGE DataKinds #-} {-# LANGUAGE DeriveAnyClass #-} module Hercules.Agent.WorkerProtocol.Command.Effect where import Data.Binary +import Hercules.API.Id (Id) import Hercules.Agent.Sensitive import Hercules.Agent.WorkerProtocol.LogSettings +import Hercules.Agent.WorkerProtocol.Orphans () import Protolude data Effect = Effect @@ -14,6 +17,8 @@ inputDerivationOutputPaths :: [ByteString], materializeDerivation :: Bool, secretsPath :: FilePath, - token :: Sensitive Text + token :: Sensitive Text, + projectId :: Id "project", + projectPath :: Text } deriving (Generic, Binary, Show, Eq) diff --git a/src/Hercules/Agent/WorkerProtocol/Orphans.hs b/src/Hercules/Agent/WorkerProtocol/Orphans.hs new file mode 100644 --- /dev/null +++ b/src/Hercules/Agent/WorkerProtocol/Orphans.hs @@ -0,0 +1,13 @@ +{-# LANGUAGE KindSignatures #-} +{-# LANGUAGE PolyKinds #-} +{-# OPTIONS_GHC -Wno-orphans #-} + +module Hercules.Agent.WorkerProtocol.Orphans where + +import Control.Applicative ((<$>)) +import Data.Binary +import Hercules.API.Id (Id (..)) + +instance Binary (Id (a :: k)) where + put (Id uuid) = put uuid + get = Id <$> get diff --git a/src/Hercules/Effect.hs b/src/Hercules/Effect.hs --- a/src/Hercules/Effect.hs +++ b/src/Hercules/Effect.hs @@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-} +{-# LANGUAGE DataKinds #-} {-# LANGUAGE ScopedTypeVariables #-} module Hercules.Effect where @@ -8,6 +9,7 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL import qualified Data.Map as M +import Hercules.API.Id (Id, idText) import Hercules.Agent.Sensitive (Sensitive (Sensitive, reveal), revealContainer) import Hercules.CNix (Derivation) import Hercules.CNix.Store (getDerivationArguments, getDerivationBuilder, getDerivationEnv) @@ -29,8 +31,8 @@ Right r -> Right r -- | Write secrets to file based on secretsMap value -writeSecrets :: (MonadIO m, KatipContext m) => FilePath -> Map Text Text -> Map Text (Sensitive Formats.Secret.Secret) -> FilePath -> m () -writeSecrets sourceFile secretsMap extraSecrets destinationDirectory = write . fmap reveal . addExtra =<< gather +writeSecrets :: (MonadIO m, KatipContext m) => Maybe FilePath -> Map Text Text -> Map Text (Sensitive Formats.Secret.Secret) -> FilePath -> m () +writeSecrets sourceFileMaybe secretsMap extraSecrets destinationDirectory = write . fmap reveal . addExtra =<< gather where addExtra = flip M.union extraSecrets write = liftIO . BS.writeFile (destinationDirectory "secrets.json") . BL.toStrict . A.encode @@ -38,15 +40,18 @@ if null secretsMap then pure mempty else do - secretsBytes <- liftIO $ BS.readFile sourceFile - r <- case A.eitherDecode $ BL.fromStrict secretsBytes of - Left e -> do - logLocM ErrorS $ "Could not parse secrets file " <> logStr sourceFile <> ": " <> logStr e - throwIO $ FatalError "Could not parse secrets file as configured on agent." - Right r -> pure (Sensitive r) + allSecrets <- + sourceFileMaybe & maybe (purer mempty) \sourceFile -> do + secretsBytes <- liftIO $ BS.readFile sourceFile + case A.eitherDecode $ BL.fromStrict secretsBytes of + Left e -> do + logLocM ErrorS $ "Could not parse secrets file " <> logStr sourceFile <> ": " <> logStr e + throwIO $ FatalError "Could not parse secrets file as configured on agent." + Right r -> pure (Sensitive r) + createDirectoryIfMissing True destinationDirectory secretsMap & M.traverseWithKey \destinationName (secretName :: Text) -> do - case revealContainer (r <&> M.lookup secretName) of + case revealContainer (allSecrets <&> M.lookup secretName) of Nothing -> liftIO $ throwIO $ @@ -62,8 +67,21 @@ { data_ = Formats.Secret.data_ secret } -runEffect :: (MonadThrow m, KatipContext m) => Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode -runEffect derivation token secretsPath apiBaseURL dir = do +data RunEffectParams = RunEffectParams + { runEffectDerivation :: Derivation, + runEffectToken :: Maybe (Sensitive Text), + runEffectSecretsConfigPath :: Maybe FilePath, + runEffectApiBaseURL :: Text, + runEffectDir :: FilePath, + runEffectProjectId :: Maybe (Id "project"), + runEffectProjectPath :: Maybe Text + } + +(=:) :: k -> a -> Map k a +(=:) = M.singleton + +runEffect :: (MonadThrow m, KatipContext m) => RunEffectParams -> m ExitCode +runEffect p@RunEffectParams {runEffectDerivation = derivation, runEffectSecretsConfigPath = secretsPath, runEffectApiBaseURL = apiBaseURL, runEffectDir = dir} = do drvBuilder <- liftIO $ getDerivationBuilder derivation drvArgs <- liftIO $ getDerivationArguments derivation drvEnv <- liftIO $ getDerivationEnv derivation @@ -74,15 +92,17 @@ secretsDir <- mkDir "secrets" runcDir <- mkDir "runc-state" let extraSecrets = - M.singleton - "hercules-ci" - ( do - tok <- token - pure $ - Formats.Secret.Secret - { data_ = M.singleton "token" $ A.String tok - } - ) + runEffectToken p + & maybe + mempty + ( \token -> + "hercules-ci" =: do + tok <- token + pure $ + Formats.Secret.Secret + { data_ = M.singleton "token" $ A.String tok + } + ) writeSecrets secretsPath drvSecretsMap extraSecrets (toS secretsDir) liftIO $ do -- Nix sandbox sets tmp to buildTopDir @@ -92,7 +112,7 @@ -- TODO: implement passAsFile? let overridableEnv, onlyImpureOverridableEnv, fixedEnv :: Map Text Text overridableEnv = - M.fromList + M.fromList $ [ ("PATH", "/path-not-set"), ("HOME", "/homeless-shelter"), ("NIX_STORE", "/nix/store"), -- TODO store.storeDir @@ -102,6 +122,9 @@ ("HERCULES_CI_API_BASE_URL", apiBaseURL), ("HERCULES_CI_SECRETS_JSON", "/secrets/secrets.json") ] + <> [("HERCULES_CI_PROJECT_ID", idText x) | x <- toList $ runEffectProjectId p] + <> [("HERCULES_CI_PROJECT_PATH", x) | x <- toList $ runEffectProjectPath p] + -- NB: this is lossy. Consider using ByteString-based process functions drvEnv' = drvEnv & M.mapKeys (decodeUtf8With lenientDecode) & fmap (decodeUtf8With lenientDecode) impureEnvVars = mempty -- TODO