packages feed

hercules-ci-agent 0.8.2 → 0.8.3

raw patch · 12 files changed

+105/−51 lines, 12 filesdep ~hercules-ci-api-agent

Dependency ranges changed: hercules-ci-api-agent

Files

CHANGELOG.md view
@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.3]++### Added++ - `hci` can now run in the effects sandbox++### Fixed++ - Interrupt handling has been improved+ ## [0.8.2]  ### Added@@ -422,6 +432,8 @@  - Initial release +[0.8.3]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.2...hercules-ci-agent-0.8.3+[0.8.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.1...hercules-ci-agent-0.8.2 [0.8.1]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.0...hercules-ci-agent-0.8.1 [0.8.0]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.5...hercules-ci-agent-0.8.0 [0.7.5]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.4...hercules-ci-agent-0.7.5
cbits/nix-2.4/hercules-store.cxx view
@@ -101,7 +101,8 @@ }  void WrappingStore::buildPaths(-      const std::vector<DerivedPath> & paths, BuildMode buildMode) {+      const std::vector<DerivedPath> & paths, BuildMode buildMode,+      std::shared_ptr<Store> evalStore) {   wrappedStore->buildPaths(paths, buildMode); } @@ -231,7 +232,7 @@       uint64_t & downloadSize, uint64_t & narSize) { }; -void HerculesStore::buildPaths(const std::vector<DerivedPath> & derivedPaths, BuildMode buildMode) {+void HerculesStore::buildPaths(const std::vector<DerivedPath> & derivedPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) {   std::exception_ptr exceptionToThrow(nullptr);    // responsibility for delete is transferred to builderCallback
cbits/nix-2.4/hercules-store.hh view
@@ -72,7 +72,8 @@    virtual void buildPaths(       const std::vector<DerivedPath> & paths,-      BuildMode buildMode = bmNormal) override;+      BuildMode buildMode = bmNormal,+      std::shared_ptr<Store> evalStore = nullptr) override;    virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,       BuildMode buildMode = bmNormal) override;@@ -134,7 +135,8 @@    virtual void buildPaths(       const std::vector<DerivedPath> & paths,-      BuildMode buildMode = bmNormal) override;+      BuildMode buildMode = bmNormal,+      std::shared_ptr<Store> evalStore = nullptr) override;    virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,       BuildMode buildMode = bmNormal) override;
hercules-ci-agent-worker/Hercules/Agent/Worker.hs view
@@ -66,7 +66,7 @@ import Hercules.CNix.Std.Vector (StdVector) import qualified Hercules.CNix.Std.Vector as Std.Vector import Hercules.CNix.Store.Context (NixStorePathWithOutputs)-import Hercules.CNix.Util (triggerInterrupt)+import Hercules.CNix.Util (installDefaultSigINTHandler) import Hercules.Error import Katip import qualified Language.C.Inline.Cpp.Exceptions as C@@ -74,7 +74,6 @@ import Protolude hiding (bracket, catch, evalState, wait, withAsync, yield) import qualified System.Environment as Environment import System.IO (BufferMode (LineBuffering), hSetBuffering)-import System.Mem.Weak (deRefWeak) import System.Posix.IO (dup, fdToHandle, stdError) import System.Posix.Signals (Handler (Catch), installHandler, raiseSignal, sigINT, sigTERM) import System.Timeout (timeout)@@ -99,30 +98,12 @@  instance Exception BuildException -extendSigINT :: IO ()-extendSigINT = do-  mainThread <- myThreadId-  weakId <- mkWeakThreadId mainThread-  _oldHandler <--    installHandler-      sigINT-      ( Catch do-          -- GHC RTS default behavior-          mt <- deRefWeak weakId-          for_ mt \t -> do-            throwTo t (toException UserInterrupt)-          -- Nix-          triggerInterrupt-      )-      Nothing-  pass- main :: IO () main = do   hSetBuffering stderr LineBuffering   Hercules.CNix.Expr.init   _ <- installHandler sigTERM (Catch $ raiseSignal sigINT) Nothing-  extendSigINT+  installDefaultSigINTHandler   Logger.initLogger   [options] <- Environment.getArgs   let allOptions =
hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs view
@@ -97,6 +97,7 @@ getDerivation (Store store) derivationPath =   nullableMoveToForeignPtrWrapper     =<< [C.throwBlock| Derivation *{+      ReceiveInterrupts _;       StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath);       std::list<nix::ref<nix::Store>> stores = getDefaultSubstituters();       stores.push_front(*$(refStore* store));@@ -132,6 +133,7 @@             alloca $ \stopTimePtr ->               alloca $ \errorMessagePtr -> do                 [C.throwBlock| void {+      ReceiveInterrupts _;       Store &store = **$(refStore* store);       bool &success = *$(bool *successPtr);       int &status = *$(int *statusPtr);
hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs view
@@ -10,6 +10,7 @@ import Hercules.CNix (Store) import qualified Hercules.CNix as CNix import Hercules.CNix.Store (Derivation)+import Hercules.Effect (RunEffectParams (..)) import qualified Hercules.Effect as Effect import Katip (KatipContext) import Protolude@@ -19,7 +20,16 @@ runEffect store command = do   derivation <- prepareDerivation store command   dir <- getCurrentDirectory-  Effect.runEffect derivation (Command.Effect.token command) (Command.Effect.secretsPath command) (Command.Effect.apiBaseURL command) dir+  Effect.runEffect+    RunEffectParams+      { runEffectDerivation = derivation,+        runEffectToken = Just $ Command.Effect.token command,+        runEffectSecretsConfigPath = Just $ Command.Effect.secretsPath command,+        runEffectApiBaseURL = Command.Effect.apiBaseURL command,+        runEffectDir = dir,+        runEffectProjectId = Just $ Command.Effect.projectId command,+        runEffectProjectPath = Just $ Command.Effect.projectPath command+      }  prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m Derivation prepareDerivation store command = do
hercules-ci-agent.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.4  name:           hercules-ci-agent-version:        0.8.2+version:        0.8.3 synopsis:       Runs Continuous Integration tasks on your machines category:       Nix, CI, Testing, DevOps homepage:       https://docs.hercules-ci.com@@ -69,6 +69,7 @@       Hercules.Agent.WorkerProtocol.Event.AttributeError       Hercules.Agent.WorkerProtocol.Event.BuildResult       Hercules.Agent.WorkerProtocol.LogSettings+      Hercules.Agent.WorkerProtocol.Orphans       Hercules.Effect       Hercules.Effect.Container       Data.Conduit.Extras@@ -189,7 +190,7 @@     , filepath     , hercules-ci-agent     , hercules-ci-api-core == 0.1.3.0-    , hercules-ci-api-agent == 0.4.0.0+    , hercules-ci-api-agent == 0.4.1.0     , hostname     , http-client     , http-client-tls
hercules-ci-agent/Hercules/Agent/Effect.hs view
@@ -64,7 +64,9 @@               materializeDerivation = materialize,               secretsPath = toS $ Config.staticSecretsDirectory config </> "secrets.json",               token = Sensitive (EffectTask.token effectTask),-              apiBaseURL = Config.herculesApiBaseURL config+              apiBaseURL = Config.herculesApiBaseURL config,+              projectId = EffectTask.projectId effectTask,+              projectPath = EffectTask.projectPath effectTask             }   exitCode <- runWorker procSpec (stderrLineHandler "Effect worker") commandChan writeEvent   logLocM DebugS $ "Worker exit: " <> logStr (show exitCode :: Text)
hercules-ci-agent/Hercules/Agent/Init.hs view
@@ -11,6 +11,7 @@ import qualified Hercules.Agent.ServiceInfo as ServiceInfo import qualified Hercules.Agent.Token as Token import qualified Hercules.CNix+import qualified Hercules.CNix.Util import qualified Katip as K import qualified Network.HTTP.Client.TLS import Protolude@@ -66,3 +67,4 @@ initCNix = do   Hercules.CNix.init   Hercules.CNix.setTalkative+  Hercules.CNix.Util.installDefaultSigINTHandler
src/Hercules/Agent/WorkerProtocol/Command/Effect.hs view
@@ -1,10 +1,13 @@+{-# LANGUAGE DataKinds #-} {-# LANGUAGE DeriveAnyClass #-}  module Hercules.Agent.WorkerProtocol.Command.Effect where  import Data.Binary+import Hercules.API.Id (Id) import Hercules.Agent.Sensitive import Hercules.Agent.WorkerProtocol.LogSettings+import Hercules.Agent.WorkerProtocol.Orphans () import Protolude  data Effect = Effect@@ -14,6 +17,8 @@     inputDerivationOutputPaths :: [ByteString],     materializeDerivation :: Bool,     secretsPath :: FilePath,-    token :: Sensitive Text+    token :: Sensitive Text,+    projectId :: Id "project",+    projectPath :: Text   }   deriving (Generic, Binary, Show, Eq)
+ src/Hercules/Agent/WorkerProtocol/Orphans.hs view
@@ -0,0 +1,13 @@+{-# LANGUAGE KindSignatures #-}+{-# LANGUAGE PolyKinds #-}+{-# OPTIONS_GHC -Wno-orphans #-}++module Hercules.Agent.WorkerProtocol.Orphans where++import Control.Applicative ((<$>))+import Data.Binary+import Hercules.API.Id (Id (..))++instance Binary (Id (a :: k)) where+  put (Id uuid) = put uuid+  get = Id <$> get
src/Hercules/Effect.hs view
@@ -1,4 +1,5 @@ {-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DataKinds #-} {-# LANGUAGE ScopedTypeVariables #-}  module Hercules.Effect where@@ -8,6 +9,7 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL import qualified Data.Map as M+import Hercules.API.Id (Id, idText) import Hercules.Agent.Sensitive (Sensitive (Sensitive, reveal), revealContainer) import Hercules.CNix (Derivation) import Hercules.CNix.Store (getDerivationArguments, getDerivationBuilder, getDerivationEnv)@@ -29,8 +31,8 @@       Right r -> Right r  -- | Write secrets to file based on secretsMap value-writeSecrets :: (MonadIO m, KatipContext m) => FilePath -> Map Text Text -> Map Text (Sensitive Formats.Secret.Secret) -> FilePath -> m ()-writeSecrets sourceFile secretsMap extraSecrets destinationDirectory = write . fmap reveal . addExtra =<< gather+writeSecrets :: (MonadIO m, KatipContext m) => Maybe FilePath -> Map Text Text -> Map Text (Sensitive Formats.Secret.Secret) -> FilePath -> m ()+writeSecrets sourceFileMaybe secretsMap extraSecrets destinationDirectory = write . fmap reveal . addExtra =<< gather   where     addExtra = flip M.union extraSecrets     write = liftIO . BS.writeFile (destinationDirectory </> "secrets.json") . BL.toStrict . A.encode@@ -38,15 +40,18 @@       if null secretsMap         then pure mempty         else do-          secretsBytes <- liftIO $ BS.readFile sourceFile-          r <- case A.eitherDecode $ BL.fromStrict secretsBytes of-            Left e -> do-              logLocM ErrorS $ "Could not parse secrets file " <> logStr sourceFile <> ": " <> logStr e-              throwIO $ FatalError "Could not parse secrets file as configured on agent."-            Right r -> pure (Sensitive r)+          allSecrets <-+            sourceFileMaybe & maybe (purer mempty) \sourceFile -> do+              secretsBytes <- liftIO $ BS.readFile sourceFile+              case A.eitherDecode $ BL.fromStrict secretsBytes of+                Left e -> do+                  logLocM ErrorS $ "Could not parse secrets file " <> logStr sourceFile <> ": " <> logStr e+                  throwIO $ FatalError "Could not parse secrets file as configured on agent."+                Right r -> pure (Sensitive r)+           createDirectoryIfMissing True destinationDirectory           secretsMap & M.traverseWithKey \destinationName (secretName :: Text) -> do-            case revealContainer (r <&> M.lookup secretName) of+            case revealContainer (allSecrets <&> M.lookup secretName) of               Nothing ->                 liftIO $                   throwIO $@@ -62,8 +67,21 @@                       { data_ = Formats.Secret.data_ secret                       } -runEffect :: (MonadThrow m, KatipContext m) => Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode-runEffect derivation token secretsPath apiBaseURL dir = do+data RunEffectParams = RunEffectParams+  { runEffectDerivation :: Derivation,+    runEffectToken :: Maybe (Sensitive Text),+    runEffectSecretsConfigPath :: Maybe FilePath,+    runEffectApiBaseURL :: Text,+    runEffectDir :: FilePath,+    runEffectProjectId :: Maybe (Id "project"),+    runEffectProjectPath :: Maybe Text+  }++(=:) :: k -> a -> Map k a+(=:) = M.singleton++runEffect :: (MonadThrow m, KatipContext m) => RunEffectParams -> m ExitCode+runEffect p@RunEffectParams {runEffectDerivation = derivation, runEffectSecretsConfigPath = secretsPath, runEffectApiBaseURL = apiBaseURL, runEffectDir = dir} = do   drvBuilder <- liftIO $ getDerivationBuilder derivation   drvArgs <- liftIO $ getDerivationArguments derivation   drvEnv <- liftIO $ getDerivationEnv derivation@@ -74,15 +92,17 @@   secretsDir <- mkDir "secrets"   runcDir <- mkDir "runc-state"   let extraSecrets =-        M.singleton-          "hercules-ci"-          ( do-              tok <- token-              pure $-                Formats.Secret.Secret-                  { data_ = M.singleton "token" $ A.String tok-                  }-          )+        runEffectToken p+          & maybe+            mempty+            ( \token ->+                "hercules-ci" =: do+                  tok <- token+                  pure $+                    Formats.Secret.Secret+                      { data_ = M.singleton "token" $ A.String tok+                      }+            )   writeSecrets secretsPath drvSecretsMap extraSecrets (toS secretsDir)   liftIO $ do     -- Nix sandbox sets tmp to buildTopDir@@ -92,7 +112,7 @@     -- TODO: implement passAsFile?     let overridableEnv, onlyImpureOverridableEnv, fixedEnv :: Map Text Text         overridableEnv =-          M.fromList+          M.fromList $             [ ("PATH", "/path-not-set"),               ("HOME", "/homeless-shelter"),               ("NIX_STORE", "/nix/store"), -- TODO store.storeDir@@ -102,6 +122,9 @@               ("HERCULES_CI_API_BASE_URL", apiBaseURL),               ("HERCULES_CI_SECRETS_JSON", "/secrets/secrets.json")             ]+              <> [("HERCULES_CI_PROJECT_ID", idText x) | x <- toList $ runEffectProjectId p]+              <> [("HERCULES_CI_PROJECT_PATH", x) | x <- toList $ runEffectProjectPath p]+         -- NB: this is lossy. Consider using ByteString-based process functions         drvEnv' = drvEnv & M.mapKeys (decodeUtf8With lenientDecode) & fmap (decodeUtf8With lenientDecode)         impureEnvVars = mempty -- TODO