diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.2] - 2022-03-30 + +### Added + + - Separate traces in the dashboard (as in `--show-trace`) + +### Fixed + + - Effects: `error: cannot open connection to remote store 'daemon': error: reading from file: Connection reset by peer` + ## [0.9.1] - 2022-03-18 ### Added @@ -543,6 +553,7 @@ - Initial release +[0.9.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.1...hercules-ci-agent-0.9.2 [0.9.1]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.9.0...hercules-ci-agent-0.9.1 [0.9.0]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.7...hercules-ci-agent-0.9.0 [0.8.7]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.8.6...hercules-ci-agent-0.8.7 diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs @@ -32,6 +32,7 @@ import Data.UUID (UUID) import Data.Vector (Vector) import qualified Data.Vector as V +import Foreign (alloca, peek) import Hercules.API.Agent.Evaluate.EvaluateEvent.OnPushHandlerEvent (OnPushHandlerEvent (OnPushHandlerEvent)) import qualified Hercules.API.Agent.Evaluate.EvaluateEvent.OnPushHandlerEvent import qualified Hercules.API.Agent.Evaluate.EvaluateTask as EvaluateTask @@ -56,7 +57,6 @@ import Hercules.Agent.Worker.HerculesStore (nixStore, setBuilderCallback, withHerculesStore) import Hercules.Agent.Worker.HerculesStore.Context (HerculesStore) import Hercules.Agent.Worker.Logging (withKatip) -import Hercules.Agent.Worker.NixDaemon (nixDaemon) import Hercules.Agent.WorkerProtocol.Command ( Command, ) @@ -87,6 +87,7 @@ import qualified Hercules.CNix.Std.Vector as Std.Vector import Hercules.CNix.Store.Context (NixStorePathWithOutputs) import Hercules.CNix.Util (installDefaultSigINTHandler) +import Hercules.CNix.Verbosity (setShowTrace) import Hercules.Error import Hercules.UserException (UserException (UserException)) import Katip @@ -106,6 +107,7 @@ C.context (C.cppCtx <> C.fptrCtx) C.include "" +C.include "" C.include "" C.include "" @@ -130,14 +132,12 @@ main = do hSetBuffering stderr LineBuffering Hercules.CNix.Expr.init + setShowTrace True _ <- installHandler sigTERM (Catch $ raiseSignal sigINT) Nothing installDefaultSigINTHandler Logger.initLogger args <- Environment.getArgs case args of - ["nix-daemon", options] -> do - setOptions options - nixDaemon [options] -> taskWorker options _ -> throwIO $ FatalError "worker: Unrecognized command line arguments" @@ -192,7 +192,7 @@ runCommand st ch command ) `safeLiftedCatch` ( \e -> liftIO $ do - e' <- renderException e + (e', _trace) <- renderException e writeChan ch (Just $ Exception e') exitFailure ) @@ -223,26 +223,41 @@ pure x ) -renderException :: SomeException -> IO Text +renderException :: SomeException -> IO (Text, Maybe Text) renderException e | Just (C.CppStdException ex _msg _ty) <- fromException e = renderStdException ex renderException e | Just (C.CppNonStdException _ex maybeType) <- fromException e = - pure $ "Unexpected C++ exception" <> foldMap (\t -> " of type " <> decodeUtf8With lenientDecode t) maybeType -renderException e | Just (FatalError msg) <- fromException e = pure msg -renderException e = pure $ toS $ displayException e + pure ("Unexpected C++ exception" <> foldMap (\t -> " of type " <> decodeUtf8With lenientDecode t) maybeType, Nothing) +renderException e | Just (FatalError msg) <- fromException e = pure (msg, Nothing) +renderException e = pure (toS $ displayException e, Nothing) -renderStdException :: C.CppExceptionPtr -> IO Text -renderStdException e = - [C.throwBlock| char * { +renderStdException :: C.CppExceptionPtr -> IO (Text, Maybe Text) +renderStdException e = alloca \traceStrPtr -> do + msg <- + [C.throwBlock| char * { + char **traceStrPtr = $(char **traceStrPtr); + *traceStrPtr = nullptr; std::string r; std::exception_ptr *e = $fptr-ptr:(std::exception_ptr *e); try { std::rethrow_exception(*e); } catch (const nix::Error &e) { - // r = e.what(); - std::stringstream s; - nix::showErrorInfo(s, e.info(), true); - r = s.str(); + { + std::stringstream s; + nix::showErrorInfo(s, e.info(), false); + r = s.str(); + } + { + std::stringstream s; + nix::showErrorInfo(s, e.info(), true); + std::string t = s.str(); + // starts with r? + if (t.rfind(r, 0) == 0) { + t.replace(0, r.size(), ""); + t = nix::trim(t); + } + *traceStrPtr = strdup(t.c_str()); + } } catch (const std::exception &e) { r = e.what(); } catch (...) { @@ -252,8 +267,12 @@ return strdup(r.c_str()); }|] - >>= unsafePackMallocCString - <&> decodeUtf8With lenientDecode + >>= unsafePackMallocCString + <&> decodeUtf8With lenientDecode + traceText <- + peek traceStrPtr >>= traverseNonNull \s -> + unsafePackMallocCString s <&> decodeUtf8With lenientDecode + pure (msg, traceText) connectCommand :: (MonadUnliftIO m, KatipContext m, MonadThrow m) => @@ -291,7 +310,7 @@ runConduitRes ( Data.Conduit.handleC ( \e -> do - yield . Event.Error =<< liftIO (renderException e) + yield . Event.Error . fst =<< liftIO (renderException e) liftIO $ throwTo mainThread e ) ( do @@ -453,17 +472,19 @@ <> ", which is required during evaluation." <> foldMap (" " <>) (buildExceptionDetail e'), AttributeError.errorDerivation = Just (buildExceptionDerivationPath e'), - AttributeError.errorType = Just "BuildException" + AttributeError.errorType = Just "BuildException", + AttributeError.trace = Nothing -- would be nice to get a trace here. Throw and catch more C++ exception types? } yieldAttributeError path e = do - e' <- liftIO $ renderException e + (e', maybeTrace) <- liftIO $ renderException e yield $ Event.AttributeError $ AttributeError.AttributeError { AttributeError.path = path, AttributeError.message = e', AttributeError.errorDerivation = Nothing, - AttributeError.errorType = Just (show (typeOf e)) + AttributeError.errorType = Just (show (typeOf e)), + AttributeError.trace = maybeTrace } maybeThrowBuildException :: MonadIO m => BuildResult.BuildStatus -> Text -> m () diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger/Context.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger/Context.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger/Context.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger/Context.hs @@ -1,8 +1,14 @@ {-# LANGUAGE EmptyDataDecls #-} {-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TemplateHaskellQuotes #-} -module Hercules.Agent.Worker.Build.Logger.Context where +module Hercules.Agent.Worker.Build.Logger.Context + ( Fields, + HerculesLoggerEntry, + LogEntryQueue, + context, + ) +where import qualified Data.Map as M import qualified Language.C.Inline.Context as C diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -8,10 +8,15 @@ -- easy to maintain approach to do decouple it with inline-c-cpp. Perhaps it's -- better to use an FFI generator instead? -module Hercules.Agent.Worker.Build.Prefetched where +module Hercules.Agent.Worker.Build.Prefetched + ( buildDerivation, + Hercules.Agent.Worker.Build.Prefetched.getDerivation, + BuildResult (..), + ) +where import qualified Data.ByteString.Char8 as C8 -import Foreign (FinalizerPtr, ForeignPtr, alloca, newForeignPtr, nullPtr, peek) +import Foreign (alloca, peek) import Foreign.C (peekCString) import Hercules.CNix.Encapsulation import Hercules.CNix.Store @@ -91,10 +96,6 @@ errorMessage :: Text } deriving (Show) - -nullableForeignPtr :: FinalizerPtr a -> Ptr a -> IO (Maybe (ForeignPtr a)) -nullableForeignPtr _ rawPtr | rawPtr == nullPtr = pure Nothing -nullableForeignPtr finalize rawPtr = Just <$> newForeignPtr finalize rawPtr getDerivation :: Store -> StorePath -> IO (Maybe Derivation) getDerivation (Store store) derivationPath = diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs @@ -3,8 +3,11 @@ {-# LANGUAGE TemplateHaskell #-} module Hercules.Agent.Worker.HerculesStore - ( module Hercules.Agent.Worker.HerculesStore, + ( withHerculesStore, HerculesStore, + nixStore, + printDiagnostics, + setBuilderCallback, ) where diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs @@ -1,8 +1,13 @@ {-# LANGUAGE EmptyDataDecls #-} {-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TemplateHaskellQuotes #-} -module Hercules.Agent.Worker.HerculesStore.Context where +module Hercules.Agent.Worker.HerculesStore.Context + ( context, + HerculesStore, + ExceptionPtr, + ) +where import qualified Data.Map as M import Hercules.CNix.Expr.Context (Ref) diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/NixDaemon.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/NixDaemon.hs deleted file mode 100644 --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/NixDaemon.hs +++ /dev/null @@ -1,123 +0,0 @@ -{-# LANGUAGE BlockArguments #-} -{-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE TemplateHaskell #-} - -module Hercules.Agent.Worker.NixDaemon where - -import Control.Concurrent.STM.TVar (modifyTVar, newTVarIO, readTVar) -import qualified Data.Binary -import qualified Data.Binary.Get -import qualified Data.ByteString as BS -import qualified Data.ByteString.Lazy as BL -import qualified Data.Map as M -import Hercules.Agent.Binary (decodeBinaryFromHandle) -import qualified Hercules.Agent.WorkerProtocol.Command.StartDaemon as StartDaemon -import Hercules.Agent.WorkerProtocol.Event.DaemonStarted (DaemonStarted (DaemonStarted, _dummy)) -import qualified Hercules.CNix as CNix -import qualified Language.C.Inline.Cpp as C -import qualified Language.C.Inline.Cpp.Exception as C -import Network.Socket -import Protolude -import System.Posix.Internals (setNonBlockingFD) -import UnliftIO (BufferMode (NoBuffering), hSetBuffering, timeout) -import UnliftIO.IO (hFlush) - -C.context C.cppCtx - -C.include "" -C.include "" -C.include "" -C.using "namespace nix" - -nixDaemon :: IO () --- nixDaemon = withKatip $ Logger.withLoggerConduit (_) $ Logger.withTappedStderr Logger.tapper $ liftIO $ do -nixDaemon = do - hSetBuffering stdin NoBuffering - startCmd <- - getStdinMessage >>= \case - Left _ -> throwIO $ FatalError "Could not decode command" - Right Nothing -> do - putErrText "warning: Exit before receiving start command" - exitSuccess -- We're done. - Right (Just r) -> pure r - let path = StartDaemon.socketPath startCmd - - CNix.init - - clientThreads <- newTVarIO mempty - - sock <- socket AF_UNIX Stream 0 - bind sock (SockAddrUnix path) - listen sock 100 - - putStdoutMessage DaemonStarted {_dummy = True} - - let socketLoop = - forever do - (clientSocket, _) <- accept sock - (pid, _uid, _gid) <- getPeerCredential clientSocket - uninterruptibleMask \_unmask -> do - let removeMe = do - t <- myThreadId - atomically do - modifyTVar clientThreads (M.delete t) - t <- - forkFinally -- TODO forkOS? - (handleClient clientSocket) - \case - Left _e -> removeMe -- >> putErrText ("Connection for pid " <> showPid pid <> " ended: " <> toS (displayException _e)) - Right _ -> removeMe - atomically do - modifyTVar clientThreads (M.insert t pid) - withAsync socketLoop \_socketLoopAsync -> - void getStdinMessage - - void $ timeout (10 * 60 * 1000 * 1000) do - hadThreads <- do - ts <- atomically do - readTVar clientThreads - let hasThreads = not (null ts) - when hasThreads do - putErrLn ("Waiting for termination of connections from pids " <> unwords (showPid <$> toList ts)) - pure hasThreads - atomically do - ts <- readTVar clientThreads - guard $ null ts - when hadThreads do - putErrText "All connections terminated; exiting" - -showPid :: Maybe C.CUInt -> Text -showPid = maybe "unknown pid" show - -putStdoutMessage :: DaemonStarted -> IO () -putStdoutMessage msg = do - BS.hPut stdout $ BL.toStrict $ Data.Binary.encode msg - hFlush stdout - -getStdinMessage :: - IO - ( Either - (ByteString, Data.Binary.Get.ByteOffset, [Char]) - (Maybe StartDaemon.StartDaemon) - ) -getStdinMessage = decodeBinaryFromHandle stdin - -handleClient :: Socket -> IO () -handleClient clientSocket = withFdSocket clientSocket \fd -> flip finally (close clientSocket) do - setNonBlockingFD fd False - [C.throwBlock| void { - ref store = openStore(); - FdSource from($(int fd)); - FdSink to($(int fd)); - daemon::TrustedFlag trusted = daemon::NotTrusted; - daemon::RecursiveFlag recursive = daemon::NotRecursive; - std::function authHook = [](Store &){}; - daemon::processConnection( - store - , from - , to - , trusted - , recursive - , authHook - ); - }|] diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.9.1 +version: 0.9.2 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -90,12 +90,10 @@ Hercules.Agent.WorkerProtocol.Command.BuildResult Hercules.Agent.WorkerProtocol.Command.Effect Hercules.Agent.WorkerProtocol.Command.Eval - Hercules.Agent.WorkerProtocol.Command.StartDaemon Hercules.Agent.WorkerProtocol.Event Hercules.Agent.WorkerProtocol.Event.Attribute Hercules.Agent.WorkerProtocol.Event.AttributeError Hercules.Agent.WorkerProtocol.Event.BuildResult - Hercules.Agent.WorkerProtocol.Event.DaemonStarted Hercules.Agent.WorkerProtocol.LogSettings Hercules.Agent.WorkerProtocol.Orphans Hercules.Effect @@ -135,6 +133,7 @@ , lifted-base , monad-control , mtl + , network , network-uri , protolude >= 0.3 , process @@ -218,7 +217,7 @@ , filepath , hercules-ci-agent , hercules-ci-api-core == 0.1.4.0 - , hercules-ci-api-agent == 0.4.3.0 + , hercules-ci-api-agent == 0.4.4.0 , hostname , http-client , http-client-tls @@ -272,7 +271,6 @@ Hercules.Agent.Worker.HerculesStore Hercules.Agent.Worker.HerculesStore.Context Hercules.Agent.Worker.Logging - Hercules.Agent.Worker.NixDaemon hs-source-dirs: hercules-ci-agent-worker @@ -398,3 +396,13 @@ , transformers-base , unliftio-core default-language: Haskell2010 + +executable hercules-ci-nix-daemon + import: cxx-opts + main-is: main.cc + cxx-sources: hercules-ci-nix-daemon/daemon.cc + default-language: Haskell2010 + hs-source-dirs: hercules-ci-nix-daemon + pkgconfig-depends: + nix-store >= 2.0 + , nix-main >= 2.0 diff --git a/hercules-ci-agent/Hercules/Agent/Evaluate.hs b/hercules-ci-agent/Hercules/Agent/Evaluate.hs --- a/hercules-ci-agent/Hercules/Agent/Evaluate.hs +++ b/hercules-ci-agent/Hercules/Agent/Evaluate.hs @@ -497,7 +497,8 @@ { AttributeErrorEvent.expressionPath = decode <$> WorkerAttributeError.path e, AttributeErrorEvent.errorMessage = WorkerAttributeError.message e, AttributeErrorEvent.errorType = WorkerAttributeError.errorType e, - AttributeErrorEvent.errorDerivation = WorkerAttributeError.errorDerivation e + AttributeErrorEvent.errorDerivation = WorkerAttributeError.errorDerivation e, + AttributeErrorEvent.trace = WorkerAttributeError.trace e } continue Event.EvaluationDone -> diff --git a/hercules-ci-nix-daemon/daemon.cc b/hercules-ci-nix-daemon/daemon.cc new file mode 100644 --- /dev/null +++ b/hercules-ci-nix-daemon/daemon.cc @@ -0,0 +1,112 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace nix; + +static void sigChldHandler(int sigNo) +{ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + // Reap all dead children. + while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; +} + +static void setSigChldAction() +{ + struct sigaction act, oact; + act.sa_handler = sigChldHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + +extern "C" int main(int argc, char **argv) { + nix::initNix(); + + for (int i = 1; i < argc && argv[i]; i++) { + std::string arg(argv[i]); + if (arg == "--option") { + if (i + 2 < argc) { + nix::globalConfig.set(argv[i+1], argv[i+2]); + i += 2; + } + else { + std::cerr << "Not enough arguments to --option" << std::endl; + return 1; + } + } + } + + // See withNixDaemonProxy for stdin (ab)use + AutoCloseFD socketFD = dup(0); + closeOnExec(socketFD.get()); + + // Put /dev/null on stdin + int devnull = open("/dev/null", O_RDONLY); + dup2(devnull, 0); + close(devnull); + + setSigChldAction(); + while (1) { + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + try { + AutoCloseFD remote = accept(socketFD.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + else throw SysError("hercules-ci-nix-daemon: accepting connection"); + } + closeOnExec(remote.get()); + ProcessOptions options; + options.errorPrefix = "hercules-ci-nix-daemon: unexpected error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess([&]() { + socketFD = -1; + + // Regular nix daemon creates a new session for connections and + // does not kill connection child processes. Those are not + // appropriate behaviors for a temporary proxy. + + FdSource from(remote.get()); + FdSink to(remote.get()); + // TODO: disable caching without interfering with user parameters + ref store = openStore(); + daemon::TrustedFlag trusted = daemon::NotTrusted; + daemon::RecursiveFlag recursive = daemon::NotRecursive; + std::function authHook = [](Store &){}; + daemon::processConnection( + store + , from + , to + , trusted + , recursive + , authHook + ); + }); + } catch (Interrupted & e) { + return 0; + } catch (Error & error) { + ErrorInfo ei = error.info(); + ei.msg = hintfmt("hercules-ci-nix-daemon: error processing connection: %1%", ei.msg.str()); + logError(ei); + } + } + return 0; +} diff --git a/hercules-ci-nix-daemon/main.cc b/hercules-ci-nix-daemon/main.cc new file mode 100644 --- /dev/null +++ b/hercules-ci-nix-daemon/main.cc @@ -0,0 +1,4 @@ +/* + main() is in daemon.cc, because cabal doesn't pass + the correct flags to the main-is file. +*/ diff --git a/src/Hercules/Agent/WorkerProcess.hs b/src/Hercules/Agent/WorkerProcess.hs --- a/src/Hercules/Agent/WorkerProcess.hs +++ b/src/Hercules/Agent/WorkerProcess.hs @@ -1,6 +1,7 @@ module Hercules.Agent.WorkerProcess ( runWorker, getWorkerExe, + getDaemonExe, WorkerEnvSettings (..), prepareEnv, modifyEnv, @@ -65,6 +66,10 @@ getWorkerExe :: MonadIO m => m [Char] getWorkerExe = do liftIO getBinDir <&> ( "hercules-ci-agent-worker") + +getDaemonExe :: MonadIO m => m [Char] +getDaemonExe = do + liftIO getBinDir <&> ( "hercules-ci-nix-daemon") -- | Control a child process by communicating over stdin and stdout -- using a 'Binary' interface. diff --git a/src/Hercules/Agent/WorkerProtocol/Command/StartDaemon.hs b/src/Hercules/Agent/WorkerProtocol/Command/StartDaemon.hs deleted file mode 100644 --- a/src/Hercules/Agent/WorkerProtocol/Command/StartDaemon.hs +++ /dev/null @@ -1,11 +0,0 @@ -{-# LANGUAGE DeriveAnyClass #-} - -module Hercules.Agent.WorkerProtocol.Command.StartDaemon where - -import Data.Binary -import Protolude - -data StartDaemon = StartDaemon - { socketPath :: FilePath - } - deriving (Generic, Binary, Show, Eq) diff --git a/src/Hercules/Agent/WorkerProtocol/Event/AttributeError.hs b/src/Hercules/Agent/WorkerProtocol/Event/AttributeError.hs --- a/src/Hercules/Agent/WorkerProtocol/Event/AttributeError.hs +++ b/src/Hercules/Agent/WorkerProtocol/Event/AttributeError.hs @@ -10,6 +10,7 @@ { path :: [ByteString], message :: Text, errorType :: Maybe Text, - errorDerivation :: Maybe Text + errorDerivation :: Maybe Text, + trace :: Maybe Text } deriving (Generic, Binary, Show, Eq) diff --git a/src/Hercules/Agent/WorkerProtocol/Event/DaemonStarted.hs b/src/Hercules/Agent/WorkerProtocol/Event/DaemonStarted.hs deleted file mode 100644 --- a/src/Hercules/Agent/WorkerProtocol/Event/DaemonStarted.hs +++ /dev/null @@ -1,10 +0,0 @@ -{-# LANGUAGE DeriveAnyClass #-} - -module Hercules.Agent.WorkerProtocol.Event.DaemonStarted where - -import Data.Binary -import Protolude - -data DaemonStarted = DaemonStarted - {_dummy :: Bool {- make its binary representation non-empty -}} - deriving (Generic, Binary, Show, Eq) diff --git a/src/Hercules/Effect.hs b/src/Hercules/Effect.hs --- a/src/Hercules/Effect.hs +++ b/src/Hercules/Effect.hs @@ -7,16 +7,11 @@ import Control.Monad.Catch (MonadThrow) import qualified Data.Aeson as A import qualified Data.ByteString as BS -import qualified Data.ByteString.Char8 as C8 import qualified Data.ByteString.Lazy as BL import qualified Data.Map as M import Hercules.API.Id (Id, idText) import Hercules.Agent.Sensitive (Sensitive (Sensitive, reveal), revealContainer) -import Hercules.Agent.WorkerProcess (runWorker) import qualified Hercules.Agent.WorkerProcess as WorkerProcess -import Hercules.Agent.WorkerProtocol.Command.StartDaemon (StartDaemon (StartDaemon)) -import qualified Hercules.Agent.WorkerProtocol.Command.StartDaemon as StartDaemon -import Hercules.Agent.WorkerProtocol.Event.DaemonStarted (DaemonStarted (DaemonStarted)) import Hercules.CNix (Derivation) import Hercules.CNix.Store (getDerivationArguments, getDerivationBuilder, getDerivationEnv) import Hercules.Effect.Container (BindMount (BindMount)) @@ -25,11 +20,12 @@ import qualified Hercules.Formats.Secret as Formats.Secret import Hercules.Secrets (SecretContext, evalCondition, evalConditionTrace) import Katip (KatipContext, Severity (..), logLocM, logStr) +import Network.Socket (Family (AF_UNIX), SockAddr (SockAddrUnix), SocketType (Stream), bind, listen, socket, withFdSocket) import Protolude import System.FilePath -import UnliftIO (timeout) -import qualified UnliftIO +import System.Posix (dup, fdToHandle) import UnliftIO.Directory (createDirectory, createDirectoryIfMissing) +import UnliftIO.Process (withCreateProcess) import qualified UnliftIO.Process as Process parseDrvSecretsMap :: Map ByteString ByteString -> Either Text (Map Text Text) @@ -206,32 +202,27 @@ withNixDaemonProxy :: [(Text, Text)] -> FilePath -> IO a -> IO a withNixDaemonProxy extraNixOptions socketPath wrappedAction = do - workerExe <- WorkerProcess.getWorkerExe - let opts = ["nix-daemon", show extraNixOptions] - procSpec = - (Process.proc workerExe opts) - { -- Process.env = Just workerEnv, - Process.close_fds = True - -- Process.cwd = Just workDir - } - daemonCmdChan <- liftIO newChan - daemonReadyVar <- liftIO newEmptyMVar - let onDaemonEvent :: MonadIO m => DaemonStarted -> m () - onDaemonEvent DaemonStarted {} = liftIO (void $ tryPutMVar daemonReadyVar ()) - liftIO $ writeChan daemonCmdChan (Just $ Just $ StartDaemon {socketPath = socketPath}) - UnliftIO.withAsync (runWorker procSpec (\_ s -> liftIO (C8.hPutStrLn stderr ("hci proxy nix-daemon: " <> s))) daemonCmdChan onDaemonEvent) $ \daemonAsync -> do - race (readMVar daemonReadyVar) (wait daemonAsync) >>= \case - Left _ -> pass - Right e -> throwIO $ FatalError $ "Hercules CI proxy nix-daemon exited before being told to; status " <> show e - - a <- wrappedAction + -- Open the socket asap, so we don't have to wait for + -- a readiness signal from the daemon, or poll, etc. + sock <- socket AF_UNIX Stream 0 + bind sock (SockAddrUnix socketPath) + listen sock 100 - timeoutResult <- timeout (60 * 1000 * 1000) $ do - writeChan daemonCmdChan (Just Nothing) - writeChan daemonCmdChan Nothing - void $ wait daemonAsync - case timeoutResult of - Nothing -> putErrText "warning: Hercules CI proxy nix-daemon did not shut down in time." - _ -> pass + -- (Ab)use stdin to transfer the socket while securely + -- closing all other fds + socketAsHandle <- withFdSocket sock \fd -> do + fd' <- dup (fromIntegral fd) + fdToHandle fd' - pure a + exe <- WorkerProcess.getDaemonExe + let opts = extraNixOptions >>= \(k, v) -> ["--option", toS k, toS v] + procSpec = + (Process.proc exe opts) + { -- close all other fds to be secure + Process.close_fds = True, + Process.std_in = Process.UseHandle socketAsHandle, + Process.std_err = Process.Inherit, + Process.std_out = Process.UseHandle stderr + } + withCreateProcess procSpec $ \_in _out _err _processHandle -> do + wrappedAction