packages feed

hercules-ci-agent 0.7.3 → 0.7.4

raw patch · 6 files changed

+106/−19 lines, 6 files

Files

CHANGELOG.md view
@@ -5,8 +5,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased]+## [0.7.4] - 2020-08-25 +### Fixed++ - Paths that are missing from the binary cache will be rebuilt. This affected+   agent 0.7 - 0.7.3 users with trusted user optimizations turned on, which is+   the default when using the NixOS or nix-darwin modules.++ - Prevent states where no progress can be made. One caused by a potential buildup+   of batched messages that may not fit within the timeout interval; the other+   a receive operation without a timeout during the initial socket handshake.++ - The log socket will remain open instead of reconnecting unnecessarily.++ - Add a safety measure to prevent unintended increases in workload in case+   Nix sees an opportunity for concurrency that was not intended by Hercules CI.++### Changed++ - The NixOS module in the hercules-ci-agent repo now disables the upcoming+   module that is packaged upstream with NixOS.++   The upstream module will configure fewer things for you, to be in line with+   normal NixOS expectations. Notably, it does not configure automatic garbage+   collection and it does not preconfigure NixOps keys deployment.++   The configuration interface will remain unchanged for `0.7` but `0.8` will+   match the upstream interface.++## [0.7.3] - 2020-07-18+ ### Added   - Evaluation log@@ -285,8 +314,12 @@  - Initial release -[0.6.6]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.4...hercules-ci-agent-0.6.6-[0.6.5]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.3...hercules-ci-agent-0.6.5+[0.7.4]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.3...hercules-ci-agent-0.7.4+[0.7.3]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.2...hercules-ci-agent-0.7.3+[0.7.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.7.1...hercules-ci-agent-0.7.2+[0.7.1]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.6...hercules-ci-agent-0.7.1+[0.6.6]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.5...hercules-ci-agent-0.6.6+[0.6.5]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.4...hercules-ci-agent-0.6.5 [0.6.4]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.3...hercules-ci-agent-0.6.4 [0.6.3]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.2...hercules-ci-agent-0.6.3 [0.6.2]: https://github.com/hercules-ci/hercules-ci-agent/compare/hercules-ci-agent-0.6.1...hercules-ci-agent-0.6.2
hercules-ci-agent-worker/Hercules/Agent/Worker.hs view
@@ -10,7 +10,6 @@ import CNix import qualified CNix.Internal.Raw import Conduit-import Control.Concurrent.Async.Lifted.Safe import Control.Concurrent.STM import qualified Control.Exception.Lifted as EL import Control.Monad.IO.Unlift@@ -64,6 +63,7 @@ import System.IO (BufferMode (LineBuffering), hSetBuffering) import System.Posix.IO (dup, fdToHandle, stdError) import System.Timeout (timeout)+import UnliftIO.Async (wait, withAsync) import UnliftIO.Exception (bracket, catch) import Prelude () import qualified Prelude@@ -92,8 +92,15 @@   CNix.init   Logger.initLogger   [options] <- Environment.getArgs-  -- narinfo-cache-negative-ttl: Always try requesting narinfos because it may have been built in the meanwhile-  let allOptions = Prelude.read options ++ [("narinfo-cache-negative-ttl", "0")]+  let allOptions =+        Prelude.read options+          ++ [+               -- narinfo-cache-negative-ttl: Always try requesting narinfos because it may have been built in the meanwhile+               ("narinfo-cache-negative-ttl", "0"),+               -- Build concurrency is controlled by hercules-ci-agent, so set it+               -- to 1 to avoid accidentally consuming too many resources at once.+               ("max-jobs", "1")+             ]   for_ allOptions $ \(k, v) -> do     setGlobalOption k v     setOption k v@@ -220,8 +227,16 @@ logger :: (MonadIO m, MonadUnliftIO m, KatipContext m) => LogSettings.LogSettings -> ConduitM () (Vector LogEntry) m () -> m () logger logSettings_ entriesSource = do   socketConfig <- liftIO $ makeSocketConfig logSettings_-  -- TODO integrate katip more-  Socket.withReliableSocket socketConfig $ \socket -> katipAddNamespace "Build" do+  let withPings socket m =+        withAsync+          ( liftIO $ forever do+              -- TODO add ping constructor to Frame or use websocket pings+              let ping = LogMessage.LogEntries mempty+              threadDelay 30_000_000+              atomically $ Socket.write socket ping+          )+          (const m)+  Socket.withReliableSocket socketConfig $ \socket -> withPings socket $ katipAddNamespace "Build" do     let conduit =           entriesSource             .| Logger.unbatch
hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs view
@@ -1,3 +1,6 @@+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE ScopedTypeVariables #-}+ module Hercules.Agent.Worker.Build where  import CNix@@ -20,13 +23,21 @@ runBuild store build = do   let extraPaths = Command.Build.inputDerivationOutputPaths build       drvPath = toS $ Command.Build.drvPath build-  for_ extraPaths $ \input ->-    liftIO $ CNix.ensurePath store input+  x <- for extraPaths $ \input -> do+    liftIO $ try $ CNix.ensurePath store input+  materialize <- case sequenceA x of+    Right _ ->+      -- no error, proceed with requested materialization setting+      pure $ Command.Build.materializeDerivation build+    Left (e :: SomeException) -> liftIO do+      CNix.logInfo $ "while retrieving dependencies: " <> toS (displayException e)+      CNix.logInfo $ "unable to retrieve dependency; will build locally"+      pure True   derivationMaybe <- liftIO $ Build.getDerivation store drvPath   derivation <- case derivationMaybe of     Just drv -> pure drv     Nothing -> panic $ "Could not retrieve derivation " <> show drvPath <> " from local store or binary caches."-  nixBuildResult <- liftIO $ buildDerivation store drvPath derivation (extraPaths <$ guard (not (Command.Build.materializeDerivation build)))+  nixBuildResult <- liftIO $ buildDerivation store drvPath derivation (extraPaths <$ guard (not materialize))   katipAddContext (sl "result" (show nixBuildResult :: Text)) $     logLocM DebugS "Build result"   buildResult <- liftIO $ enrichResult store derivation nixBuildResult
hercules-ci-agent.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.4  name:           hercules-ci-agent-version:        0.7.3+version:        0.7.4 synopsis:       Runs Continuous Integration tasks on your machines category:       Nix, CI, Testing, DevOps homepage:       https://docs.hercules-ci.com
src-cnix/CNix.hs view
@@ -106,6 +106,13 @@     settings.set($bs-cstr:optionStr, $bs-cstr:valueStr);   }|] +logInfo :: Text -> IO ()+logInfo t = do+  let bstr = encodeUtf8 t+  [C.throwBlock| void {+    printInfo($bs-cstr:bstr);+  }|]+ mkBindings :: Ptr Bindings' -> IO Bindings mkBindings p = pure $ Bindings p
src/Hercules/Agent/Socket.hs view
@@ -18,7 +18,7 @@ import Control.Monad.IO.Unlift import qualified Data.Aeson as A import Data.DList (DList, fromList)-import Data.List (dropWhileEnd)+import Data.List (dropWhileEnd, splitAt) import Data.Semigroup import Data.Time (NominalDiffTime, addUTCTime, diffUTCTime, getCurrentTime) import Data.Time.Extras@@ -34,6 +34,7 @@ import Protolude hiding (atomically, handle, race, race_) import UnliftIO.Async (race, race_) import UnliftIO.Exception (handle)+import UnliftIO.Timeout (timeout) import Wuss (runSecureClientWith)  data Socket r w@@ -114,14 +115,24 @@           & foldMap (\case Frame.Msg {n = n} -> Option $ Just $ Max n; _ -> mempty)           & traverse_ (\(Max n) -> setExpectedAck n)       send :: Connection -> [Frame ap ap] -> m ()-      send conn msgs = do-        liftIO $ WS.sendDataMessages conn (WS.Binary . A.encode <$> msgs)-        setExpectedAckForMsgs msgs+      send conn = sendSorted . sortBy (compare `on` msgN)+        where+          sendRaw :: [Frame ap ap] -> m ()+          sendRaw msgs = do+            liftIO $ WS.sendDataMessages conn (WS.Binary . A.encode <$> msgs)+            setExpectedAckForMsgs msgs+          sendSorted :: [Frame ap ap] -> m ()+          sendSorted [] = pass+          sendSorted msgs = do+            let (msgsNow, msgsLater) = Data.List.splitAt 100 msgs+            sendRaw msgsNow+            sendSorted msgsLater       recv :: Connection -> m (Frame sp sp)       recv conn = do-        (liftIO $ A.eitherDecode <$> WS.receiveData conn) >>= \case-          Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toSL e)-          Right r -> pure r+        withTimeout ackTimeout (FatalError "Hercules.Agent.Socket.recv timed out") $+          (liftIO $ A.eitherDecode <$> WS.receiveData conn) >>= \case+            Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toSL e)+            Right r -> pure r       handshake conn = katipAddNamespace "Handshake" do         siMsg <- recv conn         case siMsg of@@ -218,6 +229,10 @@         handshake conn       readThread conn `race_` writeThread conn `race_` noAckCleanupThread +msgN :: Frame o a -> Maybe Integer+msgN (Frame.Msg {n = n}) = Just n+msgN _ = Nothing+ withConnection' :: (MonadUnliftIO m) => SocketConfig any0 any1 m -> (Connection -> m a) -> m a withConnection' socketConfig m = do   UnliftIO unlift <- askUnliftIO@@ -244,3 +259,9 @@  slash :: [Char] -> [Char] -> [Char] a `slash` b = dropWhileEnd (== '/') a <> "/" <> dropWhile (== '/') b++withTimeout :: (Exception e, MonadIO m, MonadUnliftIO m) => NominalDiffTime -> e -> m a -> m a+withTimeout t e _ | t <= 0 = throwIO e+withTimeout t e m = timeout (ceiling $ t * 1_000_000) m >>= \case+  Nothing -> throwIO e+  Just a -> pure a