packages feed

kenshou-check-0.1.0.0: src/Kenshou/Check/Process.hs

module Kenshou.Check.Process
  ( ProcessSpec (..),
    ChildSignal (..),
    RestartPolicy (..),
    ProgressSnapshot (..),
    Supervisor,
    Child,
    roleProcess,
    withSupervisor,
    spawn,
    awaitReady,
    sendCommand,
    progress,
    awaitMark,
    signalChild,
    killChild,
    terminateChild,
    restartChild,
    stopGracefully,
    withRestartLoop,
    crashWindows,
    childPid,
    childProc,
    childExitCode,
    reapChild,
    readChildMessages,
    sweepOrphans,
  )
where

import Control.Concurrent (threadDelay)
import Control.Concurrent.Async (Async, async, cancel, waitCatch)
import Control.Concurrent.MVar
import Control.Concurrent.STM
import Control.Exception (SomeException, bracket, catch, evaluate)
import Control.Monad (forM_, unless, void)
import Data.Aeson
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString.Char8 qualified as ByteString
import Data.ByteString.Lazy.Char8 qualified as LazyByteString
import Data.IORef
import Data.Int (Int64)
import Data.List (foldl')
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Data.Time.Clock.POSIX (getPOSIXTime)
import Kenshou.Check.Fact
import Kenshou.Check.Ledger
import Kenshou.Check.Scenario
import Kenshou.Check.Window
import Kenshou.Core.Context (Environment (..), RunContext (..))
import Kenshou.Core.Env.Postgres (PostgresEnv (..))
import Kenshou.Core.Id (renderRunId)
import Kenshou.Core.Resource.Types (ProvisionedResource (..))
import Kenshou.Core.Role
import Kenshou.Core.RunSpec (EnvironmentSpec (..))
import System.Directory (createDirectoryIfMissing)
import System.Environment (getEnvironment, getExecutablePath)
import System.Exit (ExitCode (..))
import System.FilePath ((</>))
import System.IO
import System.Posix.Signals
import System.Posix.Types (CPid)
import System.Process
import System.Timeout (timeout)

data ProcessSpec = ProcessSpec
  { proc :: !ProcId,
    executable :: !FilePath,
    args :: ![String],
    env :: ![(String, String)],
    initial :: !(Maybe ControlMessage)
  }

data ChildSignal = Term | Kill | Stop | Cont deriving stock (Eq, Ord, Show)

data RestartPolicy = RestartPolicy
  { initialBackoffMillis :: !Int,
    maxBackoffMillis :: !Int,
    multiplier :: !Double,
    maxRestarts :: !Int
  }
  deriving stock (Eq, Show)

data ProgressSnapshot = ProgressSnapshot
  { ready :: !Bool,
    count :: !Int64,
    marks :: !(Map Text Value),
    lastMessage :: !(Maybe WorkerMessage)
  }
  deriving stock (Eq, Show)

data Supervisor = Supervisor
  { environment :: !CheckEnv,
    children :: !(MVar [Child]),
    windows :: !(IORef [DisturbanceWindow])
  }

data Child = Child
  { spec :: !ProcessSpec,
    pid :: !CPid,
    input :: !Handle,
    inputLock :: !(MVar ()),
    output :: !Handle,
    errorLog :: !Handle,
    processHandle :: !ProcessHandle,
    state :: !(TVar ProgressSnapshot),
    listener :: !(Async ()),
    controlLog :: !FilePath,
    stdoutLog :: !FilePath
  }

defaultRestartPolicy :: RestartPolicy
defaultRestartPolicy = RestartPolicy 100 5000 2 50

roleProcess :: CheckEnv -> Text -> Int -> Value -> IO ProcessSpec
roleProcess environment role index arguments = do
  executable <- getExecutablePath
  roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName role)
  let context = environment.context
      proc = ProcId role index 0
      instanceName = role <> "-" <> Text.pack (show index)
      postgres = fmap (\postgres -> PostgresConnInfo postgres.connectionString postgres.adminConnectionString) context.env.postgres
      workerInit = WorkerInit context.runId context.scenario roleName instanceName context.seed context.knobs context.dimensions postgres (fmap (.workerValue) context.env.resources) context.environmentSpec.telemetry context.outDir arguments
      appName = "kenshou-" <> Text.unpack (Text.take 8 (renderRunId context.runId)) <> "-" <> Text.unpack role <> "-" <> show index
  pure (ProcessSpec proc executable ["worker", "--role", Text.unpack role] [("PGAPPNAME", appName)] (Just (CtlInit workerInit)))

withSupervisor :: CheckEnv -> (Supervisor -> IO value) -> IO value
withSupervisor environment action = bracket acquire cleanup action
  where
    acquire = Supervisor environment <$> newMVar [] <*> newIORef []
    cleanup supervisor = readMVar supervisor.children >>= mapM_ cleanupChild

spawn :: Supervisor -> ProcessSpec -> IO Child
spawn supervisor spec = do
  let logs = supervisor.environment.context.outDir </> "logs"
      label = fileLabel spec.proc
      stderrPath = logs </> label <> ".stderr.log"
      controlPath = logs </> label <> ".control.jsonl"
      stdoutPath = logs </> label <> ".stdout.log"
  createDirectoryIfMissing True logs
  stderrHandle <- openFile stderrPath AppendMode
  baseEnvironment <- getEnvironment
  let mergedEnvironment = Map.toList (foldl' (\values (key, value) -> Map.insert key value values) (Map.fromList baseEnvironment) spec.env)
  (Just input, Just output, _, processHandle) <-
    createProcess
      (proc spec.executable spec.args)
        { std_in = CreatePipe,
          std_out = CreatePipe,
          std_err = UseHandle stderrHandle,
          env = Just mergedEnvironment,
          create_group = True
        }
  hSetBuffering input LineBuffering
  hSetBuffering output LineBuffering
  pid <- getPid processHandle >>= maybe (ioError (userError "spawned worker has no pid")) pure
  state <- newTVarIO (ProgressSnapshot False 0 Map.empty Nothing)
  inputLock <- newMVar ()
  listener <- async (listenWorker output controlPath stdoutPath state)
  let child = Child spec pid input inputLock output stderrHandle processHandle state listener controlPath stdoutPath
  modifyMVar_ supervisor.children (pure . (child :))
  appendPid supervisor child
  forM_ spec.initial (sendCommand child)
  pure child

awaitReady :: Child -> Int -> IO ()
awaitReady child timeoutMillis = awaitState timeoutMillis child (\snapshot -> snapshot.ready) "worker did not become ready"

sendCommand :: Child -> ControlMessage -> IO ()
sendCommand child message = withMVar child.inputLock \_ -> do
  LazyByteString.hPutStrLn child.input (encode message)
  hFlush child.input

progress :: Child -> STM ProgressSnapshot
progress child = readTVar child.state

awaitMark :: Child -> Text -> Int -> IO ()
awaitMark child mark timeoutMillis = awaitState timeoutMillis child (Map.member mark . (.marks)) ("worker did not report mark " <> Text.unpack mark)

signalChild :: Supervisor -> Child -> ChildSignal -> IO ()
signalChild supervisor child signal = do
  recordWindowEdge supervisor child DisturbanceStart signal
  signalProcessGroup (signalValue signal) child.pid
  case signal of
    Cont -> recordWindowEdge supervisor child DisturbanceEnd signal
    _ -> pure ()

killChild :: Supervisor -> Child -> IO ()
killChild supervisor child = do
  signalChild supervisor child Kill
  void (waitExit child)
  retireChild supervisor child

terminateChild :: Supervisor -> Child -> IO ()
terminateChild supervisor child = do
  signalChild supervisor child Term
  void (waitExit child)
  retireChild supervisor child

restartChild :: Supervisor -> Child -> IO Child
restartChild supervisor child = do
  let oldProc = child.spec.proc
      nextProc = oldProc {incarnation = oldProc.incarnation + 1}
      nextSpec = ProcessSpec nextProc child.spec.executable child.spec.args child.spec.env child.spec.initial
  replacement <- spawn supervisor nextSpec
  awaitReady replacement 10000
  recordWindowEdge supervisor replacement DisturbanceEnd Kill
  pure replacement

stopGracefully :: Supervisor -> Child -> Int -> IO ExitCode
stopGracefully supervisor child graceMillis = do
  sendCommand child (CtlStop graceMillis)
  finished <- timeout (graceMillis * 1000) (waitExit child)
  result <- case finished of
    Just code -> pure code
    Nothing -> do
      signalChild supervisor child Term
      terminated <- timeout 1000000 (waitExit child)
      case terminated of
        Just code -> pure code
        Nothing -> signalChild supervisor child Kill >> waitExit child
  retireChild supervisor child
  pure result

retireChild :: Supervisor -> Child -> IO ()
retireChild supervisor child =
  modifyMVar_ supervisor.children \children -> do
    let remaining = filter ((/= child.pid) . (.pid)) children
    -- Force the entire spine: a delayed filter retains retired Child handles
    -- and their completed listener threads until supervisor cleanup.
    _ <- evaluate (length remaining)
    pure remaining

withRestartLoop :: Supervisor -> RestartPolicy -> ProcessSpec -> (IO Child -> IO value) -> IO value
withRestartLoop supervisor _policy spec action = action (spawn supervisor spec)

crashWindows :: Supervisor -> IO [DisturbanceWindow]
crashWindows supervisor = reverse <$> readIORef supervisor.windows

childPid :: Child -> CPid
childPid = (.pid)

-- | The exit code of a child that has exited, without waiting.
childExitCode :: Child -> IO (Maybe ExitCode)
childExitCode child = getProcessExitCode child.processHandle

-- | Collect a child that has already exited (or wait for it to exit) and
-- remove it from the supervisor, without signalling it. A restart loop uses
-- this for a process that died on its own.
reapChild :: Supervisor -> Child -> IO ExitCode
reapChild supervisor child = do
  code <- waitExit child
  retireChild supervisor child
  pure code

childProc :: Child -> ProcId
childProc child = child.spec.proc

-- The control log retains every mark, including repeated marks that the
-- progress snapshot replaces with the latest payload.
readChildMessages :: Child -> IO [WorkerMessage]
readChildMessages child = do
  contents <- ByteString.readFile child.controlLog
  pure [message | line <- ByteString.lines contents, Right message <- [eitherDecodeStrict' line]]

sweepOrphans :: FilePath -> IO [CPid]
sweepOrphans path = do
  exists <- tryRead path
  case exists of
    Nothing -> pure []
    Just contents -> fmap concat . traverse killLine $ ByteString.lines contents
  where
    killLine line = case eitherDecodeStrict' line of
      Right (PidRecord pid _ _ _) -> (signalProcess sigKILL (fromIntegral pid) >> pure [fromIntegral pid]) `catch` ignoreList
      Left _ -> pure []

data PidRecord = PidRecord Int Text Int Int

instance ToJSON PidRecord where
  toJSON (PidRecord pid role index incarnation) = object ["pid" .= pid, "role" .= role, "index" .= index, "incarnation" .= incarnation]

instance FromJSON PidRecord where
  parseJSON = withObject "PidRecord" \value -> PidRecord <$> value .: "pid" <*> value .: "role" <*> value .: "index" <*> value .: "incarnation"

listenWorker :: Handle -> FilePath -> FilePath -> TVar ProgressSnapshot -> IO ()
listenWorker output controlPath stdoutPath state = go
  where
    go = do
      done <- hIsEOF output
      unless done do
        line <- ByteString.hGetLine output
        case eitherDecodeStrict' line of
          Right message -> do
            ByteString.appendFile controlPath (line <> "\n")
            atomically (modifyTVar' state (applyMessage message))
          Left _ -> ByteString.appendFile stdoutPath (line <> "\n")
        go

applyMessage :: WorkerMessage -> ProgressSnapshot -> ProgressSnapshot
applyMessage message snapshot = case message of
  WrkReady -> snapshot {ready = True, lastMessage = Just message}
  WrkProgress count _ -> snapshot {count, lastMessage = Just message}
  WrkCustom name payload -> snapshot {marks = Map.insert name payload snapshot.marks, lastMessage = Just message}
  _ -> snapshot {lastMessage = Just message}

awaitState :: Int -> Child -> (ProgressSnapshot -> Bool) -> String -> IO ()
awaitState timeoutMillis child predicate failure = do
  result <- timeout (timeoutMillis * 1000) (atomically (readTVar child.state >>= check . predicate))
  maybe (ioError (userError failure)) pure result

recordWindowEdge :: Supervisor -> Child -> FactKind -> ChildSignal -> IO ()
recordWindowEdge supervisor child kind signal = do
  now <- wallMicros
  let label = Text.toLower (Text.pack (show signal))
      target = renderProcId child.spec.proc
      attrs = KeyMap.fromList [(Key.fromText "label", String label), (Key.fromText "target", String target), (Key.fromText "pid", Number (fromIntegral child.pid))]
  recordDurable supervisor.environment.ledger kind target 0 label attrs
  window <- evaluate (DisturbanceWindow label target now (if kind == DisturbanceEnd then Just now else Nothing))
  modifyIORef' supervisor.windows (window :)

appendPid :: Supervisor -> Child -> IO ()
appendPid supervisor child = do
  let path = supervisor.environment.context.outDir </> "logs" </> "pids.jsonl"
      proc = child.spec.proc
  LazyByteString.appendFile path (encode (PidRecord (fromIntegral child.pid) proc.role proc.index proc.incarnation) <> "\n")

cleanupChild :: Child -> IO ()
cleanupChild child = do
  (signalProcessGroup sigKILL child.pid >> pure ()) `catch` ignoreUnit
  void (timeout 1000000 (waitExit child))
  closeChildHandles child

closeChildHandles :: Child -> IO ()
closeChildHandles child = do
  cancel child.listener
  hClose child.input `catch` ignoreUnit
  hClose child.output `catch` ignoreUnit
  hClose child.errorLog `catch` ignoreUnit

signalValue :: ChildSignal -> Signal
signalValue Term = sigTERM
signalValue Kill = sigKILL
signalValue Stop = sigSTOP
signalValue Cont = sigCONT

fileLabel :: ProcId -> FilePath
fileLabel proc = Text.unpack (Text.replace "/" "-" proc.role) <> "-" <> show proc.index <> "." <> show proc.incarnation

wallMicros :: IO Int64
wallMicros = round . (* 1000000) <$> getPOSIXTime

tryRead :: FilePath -> IO (Maybe ByteString.ByteString)
tryRead path = (Just <$> ByteString.readFile path) `catch` ignoreMaybe

waitExit :: Child -> IO ExitCode
waitExit child = do
  code <- waitForProcess child.processHandle
  void (timeout 1000000 (waitCatch child.listener))
  closeChildHandles child
  pure code

ignoreUnit :: SomeException -> IO ()
ignoreUnit _ = pure ()

ignoreList :: SomeException -> IO [CPid]
ignoreList _ = pure []

ignoreMaybe :: SomeException -> IO (Maybe value)
ignoreMaybe _ = pure Nothing