packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Health.hs

module Kenshou.Remote.Cell.Health (mapCellHealth, withCellHealthFile) where

import Control.Concurrent (forkIO, killThread, threadDelay)
import Control.Exception (IOException, catch, finally, mask_)
import Data.Aeson (Value, eitherDecodeStrict', encode, object, withObject, (.:), (.:?), (.=))
import Data.Aeson.Types (Parser, parseEither)
import Data.ByteString qualified as ByteString
import Data.ByteString.Lazy qualified as LazyByteString
import Data.IORef (newIORef, readIORef, writeIORef)
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as TextIO
import Data.Time (UTCTime)
import Kenshou.Core.Id (RunId)
import System.Directory (doesFileExist)
import System.IO (stderr)

-- The cell owner emits a health document for each observation window. Keep
-- only tripped gates, because an ok gate is not a run health notice.
mapCellHealth :: RunId -> Value -> Either Text [Value]
mapCellHealth expected document = either (Left . Text.pack) Right (parseEither parseDocument document)
  where
    parseDocument = withObject "cell health" \fields -> do
      schema <- fields .: "schema"
      if schema == ("cell.health/v1" :: Text) then pure () else fail "unsupported cell health schema"
      runId <- fields .: "runId"
      if runId == expected then pure () else fail "cell health run ID differs from the active cell run"
      gates <- fields .: "gates" :: Parser [Value]
      concat <$> traverse parseGate gates

    parseGate = withObject "cell health gate" \gate -> do
      name <- gate .: "name" :: Parser Text
      machine <- gate .: "machine" :: Parser Text
      status <- gate .: "status" :: Parser Text
      at <- gate .: "at" :: Parser UTCTime
      detail <- gate .:? "detail" :: Parser (Maybe Text)
      if name `elem` ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"] && not (Text.null machine)
        then pure ()
        else fail "unknown cell health gate or empty machine"
      case status of
        "ok" -> pure []
        "tripped" ->
          pure
            [ object
                [ "schema" .= ("kenshou.health-notice/v1" :: Text),
                  "source" .= ("cell-health:" <> name <> ":" <> machine),
                  "severity" .= (if name == "background-load" then "soft" else "hard" :: Text),
                  "at" .= at,
                  "detail" .= maybe ("cell health gate tripped: " <> name) (\message -> if Text.null message then "cell health gate tripped: " <> name else message) detail
                ]
            ]
        _ -> fail "unknown cell health gate status"

-- Re-read the rolling file because its producer may replace it atomically or
-- append JSON lines. Deduplicate notices across repeated health snapshots.
withCellHealthFile :: Maybe FilePath -> FilePath -> RunId -> IO result -> IO result
withCellHealthFile Nothing _ _ action = action
withCellHealthFile (Just source) destination runId action = do
  initial <- safePoll Set.empty
  latest <- newIORef initial
  thread <- forkIO (loop latest initial)
  action `finally` do
    killThread thread
    readIORef latest >>= safePoll >>= writeIORef latest
  where
    loop latest seen = do
      threadDelay 1000000
      next <- mask_ do
        updated <- safePoll seen
        writeIORef latest updated
        pure updated
      loop latest next

    safePoll seen =
      poll seen `catch` \(failure :: IOException) -> do
        TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack (show failure))
        pure seen

    poll seen = do
      exists <- doesFileExist source
      if not exists
        then pure seen
        else do
          bytes <- ByteString.readFile source
          foldLines seen (completeLines bytes)

    foldLines seen [] = pure seen
    foldLines seen (line : rest) = do
      case eitherDecodeStrict' line >>= firstText . mapCellHealth runId of
        Left problem -> TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack problem) >> foldLines seen rest
        Right notices -> do
          let new = filter (\notice -> encode notice `Set.notMember` seen) notices
          mapM_ (LazyByteString.appendFile destination . (<> "\n") . encode) new
          foldLines (foldr (Set.insert . encode) seen new) rest

    firstText = either (Left . Text.unpack) Right

completeLines :: ByteString.ByteString -> [ByteString.ByteString]
completeLines bytes | ByteString.null bytes = []
completeLines bytes = case ByteString.split 10 bytes of
  [] -> []
  parts -> filter (not . ByteString.null) (if ByteString.last bytes == 10 then parts else init parts)