packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Index.hs

module Kenshou.Remote.Cell.Index
  ( CellManifestLink (..),
    EvidenceLink (..),
    RunLink (..),
    CellRunIndex (..),
    deriveCellRunIndex,
    writeCellRunIndex,
  )
where

import Control.Exception (bracket)
import Control.Monad (forM)
import Crypto.Hash.SHA256 qualified as SHA256
import Data.Aeson (FromJSON (..), ToJSON (..), Value, eitherDecode, encode, object, withObject, (.:), (.:?), (.=))
import Data.Aeson qualified as Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString.Base16 qualified as Base16
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Int (Int64)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TextEncoding
import Data.Time (UTCTime, getCurrentTime)
import Kenshou.Core.Id (RunId, parseRunId, renderRunId)
import Kenshou.Core.Outcome (Outcome)
import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellRunResult (..), CellStatus)
import Kenshou.Remote.Cell.Fetch (VerifyProblem (..), effectiveOutcome, verifyCellRun, verifyCellRunWithStatus)
import Kenshou.Remote.Store (Bucket (..))
import System.Directory (doesFileExist, removeFile, renameFile)
import System.FilePath (takeDirectory, (</>))
import System.IO (hClose, openBinaryTempFile)

data CellManifestLink = CellManifestLink
  { path :: !Text,
    sha256 :: !Text,
    bytes :: !Int64
  }
  deriving stock (Eq, Show)

data EvidenceLink = EvidenceLink
  { kind :: !Text,
    path :: !Text,
    sha256 :: !Text,
    bytes :: !Int64,
    mediaType :: !Text
  }
  deriving stock (Eq, Show)

data RunLink = RunLink
  { runId :: !RunId,
    path :: !Text,
    scenario :: !Text,
    manifestSha256 :: !Text,
    recordedOutcome :: !Outcome,
    effectiveOutcome :: !Outcome,
    overrideReason :: !(Maybe Text)
  }
  deriving stock (Eq, Show)

data CellRunIndex = CellRunIndex
  { cellRun :: !RunId,
    cell :: !Text,
    leaseId :: !RunId,
    leaseSequence :: !Int,
    resultsBaseUri :: !Text,
    dataBaseUri :: !Text,
    cellManifest :: !CellManifestLink,
    cellOutcome :: !CellOutcome,
    reasons :: ![Text],
    entryExitCode :: !(Maybe Int),
    evidence :: ![EvidenceLink],
    runs :: ![RunLink],
    verifiedAt :: !UTCTime
  }
  deriving stock (Eq, Show)

instance ToJSON CellManifestLink where
  toJSON link = object ["path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes]

instance FromJSON CellManifestLink where
  parseJSON = withObject "cell manifest link" \value -> CellManifestLink <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes"

instance ToJSON EvidenceLink where
  toJSON link = object ["kind" .= link.kind, "path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes, "mediaType" .= link.mediaType]

instance FromJSON EvidenceLink where
  parseJSON = withObject "cell evidence link" \value -> EvidenceLink <$> value .: "kind" <*> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"

instance ToJSON RunLink where
  toJSON run = object ["runId" .= run.runId, "path" .= run.path, "scenario" .= run.scenario, "manifestSha256" .= run.manifestSha256, "recordedOutcome" .= run.recordedOutcome, "effectiveOutcome" .= run.effectiveOutcome, "overrideReason" .= run.overrideReason]

instance FromJSON RunLink where
  parseJSON = withObject "cell nested run" \value -> RunLink <$> value .: "runId" <*> value .: "path" <*> value .: "scenario" <*> value .: "manifestSha256" <*> value .: "recordedOutcome" <*> value .: "effectiveOutcome" <*> value .:? "overrideReason"

instance ToJSON CellRunIndex where
  toJSON index =
    object
      [ "schema" .= ("kenshou.cell-run/v1" :: Text),
        "cellRun" .= index.cellRun,
        "cell" .= index.cell,
        "leaseId" .= index.leaseId,
        "leaseSequence" .= index.leaseSequence,
        "resultsBaseUri" .= index.resultsBaseUri,
        "dataBaseUri" .= index.dataBaseUri,
        "cellManifest" .= index.cellManifest,
        "cellOutcome" .= index.cellOutcome,
        "reasons" .= index.reasons,
        "entryExitCode" .= index.entryExitCode,
        "evidence" .= index.evidence,
        "runs" .= index.runs,
        "verifiedAt" .= index.verifiedAt
      ]

instance FromJSON CellRunIndex where
  parseJSON = withObject "cell run index" \value -> do
    schema <- value .: "schema"
    if schema /= ("kenshou.cell-run/v1" :: Text) then fail "unsupported cell run index" else pure ()
    CellRunIndex <$> value .: "cellRun" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "resultsBaseUri" <*> value .: "dataBaseUri" <*> value .: "cellManifest" <*> value .: "cellOutcome" <*> value .: "reasons" <*> value .:? "entryExitCode" <*> value .: "evidence" <*> value .: "runs" <*> value .: "verifiedAt"

deriveCellRunIndex :: Bucket -> Maybe CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellRunIndex)
deriveCellRunIndex bucket status tree = do
  verified <- case status of
    Nothing -> verifyCellRun tree
    Just sealed -> verifyCellRunWithStatus sealed tree
  case verified of
    Left problems -> pure (Left problems)
    Right manifest -> do
      resultBytes <- LazyByteString.readFile (tree </> "cell" </> "result.json")
      case eitherDecode resultBytes :: Either String CellRunResult of
        Left failure -> pure (Left (InvalidManifest ("cell/result.json: " <> Text.pack failure) :| []))
        Right cellResult -> do
          manifestBytes <- LazyByteString.readFile (tree </> "manifest.json")
          let base = "gs://" <> bucket.unBucket <> "/runs/" <> renderRunId manifest.runId
              linked = CellManifestLink "manifest.json" (digest manifestBytes) (LazyByteString.length manifestBytes)
              evidence = [EvidenceLink kind artifact.path artifact.sha256 artifact.bytes artifact.mediaType | artifact <- manifest.artifacts, Just kind <- [evidenceKind artifact.path]]
          indexed <- forM (nestedRuns manifest.artifacts) \(identifier, nestedPath) -> do
            bytes <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "run-result.json")
            nestedManifest <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "manifest.json")
            pure case runFields bytes of
              Left failure -> Left (RunResultMismatch (nestedPath <> ": " <> failure))
              Right (scenario, recorded) ->
                let effective = effectiveOutcome manifest.outcome True recorded
                    override = if manifest.outcome == InfrastructureFailure || effective /= recorded then Just (reasonFor cellResult.reasons) else Nothing
                 in Right (RunLink identifier nestedPath scenario (digest nestedManifest) recorded effective override)
          case sequence indexed of
            Left problem -> pure (Left (problem :| []))
            Right runs -> do
              now <- getCurrentTime
              pure (Right (CellRunIndex manifest.runId manifest.cell manifest.leaseId manifest.leaseSequence base (base <> "/output") linked manifest.outcome cellResult.reasons cellResult.entryExitCode evidence runs now))

writeCellRunIndex :: FilePath -> CellRunIndex -> IO FilePath
writeCellRunIndex tree index = do
  let destination = takeDirectory tree </> "cell-run.json"
  bracket (openBinaryTempFile (takeDirectory tree) ".kenshou-cell-run-") cleanup \(temporary, handle) -> do
    hClose handle
    LazyByteString.writeFile temporary (encode index)
    renameFile temporary destination
    pure destination
  where
    cleanup (temporary, _) = do
      exists <- doesFileExist temporary
      if exists then removeFile temporary else pure ()

nestedRuns :: [Artifact] -> [(RunId, Text)]
nestedRuns artifacts =
  [ (identifier, "output/" <> renderRunId identifier)
  | artifact <- artifacts,
    ["output", rendered, "manifest.json"] <- [Text.splitOn "/" artifact.path],
    Right identifier <- [parseRunId rendered]
  ]

runFields :: LazyByteString.ByteString -> Either Text (Text, Outcome)
runFields bytes = case eitherDecode bytes :: Either String Value of
  Left failure -> Left (Text.pack failure)
  Right (Aeson.Object fields) -> case (KeyMap.lookup "scenario" fields, KeyMap.lookup "outcome" fields) of
    (Just (Aeson.String scenario), Just verdict) -> case Aeson.fromJSON verdict of
      Aeson.Success outcome -> Right (scenario, outcome)
      Aeson.Error failure -> Left (Text.pack failure)
    _ -> Left "scenario or outcome is missing"
  Right _ -> Left "run result is not an object"

evidenceKind :: Text -> Maybe Text
evidenceKind "cell/fingerprint.json" = Just "cell-fingerprint"
evidenceKind "cell/health.json" = Just "cell-health"
evidenceKind "cell/reset-evidence.json" = Just "cell-reset-evidence"
evidenceKind "metrics/export.jsonl.zst" = Just "cell-metrics-export"
evidenceKind _ = Nothing

reasonFor :: [Text] -> Text
reasonFor [] = "cell-infrastructure-failure"
reasonFor reasons = Text.intercalate "; " reasons

digest :: LazyByteString.ByteString -> Text
digest = TextEncoding.decodeUtf8 . Base16.encode . SHA256.hashlazy