packages feed

kenshou-evidence-0.1.0.0: src/Kenshou/Evidence/Source.hs

module Kenshou.Evidence.Source
  ( SourceError (..),
    VerifiedFile (..),
    RunResultView (..),
    CellEvidence (..),
    RunSource (..),
    ComparisonView (..),
    ComparisonSource (..),
    loadRunSource,
    loadComparisonSource,
  )
where

import Control.Exception (IOException, try)
import Control.Monad (forM, forM_, unless, when)
import Control.Monad.Trans.Class (lift)
import Control.Monad.Trans.Except (ExceptT, runExceptT, throwE)
import Data.Aeson (FromJSON (..), Value (..), eitherDecodeStrict', withObject, (.:), (.:?))
import Data.Aeson qualified as Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.List (sort)
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime)
import Data.Word (Word64)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Cohort (CohortIdentity)
import Kenshou.Core.Id (RunId, ScenarioId, renderRunId, unSeed)
import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))
import Kenshou.Core.Outcome (Outcome)
import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..), SpecPlacement (..))
import Kenshou.Evidence.Types (Sha256, sha256Bytes)
import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..))
import Kenshou.Remote.Cell.Fetch qualified as CellFetch
import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), RunLink (..))
import Numeric.Natural (Natural)
import System.Directory (canonicalizePath, doesDirectoryExist, doesFileExist, listDirectory, pathIsSymbolicLink)
import System.FilePath (isAbsolute, normalise, splitDirectories, takeDirectory, takeFileName, (</>))

newtype SourceError = SourceError Text deriving stock (Eq, Show)

data VerifiedFile = VerifiedFile
  { relativePath :: !FilePath,
    digest :: !Sha256,
    bytes :: !Natural,
    mediaType :: !Text
  }
  deriving stock (Eq, Show)

-- RunResult has no FromJSON instance in the kernel. This reader keeps the
-- evidence-facing fields typed while leaving the large diagnostic payload to
-- the versioned run-result document itself.
data RunResultView = RunResultView
  { resultRunId :: !RunId,
    resultScenario :: !ScenarioId,
    resultOutcome :: !Outcome,
    resultTier :: !Text,
    resultSeed :: !Word64,
    resultSpecSha256 :: !Text,
    resultStartedAt :: !UTCTime,
    resultEndedAt :: !UTCTime,
    resultCohort :: !CohortIdentity,
    resultFingerprint :: !Value,
    resultCompatibility :: !Value,
    resultSummaries :: !(Maybe Value),
    resultKnownDefect :: !(Maybe Value)
  }
  deriving stock (Eq, Show)

instance FromJSON RunResultView where
  parseJSON = withObject "RunResult" \value -> do
    schema <- value .: "schema"
    unless (schema == ("kenshou.run-result/v1" :: Text)) (fail "unsupported run-result schema")
    spec <- value .: "spec"
    timings <- value .: "timings"
    RunResultView
      <$> value .: "runId"
      <*> value .: "scenario"
      <*> value .: "outcome"
      <*> value .: "tier"
      <*> value .: "seed"
      <*> spec .: "sha256"
      <*> timings .: "startedAt"
      <*> timings .: "endedAt"
      <*> value .: "cohort"
      <*> value .: "fingerprint"
      <*> value .: "compatibility"
      <*> value .:? "summaries"
      <*> value .:? "knownDefect"

data CellEvidence = CellEvidence
  { cellManifest :: !CellManifest,
    cellManifestPath :: !FilePath,
    cellManifestFile :: !VerifiedFile,
    cellEffectiveOutcome :: !Outcome
  }
  deriving stock (Eq, Show)

data RunSource = RunSource
  { manifest :: !Manifest,
    spec :: !RunSpec,
    result :: !RunResultView,
    files :: ![VerifiedFile],
    cellEvidence :: !(Maybe CellEvidence)
  }
  deriving stock (Eq, Show)

data ComparisonView = ComparisonView
  { comparisonId :: !RunId,
    baselineRuns :: ![RunId],
    candidateRuns :: ![RunId],
    startedAt :: !UTCTime,
    finishedAt :: !UTCTime,
    harnessRevision :: !(Maybe Text),
    harnessDirty :: !(Maybe Bool),
    design :: !Text,
    variedFactors :: ![Text],
    pairCount :: !Int,
    verdict :: !Text
  }
  deriving stock (Eq, Show)

data ComparisonSource = ComparisonSource
  { view :: !ComparisonView,
    file :: !VerifiedFile
  }
  deriving stock (Eq, Show)

instance FromJSON ComparisonView where
  parseJSON = withObject "Comparison" \value -> do
    schema <- value .: "schema"
    unless (schema == ("kenshou.comparison/v1" :: Text)) (fail "unsupported comparison schema")
    ComparisonView
      <$> value .: "comparisonId"
      <*> value .: "baselineRuns"
      <*> value .: "candidateRuns"
      <*> value .: "startedAt"
      <*> value .: "finishedAt"
      <*> value .:? "harnessRevision"
      <*> value .:? "harnessDirty"
      <*> value .: "design"
      <*> value .: "variedFactors"
      <*> value .: "pairCount"
      <*> value .: "verdict"

loadComparisonSource :: FilePath -> IO (Either SourceError ComparisonSource)
loadComparisonSource path = do
  loaded <- try (runExceptT (loadComparison path)) :: IO (Either IOException (Either SourceError ComparisonSource))
  pure $ either (Left . SourceError . Text.pack . show) id loaded

loadComparison :: FilePath -> ExceptT SourceError IO ComparisonSource
loadComparison path = do
  symlink <- lift (pathIsSymbolicLink path)
  when symlink (reject "comparison document is a symbolic link")
  contents <- lift (ByteString.readFile path)
  view <- either (reject . Text.pack) pure (eitherDecodeStrict' contents)
  when (view.pairCount <= 0 || view.pairCount /= length view.baselineRuns || view.pairCount /= length view.candidateRuns) (reject "comparison pair count and arm IDs disagree")
  when (view.startedAt > view.finishedAt) (reject "comparison ends before it starts")
  when (null view.variedFactors) (reject "comparison has no varied factor")
  when (view.design `notElem` ["abba", "baab", "sequential"]) (reject "comparison has an unknown design")
  when (view.verdict `notElem` ["pass", "regression", "inconclusive", "infrastructure-failure"]) (reject "comparison has an unknown verdict")
  when (Set.size (Set.fromList (view.baselineRuns <> view.candidateRuns)) /= view.pairCount * 2) (reject "comparison repeats a run ID")
  let file = VerifiedFile "comparison.json" (sha256Bytes contents) (fromIntegral (ByteString.length contents)) "application/json"
  pure ComparisonSource {view, file}

loadRunSource :: FilePath -> IO (Either SourceError RunSource)
loadRunSource root = do
  loaded <- try (runExceptT (load root)) :: IO (Either IOException (Either SourceError RunSource))
  pure $ either (Left . SourceError . Text.pack . show) id loaded

load :: FilePath -> ExceptT SourceError IO RunSource
load root = do
  manifest <- readJson (root </> "manifest.json")
  spec <- readJson (root </> "run-spec.json")
  files <- verifyFiles root manifest (spec.environment.placement == RunOnCell)
  result <- readJson (root </> "run-result.json")
  when (manifest.runId /= result.resultRunId) (reject "manifest and run result identify different runs")
  when (spec.runId /= Just result.resultRunId) (reject "run spec and run result identify different runs")
  when (spec.scenario /= result.resultScenario) (reject "run spec and run result identify different scenarios")
  when (fmap unSeed spec.seed /= Just result.resultSeed) (reject "run spec and run result use different seeds")
  when (result.resultStartedAt > result.resultEndedAt) (reject "run result ends before it starts")
  specBytes <- lift (ByteString.readFile (root </> "run-spec.json"))
  when (result.resultSpecSha256 /= sha256Hex specBytes) (reject "run result does not match run-spec.json")
  cellEvidence <- case spec.environment.placement of
    RunLocal -> pure Nothing
    RunOnCell -> Just <$> loadCellEvidence root result
  pure RunSource {manifest, spec, result, files, cellEvidence}

loadCellEvidence :: FilePath -> RunResultView -> ExceptT SourceError IO CellEvidence
loadCellEvidence root result = do
  realRoot <- lift (canonicalizePath root)
  let sidecar = root </> "cell-manifest.json"
      fetched = takeDirectory (takeDirectory realRoot) </> "manifest.json"
  hasSidecar <- lift (doesFileExist sidecar)
  hasFetched <- lift (doesFileExist fetched)
  path <- case (hasSidecar, hasFetched) of
    (True, _) -> pure sidecar
    (_, True) -> pure fetched
    _ -> reject "cell run has no outer cell manifest"
  symlink <- lift (pathIsSymbolicLink path)
  when symlink (reject "outer cell manifest is a symbolic link")
  bytes <- lift (ByteString.readFile path)
  cellManifest <- either (reject . Text.pack) pure (eitherDecodeStrict' bytes)
  (namedCell, namedRun) <- case result.resultFingerprint of
    Object fields -> case KeyMap.lookup "cell" fields of
      Just (Object cellFields) -> do
        cell <- parseField "fingerprint.cell.cell" =<< maybe (reject "fingerprint.cell.cell is missing") pure (KeyMap.lookup "cell" cellFields)
        cellRun <- parseField "fingerprint.cell.cellRun" =<< maybe (reject "fingerprint.cell.cellRun is missing") pure (KeyMap.lookup "cellRun" cellFields)
        pure (cell, cellRun)
      _ -> reject "cell run has no fingerprint.cell"
    _ -> reject "cell run has no fingerprint object"
  when (cellManifest.cell /= namedCell || cellManifest.runId /= namedRun) (reject "outer cell manifest differs from nested cell fingerprint")
  let nestedPrefix = "output/" <> renderRunId result.resultRunId <> "/"
  forM_ ["manifest.json", "run-result.json", "run-spec.json"] \name -> do
    let nestedPath = root </> Text.unpack name
    nestedBytes <- lift (ByteString.readFile nestedPath)
    case [artifact | artifact <- cellManifest.artifacts, artifact.path == nestedPrefix <> name] of
      [artifact]
        | Just artifact.sha256 == Text.stripPrefix "sha256:" (sha256Hex nestedBytes) && artifact.bytes == fromIntegral (ByteString.length nestedBytes) -> pure ()
      _ -> reject ("outer cell manifest does not pin nested " <> name)
  let cellManifestFile = VerifiedFile "cell-manifest.json" (sha256Bytes bytes) (fromIntegral (ByteString.length bytes)) "application/json"
      cellEffectiveOutcome = CellFetch.effectiveOutcome cellManifest.outcome True result.resultOutcome
      indexPath = takeDirectory (takeDirectory (takeDirectory realRoot)) </> "cell-run.json"
  hasIndex <- lift (doesFileExist indexPath)
  when hasIndex do
    index <- readJson indexPath :: ExceptT SourceError IO CellRunIndex
    when (index.cellRun /= cellManifest.runId || index.cell /= cellManifest.cell || index.cellManifest.sha256 /= Text.drop 7 (sha256Hex bytes)) (reject "cell-run index disagrees with the outer manifest")
    case [run | run <- index.runs, run.runId == result.resultRunId] of
      [run] | run.recordedOutcome == result.resultOutcome && run.effectiveOutcome == cellEffectiveOutcome -> pure ()
      _ -> reject "cell-run index disagrees with the nested effective outcome"
  pure CellEvidence {cellManifest, cellManifestPath = path, cellManifestFile, cellEffectiveOutcome}
  where
    parseField :: (FromJSON value) => Text -> Value -> ExceptT SourceError IO value
    parseField label value = case Aeson.fromJSON value of
      Aeson.Error problem -> reject (label <> ": " <> Text.pack problem)
      Aeson.Success parsed -> pure parsed

readJson :: (FromJSON value) => FilePath -> ExceptT SourceError IO value
readJson path = do
  contents <- lift (ByteString.readFile path)
  either (\message -> reject (Text.pack (takeFileName path) <> ": " <> Text.pack message)) pure (eitherDecodeStrict' contents)

verifyFiles :: FilePath -> Manifest -> Bool -> ExceptT SourceError IO [VerifiedFile]
verifyFiles root manifest cellRun = do
  let listed = map (.path) manifest.files
  when (length listed /= Set.size (Set.fromList listed)) (reject "manifest has duplicate file paths")
  verified <- forM manifest.files $ \file -> do
    unless (safeRelative file.path) (reject ("unsafe manifest path: " <> Text.pack file.path))
    let path = root </> file.path
    symlink <- lift (pathIsSymbolicLink path)
    when symlink (reject ("manifest path is a symbolic link: " <> Text.pack file.path))
    exists <- lift (doesFileExist path)
    unless exists (reject ("manifest file is missing: " <> Text.pack file.path))
    contents <- lift (ByteString.readFile path)
    when (file.sha256 /= sha256Hex contents) (reject ("digest mismatch: " <> Text.pack file.path))
    when (file.bytes /= fromIntegral (ByteString.length contents)) (reject ("size mismatch: " <> Text.pack file.path))
    when (Text.null file.mediaType) (reject ("missing media type: " <> Text.pack file.path))
    pure VerifiedFile {relativePath = file.path, digest = sha256Bytes contents, bytes = fromIntegral (ByteString.length contents), mediaType = file.mediaType}
  actual <- scanFiles root ""
  let sidecars = if cellRun then ["cell-manifest.json"] else []
  unless (sort listed == sort (filter (`notElem` ("manifest.json" : sidecars)) actual)) (reject "manifest does not enumerate every run file exactly once")
  pure verified

scanFiles :: FilePath -> FilePath -> ExceptT SourceError IO [FilePath]
scanFiles root relative = do
  let directory = root </> relative
  names <- lift (listDirectory directory)
  concat <$> forM names (visit directory)
  where
    visit directory name = do
      let relativePath = if null relative then name else relative </> name
          path = directory </> name
      unless (safeRelative relativePath) (reject ("unsafe run file path: " <> Text.pack relativePath))
      symlink <- lift (pathIsSymbolicLink path)
      when symlink (reject ("run directory contains a symbolic link: " <> Text.pack relativePath))
      isDirectory <- lift (doesDirectoryExist path)
      if isDirectory
        then scanFiles root relativePath
        else do
          regular <- lift (doesFileExist path)
          unless regular (reject ("run directory contains a non-file: " <> Text.pack relativePath))
          pure [relativePath]

safeRelative :: FilePath -> Bool
safeRelative path =
  not (null path)
    && not (isAbsolute path)
    && normalise path == path
    && all (\segment -> segment /= "" && segment /= "." && segment /= "..") (splitDirectories path)
    && '\\' `notElem` path

reject :: Text -> ExceptT SourceError IO value
reject = throwE . SourceError