packages feed

kenshou-evidence-0.1.0.0: src/Kenshou/Evidence/Check.hs

module Kenshou.Evidence.Check
  ( CheckOptions (..),
    CheckError (..),
    Finding (..),
    checkBundle,
    checkBundleWithStore,
    checkDocument,
  )
where

import Control.Exception (IOException, try)
import Control.Monad (forM)
import Data.Aeson (Result (..), Value (..), fromJSON)
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.List (isPrefixOf, nub, sort)
import Data.Map.Strict qualified as Map
import Data.Maybe (mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text.IO
import Data.Time (UTCTime, defaultTimeLocale, formatTime, parseTimeM)
import Kenshou.Core.Id (ScenarioId (..), parseRunId, parseScenarioId, renderLayer)
import Kenshou.Evidence.Store (ObjectStat (..), ObjectStore (..), validateObjectUri)
import Kenshou.Evidence.Types (mkRevision, mkSha256, sha256Bytes)
import Okf.Actor (renderActor)
import Okf.Document (OKFDocument (..), Verification (..), frontmatterKeys, frontmatterLookup, parseDocument, readVerified, removeField)
import System.Directory (canonicalizePath, doesDirectoryExist, listDirectory)
import System.Exit (ExitCode (..))
import System.FilePath (makeRelative, splitDirectories, takeExtension, (</>))
import System.IO.Temp (withSystemTempDirectory)
import System.Process (readProcessWithExitCode)

data CheckOptions = CheckOptions
  { bundleRoot :: !FilePath,
    baseRef :: !(Maybe Text),
    network :: !Bool,
    deep :: !Bool
  }
  deriving stock (Eq, Show)

newtype CheckError = CheckError Text deriving stock (Eq, Show)

data Finding = Finding
  { concept :: !FilePath,
    rule :: !Text,
    message :: !Text
  }
  deriving stock (Eq, Show)

checkBundle :: CheckOptions -> IO (Either CheckError [Finding])
checkBundle = checkBundleWithStore Nothing

checkBundleWithStore :: Maybe ObjectStore -> CheckOptions -> IO (Either CheckError [Finding])
checkBundleWithStore store options = do
  result <- try (check store options) :: IO (Either IOException (Either CheckError [Finding]))
  pure $ either (Left . CheckError . Text.pack . show) id result

check :: Maybe ObjectStore -> CheckOptions -> IO (Either CheckError [Finding])
check store options = do
  exists <- doesDirectoryExist options.bundleRoot
  if not exists
    then pure (Left (CheckError "evidence bundle does not exist"))
    else do
      paths <- sort <$> conceptFiles options.bundleRoot
      current <- forM paths $ \path -> do
        content <- Text.IO.readFile (options.bundleRoot </> path)
        pure (path, content)
      let local =
            concatMap
              ( \(path, content) -> case parseDocument content of
                  Left err -> [Finding path "parse" (Text.pack (show err))]
                  Right document -> checkDocument path document
              )
              current
          references = checkReferences current
      historical <- checkHistory options
      remote <-
        if options.network
          then case store of
            Nothing -> pure (Left (CheckError "--network requires an object store and GCP project"))
            Just objectStore -> checkNetwork objectStore options.deep current
          else pure (Right [])
      pure $ do
        historyFindings <- historical
        remoteFindings <- remote
        pure (local <> references <> historyFindings <> remoteFindings)

checkReferences :: [(FilePath, Text)] -> [Finding]
checkReferences current = concatMap inspect parsed <> concatMap attestationConsistency parsed
  where
    parsed = [(path, document) | (path, content) <- current, Right document <- [parseDocument content]]
    byPath = Map.fromList parsed
    field document name = frontmatterLookup name document.frontmatter
    asText (Just (String value)) = Just value
    asText _ = Nothing
    nested name = \case
      Just (Object fields) -> KeyMap.lookup (Key.fromText name) fields
      _ -> Nothing
    paths = \case
      Just (Array values) -> [value | String value <- foldr (:) [] values]
      _ -> []
    target path scenario ordinaryRunOnly ref =
      let relative = Text.unpack (Text.dropWhile (== '/') ref)
       in case Map.lookup relative byPath of
            Nothing -> [Finding path "reference-targets" ("missing run target: " <> ref)]
            Just document | field document "type" /= Just (String "Verification Run") || (ordinaryRunOnly && field document "recordKind" /= Just (String "run")) -> [Finding path "reference-targets" ("target is not a recorded run: " <> ref)]
            Just document -> [Finding path "reference-targets" ("target has a different scenario: " <> ref) | scenario /= Nothing, field document "scenario" /= scenario]
    inspect (path, document)
      | field document "type" == Just (String "Verification Run") =
          let previous = maybe [] (\ref -> target path (field document "scenario") True ref <> checkPrevious path document ref) (asText (field document "previousRun"))
              comparison = field document "comparison"
              arms = paths (nested "baselineRuns" comparison) <> paths (nested "candidateRuns" comparison)
           in previous <> concatMap (target path (field document "scenario") True) arms
      | field document "type" == Just (String "Attestation") =
          maybe [] (target path Nothing False) (asText (field document "run"))
      | otherwise = []
    attestationConsistency (path, document)
      | field document "type" == Just (String "Attestation") =
          let checks = case field document "checks" of
                Just (Array values) -> foldr (:) [] values
                _ -> []
              names = mapMaybe (\value -> asText (nested "name" (Just value))) checks
              required = ["digests-match", "revisions-resolve", "cohort-matches-plan", "verdict-recomputed", "environment-captured", "clean-worktree"]
              outcomes = [(name, asText (nested "result" (Just value))) | value <- checks, Just name <- [asText (nested "name" (Just value))]]
              contradicted = any (\name -> lookup name outcomes == Just (Just "failed")) ["digests-match", "cohort-matches-plan", "verdict-recomputed"]
              complete = all (\(_, result) -> result == Just "passed") outcomes
              expected = if contradicted then "refuted" else if complete then "confirmed" else "incomplete"
              matched = sort names == sort required
              verdictIssue = [Finding path "attestation-consistency" "attestation verdict disagrees with its checks" | matched, field document "verdict" /= Just (String expected)]
              namesIssue = [Finding path "attestation-consistency" "attestation must have all six distinct check names" | not matched]
              runReference = asText (field document "run")
              digestsIssue = case runReference >>= (\ref -> Map.lookup (Text.unpack (Text.dropWhile (== '/') ref)) byPath) of
                Just runDocument
                  | lookup "digests-match" outcomes == Just (Just "passed") ->
                      let recorded = case field runDocument "data" of
                            Just (Array values) -> sort (mapMaybe (\value -> asText (nested "digest" (Just value))) (foldr (:) [] values))
                            _ -> []
                          attested = case field document "dataDigests" of
                            Just (Array values) -> sort [value | String value <- foldr (:) [] values]
                            _ -> []
                       in [Finding path "attestation-consistency" "dataDigests must cover the run's linked data" | not (all (`elem` attested) recorded)]
                _ -> []
              verifiedIssue = case runReference >>= (\ref -> Map.lookup (Text.unpack (Text.dropWhile (== '/') ref)) byPath) of
                Just runDocument
                  | field document "verdict" == Just (String "confirmed") ->
                      let actor = asText (field document "attester")
                          backed = any (\entry -> Just (renderActor entry.verificationBy) == actor) (readVerified runDocument.frontmatter)
                       in [Finding path "attestation-consistency" "confirmed attestation has no matching verified entry on the run" | not backed]
                _ -> []
           in namesIssue <> verdictIssue <> digestsIssue <> verifiedIssue
      | field document "type" == Just (String "Verification Run") =
          let processEntries = [(renderActor entry.verificationBy, entry.verificationAt) | entry <- readVerified document.frontmatter, "process:" `Text.isPrefixOf` renderActor entry.verificationBy]
              matching actor at = any (\(_, attestation) -> field attestation "type" == Just (String "Attestation") && field attestation "run" == Just (String ("/" <> Text.pack path)) && field attestation "verdict" == Just (String "confirmed") && field attestation "attester" == Just (String actor) && field attestation "attestedAt" == (String <$> at)) parsed
           in [Finding path "attestation-consistency" "verified process entry has no matching confirmed attestation" | (actor, at) <- processEntries, not (matching actor at)]
      | otherwise = []
    checkPrevious path document ref =
      let relative = Text.unpack (Text.dropWhile (== '/') ref)
       in case Map.lookup relative byPath of
            Nothing -> []
            Just previous ->
              [ Finding path "reference-targets" "previousRun must have the same scenario and compatibility key, and start earlier"
              | field document "scenario" /= field previous "scenario"
                  || field document "compatibilityKey" /= field previous "compatibilityKey"
                  || maybe
                    True
                    not
                    ( do
                        now <- asText (field document "startedAt") >>= parseUtc
                        thenTime <- asText (field previous "startedAt") >>= parseUtc
                        pure (thenTime < now)
                    )
              ]

checkNetwork :: ObjectStore -> Bool -> [(FilePath, Text)] -> IO (Either CheckError [Finding])
checkNetwork store deep current = withSystemTempDirectory "kenshou-evidence-network" $ \scratch -> do
  results <- forM current $ \(path, content) -> case parseDocument content of
    Left _ -> pure (Right [])
    Right document -> do
      let links = case frontmatterLookup "data" document.frontmatter of
            Just (Array values) -> foldr (:) [] values
            _ -> []
      fmap (fmap concat . sequence) $ forM links $ \link -> case networkLink link of
        Nothing -> pure (Right [])
        Just (uri, digest, expectedBytes) -> do
          observed <- store.statObject uri
          case observed of
            Left err -> pure (Left (CheckError (Text.pack (show err))))
            Right Nothing -> pure (Right [Finding path "network" ("object is missing: " <> uri)])
            Right (Just stat) ->
              if stat.bytes /= expectedBytes || stat.recordedSha256 /= Just digest
                then pure (Right [Finding path "network" ("object metadata differs: " <> uri)])
                else
                  if not deep
                    then pure (Right [])
                    else do
                      let destination = scratch </> "object"
                      fetched <- store.fetchObject uri destination
                      case fetched of
                        Left err -> pure (Left (CheckError (Text.pack (show err))))
                        Right () -> do
                          actual <- sha256Bytes <$> ByteString.readFile destination
                          pure (Right [Finding path "network" ("object bytes differ: " <> uri) | actual /= digest])
  pure (concat <$> sequence results)
  where
    networkLink (Object fields) = do
      String uri <- KeyMap.lookup "uri" fields
      String rawDigest <- KeyMap.lookup "digest" fields
      digest <- either (const Nothing) Just (mkSha256 rawDigest)
      rawBytes <- KeyMap.lookup "bytes" fields
      bytes <- case fromJSON rawBytes of
        Success parsed -> Just parsed
        Error _ -> Nothing
      pure (uri, digest, bytes)
    networkLink _ = Nothing

conceptFiles :: FilePath -> IO [FilePath]
conceptFiles root = do
  let descend relative = do
        let directory = root </> relative
        names <- listDirectory directory
        fmap concat $ forM names $ \name -> do
          let child = relative </> name
          isDirectory <- doesDirectoryExist (root </> child)
          if isDirectory then descend child else pure [child | takeExtension name == ".md", name /= "index.md", name /= "log.md"]
  runs <- doesDirectoryExist (root </> "runs")
  attestations <- doesDirectoryExist (root </> "attestations")
  (if runs then descend "runs" else pure []) >>= \runFiles ->
    (if attestations then descend "attestations" else pure []) >>= \attestationFiles ->
      pure (runFiles <> attestationFiles)

checkDocument :: FilePath -> OKFDocument -> [Finding]
checkDocument path document =
  let front = document.frontmatter
      field = (`frontmatterLookup` front)
      issue name detail = Finding path name detail
      textField name = case field name of
        Just (String value) -> Just value
        _ -> Nothing
      requireText name = [issue "string-typing" (name <> " must be a JSON string") | textField name == Nothing]
      digest name = case textField name of
        Nothing -> []
        Just value -> [issue "hex-shape" (name <> " must be 64 lowercase hexadecimal characters") | either (const True) (const False) (mkSha256 value)]
      revision name = case textField name of
        Nothing -> []
        Just value -> [issue "hex-shape" (name <> " must be 40 lowercase hexadecimal characters") | either (const True) (const False) (mkRevision value)]
      entries name = case field name of
        Just (Array values) -> toList values
        _ -> []
      member name value = case value of
        Object fields -> KeyMap.lookup (Key.fromText name) fields
        _ -> Nothing
      entryText name value = case member name value of
        Just (String result) -> Just result
        _ -> Nothing
      run = field "type" == Just (String "Verification Run")
      attestation = field "type" == Just (String "Attestation")
      common =
        [issue "event-keys" (name <> " is forbidden on an event") | (run || attestation), name <- ["status", "stale_after"], field name /= Nothing]
      runChecks =
        requireText "runId"
          <> requireText "scenario"
          <> requireText "startedAt"
          <> requireText "finishedAt"
          <> [issue "id-shape" "runId must be a UUIDv7" | maybe True (either (const True) (const False) . parseRunId) (textField "runId")]
          <> pathChecks textField issue
          <> timeChecks textField issue
          <> requireText "harnessRevision"
          <> revision "harnessRevision"
          <> [issue "dirty-purpose" "a dirty harness requires purpose: investigation" | field "harnessDirty" == Just (Bool True), field "purpose" /= Just (String "investigation")]
          <> concatMap
            ( \value ->
                [issue "string-typing" "data.uri must be a JSON string" | entryText "uri" value == Nothing]
                  <> [issue "data-uri" "data.uri must use a durable gs:// object URI" | Just uri <- [entryText "uri" value], either (const True) (const False) (validateObjectUri uri)]
                  <> [issue "hex-shape" "data.digest must be 64 lowercase hexadecimal characters" | maybe True (either (const True) (const False) . mkSha256) (entryText "digest" value)]
            )
            (entries "data")
          <> (if field "recordKind" == Just (String "run") then runOnly else [])
          <> (if field "recordKind" == Just (String "comparison") then comparisonOnly else [])
      runOnly =
        requireText "compatibilityKey"
          <> requireText "solverPlanHash"
          <> digest "compatibilityKey"
          <> digest "solverPlanHash"
          <> concatMap
            ( \value ->
                [issue "string-typing" "dimension.value must be a JSON string" | entryText "value" value == Nothing]
                  <> [issue "string-typing" "dimension.name must be a JSON string" | entryText "name" value == Nothing]
            )
            (entries "dimensions")
          <> concatMap
            ( \value ->
                [issue "hex-shape" "component.revision must be 40 lowercase hexadecimal characters" | maybe False (either (const True) (const False) . mkRevision) (entryText "revision" value)]
            )
            (entries "components")
          <> [issue "data-completeness" "run must link manifest, run-spec and run-result" | field "recordKind" == Just (String "run"), not (all (`elem` mapMaybe (entryText "kind") (entries "data")) ["manifest", "run-spec", "run-result"])]
          <> [issue "data-completeness" "cell run must link exactly one outer cell manifest" | field "recordKind" == Just (String "run"), field "placement" == Just (String "cell"), length (filter (== Just "cell-manifest") (map (entryText "kind") (entries "data"))) /= 1]
          <> [ issue "cell-fields" "cell placement requires environment.cell and cellRun"
             | field "placement" == Just (String "cell"),
               not
                 ( all
                     ( \name -> case field "environment" of
                         Just environment -> maybe False isString (member name environment)
                         _ -> False
                     )
                     ["cell", "cellRun"]
                 )
             ]
      comparisonOnly =
        let comparison = field "comparison"
            verdict = comparison >>= member "verdict"
            expected = case verdict of
              Just (String "pass") -> Just (String "passed")
              Just (String "regression") -> Just (String "failed")
              Just (String "inconclusive") -> Just (String "inconclusive")
              Just (String "infrastructure-failure") -> Just (String "infrastructure-failure")
              _ -> Nothing
         in [issue "comparison-outcome" "comparison verdict and outcome disagree" | expected /= field "outcome"]
              <> [issue "data-completeness" "comparison must link exactly one comparison document" | mapMaybe (entryText "kind") (entries "data") /= ["comparison"]]
      attestationChecks =
        [issue "event-keys" (name <> " belongs only on runs") | name <- ["layer", "tier"], field name /= Nothing]
          <> requireText "attestationId"
          <> requireText "attesterRevision"
          <> requireText "attestedAt"
          <> revision "attesterRevision"
          <> [issue "id-shape" "attestationId must be a UUIDv7" | maybe True (either (const True) (const False) . parseRunId) (textField "attestationId")]
          <> ( case (textField "attestationId", textField "attestedAt") of
                 (Just identifier, Just at) -> case parseUtc at of
                   Just time ->
                     let expected = "attestations/" <> formatTime defaultTimeLocale "%Y/%m" time <> "/" <> Text.unpack identifier <> ".md"
                      in [issue "path-consistency" ("expected " <> Text.pack expected) | path /= expected]
                   Nothing -> [issue "time-order" "attestedAt must be RFC 3339 UTC"]
                 _ -> []
             )
          <> [issue "hex-shape" "dataDigests must be 64 lowercase hexadecimal characters" | value <- entries "dataDigests", case value of String digestValue -> either (const True) (const False) (mkSha256 digestValue); _ -> True]
   in common <> (if run then runChecks else []) <> (if attestation then attestationChecks else [])
  where
    isString (String _) = True
    isString _ = False
    toList = foldr (:) []
    pathChecks textField issue = case (textField "scenario", textField "runId", textField "startedAt") of
      (Just scenario, Just runId, Just startedAt) -> case (parseScenarioId scenario, parseUtc startedAt) of
        (Right parsed, Just time) ->
          let expected = "runs/" <> Text.unpack (renderLayer parsed.layer) <> "/" <> formatTime defaultTimeLocale "%Y/%m" time <> "/" <> Text.unpack runId <> ".md"
           in [issue "path-consistency" ("expected " <> Text.pack expected) | path /= expected]
        _ -> []
      _ -> []
    timeChecks textField issue = case (textField "startedAt", textField "finishedAt") of
      (Just start, Just finish) -> case (parseUtc start, parseUtc finish) of
        (Just from, Just to) -> [issue "time-order" "finishedAt precedes startedAt" | to < from]
        _ -> [issue "time-order" "run timestamps must be RFC 3339 UTC"]
      _ -> []

parseUtc :: Text -> Maybe UTCTime
parseUtc = parseTimeM True defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ" . Text.unpack

checkHistory :: CheckOptions -> IO (Either CheckError [Finding])
checkHistory options = do
  gitRoot <- readProcessWithExitCode "git" ["-C", options.bundleRoot, "rev-parse", "--show-toplevel"] ""
  case gitRoot of
    (ExitFailure _, _, _) -> pure (Right [])
    (ExitSuccess, root, _) -> do
      repo <- canonicalizePath (stripNewline root)
      bundle <- canonicalizePath options.bundleRoot
      let relative = makeRelative repo bundle
      if [".."] `isPrefixOf` splitDirectories relative
        then pure (Right [])
        else do
          shallow <- readProcessWithExitCode "git" ["-C", repo, "rev-parse", "--is-shallow-repository"] ""
          if shallow /= (ExitSuccess, "false\n", "")
            then pure (Left (CheckError "immutability requires full Git history"))
            else do
              let scope = [relative </> "runs", relative </> "attestations"]
                  range = maybe "HEAD" ((<> "..HEAD") . Text.unpack) options.baseRef
              history <- readProcessWithExitCode "git" (["-C", repo, "log", "--format=%H", "--diff-filter=DMR", "--name-status", range, "--"] <> scope) ""
              case history of
                (ExitFailure _, _, err) -> pure (Left (CheckError (Text.pack err)))
                (ExitSuccess, output, _) -> do
                  let changed = [(commit, columns) | (commit, columns) <- historyLines output, any (\item -> takeExtension item == ".md" && not (isGenerated item)) (drop 1 columns)]
                  committed <- fmap concat $ forM changed $ \(commit, columns) -> case columns of
                    status : path : _ -> do
                      let relativePath = makeRelative relative path
                      if "D" `isPrefixOf` status || "R" `isPrefixOf` status
                        then pure [Finding relativePath "immutability" ("committed deletion or rename in " <> Text.pack commit)]
                        else compareVersions repo relativePath (commit <> "^:" <> path) (commit <> ":" <> path)
                    _ -> pure []
                  worktree <- readProcessWithExitCode "git" (["-C", repo, "diff", "--name-status", "HEAD", "--"] <> scope) ""
                  staged <- case worktree of
                    (ExitSuccess, changes, _) -> fmap concat $ forM (lines changes) $ \line -> case words line of
                      status : path : _
                        | takeExtension path == ".md",
                          not (isGenerated path) ->
                            let relativePath = makeRelative relative path
                             in if "A" `isPrefixOf` status
                                  then pure []
                                  else
                                    if "D" `isPrefixOf` status || "R" `isPrefixOf` status
                                      then pure [Finding relativePath "immutability" "committed event was deleted or renamed in the working tree"]
                                      else compareVersions repo relativePath ("HEAD:" <> path) path
                      _ -> pure []
                    _ -> pure []
                  pure (Right (committed <> staged))
  where
    stripNewline = reverse . dropWhile (== '\n') . reverse
    isGenerated path = any (`elem` ["index.md", "log.md"]) (take 1 (reverse (splitDirectories path)))
    historyLines = go Nothing . lines
      where
        go _ [] = []
        go current (line : rest)
          | length line == 40 && all (`elem` ("0123456789abcdef" :: String)) line = go (Just line) rest
          | otherwise = case (current, words line) of
              (Just commit, columns@(_ : _)) -> (commit, columns) : go current rest
              _ -> go current rest
    compareVersions repo relativePath oldRef newRef = do
      old <- readVersion repo oldRef
      new <- readVersion repo newRef
      pure case (old, new) of
        (Just before, Just after) -> case (parseDocument before, parseDocument after) of
          (Right a, Right b) | allowedAppend a b -> []
          (Right a, Right b) -> [Finding relativePath "immutability" ("committed event changed: " <> Text.intercalate ", " (changedParts a b))]
          _ -> [Finding relativePath "immutability" ("committed event changed: " <> Text.pack newRef)]
        _ -> [Finding relativePath "immutability" ("committed event changed: " <> Text.pack newRef)]
    readVersion repo ref
      | ':' `elem` ref = do
          (status, output, _) <- readProcessWithExitCode "git" ["-C", repo, "show", ref] ""
          pure (if status == ExitSuccess then Just (Text.pack output) else Nothing)
      | otherwise = do
          result <- try (Text.IO.readFile (repo </> ref)) :: IO (Either IOException Text)
          pure (either (const Nothing) Just result)
    allowedAppend before after =
      before.body == after.body
        && removeVerified before == removeVerified after
        && verified before `isPrefixOf` verified after
    removeVerified document = removeField "verified" document.frontmatter
    verified document = case frontmatterLookup "verified" document.frontmatter of
      Just (Array values) -> foldr (:) [] values
      _ -> []
    changedParts before after =
      [name | name <- sort (nub (frontmatterKeys before.frontmatter <> frontmatterKeys after.frontmatter)), name /= "verified", frontmatterLookup name before.frontmatter /= frontmatterLookup name after.frontmatter]
        <> ["body" | before.body /= after.body]
        <> ["verified" | not (verified before `isPrefixOf` verified after)]