packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Fetch.hs

module Kenshou.Remote.Cell.Fetch
  ( FetchError (..),
    VerifyProblem (..),
    fetchCellRun,
    verifyCellTree,
    verifyCellRun,
    verifyCellRunWithStatus,
    effectiveOutcome,
  )
where

import Control.Exception (bracket)
import Control.Monad (forM)
import Crypto.Hash.SHA256 qualified as SHA256
import Data.Aeson (FromJSON, Value (..), eitherDecode)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.ByteString.Base16 qualified as Base16
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Int (Int64)
import Data.List (sort)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Maybe (catMaybes, listToMaybe)
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TextEncoding
import Kenshou.Core.Id (RunId, parseRunId, renderRunId)
import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..), verifyManifest)
import Kenshou.Core.Outcome qualified as Outcome
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))
import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellPhase (..), CellRunResult (..), CellStatus (..), ManifestPayload (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Docs qualified as Cell
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))
import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..))
import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist, listDirectory, pathIsSymbolicLink, removeFile, renameFile)
import System.FilePath (takeDirectory, (</>))
import System.IO (IOMode (ReadMode), hClose, openBinaryTempFile, withBinaryFile)

data VerifyProblem
  = InvalidManifest !Text
  | Missing !Text
  | Extra !Text
  | DigestMismatch !Text
  | SizeMismatch !Text
  | UnsafeEntry !Text
  | PayloadMismatch !Text
  | ManifestsDisagree !Text
  | NestedManifestProblem !Text
  | RunResultMismatch !Text
  | StatusDigestMismatch
  | StatusMismatch !Text
  deriving stock (Eq, Show)

data FetchError
  = Unsealed
  | ManifestProblem !Text
  | ObjectSetMismatch ![Text] ![Text] -- missing, extra
  | ObjectChanged !Text
  | ObjectCorrupt !Text
  | VerificationFailed !(NonEmpty VerifyProblem)
  deriving stock (Eq, Show)

effectiveOutcome :: Cell.CellOutcome -> Bool -> Outcome.Outcome -> Outcome.Outcome
effectiveOutcome Cell.Completed _ recorded = recorded
effectiveOutcome Cell.InfrastructureFailure _ _ = Outcome.InfrastructureFailure
effectiveOutcome Cell.Cancelled complete recorded = if complete then recorded else Outcome.Errored
effectiveOutcome Cell.TimedOut complete recorded = if complete then recorded else Outcome.Errored

fetchCellRun :: ObjectStore -> Bucket -> RunId -> FilePath -> IO (Either FetchError FilePath)
fetchCellRun store bucket identifier outDir = do
  let prefix = "runs/" <> renderRunId identifier <> "/"
      manifestObject = ObjectName (prefix <> "manifest.json")
      tree = outDir </> Text.unpack (renderRunId identifier) </> "tree"
  sealed <- store.getObject bucket manifestObject
  case sealed of
    Nothing -> pure (Left Unsealed)
    Just (manifestBytes, manifestMeta) -> case eitherDecode manifestBytes :: Either String CellManifest of
      Left failure -> pure (Left (ManifestProblem (Text.pack failure)))
      Right manifest
        | manifest.runId /= identifier -> pure (Left (ManifestProblem "manifest run ID differs from result prefix"))
        | otherwise -> do
            listed <- store.listObjects bucket prefix
            let actual = Set.fromList [object.unObjectName | (object, _) <- listed]
                expected = Set.fromList ((prefix <> "manifest.json") : [prefix <> artifact.path | artifact <- manifest.artifacts])
                absent = Set.toList (expected Set.\\ actual)
                unexpected = Set.toList (actual Set.\\ expected)
            if not (null absent && null unexpected)
              then pure (Left (ObjectSetMismatch absent unexpected))
              else case lookup manifestObject listed of
                Nothing -> pure (Left Unsealed)
                Just listedManifest | listedManifest.generation /= manifestMeta.generation -> pure (Left (ObjectChanged "manifest.json"))
                Just _ -> do
                  createDirectoryIfMissing True tree
                  rootSymbolic <- pathIsSymbolicLink tree
                  if rootSymbolic
                    then pure (Left (VerificationFailed (UnsafeEntry "tree" :| [])))
                    else do
                      (_, unsafe) <- listTree tree
                      case unsafe of
                        first : rest -> pure (Left (VerificationFailed (UnsafeEntry first :| map UnsafeEntry rest)))
                        [] -> do
                          writeVerified tree "manifest.json" manifestBytes
                          fetched <- traverse (fetchArtifact store bucket prefix tree listed) manifest.artifacts
                          case firstFailure fetched of
                            Just failure -> pure (Left failure)
                            Nothing -> do
                              verified <- verifyCellTree tree
                              pure case verified of
                                Left problems -> Left (VerificationFailed problems)
                                Right _ -> Right tree

fetchArtifact :: ObjectStore -> Bucket -> Text -> FilePath -> [(ObjectName, ObjectMeta)] -> Artifact -> IO (Maybe FetchError)
fetchArtifact store bucket prefix tree listed artifact = do
  let name = ObjectName (prefix <> artifact.path)
      target = tree </> Text.unpack artifact.path
  case lookup name listed of
    Nothing -> pure (Just (ObjectChanged artifact.path))
    Just meta
      | meta.size /= artifact.bytes -> pure (Just (ObjectCorrupt artifact.path))
      | otherwise -> do
          existing <- doesFileExist target
          valid <- if existing then matches target artifact else pure False
          if valid
            then pure Nothing
            else withTemporary target \temporary -> do
              fetched <- store.downloadTo bucket name temporary
              case fetched of
                Nothing -> pure (Just (ObjectChanged artifact.path))
                Just got
                  | got.generation /= meta.generation -> pure (Just (ObjectChanged artifact.path))
                  | otherwise -> do
                      sound <- matches temporary artifact
                      if sound
                        then renameFile temporary target >> pure Nothing
                        else pure (Just (ObjectCorrupt artifact.path))

verifyCellTree :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)
verifyCellTree tree = do
  let manifestPath = tree </> "manifest.json"
  rootExists <- doesDirectoryExist tree
  if not rootExists
    then pure (Left (Missing "manifest.json" :| []))
    else do
      rootSymbolic <- pathIsSymbolicLink tree
      if rootSymbolic
        then pure (Left (UnsafeEntry "tree" :| []))
        else do
          exists <- doesFileExist manifestPath
          if not exists
            then pure (Left (Missing "manifest.json" :| []))
            else do
              symbolic <- pathIsSymbolicLink manifestPath
              if symbolic
                then pure (Left (UnsafeEntry "manifest.json" :| []))
                else do
                  bytes <- LazyByteString.readFile manifestPath
                  case eitherDecode bytes :: Either String CellManifest of
                    Left failure -> pure (Left (InvalidManifest (Text.pack failure) :| []))
                    Right manifest -> do
                      (files, unsafe) <- listTree tree
                      let actual = Set.fromList files
                          expected = Set.fromList ("manifest.json" : map (.path) manifest.artifacts)
                          missing = map Missing (Set.toList (expected Set.\\ actual))
                          extra = map Extra (Set.toList (actual Set.\\ expected))
                      damaged <- fmap concat $ forM manifest.artifacts \artifact -> do
                        if Set.member artifact.path actual
                          then do
                            (digest, size) <- digestFile (tree </> Text.unpack artifact.path)
                            pure ([DigestMismatch artifact.path | digest /= artifact.sha256] <> [SizeMismatch artifact.path | size /= artifact.bytes])
                          else pure []
                      let problems = map UnsafeEntry unsafe <> missing <> extra <> damaged
                      pure case problems of
                        [] -> Right manifest
                        first : rest -> Left (first :| rest)

-- This is the evidence gate after the cell-level size and digest verification.
-- Validate nested paths before calling the kernel verifier, whose standalone
-- entry point assumes its manifest came from a trusted local run directory.
verifyCellRun :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)
verifyCellRun tree = do
  verified <- verifyCellTree tree
  case verified of
    Left problems -> pure (Left problems)
    Right manifest -> do
      let byPath = Map.fromList [(artifact.path, artifact) | artifact <- manifest.artifacts]
      submission <- readRequired tree byPath "submission/submission.json" :: IO (Either VerifyProblem Submission)
      result <- readRequired tree byPath "cell/result.json" :: IO (Either VerifyProblem CellRunResult)
      case (submission, result) of
        (Right submitted, Right cellResult) -> do
          let identityProblems =
                [PayloadMismatch "submission run, lease or payload differs from cell manifest" | submitted.runId /= manifest.runId || submitted.leaseId /= manifest.leaseId || submitted.payload.bundle.sha256 /= manifest.payload.sha256 || submitted.payload.storePath /= manifest.payload.storePath]
                  <> [RunResultMismatch "cell result differs from cell manifest" | cellResult.runId /= manifest.runId || cellResult.cell /= manifest.cell || cellResult.leaseId /= manifest.leaseId || cellResult.leaseSequence /= manifest.leaseSequence || cellResult.outcome /= manifest.outcome]
          workProblems <- checkWork tree byPath submitted
          planned <- readPlannedRuns tree
          let nested = nestedManifestPaths manifest.artifacts
              nestedIds = Set.fromList (map fst nested)
              (planProblems, expectedIds) = case planned of
                Left problem -> ([problem], Set.empty)
                Right identifiers -> ([], identifiers)
              absentNested =
                [ NestedManifestProblem ("output/" <> run <> "/manifest.json: missing")
                | manifest.outcome == Cell.Completed,
                  run <- Set.toList (expectedIds Set.\\ nestedIds)
                ]
              unexpectedNested =
                [NestedManifestProblem ("output/" <> run <> "/manifest.json: run was not planned") | run <- Set.toList (nestedIds Set.\\ expectedIds)]
          nestedProblems <- concat <$> traverse (checkNested tree byPath manifest.payload.sha256) nested
          let problems = identityProblems <> workProblems <> planProblems <> absentNested <> unexpectedNested <> nestedProblems
          pure case problems of
            [] -> Right manifest
            first : rest -> Left (first :| rest)
        _ -> pure (Left (firstProblems submission result))

verifyCellRunWithStatus :: CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)
verifyCellRunWithStatus status tree = do
  verified <- verifyCellRun tree
  case verified of
    Left problems -> pure (Left problems)
    Right manifest -> do
      (digest, _) <- digestFile (tree </> "manifest.json")
      let problems =
            [StatusMismatch "status is not sealed or names another cell run" | status.phase /= Sealed || status.runId /= manifest.runId]
              <> [StatusMismatch "status sequence or outcome differs from cell manifest" | status.leaseSequence /= Just manifest.leaseSequence || status.outcome /= Just manifest.outcome]
              <> [StatusDigestMismatch | status.manifestSha256 /= Just digest]
      pure case problems of
        [] -> Right manifest
        first : rest -> Left (first :| rest)

readRequired :: (FromJSON document) => FilePath -> Map.Map Text Artifact -> Text -> IO (Either VerifyProblem document)
readRequired tree byPath relative = case Map.lookup relative byPath of
  Nothing -> pure (Left (Missing relative))
  Just _ -> do
    bytes <- LazyByteString.readFile (tree </> Text.unpack relative)
    pure case eitherDecode bytes of
      Left failure -> Left (InvalidManifest (relative <> ": " <> Text.pack failure))
      Right document -> Right document

firstProblems :: Either VerifyProblem left -> Either VerifyProblem right -> NonEmpty VerifyProblem
firstProblems left right = case catMaybes [either Just (const Nothing) left, either Just (const Nothing) right] of
  first : rest -> first :| rest
  [] -> InvalidManifest "missing required cell document" :| []

checkWork :: FilePath -> Map.Map Text Artifact -> Submission -> IO [VerifyProblem]
checkWork tree byPath submitted = case Map.lookup "submission/work" byPath of
  Nothing -> pure [Missing "submission/work"]
  Just artifact -> do
    (digest, size) <- digestFile (tree </> "submission" </> "work")
    pure [PayloadMismatch "submission work differs from its declared digest or size" | digest /= submitted.work.sha256 || size /= submitted.work.bytes || artifact.sha256 /= submitted.work.sha256 || artifact.bytes /= submitted.work.bytes]

readPlannedRuns :: FilePath -> IO (Either VerifyProblem (Set.Set Text))
readPlannedRuns tree = do
  bytes <- LazyByteString.readFile (tree </> "submission" </> "work")
  pure case eitherDecode bytes :: Either String Value of
    Left failure -> Left (InvalidManifest ("submission/work: " <> Text.pack failure))
    Right value -> case decodeWorkPlan value of
      Left failure -> Left (InvalidManifest ("submission/work: " <> failure))
      Right plan -> Right (Set.fromList (map (renderRunId . (.runId)) plan.runs))

nestedManifestPaths :: [Artifact] -> [(Text, Text)]
nestedManifestPaths artifacts =
  [(run, artifact.path) | artifact <- artifacts, ["output", run, "manifest.json"] <- [Text.splitOn "/" artifact.path], Right _ <- [parseRunId run]]

checkNested :: FilePath -> Map.Map Text Artifact -> Text -> (Text, Text) -> IO [VerifyProblem]
checkNested tree byPath bundleDigest (run, path) = do
  decoded <- readRequired tree byPath path :: IO (Either VerifyProblem Manifest)
  case decoded of
    Left problem -> pure [problem]
    Right nested
      | renderRunId nested.runId /= run -> pure [NestedManifestProblem (path <> ": run ID differs from directory")]
      | otherwise -> do
          let relativeFiles = map (Text.pack . (.path)) nested.files
          if any (not . safeRelative) relativeFiles || length relativeFiles /= Set.size (Set.fromList relativeFiles)
            then pure [NestedManifestProblem (path <> ": unsafe or duplicate nested path")]
            else do
              let prefix = "output/" <> run <> "/"
                  expected = sort (path : map (prefix <>) relativeFiles)
                  actual = sort [name | name <- Map.keys byPath, prefix `Text.isPrefixOf` name]
                  setProblem = [ManifestsDisagree (path <> ": nested file set differs from cell manifest") | expected /= actual]
                  metadataProblems =
                    [ ManifestsDisagree (full <> ": digest or size differs between manifests")
                    | file <- nested.files,
                      let full = prefix <> Text.pack file.path,
                      Just artifact <- [Map.lookup full byPath],
                      Just artifact.sha256 /= Text.stripPrefix "sha256:" file.sha256 || fromIntegral artifact.bytes /= file.bytes
                    ]
              checked <- verifyManifest (tree </> "output" </> Text.unpack run)
              let kernelProblems = case checked of
                    Left problems -> [NestedManifestProblem (path <> ": " <> Text.pack (show problems))]
                    Right () -> []
              fingerprintProblems <- checkFingerprint tree prefix run bundleDigest
              pure (setProblem <> metadataProblems <> kernelProblems <> fingerprintProblems)

checkFingerprint :: FilePath -> Text -> Text -> Text -> IO [VerifyProblem]
checkFingerprint tree prefix run expected = do
  let relative = prefix <> "run-result.json"
  present <- doesFileExist (tree </> Text.unpack relative)
  if not present
    then pure [RunResultMismatch (relative <> ": missing")]
    else do
      bytes <- LazyByteString.readFile (tree </> Text.unpack relative)
      pure case eitherDecode bytes :: Either String Value of
        Left failure -> [RunResultMismatch (relative <> ": " <> Text.pack failure)]
        Right value ->
          [RunResultMismatch (relative <> ": run ID differs from directory") | resultRunId value /= Just run]
            <> [RunResultMismatch (relative <> ": payload fingerprint differs from submission") | fingerprintDigest value /= Just expected]

resultRunId :: Value -> Maybe Text
resultRunId (Object result) = do
  String identifier <- KeyMap.lookup "runId" result
  pure identifier
resultRunId _ = Nothing

fingerprintDigest :: Value -> Maybe Text
fingerprintDigest (Object result) = do
  Object fingerprint <- KeyMap.lookup "fingerprint" result
  Object cell <- KeyMap.lookup "cell" fingerprint
  Object payload <- KeyMap.lookup "payload" cell
  String digest <- KeyMap.lookup "bundleSha256" payload
  pure digest
fingerprintDigest _ = Nothing

safeRelative :: Text -> Bool
safeRelative value =
  not (Text.null value)
    && not (Text.isPrefixOf "/" value)
    && not (Text.any (`elem` ['\\', '\0']) value)
    && all (\part -> not (Text.null part) && part /= "." && part /= "..") (Text.splitOn "/" value)

listTree :: FilePath -> IO ([Text], [Text])
listTree root = go ""
  where
    go relative = do
      let directory = if Text.null relative then root else root </> Text.unpack relative
      names <- listDirectory directory
      parts <- forM names \name -> do
        let path = if Text.null relative then Text.pack name else relative <> "/" <> Text.pack name
            full = root </> Text.unpack path
        symbolic <- pathIsSymbolicLink full
        if symbolic
          then pure ([], [path])
          else do
            directoryEntry <- doesDirectoryExist full
            if directoryEntry
              then go path
              else do
                regular <- doesFileExist full
                pure (if regular then [path] else [], if regular then [] else [path])
      pure (concatMap fst parts, concatMap snd parts)

matches :: FilePath -> Artifact -> IO Bool
matches path artifact = do
  (digest, size) <- digestFile path
  pure (digest == artifact.sha256 && size == artifact.bytes)

digestFile :: FilePath -> IO (Text, Int64)
digestFile path = withBinaryFile path ReadMode (go SHA256.init 0)
  where
    go context size handle = do
      chunk <- ByteString.hGetSome handle 65536
      if ByteString.null chunk
        then pure (TextEncoding.decodeUtf8 (Base16.encode (SHA256.finalize context)), size)
        else do
          let nextContext = SHA256.update context chunk
              nextSize = size + fromIntegral (ByteString.length chunk)
          nextContext `seq` nextSize `seq` go nextContext nextSize handle

writeVerified :: FilePath -> FilePath -> LazyByteString.ByteString -> IO ()
writeVerified root relative bytes = do
  let target = root </> relative
  withTemporary target \temporary -> LazyByteString.writeFile temporary bytes >> renameFile temporary target

withTemporary :: FilePath -> (FilePath -> IO value) -> IO value
withTemporary target action = do
  createDirectoryIfMissing True (takeDirectory target)
  bracket (openBinaryTempFile (takeDirectory target) ".kenshou-fetch-") cleanup \(temporary, handle) -> do
    hClose handle
    action temporary
  where
    cleanup (temporary, _) = do
      exists <- doesFileExist temporary
      if exists then removeFile temporary else pure ()

firstFailure :: [Maybe failure] -> Maybe failure
firstFailure = listToMaybe . catMaybes