packages feed

kenshou-remote-0.1.0.0: test/FetchSpec.hs

module FetchSpec (spec, completeTreeWith, completeTreeWithNested) where

import Data.Aeson (Value, eitherDecode, encode, object, (.=))
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (toList)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Kenshou.Core.Id (RunId, newRunId, parseScenarioId, renderRunId)
import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))
import Kenshou.Core.Outcome qualified as Outcome
import Kenshou.Core.RunSpec (minimalRunSpec)
import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellRunResult (..), CellStatus (..), LogChunks (..), ManifestPayload (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Fetch (FetchError (..), VerifyProblem (..), effectiveOutcome, fetchCellRun, verifyCellRun, verifyCellRunWithStatus, verifyCellTree)
import Kenshou.Remote.Cell.Index (CellRunIndex (..), RunLink (..), deriveCellRunIndex, writeCellRunIndex)
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))
import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))
import Kenshou.Remote.Store.File (newFileStore)
import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile)
import System.FilePath (takeDirectory, (</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "sealed cell tree fetch" do
  it "round-trips the derived cell-run index fixture" do
    bytes <- LazyByteString.readFile "test/golden/cell-run.json"
    case eitherDecode bytes :: Either String CellRunIndex of
      Left failure -> expectationFailure failure
      Right index -> eitherDecode (encode index) `shouldBe` (eitherDecode bytes :: Either String Value)

  it "maps cell failures onto effective nested-run outcomes" do
    effectiveOutcome Completed True Outcome.Failed `shouldBe` Outcome.Failed
    effectiveOutcome InfrastructureFailure True Outcome.Passed `shouldBe` Outcome.InfrastructureFailure
    effectiveOutcome Cancelled True Outcome.Passed `shouldBe` Outcome.Passed
    effectiveOutcome Cancelled False Outcome.Passed `shouldBe` Outcome.Errored
    effectiveOutcome TimedOut False Outcome.Inconclusive `shouldBe` Outcome.Errored

  it "requires a seal, verifies bytes, and repairs a damaged local artifact on replay" $ withSystemTempDirectory "kenshou-fetch" \root -> do
    store <- newFileStore (root </> "store")
    identifier <- newRunId
    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left Unsealed
    manifest <- publishFixture store identifier
    fetched <- fetchCellRun store bucket identifier (root </> "out")
    tree <- case fetched of
      Right path -> pure path
      Left failure -> expectationFailure (show failure) >> error "unreachable"
    verifyCellTree tree `shouldReturn` Right manifest
    LazyByteString.writeFile (tree </> "cell" </> "result.json") "bad"
    damaged <- verifyCellTree tree
    damaged `shouldSatisfy` hasDigestMismatch
    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Right tree
    verifyCellTree tree `shouldReturn` Right manifest
    LazyByteString.writeFile (tree </> "unexpected.txt") "extra"
    withExtra <- verifyCellTree tree
    withExtra `shouldSatisfy` hasExtra
    removeFile (tree </> "unexpected.txt")
    removeFile (tree </> "cell" </> "result.json")
    missing <- verifyCellTree tree
    missing `shouldSatisfy` hasMissing

  it "refuses an unlisted result object and a corrupt published artifact" $ withSystemTempDirectory "kenshou-fetch" \root -> do
    store <- newFileStore (root </> "store")
    identifier <- newRunId
    _ <- publishFixture store identifier
    put store identifier "stray.txt" "extra"
    result <- fetchCellRun store bucket identifier (root </> "out")
    result `shouldSatisfy` \case
      Left (ObjectSetMismatch _ [unexpected]) -> unexpected == prefix identifier <> "stray.txt"
      _ -> False
    store.deleteObject bucket (ObjectName (prefix identifier <> "stray.txt")) NoPrecondition `shouldReturn` True
    put store identifier "cell/result.json" (LazyByteString.replicate (LazyByteString.length resultBytes) 120)
    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left (ObjectCorrupt "cell/result.json")
    doesFileExist (root </> "out" </> Text.unpack (renderRunId identifier) </> "tree" </> "cell" </> "result.json") `shouldReturn` False

  it "cross-checks the submission, cell result and nested Kenshou manifest" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, manifest) <- completeTree root False False False
    verifyCellRun tree `shouldReturn` Right manifest

  it "accepts cell support directories without nested run manifests" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, manifest) <- completeTree root False False False
    let supporting = [("output/kenshou-cell/context.json", "{}"), ("output/specs/0001.json", "{}")]
        expanded = manifest {artifacts = manifest.artifacts <> [artifactFor name bytes | (name, bytes) <- supporting]}
    mapM_
      ( \(name, bytes) -> do
          let destination = tree </> Text.unpack name
          createDirectoryIfMissing True (takeDirectory destination)
          LazyByteString.writeFile destination bytes
      )
      supporting
    LazyByteString.writeFile (tree </> "manifest.json") (encode expanded)
    verifyCellRun tree `shouldReturn` Right expanded

  it "rejects a payload identity mismatch after all file digests pass" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, _) <- completeTree root True False False
    result <- verifyCellRun tree
    result `shouldSatisfy` \case
      Left problems -> any isPayloadMismatch problems
      Right _ -> False

  it "requires a nested manifest and a matching nested run identifier" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, _) <- completeTree (root </> "wrong-id") False True False
    mismatch <- verifyCellRun tree
    mismatch `shouldSatisfy` \case
      Left problems -> any isRunResultMismatch problems
      Right _ -> False
    (otherTree, manifest) <- completeTree (root </> "missing") False False False
    case [artifact.path | artifact <- manifest.artifacts, "/manifest.json" `Text.isSuffixOf` artifact.path] of
      [nestedPath] -> do
        removeFile (otherTree </> Text.unpack nestedPath)
        let withoutNested = manifest {artifacts = filter ((/= nestedPath) . (.path)) manifest.artifacts}
        LazyByteString.writeFile (otherTree </> "manifest.json") (encode withoutNested)
        absent <- verifyCellRun otherTree
        absent `shouldSatisfy` \case
          Left problems -> any isNestedManifestProblem problems
          Right _ -> False
      _ -> expectationFailure "expected one nested manifest"

  it "rejects an unsafe nested path before calling the kernel verifier" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, _) <- completeTree root False False True
    result <- verifyCellRun tree
    result `shouldSatisfy` \case
      Left problems -> any isNestedManifestProblem problems
      Right _ -> False

  it "requires the final status to agree with the fetched seal" $ withSystemTempDirectory "kenshou-verify" \root -> do
    (tree, manifest) <- completeTree root False False False
    bytes <- LazyByteString.readFile (tree </> "manifest.json")
    let sealed = CellStatus manifest.runId Sealed (Just manifest.leaseSequence) manifest.sealedAt Nothing (LogChunks 0 0) (Just manifest.outcome) (Just (digestOf bytes)) (Just [])
    verifyCellRunWithStatus sealed tree `shouldReturn` Right manifest
    let wrong = CellStatus sealed.runId sealed.phase sealed.leaseSequence sealed.updatedAt sealed.phaseStartedAt sealed.logChunks sealed.outcome (Just (Text.replicate 64 "f")) sealed.reasons
    result <- verifyCellRunWithStatus wrong tree
    result `shouldSatisfy` \case
      Left problems -> StatusDigestMismatch `elem` problems
      Right _ -> False

  it "derives a durable index beside a verified cell tree" $ withSystemTempDirectory "kenshou-index" \root -> do
    (tree, manifest) <- completeTree root False False False
    derived <- deriveCellRunIndex (Bucket "results") Nothing tree
    case derived of
      Left failure -> expectationFailure (show failure)
      Right index -> do
        index.cellRun `shouldBe` manifest.runId
        index.resultsBaseUri `shouldBe` ("gs://results/runs/" <> renderRunId manifest.runId)
        index.dataBaseUri `shouldBe` (index.resultsBaseUri <> "/output")
        length index.runs `shouldBe` 1
        map (.recordedOutcome) index.runs `shouldBe` [Outcome.Passed]
        map (.effectiveOutcome) index.runs `shouldBe` [Outcome.Passed]
        destination <- writeCellRunIndex tree index
        destination `shouldBe` root </> "cell-run.json"
        stored <- LazyByteString.readFile destination
        eitherDecode stored `shouldBe` Right index

publishFixture :: ObjectStore -> RunId -> IO CellManifest
publishFixture store identifier = do
  lease <- newRunId
  now <- getCurrentTime
  let payload = ManifestPayload "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-fixture"
      work = workObjectFor "application/json" resultBytes
      artifact = Artifact "cell/result.json" work.sha256 work.bytes "application/json"
      manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" payload Completed 3600 [artifact]
  put store identifier "cell/result.json" resultBytes
  put store identifier "manifest.json" (encode manifest)
  pure manifest

put :: ObjectStore -> RunId -> Text -> LazyByteString.ByteString -> IO ()
put store identifier suffix bytes = do
  _ <- store.putObject bucket (ObjectName (prefix identifier <> suffix)) "application/json" NoPrecondition bytes
  pure ()

prefix :: RunId -> Text
prefix identifier = "runs/" <> renderRunId identifier <> "/"

bucket :: Bucket
bucket = Bucket "results"

resultBytes :: LazyByteString.ByteString
resultBytes = "{\"schema\":\"cell.run-result/v1\"}"

completeTree :: FilePath -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)
completeTree root wrongPayload wrongRunId unsafeNested = do
  identifier <- newRunId
  lease <- newRunId
  completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested

completeTreeWith :: FilePath -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)
completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested = do
  nestedId <- newRunId
  completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested

completeTreeWithNested :: FilePath -> RunId -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)
completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested = do
  otherId <- newRunId
  now <- getCurrentTime
  scenario <- either (ioError . userError . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")
  source <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"
  fixture <- either (ioError . userError) pure (eitherDecode source :: Either String Submission)
  let workBytes = encode (object ["schema" .= ("kenshou.run-plan/v1" :: Text), "planId" .= identifier, "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= nestedId, "estimateMinutes" .= (1 :: Int), "spec" .= minimalRunSpec scenario]]])
      workInfo = workObjectFor "application/json" workBytes
      submitted = Submission identifier lease fixture.payload workInfo fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels
      payloadDigest = fixture.payload.bundle.sha256
      manifestDigest = if wrongPayload then Text.replicate 64 "b" else payloadDigest
      manifestPayload = ManifestPayload manifestDigest fixture.payload.storePath
      cellResult = CellRunResult identifier "alpha" lease 1 Completed (Just 0) Nothing []
      runBytes = encode (object ["schema" .= ("kenshou.run-result/v1" :: Text), "runId" .= (if wrongRunId then otherId else nestedId), "scenario" .= ("selftest/kernel/correctness/always-pass" :: Text), "outcome" .= Outcome.Passed, "fingerprint" .= object ["cell" .= object ["payload" .= object ["bundleSha256" .= payloadDigest]]]])
      nestedFile = ManifestFile (if unsafeNested then "../escape" else "run-result.json") ("sha256:" <> digestOf runBytes) (fromIntegral (LazyByteString.length runBytes)) "application/json"
      nestedManifest = Manifest nestedId now [nestedFile]
      files =
        [ ("submission/work", workBytes),
          ("submission/submission.json", encode submitted),
          ("cell/result.json", encode cellResult),
          ("output/" <> renderRunId nestedId <> "/run-result.json", runBytes),
          ("output/" <> renderRunId nestedId <> "/manifest.json", encode nestedManifest)
        ]
      artifacts = [artifactFor name bytes | (name, bytes) <- files]
      manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" manifestPayload Completed 3600 artifacts
      tree = root </> "tree"
  mapM_
    ( \(name, bytes) -> do
        let destination = tree </> Text.unpack name
        createDirectoryIfMissing True (takeDirectory destination)
        LazyByteString.writeFile destination bytes
    )
    files
  LazyByteString.writeFile (tree </> "manifest.json") (encode manifest)
  pure (tree, manifest)

artifactFor :: Text -> LazyByteString.ByteString -> Artifact
artifactFor name bytes = let info = workObjectFor "application/json" bytes in Artifact name info.sha256 info.bytes "application/json"

digestOf :: LazyByteString.ByteString -> Text
digestOf bytes = (workObjectFor "application/json" bytes).sha256

isPayloadMismatch :: VerifyProblem -> Bool
isPayloadMismatch (PayloadMismatch _) = True
isPayloadMismatch _ = False

isRunResultMismatch :: VerifyProblem -> Bool
isRunResultMismatch (RunResultMismatch _) = True
isRunResultMismatch _ = False

isNestedManifestProblem :: VerifyProblem -> Bool
isNestedManifestProblem (NestedManifestProblem _) = True
isNestedManifestProblem _ = False

hasDigestMismatch :: Either (NonEmpty VerifyProblem) CellManifest -> Bool
hasDigestMismatch (Left problems) = DigestMismatch "cell/result.json" `elem` toList problems
hasDigestMismatch _ = False

hasExtra :: Either (NonEmpty VerifyProblem) CellManifest -> Bool
hasExtra (Left problems) = Extra "unexpected.txt" `elem` toList problems
hasExtra _ = False

hasMissing :: Either (NonEmpty VerifyProblem) CellManifest -> Bool
hasMissing (Left problems) = Missing "cell/result.json" `elem` toList problems
hasMissing _ = False