kenshou-remote-0.1.0.0: test/SessionSpec.hs
module SessionSpec (spec) where
import Control.Concurrent (forkIO, threadDelay)
import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)
import Control.Exception (SomeException, try)
import Control.Monad (forM_)
import Data.Aeson (eitherDecode, encode)
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Either (isLeft)
import Data.IORef (newIORef, readIORef, writeIORef)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import FetchSpec (completeTreeWithNested)
import Kenshou.Core.Id (RunId, newRunId, renderRunId)
import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..))
import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), acquireLease, leaseSnapshot, releaseLease, requestCancel)
import Kenshou.Remote.Cell.Session (SessionError (..), runSubmission)
import Kenshou.Remote.Cell.Session qualified as Session
import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)
import Kenshou.Remote.Cell.Session.Rebind (RebindError (RemoteMarkerExists), rebindPlannedSlices)
import Kenshou.Remote.Cell.Session.Rebind qualified as Rebind
import Kenshou.Remote.Cell.Session.Resume (HeldResumeError (..), resumeHeldSession, resumeObservedSlices)
import Kenshou.Remote.Cell.Session.Runner (SessionRunError (..), runPendingSlices, runPlannedSlices)
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))
import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))
import Kenshou.Remote.Store.File (newFileStore)
import System.Directory (createDirectoryIfMissing, doesFileExist)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import System.Timeout (timeout)
import Test.Hspec
spec :: Spec
spec = describe "one leased cell submission" do
it "round-trips a durable journal and rejects skipped transitions" do
journal <- readSessionJournal "test/golden/cell-session.json" >>= expectRight
now <- getCurrentTime
identifier <- case journal.slices of
[slice] -> pure slice.cellRun
_ -> expectationFailure "expected one golden slice" >> error "unreachable"
applyTransition now identifier (Session.ResultsFetched "tree") journal `shouldSatisfy` isLeft
submitted <- expectRight (applyTransition now identifier Session.SubmissionPublished journal)
fmap (.state) submitted.slices `shouldBe` [SliceSubmitted]
applyTransition now identifier Session.SubmissionPublished submitted `shouldSatisfy` isLeft
it "reconciles an exact marker after interruption before its publication checkpoint without writing another marker" $ withSystemTempDirectory "kenshou-marker-recovery" \root -> do
store <- newFileStore root
handle <- acquireLease store cellRef request >>= expectAcquired
submission <- fixtureFor handle
let path = root </> "out" </> "session.json"
marker = ObjectName (submissionPrefix submission <> "submission.json")
(_, _) <- journalCheckpoint path submission
absent <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure
fmap (.state) absent.slices `shouldBe` [SlicePlanned]
store.getObject cellRef.controlBucket marker `shouldReturn` Nothing
_ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)
retained <- store.getObject cellRef.controlBucket marker
observed <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure
fmap (.state) observed.slices `shouldBe` [SliceSubmitted]
store.getObject cellRef.controlBucket marker `shouldReturn` retained
releaseLease store cellRef handle `shouldReturn` True
it "stops before publication when the lease was cancelled" $ withSystemTempDirectory "kenshou-session" \root -> do
store <- newFileStore root
handle <- acquireLease store cellRef request >>= expectAcquired
submission <- fixtureFor handle
requestCancel store cellRef (Just submission.leaseId) False `shouldReturn` True
runSubmission store cellRef (Bucket "results") handle submission workBytes (root <> "/out") (const (pure ())) `shouldReturn` Left LeaseLost
store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []
it "returns the cell's rejection reason after publishing the marker" $ withSystemTempDirectory "kenshou-session" \root -> do
store <- newFileStore root
handle <- acquireLease store cellRef request >>= expectAcquired
submission <- fixtureFor handle
let journalPath = root </> "out" </> "session.json"
(_, checkpoint) <- journalCheckpoint journalPath submission
_ <- forkIO do
let marker = ObjectName (submissionPrefix submission <> "submission.json")
let awaitMarker = do
visible <- store.statObject cellRef.controlBucket marker
case visible of
Nothing -> threadDelay 10000 >> awaitMarker
Just _ -> pure ()
awaitMarker
now <- getCurrentTime
_ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "rejected.json")) "application/json" DoesNotExist (encode (Rejected submission.runId "unsafe-command" now))
pure ()
Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint `shouldReturn` Left (SubmissionRejected "unsafe-command")
journal <- readSessionJournal journalPath >>= expectRight
fmap (.state) journal.slices `shouldBe` [SliceRejected]
fmap (.rejectionReason) journal.slices `shouldBe` [Just "unsafe-command"]
it "takes one submission through a sealed local protocol agent and writes the index" $ withSystemTempDirectory "kenshou-session" \root -> do
store <- newFileStore root
handle <- acquireLease store cellRef request >>= expectAcquired
submission <- fixtureFor handle
let journalPath = root </> "out" </> "session.json"
(nested, checkpoint) <- journalCheckpoint journalPath submission
workerResult <- newEmptyMVar
_ <- forkIO do
completed <- try (publishSealedFixture store root submission nested) :: IO (Either SomeException ())
putMVar workerResult completed
result <- timeout 10000000 (Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint)
case result of
Nothing -> expectationFailure "session did not observe the sealed result within ten seconds"
Just (Left failure) -> expectationFailure (show failure)
Just (Right index) -> do
index.cellRun `shouldBe` submission.runId
indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")
indexed `shouldBe` True
journal <- readSessionJournal journalPath >>= expectRight
fmap (.state) journal.slices `shouldBe` [SliceVerified]
fmap (.manifestSha256) journal.slices `shouldBe` [Just index.cellManifest.sha256]
otherRun <- newRunId
now <- getCurrentTime
case journal.slices of
[slice] -> do
let mismatched = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, runIds = [otherRun]}]}
applyTransition now submission.runId (Session.ResultsVerified index) mismatched `shouldBe` Left "verified index nested runs differ from planned slice"
let preEntryFailure = index {cellOutcome = InfrastructureFailure, runs = []}
fetchedFailure = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, cellOutcome = Just InfrastructureFailure}]}
case applyTransition now submission.runId (Session.ResultsVerified preEntryFailure) fetchedFailure of
Right completed -> fmap (.state) completed.slices `shouldBe` [SliceVerified]
Left problem -> expectationFailure (Text.unpack problem)
_ -> expectationFailure "expected one journal slice"
worker <- takeMVar workerResult
case worker of
Left failure -> expectationFailure (show failure)
Right () -> pure ()
it "runs two prepared slices under one lease and checkpoints both results" $ withSystemTempDirectory "kenshou-session" \root -> do
store <- newFileStore root
handle <- acquireLease store cellRef request >>= expectAcquired
first <- fixtureFor handle
second <- fixtureFor handle
firstNested <- newRunId
secondNested <- newRunId
sessionId <- newRunId
now <- getCurrentTime
let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)
journalPath = sessionDir </> "session.json"
firstWork = "slice-0/work.json"
secondWork = "slice-1/work.json"
makeSlice index submission nested workPath = SliceJournal index submission.runId [index] [nested] submission.reset submission workPath SlicePlanned Nothing Nothing Nothing Nothing Nothing
initial = SessionJournal sessionId "alpha" "file" Nothing "control" "results" first.leaseId Held Map.empty (Text.replicate 64 "a") [] [makeSlice 0 first firstNested firstWork, makeSlice 1 second secondNested secondWork] now now
createDirectoryIfMissing True (sessionDir </> "slice-0")
createDirectoryIfMissing True (sessionDir </> "slice-1")
LazyByteString.writeFile (sessionDir </> firstWork) workBytes
LazyByteString.writeFile (sessionDir </> secondWork) "wrong"
runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (InvalidSession "prepared work digest or size differs: slice-1/work.json")
doesFileExist journalPath `shouldReturn` False
LazyByteString.writeFile (sessionDir </> secondWork) workBytes
workerResult <- newEmptyMVar
_ <- forkIO do
completed <- try (forM_ [(first, firstNested), (second, secondNested)] \(submission, nested) -> publishSealedFixture store root submission nested) :: IO (Either SomeException ())
putMVar workerResult completed
result <- timeout 20000000 (runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())))
case result of
Nothing -> expectationFailure "two-slice session did not finish within twenty seconds"
Just (Left failure) -> expectationFailure (show failure)
Just (Right journal) -> do
fmap (.state) journal.slices `shouldBe` [SliceVerified, SliceVerified]
persisted <- readSessionJournal journalPath >>= expectRight
persisted `shouldBe` journal
forM_ [first, second] \submission -> do
indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")
indexed `shouldBe` True
worker <- takeMVar workerResult
case worker of
Left failure -> expectationFailure (show failure)
Right () -> pure ()
runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (SessionFileExists journalPath)
case journal.slices of
[firstSlice, secondSlice] -> do
let interrupted =
journal
{ slices =
[ firstSlice {state = SliceSealed, entryExitCode = Nothing, fetchedPath = Nothing},
secondSlice {state = SliceSubmitted, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}
]
}
writeSessionJournal journalPath interrupted
runPendingSlices store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Left (UnresolvedSlice firstSlice.cellRun SliceSealed)
resumed <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))
recovered <- case resumed of
Left failure -> expectationFailure (show failure) >> error "unreachable"
Right value -> pure value
fmap (.state) recovered.slices `shouldBe` [SliceVerified, SliceVerified]
readSessionJournal journalPath `shouldReturn` Right recovered
resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right recovered
priorLeaseId <- newRunId
let completedPriorLease = (recovered :: SessionJournal) {leaseId = priorLeaseId}
writeSessionJournal journalPath completedPriorLease
resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right completedPriorLease
readSessionJournal journalPath `shouldReturn` Right completedPriorLease
writeSessionJournal journalPath recovered
case recovered.slices of
[firstRecovered, secondRecovered] -> do
writeSessionJournal journalPath (recovered {slices = [firstRecovered {state = SliceFetched, entryExitCode = Nothing}, secondRecovered]})
fetchedAgain <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))
case fetchedAgain of
Left failure -> expectationFailure (show failure)
Right value -> fmap (.state) value.slices `shouldBe` [SliceVerified, SliceVerified]
let oldMarker = secondRecovered {state = SlicePlanned, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}
writeSessionJournal journalPath (recovered {slices = [firstRecovered, oldMarker]})
rebindPlannedSlices store cellRef handle journalPath `shouldReturn` Left (RemoteMarkerExists secondRecovered.cellRun)
observedMarker <- resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ()))
case observedMarker of
Left failure -> expectationFailure (show failure)
Right collected -> fmap (.state) collected.slices `shouldBe` [SliceVerified, SliceVerified]
_ -> expectationFailure "expected two recovered slices"
_ -> expectationFailure "expected two journal slices"
fresh <- fixtureFor handle
freshNested <- newRunId
freshSessionId <- newRunId
at <- getCurrentTime
let freshDir = root </> "out" </> Text.unpack (renderRunId freshSessionId)
freshPath = freshDir </> "session.json"
freshSlice = SliceJournal 0 fresh.runId [0] [freshNested] fresh.reset fresh "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing
freshJournal = SessionJournal freshSessionId "alpha" "file" Nothing "control" "results" fresh.leaseId Held Map.empty (Text.replicate 64 "a") [] [freshSlice] at at
writeSessionJournal freshPath freshJournal
rebound <- rebindPlannedSlices store cellRef handle freshPath
case rebound of
Left failure -> expectationFailure (show failure)
Right updated -> do
updated.leaseId `shouldBe` fresh.leaseId
fmap (.state) updated.slices `shouldBe` [SlicePlanned]
fmap (.cellRun) updated.slices `shouldNotBe` [fresh.runId]
readSessionJournal freshPath `shouldReturn` Right updated
case updated.slices of
[pendingSlice] -> do
createDirectoryIfMissing True (freshDir </> "slice-0")
LazyByteString.writeFile (freshDir </> pendingSlice.workPath) workBytes
pendingWorker <- newEmptyMVar
_ <- forkIO do
completed <- try (publishSealedFixture store root pendingSlice.submission freshNested) :: IO (Either SomeException ())
putMVar pendingWorker completed
continued <- timeout 10000000 (runPendingSlices store cellRef (Bucket "results") handle freshPath (const (pure ())))
case continued of
Nothing -> expectationFailure "rebound slice did not finish within ten seconds"
Just (Left failure) -> expectationFailure (show failure)
Just (Right finalJournal) -> do
fmap (.state) finalJournal.slices `shouldBe` [SliceVerified]
readSessionJournal freshPath `shouldReturn` Right finalJournal
pendingOutcome <- takeMVar pendingWorker
case pendingOutcome of
Left failure -> expectationFailure (show failure)
Right () -> pure ()
_ -> expectationFailure "expected one rebound slice"
it "retries an unaccepted submitted slice only after a fresh lease fences it" $ withSystemTempDirectory "kenshou-session" \root -> do
store <- newFileStore root
old <- acquireLease store cellRef request >>= expectAcquired
submission <- fixtureFor old
nested <- newRunId
sessionId <- newRunId
now <- getCurrentTime
let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)
journalPath = sessionDir </> "session.json"
slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SliceSubmitted Nothing Nothing Nothing Nothing Nothing
journal = SessionJournal sessionId "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now
marker = ObjectName (submissionPrefix submission <> "submission.json")
status = ObjectName (submissionPrefix submission <> "status.json")
result = ObjectName ("runs/" <> renderRunId submission.runId <> "/cell/result.json")
createDirectoryIfMissing True (sessionDir </> "slice-0")
LazyByteString.writeFile (sessionDir </> slice.workPath) workBytes
writeSessionJournal journalPath journal
_ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)
rebindPlannedSlices store cellRef old journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)
releaseLease store cellRef old `shouldReturn` True
fresh <- acquireLease store cellRef request >>= expectAcquired
_ <- store.putObject cellRef.controlBucket status "application/json" DoesNotExist "{}"
rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)
store.deleteObject cellRef.controlBucket status NoPrecondition `shouldReturn` True
_ <- store.putObject (Bucket "results") result "application/json" DoesNotExist "{}"
rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)
store.deleteObject (Bucket "results") result NoPrecondition `shouldReturn` True
rebound <- rebindPlannedSlices store cellRef fresh journalPath
updated <- case rebound of
Left failure -> expectationFailure (show failure) >> error "unreachable"
Right value -> pure value
fmap (.state) updated.slices `shouldBe` [SlicePlanned]
updated.leaseId `shouldNotBe` submission.leaseId
case updated.slices of
[reboundSlice] -> do
reboundSlice.cellRun `shouldNotBe` submission.runId
reboundSlice.submission.leaseId `shouldBe` updated.leaseId
store.statObject cellRef.controlBucket marker >>= (`shouldSatisfy` (maybe False (const True)))
_ -> expectationFailure "expected one rebound slice"
publishSealedFixture :: ObjectStore -> FilePath -> Submission -> RunId -> IO ()
publishSealedFixture store root submission nested = do
let marker = ObjectName (submissionPrefix submission <> "submission.json")
results = Bucket "results"
resultPrefix = "runs/" <> renderRunId submission.runId <> "/"
awaitMarker = do
visible <- store.statObject cellRef.controlBucket marker
case visible of
Nothing -> threadDelay 10000 >> awaitMarker
Just _ -> pure ()
awaitMarker
(sourceTree, manifest) <- completeTreeWithNested (root </> "fixture" </> Text.unpack (renderRunId submission.runId)) submission.runId submission.leaseId nested False False False
forM_ manifest.artifacts \artifact -> do
bytes <- LazyByteString.readFile (sourceTree </> Text.unpack artifact.path)
_ <- store.putObject results (ObjectName (resultPrefix <> artifact.path)) artifact.mediaType DoesNotExist bytes
pure ()
bytes <- LazyByteString.readFile (sourceTree </> "manifest.json")
_ <- store.putObject results (ObjectName (resultPrefix <> "manifest.json")) "application/json" DoesNotExist bytes
now <- getCurrentTime
let digest = (workObjectFor "application/json" bytes).sha256
status = CellStatus submission.runId Sealed (Just manifest.leaseSequence) now Nothing (LogChunks 0 0) (Just Completed) (Just digest) (Just [])
_ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "status.json")) "application/json" DoesNotExist (encode status)
pure ()
fixtureFor :: LeaseHandle -> IO Submission
fixtureFor handle = do
bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"
fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String Submission)
identifier <- newRunId
lease <- leaseSnapshot handle
let bundle = fixture.payload.bundle
payload = fixture.payload {bundle = bundle {uri = "gs://control/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}
pure (Submission identifier lease.leaseId payload (workObjectFor "application/json" workBytes) fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels)
submissionPrefix :: Submission -> Text
submissionPrefix submission = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"
workBytes :: LazyByteString.ByteString
workBytes = "{}"
cellRef :: CellRef
cellRef = CellRef "alpha" (Bucket "control")
request :: LeaseRequest
request = LeaseRequest "tester@workstation" "verification" 120
expectAcquired :: AcquireOutcome -> IO LeaseHandle
expectAcquired (Acquired handle) = pure handle
expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"
expectRight :: Either Text value -> IO value
expectRight = either (\failure -> expectationFailure (Text.unpack failure) >> error "unreachable") pure
journalCheckpoint :: FilePath -> Submission -> IO (RunId, Session.SessionTransition -> IO ())
journalCheckpoint path submission = do
now <- getCurrentTime
identifier <- newRunId
nested <- newRunId
let slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing
initial = SessionJournal identifier "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now
writeSessionJournal path initial
journalRef <- newIORef initial
pure
( nested,
\transition -> do
current <- readIORef journalRef
at <- getCurrentTime
changed <- either (ioError . userError . Text.unpack) pure (applyTransition at submission.runId transition current)
writeSessionJournal path changed
writeIORef journalRef changed
)