kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Session/Resume.hs
module Kenshou.Remote.Cell.Session.Resume
( ResumeError (..),
HeldResumeError (..),
resumeObservedSlices,
resumeHeldSession,
)
where
import Data.Aeson (eitherDecode)
import Data.IORef (newIORef, readIORef, writeIORef)
import Data.List.NonEmpty (NonEmpty)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Kenshou.Core.Id (RunId, renderRunId)
import Kenshou.Remote.Cell.Docs (Submission)
import Kenshou.Remote.Cell.Fetch (FetchError, VerifyProblem, fetchCellRun)
import Kenshou.Remote.Cell.Index (deriveCellRunIndex, writeCellRunIndex)
import Kenshou.Remote.Cell.Lease (CellRef (..), LeaseHandle)
import Kenshou.Remote.Cell.Session (SessionTransition (..))
import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)
import Kenshou.Remote.Cell.Session.Rebind (RebindError, rebindPlannedSlices)
import Kenshou.Remote.Cell.Session.Runner (SessionRunError, runPendingSlices)
import Kenshou.Remote.Cell.Watch (WatchEvent, WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)
import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))
import System.FilePath (takeDirectory)
data ResumeError
= InvalidJournal !Text
| ResultFetchFailed !RunId !FetchError
| ResultVerificationFailed !RunId !(NonEmpty VerifyProblem)
deriving stock (Eq, Show)
data HeldResumeError
= ObservationError !ResumeError
| RebindingError !RebindError
| ContinuationError !SessionRunError
deriving stock (Eq, Show)
resumeHeldSession :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> (WatchEvent -> IO ()) -> IO (Either HeldResumeError SessionJournal)
resumeHeldSession store ref resultsBucket handle journalPath emit = do
observed <- resumeObservedSlices store ref resultsBucket journalPath emit
case observed of
Left failure -> pure (Left (ObservationError failure))
Right observed | not (null observed.slices) && all (\slice -> slice.state `elem` [SliceVerified, SliceRejected]) observed.slices -> pure (Right observed)
Right _ -> do
rebound <- rebindPlannedSlices store ref handle journalPath
case rebound of
Left failure -> pure (Left (RebindingError failure))
Right _ -> do
continued <- runPendingSlices store ref resultsBucket handle journalPath emit
pure (either (Left . ContinuationError) Right continued)
-- Reconcile observations and collect already sealed results. Planned or still
-- running slices remain untouched for a later lease-aware resume step.
resumeObservedSlices :: ObjectStore -> CellRef -> Bucket -> FilePath -> (WatchEvent -> IO ()) -> IO (Either ResumeError SessionJournal)
resumeObservedSlices store ref resultsBucket journalPath emit = do
decoded <- readSessionJournal journalPath
case decoded of
Left problem -> pure (Left (InvalidJournal problem))
Right journal
| journal.cell /= ref.cellName || journal.controlBucket /= ref.controlBucket.unBucket || journal.resultsBucket /= resultsBucket.unBucket -> pure (Left (InvalidJournal "session cell or buckets differ from the requested store"))
| otherwise -> do
current <- newIORef journal
reconcile current journal.slices
where
outRoot = takeDirectory (takeDirectory journalPath)
checkpoint current identifier transition = do
before <- readIORef current
now <- getCurrentTime
changed <- either (ioError . userError . Text.unpack) pure (applyTransition now identifier transition before)
writeSessionJournal journalPath changed
writeIORef current changed
reconcile current [] = Right <$> readIORef current
reconcile current (slice : rest) = do
outcome <- case slice.state of
SlicePlanned -> do
let name = ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/submission.json")
marker <- store.getObject ref.controlBucket name
case marker of
Nothing -> pure (Right ())
Just (bytes, _) -> case eitherDecode bytes :: Either String Submission of
Right submission | submission == slice.submission -> do
checkpoint current slice.cellRun SubmissionPublished
observeSubmitted current slice
_ -> pure (Left (InvalidJournal "retained planned submission differs from the visible marker"))
SliceRejected -> pure (Right ())
SliceVerified -> pure (Right ())
SliceSubmitted -> observeSubmitted current slice
SliceSealed -> do
status <- sealedStatus slice.cellRun
collect current slice.cellRun status True
SliceFetched -> do
status <- sealedStatus slice.cellRun
collect current slice.cellRun status False
case outcome of
Left failure -> pure (Left failure)
Right () -> reconcile current rest
observeSubmitted current slice = do
observed <- observe slice.cellRun
case observed of
Nothing -> pure (Right ())
Just (RunRejected rejected) -> checkpoint current slice.cellRun (SubmissionRejectedByCell rejected) >> pure (Right ())
Just (RunSealed status) -> do
checkpoint current slice.cellRun (SubmissionSealed status)
collect current slice.cellRun (Just status) True
observe identifier = do
snapshot <- pollCellRun store ref identifier newWatchCursor
mapM_ emit snapshot.events
pure snapshot.terminal
sealedStatus identifier = do
terminal <- observe identifier
pure case terminal of
Just (RunSealed status) -> Just status
_ -> Nothing
collect current identifier status shouldCheckpointFetch = do
fetched <- fetchCellRun store resultsBucket identifier outRoot
case fetched of
Left failure -> pure (Left (ResultFetchFailed identifier failure))
Right tree -> do
if shouldCheckpointFetch then checkpoint current identifier (ResultsFetched tree) else pure ()
verified <- deriveCellRunIndex resultsBucket status tree
case verified of
Left problems -> pure (Left (ResultVerificationFailed identifier problems))
Right index -> do
_ <- writeCellRunIndex tree index
checkpoint current identifier (ResultsVerified index)
pure (Right ())