packages feed

kenshou-remote-0.1.0.0: test/PerfRunnerSpec.hs

module PerfRunnerSpec (spec) where

import Control.Concurrent (threadDelay)
import Data.Aeson
import Data.IORef
import Data.Map.Strict qualified as Map
import Data.Maybe (isJust)
import Data.Text (Text)
import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)
import Kenshou.Core.Id (newRunId)
import Kenshou.Core.Knob (mkKnobName)
import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)
import Kenshou.Perf.Plan
import Kenshou.Perf.Prepared qualified as Prepared
import Kenshou.Perf.Result
import Kenshou.Perf.Result qualified as Result
import Kenshou.Remote.Cell.Lease qualified as Lease
import Kenshou.Remote.Perf.Journal
import Kenshou.Remote.Perf.Runner
import Kenshou.Remote.Store (Bucket (..))
import Kenshou.Remote.Store.File (newFileStore)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "performance runner orchestration" do
  it "persists a claim before CAS and reconciles a timed-out accepted claim without redispatch" $ withSystemTempDirectory "perf-admission-ambiguous" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    store <- newFileStore (directory </> "store")
    identifier <- newRunId
    writes <- newIORef (0 :: Int)
    let identity = LeaseIdentity "alpha" identifier
        ref = Lease.CellRef "alpha" (Bucket "control")
        stalled =
          hooks
            { admit = \entry _ pending callbacks -> do
                pending `shouldBe` Nothing
                callbacks.proposeClaim identity
                state <- readPerfSession path >>= right >>= right . journalState
                state.pendingClaim `shouldBe` Just (entry.profile.name, identity, now)
                claimed <- Lease.acquireLeaseWithId store ref (Lease.LeaseRequest "fixture" "ambiguous transport control" 180) identifier
                case claimed of Lease.Acquired _ -> pure (); _ -> fail "fixture claim failed"
                modifyIORef' writes (+ 1)
                threadDelay 60000000
                pure (Admitted identity),
              releaseOwnedLease = \_ -> pure False
            }
    failed <- runPerformance stalled prepared path
    failed.exitCode `shouldBe` 4
    retained <- readPerfSession path >>= right >>= right . journalState
    retained.pendingClaim `shouldSatisfy` isJust
    readIORef calls `shouldReturn` []
    let resumed =
          hooks
            { admit = \_ _ pending _ -> do
                case pending of Just (_, proposed, _) -> proposed `shouldBe` identity; _ -> fail "lost retained proposal"
                active <- Lease.reattachActiveLease store ref identifier
                isJust active `shouldBe` True
                pure (Admitted identity),
              ownsLease = \_ -> isJust <$> Lease.reattachActiveLease store ref identifier,
              releaseOwnedLease = \_ -> Lease.reattachLease store ref identifier >>= maybe (pure False) (Lease.releaseLease store ref)
            }
    recovered <- runPerformance resumed prepared path
    recovered.exitCode `shouldBe` 0
    readIORef writes `shouldReturn` 1
    final <- readPerfSession path >>= right >>= right . journalState
    final.pendingClaim `shouldBe` Nothing
    final.executionDeadline `shouldBe` Just (addUTCTime 1800 now)
  it "pins the deadline when cleanup confirms release of an unacknowledged claim" $ withSystemTempDirectory "perf-admission-release" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    identifier <- newRunId
    clockRef <- newIORef now
    let late =
          hooks
            { clock = readIORef clockRef,
              admit = \_ _ _ callbacks -> do
                callbacks.proposeClaim (LeaseIdentity "alpha" identifier)
                writeIORef clockRef (addUTCTime 11 now)
                pure (Admitted (LeaseIdentity "alpha" identifier))
            }
    result <- runPerformance late prepared path
    result.exitCode `shouldBe` 4
    state <- readPerfSession path >>= right >>= right . journalState
    state.pendingClaim `shouldBe` Nothing
    state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)
    readIORef calls `shouldReturn` []
  it "collects sealed runtime failures under a lost lease before starting a fresh attempt" $ withSystemTempDirectory "perf-lost-seal" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    first <- newRunId
    session <- readPerfSession path >>= right
    entry <- case prepared.plan.selected of value : _ -> pure value; _ -> fail "empty fixture"
    acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" first))) session)
    started <- right (appendEvent (SessionEvent now (ComparisonStarted entry.profile.name 0)) acquired)
    writePerfSession path started
    let recovered =
          hooks
            { ownsLease = \identity -> pure (identity.leaseId /= first),
              recoverRuntimeFailures = \identity -> pure [(entry.profile.name, "sealed exit-zero defect") | identity.leaseId == first]
            }
    result <- runPerformance recovered prepared path
    result.exitCode `shouldBe` 1
    result.blockingRuntimeFailures `shouldBe` ["sealed exit-zero defect"]
    readIORef calls >>= \executed -> map (\(_, attempt, _) -> attempt) executed `shouldBe` [1, 0]
  it "executes both pinned configurations and reuses completed evidence on resume" $ withSystemTempDirectory "perf-runner" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    result <- runPerformance hooks prepared path
    result.exitCode `shouldBe` 0
    length result.configurations `shouldBe` 2
    result.gatingEligible `shouldBe` False
    resumed <- runPerformance hooks prepared path
    resumed `shouldBe` result
    readIORef calls >>= \executed -> length executed `shouldBe` 2
  it "persists runtime blockers and retries a whole configuration after interruption" $ withSystemTempDirectory "perf-runner" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    let interrupted =
          hooks
            { compareConfiguration = \_ _ _ _ callbacks -> do
                callbacks.reserveReplacement 5
                callbacks.recordRuntimeFailure "known defect with exit zero"
                ioError (userError "simulated lost connection")
            }
    failed <- runPerformance interrupted prepared path
    failed.exitCode `shouldBe` 4
    restored <- readPerfSession path >>= right
    interruptedState <- right (journalState restored)
    Map.elems interruptedState.replacements `shouldBe` [1]
    resumed <- runPerformance hooks prepared path
    resumed.exitCode `shouldBe` 1
    resumed.blockingRuntimeFailures `shouldBe` ["known defect with exit zero"]
    resumedState <- readPerfSession path >>= right >>= right . journalState
    resumedState.executionDeadline `shouldBe` interruptedState.executionDeadline
    Map.elems resumedState.replacements `shouldBe` [1]
    readIORef calls >>= \executed -> map (\(_, index, _) -> index) executed `shouldBe` [1, 0]
  it "returns infrastructure failure when durable publication throws or rejects verification" $ withSystemTempDirectory "perf-runner" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    result <- runPerformance hooks {publishAndVerify = \_ _ -> ioError (userError "storage unavailable")} prepared path
    result.exitCode `shouldBe` 4
    recovered <- runPerformance hooks prepared path
    recovered.exitCode `shouldBe` 0
    invalid <- runPerformance hooks {publishAndVerify = \_ _ -> pure False} prepared path
    invalid.exitCode `shouldBe` 4
  it "rejects a report decorator that changes a performance decision" $ withSystemTempDirectory "perf-runner-report" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    published <- newIORef False
    result <- runPerformance hooks {reportMetadata = \_ report -> pure report {Result.changeCoverage = "complete"}, publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path
    result.exitCode `shouldBe` 4
    readIORef published `shouldReturn` False
  it "rejects changed immutable preparation before any admission" $ withSystemTempDirectory "perf-runner" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    runPerformance hooks prepared {Prepared.harnessRevision = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} path `shouldThrow` anyIOException
    readIORef calls `shouldReturn` []
  it "does not extend cleanup or publication time when a finished session is resumed" $ withSystemTempDirectory "perf-runner-deadline" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    original <- runPerformance hooks prepared path
    original.exitCode `shouldBe` 0
    published <- newIORef False
    result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path
    result.exitCode `shouldBe` 4
    readIORef published `shouldReturn` False
  it "does not publish during cleanup grace after the execution deadline" $ withSystemTempDirectory "perf-publication-grace" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    runPerformance hooks prepared path >>= \result -> result.exitCode `shouldBe` 0
    published <- newIORef False
    result <- runPerformance hooks {clock = pure (addUTCTime 1830 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path
    result.exitCode `shouldBe` 4
    readIORef published `shouldReturn` False
  it "rejects publication that completes beyond the original execution deadline" $ withSystemTempDirectory "perf-publication-late" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    current <- newIORef now
    result <- runPerformance hooks {clock = readIORef current, publishAndVerify = \_ _ -> writeIORef current (addUTCTime 1801 now) >> pure True} prepared path
    result.exitCode `shouldBe` 4
    result.evidenceVerified `shouldBe` False
  it "reserves an identified replacement once when the same callback is reconciled" $ withSystemTempDirectory "perf-runner-reservation" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    let reconciled =
          hooks
            { compareConfiguration = \lease entry index resuming callbacks -> do
                callbacks.reserveReplacement entry.profile.minimumPairs
                callbacks.reserveReplacement entry.profile.minimumPairs
                hooks.compareConfiguration lease entry index resuming callbacks
            }
    result <- runPerformance reconciled prepared path
    result.exitCode `shouldBe` 0
    state <- readPerfSession path >>= right >>= right . journalState
    Map.elems state.replacements `shouldBe` [1, 1]
    length state.replacementIdentities `shouldBe` 2
  it "does not restart cancellation or release grace for an expired interrupted session" $ withSystemTempDirectory "perf-expired-cleanup" \directory -> do
    (prepared, path) <- fixture directory
    session <- readPerfSession path >>= right
    identifier <- newRunId
    acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" identifier))) session)
    active <- right (appendEvent (SessionEvent now (ComparisonStarted (firstProfile prepared) 0)) acquired)
    writePerfSession path active
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    cleanupCalls <- newIORef (0 :: Int)
    hooks <- fixtureHooks calls
    result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), cancelOwnedWork = \_ -> modifyIORef' cleanupCalls (+ 1), releaseOwnedLease = \_ -> modifyIORef' cleanupCalls (+ 1) >> pure True} prepared path
    result.exitCode `shouldBe` 4
    readIORef calls `shouldReturn` []
    readIORef cleanupCalls `shouldReturn` 0
  it "releases an incompatible lease before admitting the next configuration" $ withSystemTempDirectory "perf-runner-cells" \directory -> do
    (prepared, path) <- fixture directory
    calls <- newIORef ([] :: [(Text, Int, Bool)])
    hooks <- fixtureHooks calls
    admissions <- newIORef (0 :: Int)
    releases <- newIORef (0 :: Int)
    first <- newRunId
    second <- newRunId
    let changed =
          hooks
            { admit = \_ _ _ _ -> do
                count <- atomicModifyIORef' admissions (\n -> (n + 1, n))
                pure (Admitted (LeaseIdentity "alpha" (if count == 0 then first else second))),
              canRunConfiguration = \lease entry -> pure (lease.leaseId /= first || entry.profile.name == firstProfile prepared),
              releaseOwnedLease = \_ -> modifyIORef' releases (+ 1) >> pure True
            }
    result <- runPerformance changed prepared path
    result.exitCode `shouldBe` 0
    readIORef admissions `shouldReturn` 2
    readIORef releases `shouldReturn` 2
    state <- readPerfSession path >>= right >>= right . journalState
    state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)

fixture :: FilePath -> IO (Prepared.PerfPrepared, FilePath)
fixture directory = do
  prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= either fail pure
  identifier <- newRunId
  session <- right (initialSession identifier (Prepared.preparedDigest prepared) prepared.plan "gs://fixture-evidence/performance" now)
  let path = directory </> "session.json"
  writePerfSession path session
  pure (prepared, path)

fixtureHooks :: IORef [(Text, Int, Bool)] -> IO RunnerHooks
fixtureHooks calls = do
  contracts <- fixtureContracts
  lease <- newRunId
  pure
    RunnerHooks
      { contracts,
        clock = pure now,
        admit = \_ _ _ _ -> pure (Admitted (LeaseIdentity "alpha" lease)),
        ownsLease = \_ -> pure True,
        canRunConfiguration = \_ _ -> pure True,
        compareConfiguration = \_ entry index resuming _ -> do
          modifyIORef' calls (<> [(entry.profile.name, index, resuming)])
          let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= entry.profile.minimumPairs]
          pure (ConfigurationResult entry.profile.name entry.workloadSha256 "pass" "healthy" [] entry.profile.minimumPairs entry.profile.minimumPairs [] (Just comparison)),
        verifyCompleted = \_ -> pure True,
        cancelOwnedWork = \_ -> pure (),
        releaseOwnedLease = \_ -> pure True,
        recoverRuntimeFailures = \_ -> pure [],
        reportMetadata = \_ result -> pure result,
        publishAndVerify = \_ _ -> pure True
      }

now :: UTCTime
now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)

right :: (Show e) => Either e a -> IO a
right = either (fail . show) pure

-- The captured preparation pins the first consumer's reviewed benchmark revisions;
-- its contracts are data in the consumer-baseline bridge.
fixtureContracts :: IO PerfContracts
fixtureContracts = do
  document <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/perf-contracts.json" >>= either fail pure
  let entries = [(entry.scenario, entry.revision) | entry <- (document :: ContractsDocument).reviewed]
  countKnob <- either (fail . show) pure (mkKnobName document.countKnob)
  pure noPerfContracts {reviewedPhaseContracts = entries, steadyCountKnob = Just countKnob}

data ContractsDocument = ContractsDocument {reviewed :: [ReviewedContract], countKnob :: Text}

data ReviewedContract = ReviewedContract {scenario :: Text, revision :: Int}

instance FromJSON ContractsDocument where
  parseJSON = withObject "PerfContracts" \value -> ContractsDocument <$> value .: "reviewedPhaseContracts" <*> value .: "steadyCountKnob"

instance FromJSON ReviewedContract where
  parseJSON = withObject "ReviewedPhaseContract" \value -> ReviewedContract <$> value .: "scenario" <*> value .: "revision"

-- The first profile the captured preparation selects.
firstProfile :: Prepared.PerfPrepared -> Text
firstProfile prepared = case prepared.plan.selected of
  entry : _ -> entry.profile.name
  [] -> error "the captured preparation selects no profile"