packages feed

kenshou-remote-0.1.0.0: test/PairSpec.hs

module PairSpec (pairSpec) where

import Data.Aeson (Value, eitherDecodeFileStrict', toJSON)
import Data.List (nub)
import Data.Text qualified as Text
import Kenshou.Core.Id (parseScenarioId)
import Kenshou.Core.RunSpec (RunSpec (..))
import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering (..))
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Pair (PairRequest (..), PairState (..), SliceResult (..), judgePairs, pairPlan)
import Test.Hspec

pairSpec :: Spec
pairSpec = describe "cell paired schedule" do
  it "interleaves fresh same-seed benchmark arms and reserves replacement pairs" do
    source <- fixture
    let request = PairRequest "candidate" "baseline" 3 ABBA 7 1
    paired <- case pairPlan request source of
      Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"
      Right result -> pure result
    length paired.runs `shouldBe` 8
    paired.estimateMinutes `shouldBe` 8
    fmap (fmap (.arm) . (.trial)) paired.runs `shouldBe` fmap Just ["baseline", "candidate", "candidate", "baseline", "baseline", "candidate", "candidate", "baseline"]
    length (nub (fmap (.runId) paired.runs)) `shouldBe` 8
    fmap (.spec.seed) (take 2 paired.runs) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False
    fmap (.spec.seed) (take 2 (drop 2 paired.runs)) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False
    fmap (.comparison) (fmap (.spec) paired.runs) `shouldSatisfy` all (/= Nothing)
    pairPlan request source `shouldBe` Right paired
    decodeWorkPlan (toJSON paired) `shouldBe` Right paired

  it "keeps one failed arm out of a valid pair" do
    let result index arm passed = SliceResult "group" index arm passed
    judgePairs [result 0 Baseline True, result 0 Candidate True, result 1 Candidate False, result 1 Baseline True]
      `shouldBe` [(("group", 0), PairValid), (("group", 1), PairInvalid "one or both cell trials failed or are missing")]

  it "rejects a correctness plan or fewer than three pairs" do
    source <- sourceFixture
    pairPlan (PairRequest "candidate" "baseline" 3 ABBA 7 0) source `shouldBe` Left "paired comparisons require benchmark runs only"
    benchmark <- fixture
    pairPlan (PairRequest "candidate" "baseline" 2 ABBA 7 0) benchmark `shouldBe` Left "paired comparisons need at least three pairs"

fixture :: IO RunPlan
fixture = do
  source <- sourceFixture
  scenario <- case parseScenarioId "selftest/measure/benchmark/sleep-service" of
    Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"
    Right identifier -> pure identifier
  pure source {runs = [entry {spec = entry.spec {scenario = scenario}} | entry <- source.runs]}

sourceFixture :: IO RunPlan
sourceFixture = do
  decoded <- eitherDecodeFileStrict' "../kenshou-core/test/golden/run-plan.minimal.json" :: IO (Either String Value)
  document <- case decoded of
    Left problem -> expectationFailure problem >> error "unreachable"
    Right value -> pure value
  case decodeWorkPlan document of
    Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"
    Right plan -> pure plan