packages feed

shikumi-eval (empty) → 0.1.0.0

raw patch · 20 files changed

+1736/−0 lines, 20 filesdep +aesondep +baikaidep +base

Dependencies added: aeson, baikai, base, bytestring, containers, effectful, generic-lens, lens, shikumi, shikumi-eval, tasty, tasty-golden, tasty-hunit, text, vector

Files

+ CHANGELOG.md view
@@ -0,0 +1,10 @@+# Changelog++## Unreleased++## 0.1.0.0 - 2026-06-13++### Added++- Initial Hackage release of typed evaluation for shikumi programs.+- Dataset, example, prediction, metric, score, report, embedding, golden-test, usage, and bounded evaluation APIs.
+ shikumi-eval.cabal view
@@ -0,0 +1,92 @@+cabal-version:   3.4+name:            shikumi-eval+version:         0.1.0.0+synopsis:        Typed evaluation framework for shikumi LM programs (EP-8)+category:        AI+description:+  The evaluation framework for shikumi: the owned data model (@Example@,+  @Prediction@, @Dataset@, @Metric@, @Score@, @Report@ — MasterPlan integration+  point #5), built-in pure and LM-backed metrics, an @evaluate@ runner that+  scores a 'Shikumi.Program.Program' over a typed dataset with bounded+  parallelism and per-example error boundaries, and golden testing that pins a+  program's behaviour deterministically under a mock or replayed LM.++license:         BSD-3-Clause+author:          Nadeem Bitar+maintainer:      nadeem@gmail.com+build-type:      Simple+extra-doc-files: CHANGELOG.md++common common-options+  ghc-options:+    -Wall -Wcompat -Widentities -Wincomplete-uni-patterns+    -Wincomplete-record-updates -Wredundant-constraints+    -fhide-source-paths -Wmissing-export-lists -Wpartial-fields+    -Wmissing-deriving-strategies++  default-language:   GHC2024+  default-extensions:+    DeriveAnyClass+    DuplicateRecordFields+    OverloadedLabels+    OverloadedStrings++library+  import:          common-options+  hs-source-dirs:  src+  exposed-modules:+    Shikumi.Eval+    Shikumi.Eval.Embedding+    Shikumi.Eval.Evaluate+    Shikumi.Eval.Golden+    Shikumi.Eval.Metric+    Shikumi.Eval.Report+    Shikumi.Eval.Types+    Shikumi.Eval.Usage++  build-depends:+    , aeson+    , baikai+    , base          >=4.20     && <5+    , bytestring+    , containers+    , effectful+    , generic-lens+    , lens          ^>=5.3+    , shikumi       ^>=0.1.0.0+    , tasty+    , tasty-golden+    , text          ^>=2.1+    , vector++test-suite shikumi-eval-test+  import:         common-options+  type:           exitcode-stdio-1.0+  hs-source-dirs: test+  main-is:        Main.hs+  ghc-options:    -threaded -with-rtsopts=-N+  other-modules:+    DocSpec+    EmbeddingSpec+    EvalFixtures+    EvaluateSpec+    GoldenSpec+    MetricLMSpec+    MetricSpec+    ReportSpec+    TypesSpec++  build-depends:+    , aeson+    , baikai+    , base+    , effectful+    , generic-lens+    , lens+    , shikumi       ^>=0.1.0.0+    , shikumi-eval  ^>=0.1.0.0+    , tasty+    , tasty-golden+    , tasty-hunit+    , text+    , vector
+ src/Shikumi/Eval.hs view
@@ -0,0 +1,44 @@+-- | The shikumi evaluation framework — one import for the whole public surface+-- (MasterPlan integration point #5). Re-exports the owned data model+-- ('Shikumi.Eval.Types'), the metrics ('Shikumi.Eval.Metric'), the report+-- ('Shikumi.Eval.Report'), the 'evaluate' runner ('Shikumi.Eval.Evaluate'), and+-- golden testing ('Shikumi.Eval.Golden').+--+-- Worked example: measure a @Question -> Answer@ program over a small typed+-- dataset with the exact-match metric, then read the aggregate score. (The+-- @runProgram@/@evaluate@ effect stack is discharged under a mock or replayed LM+-- in tests; against a live provider it is the substrate's @LLM@ interpreter.)+--+-- @+-- import Shikumi.Eval+--+-- qaData :: 'Dataset' Question Answer+-- qaData = 'dataset'+--   [ 'example' (Question \"What is 2+2?\")     (Answer \"4\")+--   , 'example' (Question \"Capital of France?\") (Answer \"Paris\")+--   ]+--+-- -- inside an Eff stack with (LLM, Concurrent, Error ShikumiError, IOE):+-- run :: ('LLM' :> es, 'Concurrent' :> es, 'Error' 'ShikumiError' :> es, 'IOE' :> es)+--     => 'Program' Question Answer -> Eff es Double+-- run prog = do+--   report <- 'evaluatePure' qaData 'exactMatch' prog+--   pure ('aggregateScore' report)+-- @+--+-- The same snippet is exercised end-to-end under a mock in the test suite's+-- @Doc@ group, so it cannot rot.+module Shikumi.Eval+  ( module Shikumi.Eval.Types,+    module Shikumi.Eval.Metric,+    module Shikumi.Eval.Report,+    module Shikumi.Eval.Evaluate,+    module Shikumi.Eval.Golden,+  )+where++import Shikumi.Eval.Evaluate+import Shikumi.Eval.Golden+import Shikumi.Eval.Metric+import Shikumi.Eval.Report+import Shikumi.Eval.Types
+ src/Shikumi/Eval/Embedding.hs view
@@ -0,0 +1,68 @@+{-# LANGUAGE TypeApplications #-}++-- | A real embeddings interpreter for the existing 'Embedding' effect (EP-15).+--+-- 'Shikumi.Eval.Metric' ships only the pure 'Shikumi.Eval.Metric.runEmbedding'+-- (a @Text -> Vector Double@ table), so @semanticSimilarity@ has no way to call a+-- real backend. This module adds interpreters that drive the upstream+-- 'Baikai.Embedding' client (an OpenAI-compatible @\/v1\/embeddings@ endpoint),+-- so @semanticSimilarity@ runs end to end. The @Embedding@ effect itself is+-- unchanged (integration point #5); only new interpreters are added.+--+-- 'runEmbeddingBy' injects the effectful boundary (the same batching shape as+-- 'Baikai.Embedding.embed') so the production interpreter ('runEmbeddingWith') and+-- a hermetic stub share identical effect-handling, error-wrapping, and+-- vector-extraction logic — only the embedder differs. A transport failure becomes+-- a typed 'Shikumi.Error.ProviderFailure' (hence the @Error ShikumiError@ row),+-- and @IOE@ is introduced only here, at the bottom of the stack.+module Shikumi.Eval.Embedding+  ( runEmbeddingBy,+    runEmbeddingWith,+    runEmbeddingLLM,+  )+where++import Baikai.Embedding (EmbeddingModel, embed, openAIEmbeddingModel)+import Control.Exception (SomeException, displayException, try)+import Data.Text (Text)+import Data.Text qualified as T+import Data.Vector (Vector)+import Data.Vector qualified as V+import Effectful (Eff, IOE, liftIO, (:>))+import Effectful.Dispatch.Dynamic (interpret)+import Effectful.Error.Static (Error, throwError)+import Shikumi.Error (ShikumiError (..))+import Shikumi.Eval.Metric (Embedding (..))++-- | Interpret 'Embedding' with an explicit batching embedder (the same shape as+-- 'Baikai.Embedding.embed'). The production interpreter fixes it to the baikai+-- client; a deterministic stub drives it in hermetic tests. A transport exception+-- is caught and rethrown as a typed 'ProviderFailure'.+runEmbeddingBy ::+  (IOE :> es, Error ShikumiError :> es) =>+  ([Text] -> IO (Vector (Vector Double))) ->+  Eff (Embedding : es) a ->+  Eff es a+runEmbeddingBy embedder = interpret $ \_ -> \case+  EmbedText t -> do+    r <- liftIO (try @SomeException (embedder [t]))+    case r of+      Left e -> throwError (ProviderFailure (T.pack (displayException e)))+      Right vs -> case V.toList vs of+        (v : _) -> pure v+        [] -> throwError (ProviderFailure "embedding backend returned no vector")++-- | Interpret 'Embedding' against a concrete embeddings backend.+runEmbeddingWith ::+  (IOE :> es, Error ShikumiError :> es) =>+  EmbeddingModel ->+  Eff (Embedding : es) a ->+  Eff es a+runEmbeddingWith m = runEmbeddingBy (embed m)++-- | Interpret 'Embedding' against OpenAI's @text-embedding-3-small@ (the default).+runEmbeddingLLM ::+  (IOE :> es, Error ShikumiError :> es) =>+  Eff (Embedding : es) a ->+  Eff es a+runEmbeddingLLM = runEmbeddingWith (openAIEmbeddingModel "text-embedding-3-small")
+ src/Shikumi/Eval/Evaluate.hs view
@@ -0,0 +1,153 @@+-- | The core deliverable: @evaluate@ runs a 'Program' over a typed 'Dataset',+-- scores each example with a metric, and returns a 'Report' summarising the+-- aggregate score, the per-example breakdown, token usage, cost, and latency.+--+-- Examples run with __bounded__ concurrency ('pooledForConcurrentlyN', worker+-- count from 'concurrency'), which preserves dataset order in the result. Each+-- example sits inside a per-example error boundary: a 'ShikumiError' from+-- 'runProgram' becomes a 'ProgramError', one from an effectful metric becomes a+-- 'MetricError', and the 'FailurePolicy' decides whether that example is scored+-- and the run continues ('FailScore') or the whole run aborts ('FailAbort').+-- Usage/cost is accumulated by wrapping the pooled run in+-- 'Shikumi.Eval.Usage.withUsageTotals'; per-example latency is measured with a+-- monotonic clock around each example.+module Shikumi.Eval.Evaluate+  ( evaluate,+    evaluatePure,+    evaluateWith,+  )+where++import Control.Monad (replicateM)+import Data.List.NonEmpty (NonEmpty ((:|)))+import Data.Text qualified as T+import Effectful (Eff, (:>))+import Effectful.Concurrent (Concurrent)+import Effectful.Concurrent.Async (pooledForConcurrentlyN)+import Effectful.Error.Static (Error, catchError, throwError)+import Effectful.Prim (Prim)+import Shikumi.Effect.Time (Time, getMonotonicTimeNSec)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Metric (Metric, MetricM, liftMetric)+import Shikumi.Eval.Report+  ( EvalConfig (..),+    ExampleResult (..),+    FailurePolicy (..),+    FailureReason (..),+    Report,+    defaultEvalConfig,+    mkReport,+  )+import Shikumi.Eval.Types+  ( Dataset,+    Example (..),+    Prediction (..),+    Score,+    datasetExamples,+    prediction,+  )+import Shikumi.Eval.Usage (withUsageTotals)+import Shikumi.LLM (LLM)+import Shikumi.Program (Program, runProgram)++-- | Evaluate a program over a dataset with an effectful metric, using the+-- default configuration.+evaluate ::+  (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) =>+  Dataset i o ->+  MetricM es o ->+  Program i o ->+  Eff es Report+evaluate = evaluateWith defaultEvalConfig++-- | Convenience for a pure metric.+evaluatePure ::+  (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) =>+  Dataset i o ->+  Metric o ->+  Program i o ->+  Eff es Report+evaluatePure ds m = evaluate ds (liftMetric m)++-- | Evaluate with an explicit configuration.+evaluateWith ::+  (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) =>+  EvalConfig ->+  Dataset i o ->+  MetricM es o ->+  Program i o ->+  Eff es Report+evaluateWith cfg ds metric prog = do+  let indexed = zip [0 ..] (datasetExamples ds)+  (rs, totals) <-+    withUsageTotals $+      pooledForConcurrentlyN (max 1 (concurrency cfg)) indexed (evalOne cfg metric prog)+  pure (mkReport rs totals)++-- | Evaluate a single indexed example inside its error boundary, timing it with a+-- monotonic clock.+evalOne ::+  (LLM :> es, Error ShikumiError :> es, Time :> es) =>+  EvalConfig ->+  MetricM es o ->+  Program i o ->+  (Int, Example i o) ->+  Eff es ExampleResult+evalOne cfg metric prog (ix, Example inp expd) = do+  start <- getMonotonicTimeNSec+  (s, mFail) <- scoreExample cfg metric prog inp expd+  end <- getMonotonicTimeNSec+  let ms = fromIntegral ((end - start) `div` 1_000_000)+  pure ExampleResult {index = ix, score = s, failure = mFail, latencyMs = ms}++-- | Run the program, apply the metric, and resolve any failure through the+-- configured 'FailurePolicy'. Returns the score and an optional failure reason+-- (or re-throws under 'FailAbort').+scoreExample ::+  (LLM :> es, Error ShikumiError :> es) =>+  EvalConfig ->+  MetricM es o ->+  Program i o ->+  i ->+  o ->+  Eff es (Score, Maybe FailureReason)+scoreExample cfg metric prog inp expd = do+  predOrErr <- tryShikumi (buildPrediction cfg prog inp)+  case predOrErr of+    Left e -> boundary (ProgramError (renderErr e)) e+    Right pr -> do+      scoreOrErr <- tryShikumi (metric expd pr)+      case scoreOrErr of+        Left e -> boundary (MetricError (renderErr e)) e+        Right s -> pure (s, Nothing)+  where+    boundary reason e = case failurePolicy cfg of+      FailAbort -> throwError e+      FailScore s -> pure (s, Just reason)++-- | Build a 'Prediction' by running the program @numSamples@ times (at least+-- once); a single sample yields a one-element prediction.+buildPrediction ::+  (LLM :> es, Error ShikumiError :> es) =>+  EvalConfig ->+  Program i o ->+  i ->+  Eff es (Prediction o)+buildPrediction cfg prog inp =+  if n <= 1+    then prediction <$> runProgram prog inp+    else do+      outs <- replicateM n (runProgram prog inp)+      case outs of+        (x : xs) -> pure (Prediction x (x :| xs) Nothing)+        [] -> prediction <$> runProgram prog inp -- unreachable: n >= 1+  where+    n = max 1 (numSamples cfg)++-- | Run an action, catching a 'ShikumiError' into a 'Left'.+tryShikumi :: (Error ShikumiError :> es) => Eff es a -> Eff es (Either ShikumiError a)+tryShikumi act = (Right <$> act) `catchError` \_ e -> pure (Left e)++-- | Render a shikumi error for a 'FailureReason'.+renderErr :: ShikumiError -> T.Text+renderErr = T.pack . show
+ src/Shikumi/Eval/Golden.hs view
@@ -0,0 +1,87 @@+{-# LANGUAGE RankNTypes #-}++-- | Golden testing for programs. A golden test pins a program's observable+-- behaviour to a checked-in file; the test fails if that behaviour changes.+-- Determinism comes from the caller supplying a runner that discharges the effect+-- stack under a /mock or replayed/ LM (never a live one), so the only thing that+-- can change the output is a genuine change in program behaviour.+--+-- 'goldenProgram' compares a per-example transcript (one @index\\t<output>@ line+-- per example, in dataset order); 'goldenReport' compares the rendered 'Report'+-- from 'evaluate'. Both are built on @tasty-golden@'s 'goldenVsString' —+-- regenerate the golden file with @cabal test --test-options=--accept@.+--+-- The runner is a single rank-2 function @forall a. Eff es a -> IO a@, so the+-- helper stays agnostic to the exact effect stack: the caller picks a concrete+-- @es@ (e.g. @'[LLM, Error ShikumiError, IOE]@) and a function collapsing it to+-- 'IO' (throwing on a 'ShikumiError').+module Shikumi.Eval.Golden+  ( goldenProgram,+    goldenReport,+  )+where++import Data.ByteString.Lazy qualified as LBS+import Data.Text (Text)+import Data.Text qualified as T+import Data.Text.Encoding qualified as TE+import Effectful (Eff, (:>))+import Effectful.Concurrent (Concurrent)+import Effectful.Error.Static (Error)+import Effectful.Prim (Prim)+import Shikumi.Effect.Time (Time)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Evaluate (evaluate)+import Shikumi.Eval.Metric (MetricM)+import Shikumi.Eval.Report (renderReportText)+import Shikumi.Eval.Types (Dataset, Example (..), datasetExamples)+import Shikumi.LLM (LLM)+import Shikumi.Program (Program, runProgram)+import Test.Tasty (TestName, TestTree)+import Test.Tasty.Golden (goldenVsString)++-- | Turn a program plus a dataset into a golden test over a deterministic+-- per-example transcript. Each example's input is run through the program (under+-- the supplied offline runner), its output rendered, and the lines compared to+-- the golden file at @path@.+goldenProgram ::+  (LLM :> es, Error ShikumiError :> es) =>+  TestName ->+  -- | golden file path (relative to the package directory)+  FilePath ->+  -- | how to run the effect stack (mock/replay), throwing on a 'ShikumiError'+  (forall a. Eff es a -> IO a) ->+  Dataset i o ->+  Program i o ->+  -- | render an output line for the transcript+  (o -> Text) ->+  TestTree+goldenProgram name path runner ds prog render =+  goldenVsString name path $ do+    let inputs = map input (datasetExamples ds)+    outs <- runner (traverse (runProgram prog) inputs)+    pure (encodeText (renderTranscript (zip [0 ..] (map render outs))))++-- | Like 'goldenProgram' but compares the rendered 'Report' produced by+-- 'evaluate' with the given metric, rather than a raw transcript.+goldenReport ::+  (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) =>+  TestName ->+  FilePath ->+  (forall a. Eff es a -> IO a) ->+  Dataset i o ->+  MetricM es o ->+  Program i o ->+  TestTree+goldenReport name path runner ds metric prog =+  goldenVsString name path $ do+    report <- runner (evaluate ds metric prog)+    pure (encodeText (renderReportText report))++-- | One @index\\t<rendered output>@ line per example, in dataset order.+renderTranscript :: [(Int, Text)] -> Text+renderTranscript = T.unlines . map (\(i, t) -> T.pack (show i) <> "\t" <> t)++-- | Encode text as a UTF-8 lazy 'LBS.ByteString'.+encodeText :: Text -> LBS.ByteString+encodeText = LBS.fromStrict . TE.encodeUtf8
+ src/Shikumi/Eval/Metric.hs view
@@ -0,0 +1,263 @@+{-# LANGUAGE GADTs #-}+{-# LANGUAGE TypeFamilies #-}++-- | Metrics: functions that score how close a program's 'Prediction' is to the+-- expected output. There are two type aliases — a /pure/ 'Metric' and an+-- /effectful/ 'MetricM' (for metrics that must themselves call a model, e.g.+-- embedding similarity or an LLM judge) — with 'liftMetric' embedding the former+-- into the latter so @evaluate@ (defined over 'MetricM') covers both.+--+-- This module ships the pure, offline, deterministic built-ins ('exactMatch',+-- 'normalizedStringSimilarity', the 'customMetric' escape hatch), the combinators+-- ('weightedMean', 'threshold', 'invert'), and the two LM-backed metrics:+-- 'semanticSimilarity' (cosine of embedding vectors, over the 'Embedding' effect)+-- and 'modelJudge' (an LLM-as-grader that runs a tiny shikumi program returning a+-- numeric grade).+module Shikumi.Eval.Metric+  ( -- * Metric types+    Metric,+    MetricM,+    liftMetric,++    -- * Built-in pure metrics+    exactMatch,+    normalizedStringSimilarity,+    customMetric,++    -- * Combinators+    weightedMean,+    threshold,+    invert,++    -- * The embedding capability (for 'semanticSimilarity')+    Embedding (..),+    embedText,+    runEmbedding,++    -- * LM-backed metrics+    semanticSimilarity,+    modelJudge,+  )+where++import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as T+import Data.Vector (Vector)+import Data.Vector qualified as V+import Effectful (Dispatch (Dynamic), DispatchOf, Eff, Effect, (:>))+import Effectful.Dispatch.Dynamic (interpret, send)+import Effectful.Error.Static (Error)+import GHC.Generics (Generic)+import Shikumi.Adapter (ToPrompt)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Types+  ( Prediction,+    Score,+    boolScore,+    mkScore,+    predictionPrimary,+    scoreZero,+    unScore,+  )+import Shikumi.LLM (LLM)+import Shikumi.Module (predict)+import Shikumi.Program (Program, runProgram)+import Shikumi.Schema (FromModel, ToSchema, Validatable)+import Shikumi.Signature (Signature, mkSignature)++-- | A pure metric: the expected output, then the program's prediction, yields a+-- score. Pure metrics are deterministic and offline — ideal for golden tests.+type Metric o = o -> Prediction o -> Score++-- | An effectful metric, for metrics that must call a model (embedding+-- similarity, LLM-as-judge). The effect row @es@ carries whatever capability the+-- metric needs (e.g. @LLM@ or an embedding effect).+type MetricM es o = o -> Prediction o -> Eff es Score++-- | Every pure metric is trivially an effectful one.+liftMetric :: Metric o -> MetricM es o+liftMetric m e p = pure (m e p)++-- | Exact equality of the expected output and the prediction's primary output.+exactMatch :: (Eq o) => Metric o+exactMatch e p = boolScore (e == predictionPrimary p)++-- | The custom-metric escape hatch. It is the identity at the type level; its+-- only purpose is to read intent at the call site (@customMetric $ \\e p -> ...@).+customMetric :: (o -> Prediction o -> Score) -> Metric o+customMetric = id++-- | A pure, offline text-similarity metric. The projection @proj@ points at the+-- text field to compare (so it works on any record), and the score blends a+-- token-set Jaccard similarity with a normalized character-level (Levenshtein)+-- similarity over case-folded, whitespace-collapsed text. Identical strings score+-- 1; disjoint strings score near 0.+normalizedStringSimilarity :: (o -> Text) -> Metric o+normalizedStringSimilarity proj e p =+  textSimilarity (proj e) (proj (predictionPrimary p))++-- | The similarity of two strings: the mean of token-set Jaccard and normalized+-- character-level similarity, both computed over normalized (case-folded,+-- whitespace-collapsed, trimmed) text.+textSimilarity :: Text -> Text -> Score+textSimilarity a0 b0 = mkScore ((jaccard + charSim) / 2)+  where+    a = normalize a0+    b = normalize b0+    toksA = Set.fromList (T.words a)+    toksB = Set.fromList (T.words b)+    union = Set.size (Set.union toksA toksB)+    inter = Set.size (Set.intersection toksA toksB)+    jaccard+      | union == 0 = 1 -- two empty strings are identical+      | otherwise = fromIntegral inter / fromIntegral union+    maxLen = max (T.length a) (T.length b)+    charSim+      | maxLen == 0 = 1+      | otherwise = 1 - fromIntegral (levenshtein a b) / fromIntegral maxLen++-- | Case-fold, collapse runs of whitespace to single spaces, and trim.+normalize :: Text -> Text+normalize = T.unwords . T.words . T.toCaseFold++-- | The Levenshtein edit distance between two strings (the canonical single-row+-- dynamic program, folded over the characters of @t@). Self-contained — no extra+-- dependency.+levenshtein :: Text -> Text -> Int+levenshtein s t = lastElem (foldl transform [0 .. length a] b)+  where+    a = T.unpack s+    b = T.unpack t+    -- Given the previous DP row and the next character of @t@, compute the next+    -- row. @diag@/@up@ are the previous row's cells to the upper-left and above+    -- the cell being filled; @left@ is the cell just produced.+    transform prevRow charB = case prevRow of+      (p0 : rest) -> scanl fill (p0 + 1) (zip3 a prevRow rest)+      [] -> [] -- unreachable: the row always has length (length a + 1) >= 1+      where+        fill left (charA, diag, up) =+          minimum [left + 1, up + 1, diag + (if charA == charB then 0 else 1)]+    -- The last cell of the final row (0 for an empty row, which cannot occur).+    lastElem = foldl (\_ x -> x) 0++-- | Combine several metrics by a weighted average of their scores. An empty list+-- yields the constant zero metric.+weightedMean :: [(Double, Metric o)] -> Metric o+weightedMean [] = \_ _ -> mkScore 0+weightedMean ws = \e p ->+  let totalW = sum (map fst ws)+      weighted = sum [w * unScore (m e p) | (w, m) <- ws]+   in if totalW == 0 then mkScore 0 else mkScore (weighted / totalW)++-- | Turn a graded metric into pass/fail: score 1 when @m@ meets the threshold,+-- else 0.+threshold :: Double -> Metric o -> Metric o+threshold t m = \e p -> boolScore (unScore (m e p) >= t)++-- | Invert a metric (@1 - s@), useful for "lower is better" wrapped metrics.+invert :: Metric o -> Metric o+invert m = \e p -> mkScore (1 - unScore (m e p))++-- ---------------------------------------------------------------------------+-- The embedding capability+-- ---------------------------------------------------------------------------++-- | A minimal embedding effect: turn text into a dense vector. EP-1's @LLM@+-- effect exposes no embedding operation, so 'semanticSimilarity' is expressed+-- over this capability; its interpreter is supplied by the caller (a real+-- embedding backend, or 'runEmbedding' over a pure function for tests).+data Embedding :: Effect where+  EmbedText :: Text -> Embedding m (Vector Double)++type instance DispatchOf Embedding = 'Dynamic++-- | Embed a piece of text.+embedText :: (Embedding :> es) => Text -> Eff es (Vector Double)+embedText t = send (EmbedText t)++-- | Interpret 'Embedding' with a pure function (deterministic; good for tests and+-- for backends that precompute embeddings).+runEmbedding :: (Text -> Vector Double) -> Eff (Embedding : es) a -> Eff es a+runEmbedding f = interpret $ \_ -> \case+  EmbedText t -> pure (f t)++-- | Embedding-based semantic similarity: the cosine similarity of the embeddings+-- of the projected expected and predicted text, mapped from @[-1, 1]@ into+-- @[0, 1]@. A zero-norm vector yields 'scoreZero' (no signal).+semanticSimilarity :: (Embedding :> es) => (o -> Text) -> MetricM es o+semanticSimilarity proj expd predd = do+  a <- embedText (proj expd)+  b <- embedText (proj (predictionPrimary predd))+  pure (cosineScore a b)++-- | Map the cosine similarity of two vectors into a 'Score'.+cosineScore :: Vector Double -> Vector Double -> Score+cosineScore a b+  | na == 0 || nb == 0 = scoreZero+  | otherwise = mkScore ((dot / (na * nb) + 1) / 2)+  where+    dot = V.sum (V.zipWith (*) a b)+    na = sqrt (V.sum (V.map (\x -> x * x) a))+    nb = sqrt (V.sum (V.map (\x -> x * x) b))++-- ---------------------------------------------------------------------------+-- LLM-as-judge+-- ---------------------------------------------------------------------------++-- | The judge's input: the grading rubric plus the reference and candidate texts+-- (plain fields so the rendered prompt stays clean).+data JudgeInput = JudgeInput+  { rubric :: Text,+    reference :: Text,+    candidate :: Text+  }+  deriving stock (Generic, Show)++instance FromModel JudgeInput++instance ToPrompt JudgeInput++-- | The judge's output: a single numeric grade, intended to be in @[0, 1]@+-- (clamped by 'mkScore' regardless).+newtype Grade = Grade {grade :: Double}+  deriving stock (Generic, Show)++instance ToSchema Grade++instance FromModel Grade++instance ToPrompt Grade++instance Validatable Grade++-- | The judge program: a single structured-output predictor that returns a+-- 'Grade'. The @rubricText@ becomes the signature instruction.+judgeProgram :: Text -> Program JudgeInput Grade+judgeProgram rubricText = predict (judgeSig rubricText)++judgeSig :: Text -> Signature JudgeInput Grade+judgeSig rubricText =+  mkSignature $+    rubricText+      <> " Compare the candidate answer to the reference answer and return a single field"+      <> " `grade`: a number in [0,1] where 1 means the candidate fully matches the"+      <> " reference and 0 means it is entirely wrong."++-- | LLM-as-judge: ask a model to grade the prediction against the expected+-- output. The rubric is supplied as text; the model returns a numeric grade,+-- decoded through the structured-output path. A decode failure surfaces as a+-- 'ShikumiError' (the per-example boundary in @evaluate@ turns it into a+-- 'Shikumi.Eval.Report.MetricError'). Threads @Error ShikumiError@ because+-- 'runProgram' does (EP-4): decode failures throw typed errors.+modelJudge ::+  (LLM :> es, Error ShikumiError :> es) =>+  -- | rubric / grading instruction+  Text ->+  -- | render an output as text for the judge to read+  (o -> Text) ->+  MetricM es o+modelJudge rubricText render expd predd = do+  let inp = JudgeInput rubricText (render expd) (render (predictionPrimary predd))+  Grade g <- runProgram (judgeProgram rubricText) inp+  pure (mkScore g)
+ src/Shikumi/Eval/Report.hs view
@@ -0,0 +1,203 @@+-- | The evaluation report and its aggregation. A 'Report' summarises a run over+-- a dataset: the mean per-example score, pass/fail counts, summed token usage and+-- cost, total latency, and a per-example breakdown ('ExampleResult', retained in+-- dataset order for failure analysis). 'mkReport' is the pure aggregator;+-- 'renderReportText' is a deterministic human-readable rendering reused by the+-- CLI (@docs/plans/12-cli-and-developer-experience.md@) and by golden tests.+--+-- 'EvalConfig' carries the run knobs (bounded 'concurrency', the 'FailurePolicy'+-- for per-example errors, and 'numSamples' per example for multi-sample metrics);+-- 'defaultEvalConfig' scores failures @0@ and keeps going.+module Shikumi.Eval.Report+  ( -- * Per-example outcome+    ExampleResult (..),+    FailureReason (..),+    FailurePolicy (..),++    -- * Run configuration+    EvalConfig (..),+    defaultEvalConfig,++    -- * Usage totals+    UsageTotals (..),+    emptyUsageTotals,++    -- * The report+    Report (..),+    mkReport,+    renderReportText,+  )+where++import Data.Maybe (isJust, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as T+import Numeric (showFFloat)+import Numeric.Natural (Natural)+import Shikumi.Eval.Types (Score, scoreZero, unScore)++-- | The reason an example did not complete normally.+data FailureReason+  = -- | @runProgram@ threw a 'Shikumi.Error.ShikumiError' (rendered to text)+    ProgramError !Text+  | -- | the metric itself failed (an effectful metric raised an error)+    MetricError !Text+  | -- | the example exceeded its time budget+    TimedOut+  deriving stock (Eq, Show)++-- | What to do when an example fails.+data FailurePolicy+  = -- | record the failure, assign this score, continue+    FailScore !Score+  | -- | re-throw, ending the whole run+    FailAbort+  deriving stock (Eq, Show)++-- | Per-example outcome retained in the report for failure analysis.+data ExampleResult = ExampleResult+  { -- | position in the dataset (0-based)+    index :: !Int,+    score :: !Score,+    failure :: !(Maybe FailureReason),+    latencyMs :: !Integer+  }+  deriving stock (Eq, Show)++-- | Summed token/cost usage across the run.+data UsageTotals = UsageTotals+  { totalInputTokens :: !Natural,+    totalOutputTokens :: !Natural,+    totalTokens :: !Natural,+    totalCostUsd :: !Rational+  }+  deriving stock (Eq, Show)++-- | The zero of 'UsageTotals'.+emptyUsageTotals :: UsageTotals+emptyUsageTotals = UsageTotals 0 0 0 0++instance Semigroup UsageTotals where+  a <> b =+    UsageTotals+      (totalInputTokens a + totalInputTokens b)+      (totalOutputTokens a + totalOutputTokens b)+      (totalTokens a + totalTokens b)+      (totalCostUsd a + totalCostUsd b)++instance Monoid UsageTotals where+  mempty = emptyUsageTotals++-- | Knobs for an evaluation run.+data EvalConfig = EvalConfig+  { -- | max examples evaluated at once (forced to @>= 1@ by 'evaluate')+    concurrency :: !Int,+    failurePolicy :: !FailurePolicy,+    -- | samples per example for multi-sample metrics (forced to @>= 1@)+    numSamples :: !Int+  }+  deriving stock (Eq, Show)++-- | The default: four-way concurrency, score failures @0@ and keep going, one+-- sample per example.+defaultEvalConfig :: EvalConfig+defaultEvalConfig =+  EvalConfig+    { concurrency = 4,+      failurePolicy = FailScore scoreZero,+      numSamples = 1+    }++-- | The aggregate result of an evaluation run.+data Report = Report+  { -- | mean of the per-example scores, in @[0, 1]@ (0 when there are no examples)+    aggregateScore :: !Double,+    -- | examples whose score is exactly @1.0@+    passCount :: !Int,+    -- | examples carrying a 'FailureReason'+    failCount :: !Int,+    total :: !Int,+    -- | per-example outcomes, retained in dataset order+    results :: ![ExampleResult],+    usage :: !UsageTotals,+    totalLatencyMs :: !Integer+  }+  deriving stock (Eq, Show)++-- | Build a report from per-example results and the run's usage totals.+-- 'aggregateScore' is the arithmetic mean of the per-example scores (0 if there+-- are no examples). 'passCount' counts perfect scores; 'failCount' counts+-- examples with a 'FailureReason'.+mkReport :: [ExampleResult] -> UsageTotals -> Report+mkReport rs u =+  Report+    { aggregateScore =+        if null rs+          then 0+          else sum (map (unScore . score) rs) / fromIntegral (length rs),+      passCount = length (filter ((== 1.0) . unScore . score) rs),+      failCount = length (filter (isJust . failure) rs),+      total = length rs,+      results = rs,+      usage = u,+      totalLatencyMs = sum (map latencyMs rs)+    }++-- | A deterministic, human-readable multi-line summary of a report. The format+-- is stable (fixed 4-decimal score and cost, examples in index order) so the CLI+-- and golden tests can rely on it. The exact shape, for a 3-example run with one+-- failure:+--+-- > score=0.6667  pass=2/3  fail=1+-- > tokens: in=120 out=45 total=165+-- > cost: $0.0023+-- > latency: 1234 ms+-- > failures:+-- >   [2] ProgramError: decode failed+--+-- When there are no failures the trailing @failures:@ block is omitted.+renderReportText :: Report -> Text+renderReportText r =+  T.intercalate "\n" (header ++ failureLines)+  where+    header =+      [ "score="+          <> fixed4 (aggregateScore r)+          <> "  pass="+          <> tshow (passCount r)+          <> "/"+          <> tshow (total r)+          <> "  fail="+          <> tshow (failCount r),+        "tokens: in="+          <> tshowNat (totalInputTokens (usage r))+          <> " out="+          <> tshowNat (totalOutputTokens (usage r))+          <> " total="+          <> tshowNat (totalTokens (usage r)),+        "cost: $" <> fixed4 (fromRational (totalCostUsd (usage r))),+        "latency: " <> tshow (totalLatencyMs r) <> " ms"+      ]+    failingResults = mapMaybe asFailure (results r)+    asFailure er = (\fr -> (index er, fr)) <$> failure er+    failureLines+      | null failingResults = []+      | otherwise = "failures:" : map renderFailure failingResults+    renderFailure (i, fr) = "  [" <> tshow i <> "] " <> renderReason fr++-- | Render a failure reason for the report.+renderReason :: FailureReason -> Text+renderReason = \case+  ProgramError t -> "ProgramError: " <> t+  MetricError t -> "MetricError: " <> t+  TimedOut -> "TimedOut"++-- | A 'Double' shown with exactly four decimal places.+fixed4 :: Double -> Text+fixed4 d = T.pack (showFFloat (Just 4) d "")++tshow :: (Show a) => a -> Text+tshow = T.pack . show++tshowNat :: Natural -> Text+tshowNat = T.pack . show
+ src/Shikumi/Eval/Types.hs view
@@ -0,0 +1,111 @@+-- | The evaluation data model — MasterPlan integration point #5. The optimizer+-- (@docs/plans/10-optimizer-framework.md@) and the CLI+-- (@docs/plans/12-cli-and-developer-experience.md@) /consume/ these types and+-- must not redefine them.+--+-- A 'Score' is a 'Double' clamped to the closed interval @[0, 1]@ at+-- construction, so downstream ranking code can assume @0 <= s <= 1@ without+-- re-checking (a @Bool@ metric maps @True -> 1@, @False -> 0@). An 'Example' is+-- one labelled datum (input + expected output); a 'Dataset' is a list of them. A+-- 'Prediction' is a program's actual output for one example, carrying a primary+-- output plus the non-empty set of sampled outputs (a single-output program+-- yields a one-element 'Prediction'), so agreement-rewarding metrics+-- (majority-vote, ensemble) can inspect every sample.+module Shikumi.Eval.Types+  ( -- * Scores+    Score (..),+    mkScore,+    scoreZero,+    scoreOne,+    boolScore,+    unScore,++    -- * Examples and datasets+    Example (..),+    example,+    Dataset (..),+    dataset,+    datasetExamples,+    datasetSize,++    -- * Predictions (program outputs)+    Prediction (..),+    prediction,+    predictionPrimary,+    predictionSamples,+  )+where++import Data.Aeson (Value)+import Data.List.NonEmpty (NonEmpty (..))++-- | A metric score, always in the closed interval @[0, 1]@.+newtype Score = Score Double+  deriving stock (Eq, Ord, Show)++-- | Clamp any 'Double' into @[0, 1]@.+mkScore :: Double -> Score+mkScore d = Score (max 0 (min 1 d))++-- | The two interval endpoints as scores.+scoreZero, scoreOne :: Score+scoreZero = Score 0+scoreOne = Score 1++-- | Map a boolean outcome to a score: @True -> 1@, @False -> 0@.+boolScore :: Bool -> Score+boolScore True = scoreOne+boolScore False = scoreZero++-- | Project the underlying 'Double' (always in @[0, 1]@).+unScore :: Score -> Double+unScore (Score d) = d++-- | One labelled datum: an input of type @i@ and its expected output of type @o@.+data Example i o = Example+  { input :: !i,+    expected :: !o+  }+  deriving stock (Eq, Show)++-- | Build an 'Example' from an input and its expected output.+example :: i -> o -> Example i o+example = Example++-- | A typed evaluation dataset.+newtype Dataset i o = Dataset [Example i o]+  deriving stock (Eq, Show)++-- | Build a 'Dataset' from a list of examples.+dataset :: [Example i o] -> Dataset i o+dataset = Dataset++-- | The examples of a dataset, in order.+datasetExamples :: Dataset i o -> [Example i o]+datasetExamples (Dataset xs) = xs++-- | The number of examples in a dataset.+datasetSize :: Dataset i o -> Int+datasetSize (Dataset xs) = length xs++-- | A program's actual output for one example. Carries a primary output and the+-- non-empty set of sampled outputs (a single-output program yields one sample).+-- An optional raw JSON @detail@ lets adapters or judges attach structure.+data Prediction o = Prediction+  { primary :: !o,+    samples :: !(NonEmpty o),+    detail :: !(Maybe Value)+  }+  deriving stock (Eq, Show)++-- | Build a single-sample prediction (primary output, one sample, no detail).+prediction :: o -> Prediction o+prediction o = Prediction o (o :| []) Nothing++-- | The designated primary output of a prediction.+predictionPrimary :: Prediction o -> o+predictionPrimary = primary++-- | The non-empty set of sampled outputs.+predictionSamples :: Prediction o -> NonEmpty o+predictionSamples = samples
+ src/Shikumi/Eval/Usage.hs view
@@ -0,0 +1,51 @@+-- | Usage accounting for an evaluation run. 'withUsageTotals' runs an 'Eff'+-- action while accumulating the token usage and cost of every @LLM@ call into a+-- 'UsageTotals', returning both the action's result and the totals.+--+-- The accumulation is a local 'interpose' on EP-1's @LLM@ effect — the same seam+-- EP-6's @cachedLLM@ and EP-7's @tracedLLM@ use. Because @LLM.complete@ returns+-- the full 'Response' (which already carries 'Baikai.Usage.Usage' and+-- 'Baikai.Cost.Cost'), no substrate hook or writer effect is needed; the+-- interposed handler reads usage straight off each response. A shared 'IORef'+-- mutated with 'atomicModifyIORef'' makes the accumulation safe under the bounded+-- concurrency @evaluate@ uses (each pooled worker gets a cloned env that still+-- references the one ref).+module Shikumi.Eval.Usage+  ( withUsageTotals,+  )+where++import Control.Lens ((^.))+import Data.Generics.Labels ()+import Effectful (Eff, (:>))+import Effectful.Dispatch.Dynamic (interpose)+import Effectful.Prim.IORef (Prim, atomicModifyIORef', newIORef, readIORef)+import Shikumi.Eval.Report (UsageTotals (..), emptyUsageTotals)+import Shikumi.LLM (LLM (..), Response, complete, stream)++-- | Run @act@, accumulating every @LLM@ call's usage/cost into a 'UsageTotals'.+withUsageTotals :: (LLM :> es, Prim :> es) => Eff es a -> Eff es (a, UsageTotals)+withUsageTotals act = do+  ref <- newIORef emptyUsageTotals+  result <-+    interpose+      ( \_ -> \case+          Complete m c o -> do+            resp <- complete m c o+            atomicModifyIORef' ref (\u -> (u <> usageOf resp, ()))+            pure resp+          Stream m c o -> stream m c o+      )+      act+  totals <- readIORef ref+  pure (result, totals)++-- | Project a response's token usage and cost into a 'UsageTotals'.+usageOf :: Response -> UsageTotals+usageOf resp =+  UsageTotals+    { totalInputTokens = resp ^. #message . #usage . #inputTokens,+      totalOutputTokens = resp ^. #message . #usage . #outputTokens,+      totalTokens = resp ^. #message . #usage . #totalTokens,+      totalCostUsd = resp ^. #message . #usage . #cost . #usd+    }
+ test/DocSpec.hs view
@@ -0,0 +1,39 @@+-- | Exercises the worked example from the 'Shikumi.Eval' module header end-to-end+-- under a mock LM, so the documented usage is guaranteed to compile and run. The+-- single import 'Shikumi.Eval' must expose everything the snippet needs.+module DocSpec (tests) where++import Effectful (runEff)+import Effectful.Concurrent (runConcurrent)+import Effectful.Error.Static (runErrorNoCallStack)+import Effectful.Prim (runPrim)+import EvalFixtures (Answer (..), Question (..), answerResponse, qaProg, runConstLLM)+import Shikumi.Effect.Time (runTime)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval (aggregateScore, dataset, evaluatePure, exactMatch, example)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertBool, assertFailure, testCase)++tests :: TestTree+tests =+  testGroup+    "Doc"+    [ testCase "module-header example runs under a mock" $ do+        -- The mock answers "4" for every call, so one of the two examples+        -- matches its expected answer: an aggregate score of 0.5.+        let qaData =+              dataset+                [ example (Question "What is 2+2?") (Answer "4"),+                  example (Question "Capital of France?") (Answer "Paris")+                ]+        result <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runConstLLM (answerResponse "4") $+              evaluatePure qaData exactMatch qaProg+        case result of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right report ->+            assertBool+              ("expected ~0.5, got " <> show (aggregateScore report))+              (abs (aggregateScore report - 0.5) < 1e-9)+    ]
+ test/EmbeddingSpec.hs view
@@ -0,0 +1,64 @@+{-# LANGUAGE TypeApplications #-}++-- | End-to-end tests for the embeddings path (EP-15, M3).+--+-- The hermetic test drives @semanticSimilarity@ through the /real/ interpreter+-- shape ('runEmbeddingBy') with a deterministic stub embedder — exercising the new+-- code path without a network — and asserts a semantically-close pair scores+-- strictly higher than a distant pair. The live test is gated on+-- @SHIKUMI_EMBEDDING_LIVE=1@ and asserts a real vector's dimensionality.+module EmbeddingSpec (tests) where++import Data.Text (Text)+import Data.Vector (Vector)+import Data.Vector qualified as V+import Effectful (runEff)+import Effectful.Error.Static (runErrorNoCallStack)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Embedding (runEmbeddingBy, runEmbeddingLLM)+import Shikumi.Eval.Metric (embedText, semanticSimilarity)+import Shikumi.Eval.Types (Score, prediction, unScore)+import System.Environment (lookupEnv)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))++-- | A deterministic stub embedder. The two cat sentences point in nearly the same+-- direction (cosine ≈ 0.995); the revenue sentence is near-orthogonal/opposite to+-- them, so the close pair must outscore the distant pair under @cosineScore@.+stubEmbed :: [Text] -> IO (Vector (Vector Double))+stubEmbed = pure . V.fromList . map vecFor+  where+    vecFor "a cat sat on the mat" = V.fromList [1, 0.9]+    vecFor "the cat is on the rug" = V.fromList [0.9, 1]+    vecFor "quarterly revenue rose" = V.fromList [1, -1]+    vecFor _ = V.fromList [0, 0]++-- | Score the similarity of two texts through the real interpreter over the stub.+runSim :: Text -> Text -> IO (Either ShikumiError Score)+runSim expd predd =+  runEff . runErrorNoCallStack @ShikumiError . runEmbeddingBy stubEmbed $+    semanticSimilarity id expd (prediction predd)++tests :: TestTree+tests =+  testGroup+    "Embedding"+    [ testCase "semanticSimilarity scores close pair above distant pair (hermetic)" $ do+        close <- runSim "a cat sat on the mat" "the cat is on the rug"+        distant <- runSim "a cat sat on the mat" "quarterly revenue rose"+        case (close, distant) of+          (Right c, Right d) ->+            assertBool+              ("close (" <> show (unScore c) <> ") must exceed distant (" <> show (unScore d) <> ")")+              (unScore c > unScore d)+          _ -> assertFailure "stub-backed semanticSimilarity errored",+      testCase "live embedding returns expected dimensionality" $ do+        live <- lookupEnv "SHIKUMI_EMBEDDING_LIVE"+        case live of+          Just "1" -> do+            r <- runEff . runErrorNoCallStack @ShikumiError . runEmbeddingLLM $ embedText "hello"+            case r of+              Right v -> V.length v @?= 1536+              Left e -> assertFailure ("live embedding failed: " <> show e)+          _ -> putStrLn "SHIKUMI_EMBEDDING_LIVE not set; skipping live test"+    ]
+ test/EvalFixtures.hs view
@@ -0,0 +1,146 @@+-- | Shared, network-free fixtures for the @evaluate@ / golden specs: a+-- @Question -> Answer@ signature and program, canned fallback-adapter responses,+-- and two mock @LLM@ interpreters — a constant one that returns a fixed response+-- for every call (order-independent, used for the aggregate-score acceptance) and+-- a scripted one that pops replies in order and can inject a typed failure (used+-- for the failure-policy cases under @concurrency = 1@).+module EvalFixtures+  ( -- * Records+    Question (..),+    Answer (..),++    -- * Signature and program+    qaSig,+    qaProg,++    -- * Canned responses+    answerResponse,+    markerResponse,++    -- * Mock LLM interpreters+    MockReply (..),+    runConstLLM,+    runScriptedLLM,+  )+where++import Baikai (AssistantContent (..), Response, _Response, _TextContent)+import Control.Lens ((&), (.~))+import Data.Generics.Labels ()+import Data.IORef (atomicModifyIORef', newIORef)+import Data.Text (Text)+import Data.Text qualified as T+import Data.Vector qualified as V+import Effectful (Eff, IOE, liftIO, type (:>))+import Effectful.Dispatch.Dynamic (interpret)+import Effectful.Error.Static (Error, throwError)+import GHC.Generics (Generic)+import Shikumi.Adapter (ToPrompt)+import Shikumi.Error (ShikumiError (..))+import Shikumi.LLM (LLM (..))+import Shikumi.Module (predict)+import Shikumi.Program (Program)+import Shikumi.Schema (FromModel, ToSchema, Validatable)+import Shikumi.Signature (Signature, mkSignature)++-- ---------------------------------------------------------------------------+-- Records+-- ---------------------------------------------------------------------------++newtype Question = Question {question :: Text}+  deriving stock (Generic, Show, Eq)++instance ToSchema Question++instance FromModel Question++instance ToPrompt Question++instance Validatable Question++newtype Answer = Answer {answer :: Text}+  deriving stock (Generic, Show, Eq)++instance ToSchema Answer++instance FromModel Answer++instance ToPrompt Answer++instance Validatable Answer++-- ---------------------------------------------------------------------------+-- Signature and program+-- ---------------------------------------------------------------------------++qaSig :: Signature Question Answer+qaSig = mkSignature "Answer the question."++-- | The simplest program: a single predict node. Its behaviour is fully+-- determined by the mock LLM the test wires underneath it.+qaProg :: Program Question Answer+qaProg = predict qaSig++-- ---------------------------------------------------------------------------+-- Canned responses (rendered as the fallback adapter's [[ ## field ## ]] form)+-- ---------------------------------------------------------------------------++-- | An assistant 'Response' whose body decodes (via the fallback adapter) to+-- @Answer t@.+answerResponse :: Text -> Response+answerResponse t = mkResponse (markerBody [("answer", t)])++-- | An assistant 'Response' whose body decodes (via the fallback adapter) to a+-- record with the given @(field, value)@ sections.+markerResponse :: [(Text, Text)] -> Response+markerResponse = mkResponse . markerBody++-- | Build a fallback-style response body from @(field, value)@ sections.+markerBody :: [(Text, Text)] -> Text+markerBody fields = T.unlines (concatMap sect fields ++ ["[[ ## completed ## ]]"])+  where+    sect (k, v) = ["[[ ## " <> k <> " ## ]]", v]++-- | An assistant 'Response' carrying @t@ as its single text block.+mkResponse :: Text -> Response+mkResponse t =+  _Response & #message . #content .~ V.singleton (AssistantText (_TextContent & #text .~ t))++-- ---------------------------------------------------------------------------+-- Mock LLM interpreters+-- ---------------------------------------------------------------------------++-- | One scripted reply: a canned success or a typed failure.+data MockReply+  = MockOk Response+  | MockFail ShikumiError++-- | Return @resp@ for every completion, ignoring the request. Order-independent,+-- so it is safe under any concurrency.+runConstLLM :: Response -> Eff (LLM : es) a -> Eff es a+runConstLLM resp = interpret $ \_ -> \case+  Complete {} -> pure resp+  Stream {} -> pure []++-- | Pop scripted replies in order; a 'MockFail' is thrown through+-- @Error ShikumiError@. Deterministic only under @concurrency = 1@.+runScriptedLLM ::+  (IOE :> es, Error ShikumiError :> es) =>+  [MockReply] ->+  Eff (LLM : es) a ->+  Eff es a+runScriptedLLM replies act = do+  ref <- liftIO (newIORef replies)+  interpret+    ( \_ -> \case+        Complete {} -> do+          r <- liftIO (atomicModifyIORef' ref pop)+          case r of+            MockOk resp -> pure resp+            MockFail e -> throwError e+        Stream {} -> pure []+    )+    act+  where+    pop (x : xs) = (xs, x)+    pop [] = ([], MockFail (ProviderFailure "mock: script exhausted"))
+ test/EvaluateSpec.hs view
@@ -0,0 +1,106 @@+-- | The plan's primary acceptance: @evaluate@ end-to-end against a mock LM, with+-- no network. Three cases — a four-of-five exact-match run scoring 0.8; a+-- program error scored 0 under the default 'FailScore' policy while the run+-- completes; and the same error aborting the run under 'FailAbort'.+module EvaluateSpec (tests) where++import Effectful (runEff)+import Effectful.Concurrent (runConcurrent)+import Effectful.Error.Static (runErrorNoCallStack)+import Effectful.Prim (runPrim)+import EvalFixtures+  ( Answer (..),+    MockReply (..),+    Question (..),+    answerResponse,+    qaProg,+    runConstLLM,+    runScriptedLLM,+  )+import Shikumi.Effect.Time (runTime)+import Shikumi.Error (ShikumiError (..))+import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith)+import Shikumi.Eval.Metric (exactMatch, liftMetric)+import Shikumi.Eval.Report+  ( EvalConfig (..),+    ExampleResult (..),+    FailurePolicy (..),+    FailureReason (..),+    Report (..),+    defaultEvalConfig,+  )+import Shikumi.Eval.Types (Score, dataset, example, scoreZero)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))++-- | Five questions; the program (driven by the constant mock) always answers+-- "yes", so the four examples whose expected answer is "yes" pass exactMatch and+-- the one whose expected answer is "no" fails — a 0.8 aggregate, independent of+-- evaluation order.+aggregateData :: [(Question, Answer)]+aggregateData =+  [ (Question "q1", Answer "yes"),+    (Question "q2", Answer "yes"),+    (Question "q3", Answer "yes"),+    (Question "q4", Answer "yes"),+    (Question "q5", Answer "no")+  ]++tests :: TestTree+tests =+  testGroup+    "Evaluate"+    [ testCase "four-of-five exact match -> aggregateScore 0.8" $ do+        let ds = dataset [example q a | (q, a) <- aggregateData]+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runConstLLM (answerResponse "yes") $+              evaluatePure ds exactMatch qaProg+        case report of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right r -> do+            assertBool+              ("expected ~0.8, got " <> show (aggregateScore r))+              (abs (aggregateScore r - 0.8) < 1e-9)+            passCount r @?= 4+            failCount r @?= 0+            total r @?= 5+            length (results r) @?= 5+            map index (results r) @?= [0, 1, 2, 3, 4],+      testCase "program error -> FailScore scoreZero, run completes" $ do+        -- concurrency = 1 so the scripted replies pop in dataset order: the first+        -- example fails, the rest succeed.+        let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")]+            replies = [MockFail providerErr, MockOk (answerResponse "ok")]+            cfg = defaultEvalConfig {concurrency = 1}+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runScriptedLLM replies $+              evaluateWith cfg ds (liftMetric exactMatch) qaProg+        case report of+          Left e -> assertFailure ("run should not abort, but got: " <> show e)+          Right r -> do+            total r @?= 2+            failCount r @?= 1+            case results r of+              (er0 : _) -> do+                index er0 @?= 0+                score er0 @?= (scoreZero :: Score)+                case failure er0 of+                  Just (ProgramError _) -> pure ()+                  other -> assertFailure ("expected ProgramError, got " <> show other)+              [] -> assertFailure "expected two results",+      testCase "FailAbort surfaces the error" $ do+        let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")]+            replies = [MockFail providerErr, MockOk (answerResponse "ok")]+            cfg = defaultEvalConfig {concurrency = 1, failurePolicy = FailAbort}+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runScriptedLLM replies $+              evaluateWith cfg ds (liftMetric exactMatch) qaProg+        case report of+          Left _ -> pure ()+          Right _ -> assertFailure "expected FailAbort to surface a Left"+    ]+  where+    providerErr = ProviderFailure "boom"
+ test/GoldenSpec.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE RankNTypes #-}++-- | A golden test for a stub program, run deterministically and offline under a+-- constant mock LM. The committed @test/golden/qa-program.golden@ pins the+-- transcript; the plan records the fail-before/pass-after demonstration.+module GoldenSpec (tests) where++import Effectful (Eff, IOE, runEff)+import Effectful.Error.Static (Error, runErrorNoCallStack)+import EvalFixtures (Answer (..), Question (..), answerResponse, qaProg, runConstLLM)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Golden (goldenProgram)+import Shikumi.Eval.Types (Dataset, dataset, example)+import Shikumi.LLM (LLM)+import Test.Tasty (TestTree, testGroup)++-- | Two questions; the constant mock always answers "yes", so the transcript is+-- deterministic regardless of the inputs.+goldenData :: Dataset Question Answer+goldenData =+  dataset+    [ example (Question "what is 2+2?") (Answer "ignored"),+      example (Question "capital of France?") (Answer "ignored")+    ]++-- | Discharge @'[LLM, Error ShikumiError, IOE]@ to 'IO' under the constant mock,+-- failing on a 'ShikumiError'.+runMock :: forall a. Eff '[LLM, Error ShikumiError, IOE] a -> IO a+runMock act = do+  r <- runEff . runErrorNoCallStack @ShikumiError . runConstLLM (answerResponse "yes") $ act+  either (fail . show) pure r++tests :: TestTree+tests =+  testGroup+    "Golden"+    [ goldenProgram+        "qa-program golden"+        "test/golden/qa-program.golden"+        runMock+        goldenData+        qaProg+        answer+    ]
+ test/Main.hs view
@@ -0,0 +1,30 @@+-- | EP-8 acceptance suite for @shikumi-eval@: the data model (@Types@), pure and+-- LM-backed metrics (@Metric@), report aggregation (@Report@), the end-to-end+-- @evaluate@ runner (@Evaluate@), golden program tests (@Golden@), and the+-- documented-usage doctest. Every group is hermetic — no network, no API key.+module Main (main) where++import DocSpec qualified+import EmbeddingSpec qualified+import EvaluateSpec qualified+import GoldenSpec qualified+import MetricLMSpec qualified+import MetricSpec qualified+import ReportSpec qualified+import Test.Tasty (defaultMain, testGroup)+import TypesSpec qualified++main :: IO ()+main =+  defaultMain $+    testGroup+      "shikumi-eval"+      [ TypesSpec.tests,+        MetricSpec.tests,+        ReportSpec.tests,+        EvaluateSpec.tests,+        MetricLMSpec.tests,+        GoldenSpec.tests,+        EmbeddingSpec.tests,+        DocSpec.tests+      ]
+ test/MetricLMSpec.hs view
@@ -0,0 +1,65 @@+-- | Unit tests for the LM-backed metrics, run against mock interpreters with no+-- network: 'semanticSimilarity' over a pure 'Embedding' interpreter (identical /+-- orthogonal / opposite vectors), and 'modelJudge' over a constant mock LLM that+-- returns a fixed grade.+module MetricLMSpec (tests) where++import Data.Text (Text)+import Data.Vector (Vector)+import Data.Vector qualified as V+import Effectful (runEff, runPureEff)+import Effectful.Error.Static (runErrorNoCallStack)+import EvalFixtures (markerResponse, runConstLLM)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Metric+  ( modelJudge,+    runEmbedding,+    semanticSimilarity,+  )+import Shikumi.Eval.Types+  ( Prediction,+    Score,+    mkScore,+    prediction,+    scoreOne,+    scoreZero,+  )+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertFailure, testCase, (@?=))++-- | A fixed embedding table: each label maps to a 2-D vector. "a" and "b" are+-- orthogonal (cosine 0 -> score 0.5); "c" is the opposite of "a" (cosine -1 ->+-- score 0); identical labels share a vector (cosine 1 -> score 1); "z" is the+-- zero vector (no signal -> score 0).+embedTable :: Text -> Vector Double+embedTable "a" = V.fromList [1, 0]+embedTable "b" = V.fromList [0, 1]+embedTable "c" = V.fromList [-1, 0]+embedTable _ = V.fromList [0, 0]++-- | Score the similarity of two text labels under the fixed embedding table.+sim :: Text -> Prediction Text -> Score+sim expd predd =+  runPureEff . runEmbedding embedTable $ semanticSimilarity id expd predd++tests :: TestTree+tests =+  testGroup+    "Metric (LM)"+    [ testCase "semanticSimilarity identical -> 1" $+        sim "a" (prediction "a") @?= scoreOne,+      testCase "semanticSimilarity orthogonal -> 0.5" $+        sim "a" (prediction "b") @?= mkScore 0.5,+      testCase "semanticSimilarity opposite -> 0" $+        sim "a" (prediction "c") @?= scoreZero,+      testCase "semanticSimilarity zero-norm -> 0" $+        sim "z" (prediction "z") @?= scoreZero,+      testCase "modelJudge returns the model's grade" $ do+        result <-+          runEff . runErrorNoCallStack @ShikumiError $+            runConstLLM (markerResponse [("grade", "0.7")]) $+              modelJudge "Grade the answer." id ("ref" :: Text) (prediction "cand")+        case result of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right s -> s @?= mkScore 0.7+    ]
+ test/MetricSpec.hs view
@@ -0,0 +1,60 @@+-- | Unit tests for the pure metrics and combinators: exact match, normalized+-- string similarity (identical / near / disjoint), and the @threshold@,+-- @weightedMean@, and @invert@ combinators. All offline and deterministic.+module MetricSpec (tests) where++import Data.Text (Text)+import Shikumi.Eval.Metric+  ( exactMatch,+    invert,+    normalizedStringSimilarity,+    threshold,+    weightedMean,+  )+import Shikumi.Eval.Types+  ( Prediction,+    Score,+    mkScore,+    prediction,+    scoreOne,+    scoreZero,+    unScore,+  )+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertBool, testCase, (@?=))++-- | A constant metric ignoring its inputs, for combinator tests.+constMetric :: Double -> (o -> Prediction o -> Score)+constMetric d _ _ = mkScore d++tests :: TestTree+tests =+  testGroup+    "Metric"+    [ testCase "exactMatch equal" $+        exactMatch ("a" :: Text) (prediction "a") @?= scoreOne,+      testCase "exactMatch unequal" $+        exactMatch ("a" :: Text) (prediction "b") @?= scoreZero,+      testCase "normalizedStringSimilarity identical" $+        normalizedStringSimilarity id ("the cat" :: Text) (prediction "the cat") @?= scoreOne,+      testCase "normalizedStringSimilarity case/space-insensitive" $+        normalizedStringSimilarity id ("The  Cat" :: Text) (prediction "the cat") @?= scoreOne,+      testCase "normalizedStringSimilarity near is strictly between" $+        let s = unScore (normalizedStringSimilarity id ("the cat sat" :: Text) (prediction "the cat ran"))+         in assertBool ("expected 0 < " <> show s <> " < 1") (s > 0 && s < 1),+      testCase "normalizedStringSimilarity disjoint is low" $+        let s = unScore (normalizedStringSimilarity id ("alpha" :: Text) (prediction "zzzzz"))+         in assertBool ("expected " <> show s <> " < 0.2") (s < 0.2),+      testCase "threshold passes a high score" $+        threshold 0.5 (constMetric 0.7) () (prediction ()) @?= scoreOne,+      testCase "threshold fails a low score" $+        threshold 0.5 (constMetric 0.3) () (prediction ()) @?= scoreZero,+      testCase "weightedMean averages by weight" $+        -- (1*0.2 + 3*0.6) / (1+3) = 2.0/4 = 0.5 (within float tolerance)+        let s = unScore (weightedMean [(1, constMetric 0.2), (3, constMetric 0.6)] () (prediction ()))+         in assertBool ("expected ~0.5, got " <> show s) (abs (s - 0.5) < 1e-9),+      testCase "weightedMean of empty is zero" $+        weightedMean [] () (prediction ()) @?= scoreZero,+      testCase "invert flips" $+        invert (constMetric 0.25) () (prediction ()) @?= mkScore 0.75+    ]
+ test/ReportSpec.hs view
@@ -0,0 +1,69 @@+-- | Unit tests for report aggregation: 'mkReport' over synthetic per-example+-- results, the 'UsageTotals' 'Monoid', and the deterministic 'renderReportText'+-- format (an inline golden-by-equality check).+module ReportSpec (tests) where++import Data.Ratio ((%))+import Data.Text (Text)+import Shikumi.Eval.Report+  ( ExampleResult (..),+    FailureReason (..),+    Report (..),+    UsageTotals (..),+    emptyUsageTotals,+    mkReport,+    renderReportText,+  )+import Shikumi.Eval.Types (mkScore, scoreOne, scoreZero)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertBool, testCase, (@?=))++-- | Three synthetic outcomes: a perfect pass, a partial, and a program failure.+fixtureResults :: [ExampleResult]+fixtureResults =+  [ ExampleResult {index = 0, score = scoreOne, failure = Nothing, latencyMs = 100},+    ExampleResult {index = 1, score = mkScore 0.5, failure = Nothing, latencyMs = 200},+    ExampleResult+      { index = 2,+        score = scoreZero,+        failure = Just (ProgramError "decode failed"),+        latencyMs = 934+      }+  ]++fixtureUsage :: UsageTotals+fixtureUsage = UsageTotals {totalInputTokens = 120, totalOutputTokens = 45, totalTokens = 165, totalCostUsd = 23 % 10000}++fixtureReport :: Report+fixtureReport = mkReport fixtureResults fixtureUsage++expectedRender :: Text+expectedRender =+  "score=0.5000  pass=1/3  fail=1\n\+  \tokens: in=120 out=45 total=165\n\+  \cost: $0.0023\n\+  \latency: 1234 ms\n\+  \failures:\n\+  \  [2] ProgramError: decode failed"++tests :: TestTree+tests =+  testGroup+    "Report"+    [ testCase "aggregateScore is the mean" $+        assertBool+          ("expected ~0.5, got " <> show (aggregateScore fixtureReport))+          (abs (aggregateScore fixtureReport - 0.5) < 1e-9),+      testCase "passCount counts perfect scores" $ passCount fixtureReport @?= 1,+      testCase "failCount counts failures" $ failCount fixtureReport @?= 1,+      testCase "total counts examples" $ total fixtureReport @?= 3,+      testCase "totalLatencyMs sums" $ totalLatencyMs fixtureReport @?= 1234,+      testCase "results retained in order" $ map index (results fixtureReport) @?= [0, 1, 2],+      testCase "empty report is zero" $ aggregateScore (mkReport [] emptyUsageTotals) @?= 0,+      testCase "UsageTotals Monoid sums" $+        UsageTotals 10 20 30 (1 % 100) <> UsageTotals 1 2 3 (2 % 100)+          @?= UsageTotals 11 22 33 (3 % 100),+      testCase "UsageTotals mempty is empty" $ (mempty :: UsageTotals) @?= emptyUsageTotals,+      testCase "renderReportText matches the documented format" $+        renderReportText fixtureReport @?= expectedRender+    ]
+ test/TypesSpec.hs view
@@ -0,0 +1,31 @@+-- | Unit tests for the owned data model: 'Score' clamping, dataset+-- construction, and single-sample 'Prediction' construction.+module TypesSpec (tests) where++import Data.List.NonEmpty (NonEmpty (..))+import Shikumi.Eval.Types+  ( Score (..),+    dataset,+    datasetSize,+    example,+    mkScore,+    prediction,+    predictionSamples,+    scoreOne,+    scoreZero,+  )+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (testCase, (@?=))++tests :: TestTree+tests =+  testGroup+    "Types"+    [ testCase "mkScore clamps high" $ mkScore 1.5 @?= scoreOne,+      testCase "mkScore clamps low" $ mkScore (-0.2) @?= scoreZero,+      testCase "mkScore passes through in-range" $ mkScore 0.5 @?= Score 0.5,+      testCase "datasetSize counts examples" $+        datasetSize (dataset [example (1 :: Int) 'a', example 2 'b']) @?= 2,+      testCase "prediction is single-sample" $+        predictionSamples (prediction ("x" :: String)) @?= ("x" :| [])+    ]