diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## Unreleased + +## 0.1.0.0 - 2026-06-13 + +### Added + +- Initial Hackage release of typed evaluation for shikumi programs. +- Dataset, example, prediction, metric, score, report, embedding, golden-test, usage, and bounded evaluation APIs. diff --git a/shikumi-eval.cabal b/shikumi-eval.cabal new file mode 100644 --- /dev/null +++ b/shikumi-eval.cabal @@ -0,0 +1,92 @@ +cabal-version: 3.4 +name: shikumi-eval +version: 0.1.0.0 +synopsis: Typed evaluation framework for shikumi LM programs (EP-8) +category: AI +description: + The evaluation framework for shikumi: the owned data model (@Example@, + @Prediction@, @Dataset@, @Metric@, @Score@, @Report@ — MasterPlan integration + point #5), built-in pure and LM-backed metrics, an @evaluate@ runner that + scores a 'Shikumi.Program.Program' over a typed dataset with bounded + parallelism and per-example error boundaries, and golden testing that pins a + program's behaviour deterministically under a mock or replayed LM. + +license: BSD-3-Clause +author: Nadeem Bitar +maintainer: nadeem@gmail.com +build-type: Simple +extra-doc-files: CHANGELOG.md + +common common-options + ghc-options: + -Wall -Wcompat -Widentities -Wincomplete-uni-patterns + -Wincomplete-record-updates -Wredundant-constraints + -fhide-source-paths -Wmissing-export-lists -Wpartial-fields + -Wmissing-deriving-strategies + + default-language: GHC2024 + default-extensions: + DeriveAnyClass + DuplicateRecordFields + OverloadedLabels + OverloadedStrings + +library + import: common-options + hs-source-dirs: src + exposed-modules: + Shikumi.Eval + Shikumi.Eval.Embedding + Shikumi.Eval.Evaluate + Shikumi.Eval.Golden + Shikumi.Eval.Metric + Shikumi.Eval.Report + Shikumi.Eval.Types + Shikumi.Eval.Usage + + build-depends: + , aeson + , baikai + , base >=4.20 && <5 + , bytestring + , containers + , effectful + , generic-lens + , lens ^>=5.3 + , shikumi ^>=0.1.0.0 + , tasty + , tasty-golden + , text ^>=2.1 + , vector + +test-suite shikumi-eval-test + import: common-options + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: Main.hs + ghc-options: -threaded -with-rtsopts=-N + other-modules: + DocSpec + EmbeddingSpec + EvalFixtures + EvaluateSpec + GoldenSpec + MetricLMSpec + MetricSpec + ReportSpec + TypesSpec + + build-depends: + , aeson + , baikai + , base + , effectful + , generic-lens + , lens + , shikumi ^>=0.1.0.0 + , shikumi-eval ^>=0.1.0.0 + , tasty + , tasty-golden + , tasty-hunit + , text + , vector diff --git a/src/Shikumi/Eval.hs b/src/Shikumi/Eval.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval.hs @@ -0,0 +1,44 @@ +-- | The shikumi evaluation framework — one import for the whole public surface +-- (MasterPlan integration point #5). Re-exports the owned data model +-- ('Shikumi.Eval.Types'), the metrics ('Shikumi.Eval.Metric'), the report +-- ('Shikumi.Eval.Report'), the 'evaluate' runner ('Shikumi.Eval.Evaluate'), and +-- golden testing ('Shikumi.Eval.Golden'). +-- +-- Worked example: measure a @Question -> Answer@ program over a small typed +-- dataset with the exact-match metric, then read the aggregate score. (The +-- @runProgram@/@evaluate@ effect stack is discharged under a mock or replayed LM +-- in tests; against a live provider it is the substrate's @LLM@ interpreter.) +-- +-- @ +-- import Shikumi.Eval +-- +-- qaData :: 'Dataset' Question Answer +-- qaData = 'dataset' +-- [ 'example' (Question \"What is 2+2?\") (Answer \"4\") +-- , 'example' (Question \"Capital of France?\") (Answer \"Paris\") +-- ] +-- +-- -- inside an Eff stack with (LLM, Concurrent, Error ShikumiError, IOE): +-- run :: ('LLM' :> es, 'Concurrent' :> es, 'Error' 'ShikumiError' :> es, 'IOE' :> es) +-- => 'Program' Question Answer -> Eff es Double +-- run prog = do +-- report <- 'evaluatePure' qaData 'exactMatch' prog +-- pure ('aggregateScore' report) +-- @ +-- +-- The same snippet is exercised end-to-end under a mock in the test suite's +-- @Doc@ group, so it cannot rot. +module Shikumi.Eval + ( module Shikumi.Eval.Types, + module Shikumi.Eval.Metric, + module Shikumi.Eval.Report, + module Shikumi.Eval.Evaluate, + module Shikumi.Eval.Golden, + ) +where + +import Shikumi.Eval.Evaluate +import Shikumi.Eval.Golden +import Shikumi.Eval.Metric +import Shikumi.Eval.Report +import Shikumi.Eval.Types diff --git a/src/Shikumi/Eval/Embedding.hs b/src/Shikumi/Eval/Embedding.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Embedding.hs @@ -0,0 +1,68 @@ +{-# LANGUAGE TypeApplications #-} + +-- | A real embeddings interpreter for the existing 'Embedding' effect (EP-15). +-- +-- 'Shikumi.Eval.Metric' ships only the pure 'Shikumi.Eval.Metric.runEmbedding' +-- (a @Text -> Vector Double@ table), so @semanticSimilarity@ has no way to call a +-- real backend. This module adds interpreters that drive the upstream +-- 'Baikai.Embedding' client (an OpenAI-compatible @\/v1\/embeddings@ endpoint), +-- so @semanticSimilarity@ runs end to end. The @Embedding@ effect itself is +-- unchanged (integration point #5); only new interpreters are added. +-- +-- 'runEmbeddingBy' injects the effectful boundary (the same batching shape as +-- 'Baikai.Embedding.embed') so the production interpreter ('runEmbeddingWith') and +-- a hermetic stub share identical effect-handling, error-wrapping, and +-- vector-extraction logic — only the embedder differs. A transport failure becomes +-- a typed 'Shikumi.Error.ProviderFailure' (hence the @Error ShikumiError@ row), +-- and @IOE@ is introduced only here, at the bottom of the stack. +module Shikumi.Eval.Embedding + ( runEmbeddingBy, + runEmbeddingWith, + runEmbeddingLLM, + ) +where + +import Baikai.Embedding (EmbeddingModel, embed, openAIEmbeddingModel) +import Control.Exception (SomeException, displayException, try) +import Data.Text (Text) +import Data.Text qualified as T +import Data.Vector (Vector) +import Data.Vector qualified as V +import Effectful (Eff, IOE, liftIO, (:>)) +import Effectful.Dispatch.Dynamic (interpret) +import Effectful.Error.Static (Error, throwError) +import Shikumi.Error (ShikumiError (..)) +import Shikumi.Eval.Metric (Embedding (..)) + +-- | Interpret 'Embedding' with an explicit batching embedder (the same shape as +-- 'Baikai.Embedding.embed'). The production interpreter fixes it to the baikai +-- client; a deterministic stub drives it in hermetic tests. A transport exception +-- is caught and rethrown as a typed 'ProviderFailure'. +runEmbeddingBy :: + (IOE :> es, Error ShikumiError :> es) => + ([Text] -> IO (Vector (Vector Double))) -> + Eff (Embedding : es) a -> + Eff es a +runEmbeddingBy embedder = interpret $ \_ -> \case + EmbedText t -> do + r <- liftIO (try @SomeException (embedder [t])) + case r of + Left e -> throwError (ProviderFailure (T.pack (displayException e))) + Right vs -> case V.toList vs of + (v : _) -> pure v + [] -> throwError (ProviderFailure "embedding backend returned no vector") + +-- | Interpret 'Embedding' against a concrete embeddings backend. +runEmbeddingWith :: + (IOE :> es, Error ShikumiError :> es) => + EmbeddingModel -> + Eff (Embedding : es) a -> + Eff es a +runEmbeddingWith m = runEmbeddingBy (embed m) + +-- | Interpret 'Embedding' against OpenAI's @text-embedding-3-small@ (the default). +runEmbeddingLLM :: + (IOE :> es, Error ShikumiError :> es) => + Eff (Embedding : es) a -> + Eff es a +runEmbeddingLLM = runEmbeddingWith (openAIEmbeddingModel "text-embedding-3-small") diff --git a/src/Shikumi/Eval/Evaluate.hs b/src/Shikumi/Eval/Evaluate.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Evaluate.hs @@ -0,0 +1,153 @@ +-- | The core deliverable: @evaluate@ runs a 'Program' over a typed 'Dataset', +-- scores each example with a metric, and returns a 'Report' summarising the +-- aggregate score, the per-example breakdown, token usage, cost, and latency. +-- +-- Examples run with __bounded__ concurrency ('pooledForConcurrentlyN', worker +-- count from 'concurrency'), which preserves dataset order in the result. Each +-- example sits inside a per-example error boundary: a 'ShikumiError' from +-- 'runProgram' becomes a 'ProgramError', one from an effectful metric becomes a +-- 'MetricError', and the 'FailurePolicy' decides whether that example is scored +-- and the run continues ('FailScore') or the whole run aborts ('FailAbort'). +-- Usage/cost is accumulated by wrapping the pooled run in +-- 'Shikumi.Eval.Usage.withUsageTotals'; per-example latency is measured with a +-- monotonic clock around each example. +module Shikumi.Eval.Evaluate + ( evaluate, + evaluatePure, + evaluateWith, + ) +where + +import Control.Monad (replicateM) +import Data.List.NonEmpty (NonEmpty ((:|))) +import Data.Text qualified as T +import Effectful (Eff, (:>)) +import Effectful.Concurrent (Concurrent) +import Effectful.Concurrent.Async (pooledForConcurrentlyN) +import Effectful.Error.Static (Error, catchError, throwError) +import Effectful.Prim (Prim) +import Shikumi.Effect.Time (Time, getMonotonicTimeNSec) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Metric (Metric, MetricM, liftMetric) +import Shikumi.Eval.Report + ( EvalConfig (..), + ExampleResult (..), + FailurePolicy (..), + FailureReason (..), + Report, + defaultEvalConfig, + mkReport, + ) +import Shikumi.Eval.Types + ( Dataset, + Example (..), + Prediction (..), + Score, + datasetExamples, + prediction, + ) +import Shikumi.Eval.Usage (withUsageTotals) +import Shikumi.LLM (LLM) +import Shikumi.Program (Program, runProgram) + +-- | Evaluate a program over a dataset with an effectful metric, using the +-- default configuration. +evaluate :: + (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) => + Dataset i o -> + MetricM es o -> + Program i o -> + Eff es Report +evaluate = evaluateWith defaultEvalConfig + +-- | Convenience for a pure metric. +evaluatePure :: + (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) => + Dataset i o -> + Metric o -> + Program i o -> + Eff es Report +evaluatePure ds m = evaluate ds (liftMetric m) + +-- | Evaluate with an explicit configuration. +evaluateWith :: + (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) => + EvalConfig -> + Dataset i o -> + MetricM es o -> + Program i o -> + Eff es Report +evaluateWith cfg ds metric prog = do + let indexed = zip [0 ..] (datasetExamples ds) + (rs, totals) <- + withUsageTotals $ + pooledForConcurrentlyN (max 1 (concurrency cfg)) indexed (evalOne cfg metric prog) + pure (mkReport rs totals) + +-- | Evaluate a single indexed example inside its error boundary, timing it with a +-- monotonic clock. +evalOne :: + (LLM :> es, Error ShikumiError :> es, Time :> es) => + EvalConfig -> + MetricM es o -> + Program i o -> + (Int, Example i o) -> + Eff es ExampleResult +evalOne cfg metric prog (ix, Example inp expd) = do + start <- getMonotonicTimeNSec + (s, mFail) <- scoreExample cfg metric prog inp expd + end <- getMonotonicTimeNSec + let ms = fromIntegral ((end - start) `div` 1_000_000) + pure ExampleResult {index = ix, score = s, failure = mFail, latencyMs = ms} + +-- | Run the program, apply the metric, and resolve any failure through the +-- configured 'FailurePolicy'. Returns the score and an optional failure reason +-- (or re-throws under 'FailAbort'). +scoreExample :: + (LLM :> es, Error ShikumiError :> es) => + EvalConfig -> + MetricM es o -> + Program i o -> + i -> + o -> + Eff es (Score, Maybe FailureReason) +scoreExample cfg metric prog inp expd = do + predOrErr <- tryShikumi (buildPrediction cfg prog inp) + case predOrErr of + Left e -> boundary (ProgramError (renderErr e)) e + Right pr -> do + scoreOrErr <- tryShikumi (metric expd pr) + case scoreOrErr of + Left e -> boundary (MetricError (renderErr e)) e + Right s -> pure (s, Nothing) + where + boundary reason e = case failurePolicy cfg of + FailAbort -> throwError e + FailScore s -> pure (s, Just reason) + +-- | Build a 'Prediction' by running the program @numSamples@ times (at least +-- once); a single sample yields a one-element prediction. +buildPrediction :: + (LLM :> es, Error ShikumiError :> es) => + EvalConfig -> + Program i o -> + i -> + Eff es (Prediction o) +buildPrediction cfg prog inp = + if n <= 1 + then prediction <$> runProgram prog inp + else do + outs <- replicateM n (runProgram prog inp) + case outs of + (x : xs) -> pure (Prediction x (x :| xs) Nothing) + [] -> prediction <$> runProgram prog inp -- unreachable: n >= 1 + where + n = max 1 (numSamples cfg) + +-- | Run an action, catching a 'ShikumiError' into a 'Left'. +tryShikumi :: (Error ShikumiError :> es) => Eff es a -> Eff es (Either ShikumiError a) +tryShikumi act = (Right <$> act) `catchError` \_ e -> pure (Left e) + +-- | Render a shikumi error for a 'FailureReason'. +renderErr :: ShikumiError -> T.Text +renderErr = T.pack . show diff --git a/src/Shikumi/Eval/Golden.hs b/src/Shikumi/Eval/Golden.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Golden.hs @@ -0,0 +1,87 @@ +{-# LANGUAGE RankNTypes #-} + +-- | Golden testing for programs. A golden test pins a program's observable +-- behaviour to a checked-in file; the test fails if that behaviour changes. +-- Determinism comes from the caller supplying a runner that discharges the effect +-- stack under a /mock or replayed/ LM (never a live one), so the only thing that +-- can change the output is a genuine change in program behaviour. +-- +-- 'goldenProgram' compares a per-example transcript (one @index\\t@ line +-- per example, in dataset order); 'goldenReport' compares the rendered 'Report' +-- from 'evaluate'. Both are built on @tasty-golden@'s 'goldenVsString' — +-- regenerate the golden file with @cabal test --test-options=--accept@. +-- +-- The runner is a single rank-2 function @forall a. Eff es a -> IO a@, so the +-- helper stays agnostic to the exact effect stack: the caller picks a concrete +-- @es@ (e.g. @'[LLM, Error ShikumiError, IOE]@) and a function collapsing it to +-- 'IO' (throwing on a 'ShikumiError'). +module Shikumi.Eval.Golden + ( goldenProgram, + goldenReport, + ) +where + +import Data.ByteString.Lazy qualified as LBS +import Data.Text (Text) +import Data.Text qualified as T +import Data.Text.Encoding qualified as TE +import Effectful (Eff, (:>)) +import Effectful.Concurrent (Concurrent) +import Effectful.Error.Static (Error) +import Effectful.Prim (Prim) +import Shikumi.Effect.Time (Time) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Evaluate (evaluate) +import Shikumi.Eval.Metric (MetricM) +import Shikumi.Eval.Report (renderReportText) +import Shikumi.Eval.Types (Dataset, Example (..), datasetExamples) +import Shikumi.LLM (LLM) +import Shikumi.Program (Program, runProgram) +import Test.Tasty (TestName, TestTree) +import Test.Tasty.Golden (goldenVsString) + +-- | Turn a program plus a dataset into a golden test over a deterministic +-- per-example transcript. Each example's input is run through the program (under +-- the supplied offline runner), its output rendered, and the lines compared to +-- the golden file at @path@. +goldenProgram :: + (LLM :> es, Error ShikumiError :> es) => + TestName -> + -- | golden file path (relative to the package directory) + FilePath -> + -- | how to run the effect stack (mock/replay), throwing on a 'ShikumiError' + (forall a. Eff es a -> IO a) -> + Dataset i o -> + Program i o -> + -- | render an output line for the transcript + (o -> Text) -> + TestTree +goldenProgram name path runner ds prog render = + goldenVsString name path $ do + let inputs = map input (datasetExamples ds) + outs <- runner (traverse (runProgram prog) inputs) + pure (encodeText (renderTranscript (zip [0 ..] (map render outs)))) + +-- | Like 'goldenProgram' but compares the rendered 'Report' produced by +-- 'evaluate' with the given metric, rather than a raw transcript. +goldenReport :: + (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es, Prim :> es) => + TestName -> + FilePath -> + (forall a. Eff es a -> IO a) -> + Dataset i o -> + MetricM es o -> + Program i o -> + TestTree +goldenReport name path runner ds metric prog = + goldenVsString name path $ do + report <- runner (evaluate ds metric prog) + pure (encodeText (renderReportText report)) + +-- | One @index\\t@ line per example, in dataset order. +renderTranscript :: [(Int, Text)] -> Text +renderTranscript = T.unlines . map (\(i, t) -> T.pack (show i) <> "\t" <> t) + +-- | Encode text as a UTF-8 lazy 'LBS.ByteString'. +encodeText :: Text -> LBS.ByteString +encodeText = LBS.fromStrict . TE.encodeUtf8 diff --git a/src/Shikumi/Eval/Metric.hs b/src/Shikumi/Eval/Metric.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Metric.hs @@ -0,0 +1,263 @@ +{-# LANGUAGE GADTs #-} +{-# LANGUAGE TypeFamilies #-} + +-- | Metrics: functions that score how close a program's 'Prediction' is to the +-- expected output. There are two type aliases — a /pure/ 'Metric' and an +-- /effectful/ 'MetricM' (for metrics that must themselves call a model, e.g. +-- embedding similarity or an LLM judge) — with 'liftMetric' embedding the former +-- into the latter so @evaluate@ (defined over 'MetricM') covers both. +-- +-- This module ships the pure, offline, deterministic built-ins ('exactMatch', +-- 'normalizedStringSimilarity', the 'customMetric' escape hatch), the combinators +-- ('weightedMean', 'threshold', 'invert'), and the two LM-backed metrics: +-- 'semanticSimilarity' (cosine of embedding vectors, over the 'Embedding' effect) +-- and 'modelJudge' (an LLM-as-grader that runs a tiny shikumi program returning a +-- numeric grade). +module Shikumi.Eval.Metric + ( -- * Metric types + Metric, + MetricM, + liftMetric, + + -- * Built-in pure metrics + exactMatch, + normalizedStringSimilarity, + customMetric, + + -- * Combinators + weightedMean, + threshold, + invert, + + -- * The embedding capability (for 'semanticSimilarity') + Embedding (..), + embedText, + runEmbedding, + + -- * LM-backed metrics + semanticSimilarity, + modelJudge, + ) +where + +import Data.Set qualified as Set +import Data.Text (Text) +import Data.Text qualified as T +import Data.Vector (Vector) +import Data.Vector qualified as V +import Effectful (Dispatch (Dynamic), DispatchOf, Eff, Effect, (:>)) +import Effectful.Dispatch.Dynamic (interpret, send) +import Effectful.Error.Static (Error) +import GHC.Generics (Generic) +import Shikumi.Adapter (ToPrompt) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Types + ( Prediction, + Score, + boolScore, + mkScore, + predictionPrimary, + scoreZero, + unScore, + ) +import Shikumi.LLM (LLM) +import Shikumi.Module (predict) +import Shikumi.Program (Program, runProgram) +import Shikumi.Schema (FromModel, ToSchema, Validatable) +import Shikumi.Signature (Signature, mkSignature) + +-- | A pure metric: the expected output, then the program's prediction, yields a +-- score. Pure metrics are deterministic and offline — ideal for golden tests. +type Metric o = o -> Prediction o -> Score + +-- | An effectful metric, for metrics that must call a model (embedding +-- similarity, LLM-as-judge). The effect row @es@ carries whatever capability the +-- metric needs (e.g. @LLM@ or an embedding effect). +type MetricM es o = o -> Prediction o -> Eff es Score + +-- | Every pure metric is trivially an effectful one. +liftMetric :: Metric o -> MetricM es o +liftMetric m e p = pure (m e p) + +-- | Exact equality of the expected output and the prediction's primary output. +exactMatch :: (Eq o) => Metric o +exactMatch e p = boolScore (e == predictionPrimary p) + +-- | The custom-metric escape hatch. It is the identity at the type level; its +-- only purpose is to read intent at the call site (@customMetric $ \\e p -> ...@). +customMetric :: (o -> Prediction o -> Score) -> Metric o +customMetric = id + +-- | A pure, offline text-similarity metric. The projection @proj@ points at the +-- text field to compare (so it works on any record), and the score blends a +-- token-set Jaccard similarity with a normalized character-level (Levenshtein) +-- similarity over case-folded, whitespace-collapsed text. Identical strings score +-- 1; disjoint strings score near 0. +normalizedStringSimilarity :: (o -> Text) -> Metric o +normalizedStringSimilarity proj e p = + textSimilarity (proj e) (proj (predictionPrimary p)) + +-- | The similarity of two strings: the mean of token-set Jaccard and normalized +-- character-level similarity, both computed over normalized (case-folded, +-- whitespace-collapsed, trimmed) text. +textSimilarity :: Text -> Text -> Score +textSimilarity a0 b0 = mkScore ((jaccard + charSim) / 2) + where + a = normalize a0 + b = normalize b0 + toksA = Set.fromList (T.words a) + toksB = Set.fromList (T.words b) + union = Set.size (Set.union toksA toksB) + inter = Set.size (Set.intersection toksA toksB) + jaccard + | union == 0 = 1 -- two empty strings are identical + | otherwise = fromIntegral inter / fromIntegral union + maxLen = max (T.length a) (T.length b) + charSim + | maxLen == 0 = 1 + | otherwise = 1 - fromIntegral (levenshtein a b) / fromIntegral maxLen + +-- | Case-fold, collapse runs of whitespace to single spaces, and trim. +normalize :: Text -> Text +normalize = T.unwords . T.words . T.toCaseFold + +-- | The Levenshtein edit distance between two strings (the canonical single-row +-- dynamic program, folded over the characters of @t@). Self-contained — no extra +-- dependency. +levenshtein :: Text -> Text -> Int +levenshtein s t = lastElem (foldl transform [0 .. length a] b) + where + a = T.unpack s + b = T.unpack t + -- Given the previous DP row and the next character of @t@, compute the next + -- row. @diag@/@up@ are the previous row's cells to the upper-left and above + -- the cell being filled; @left@ is the cell just produced. + transform prevRow charB = case prevRow of + (p0 : rest) -> scanl fill (p0 + 1) (zip3 a prevRow rest) + [] -> [] -- unreachable: the row always has length (length a + 1) >= 1 + where + fill left (charA, diag, up) = + minimum [left + 1, up + 1, diag + (if charA == charB then 0 else 1)] + -- The last cell of the final row (0 for an empty row, which cannot occur). + lastElem = foldl (\_ x -> x) 0 + +-- | Combine several metrics by a weighted average of their scores. An empty list +-- yields the constant zero metric. +weightedMean :: [(Double, Metric o)] -> Metric o +weightedMean [] = \_ _ -> mkScore 0 +weightedMean ws = \e p -> + let totalW = sum (map fst ws) + weighted = sum [w * unScore (m e p) | (w, m) <- ws] + in if totalW == 0 then mkScore 0 else mkScore (weighted / totalW) + +-- | Turn a graded metric into pass/fail: score 1 when @m@ meets the threshold, +-- else 0. +threshold :: Double -> Metric o -> Metric o +threshold t m = \e p -> boolScore (unScore (m e p) >= t) + +-- | Invert a metric (@1 - s@), useful for "lower is better" wrapped metrics. +invert :: Metric o -> Metric o +invert m = \e p -> mkScore (1 - unScore (m e p)) + +-- --------------------------------------------------------------------------- +-- The embedding capability +-- --------------------------------------------------------------------------- + +-- | A minimal embedding effect: turn text into a dense vector. EP-1's @LLM@ +-- effect exposes no embedding operation, so 'semanticSimilarity' is expressed +-- over this capability; its interpreter is supplied by the caller (a real +-- embedding backend, or 'runEmbedding' over a pure function for tests). +data Embedding :: Effect where + EmbedText :: Text -> Embedding m (Vector Double) + +type instance DispatchOf Embedding = 'Dynamic + +-- | Embed a piece of text. +embedText :: (Embedding :> es) => Text -> Eff es (Vector Double) +embedText t = send (EmbedText t) + +-- | Interpret 'Embedding' with a pure function (deterministic; good for tests and +-- for backends that precompute embeddings). +runEmbedding :: (Text -> Vector Double) -> Eff (Embedding : es) a -> Eff es a +runEmbedding f = interpret $ \_ -> \case + EmbedText t -> pure (f t) + +-- | Embedding-based semantic similarity: the cosine similarity of the embeddings +-- of the projected expected and predicted text, mapped from @[-1, 1]@ into +-- @[0, 1]@. A zero-norm vector yields 'scoreZero' (no signal). +semanticSimilarity :: (Embedding :> es) => (o -> Text) -> MetricM es o +semanticSimilarity proj expd predd = do + a <- embedText (proj expd) + b <- embedText (proj (predictionPrimary predd)) + pure (cosineScore a b) + +-- | Map the cosine similarity of two vectors into a 'Score'. +cosineScore :: Vector Double -> Vector Double -> Score +cosineScore a b + | na == 0 || nb == 0 = scoreZero + | otherwise = mkScore ((dot / (na * nb) + 1) / 2) + where + dot = V.sum (V.zipWith (*) a b) + na = sqrt (V.sum (V.map (\x -> x * x) a)) + nb = sqrt (V.sum (V.map (\x -> x * x) b)) + +-- --------------------------------------------------------------------------- +-- LLM-as-judge +-- --------------------------------------------------------------------------- + +-- | The judge's input: the grading rubric plus the reference and candidate texts +-- (plain fields so the rendered prompt stays clean). +data JudgeInput = JudgeInput + { rubric :: Text, + reference :: Text, + candidate :: Text + } + deriving stock (Generic, Show) + +instance FromModel JudgeInput + +instance ToPrompt JudgeInput + +-- | The judge's output: a single numeric grade, intended to be in @[0, 1]@ +-- (clamped by 'mkScore' regardless). +newtype Grade = Grade {grade :: Double} + deriving stock (Generic, Show) + +instance ToSchema Grade + +instance FromModel Grade + +instance ToPrompt Grade + +instance Validatable Grade + +-- | The judge program: a single structured-output predictor that returns a +-- 'Grade'. The @rubricText@ becomes the signature instruction. +judgeProgram :: Text -> Program JudgeInput Grade +judgeProgram rubricText = predict (judgeSig rubricText) + +judgeSig :: Text -> Signature JudgeInput Grade +judgeSig rubricText = + mkSignature $ + rubricText + <> " Compare the candidate answer to the reference answer and return a single field" + <> " `grade`: a number in [0,1] where 1 means the candidate fully matches the" + <> " reference and 0 means it is entirely wrong." + +-- | LLM-as-judge: ask a model to grade the prediction against the expected +-- output. The rubric is supplied as text; the model returns a numeric grade, +-- decoded through the structured-output path. A decode failure surfaces as a +-- 'ShikumiError' (the per-example boundary in @evaluate@ turns it into a +-- 'Shikumi.Eval.Report.MetricError'). Threads @Error ShikumiError@ because +-- 'runProgram' does (EP-4): decode failures throw typed errors. +modelJudge :: + (LLM :> es, Error ShikumiError :> es) => + -- | rubric / grading instruction + Text -> + -- | render an output as text for the judge to read + (o -> Text) -> + MetricM es o +modelJudge rubricText render expd predd = do + let inp = JudgeInput rubricText (render expd) (render (predictionPrimary predd)) + Grade g <- runProgram (judgeProgram rubricText) inp + pure (mkScore g) diff --git a/src/Shikumi/Eval/Report.hs b/src/Shikumi/Eval/Report.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Report.hs @@ -0,0 +1,203 @@ +-- | The evaluation report and its aggregation. A 'Report' summarises a run over +-- a dataset: the mean per-example score, pass/fail counts, summed token usage and +-- cost, total latency, and a per-example breakdown ('ExampleResult', retained in +-- dataset order for failure analysis). 'mkReport' is the pure aggregator; +-- 'renderReportText' is a deterministic human-readable rendering reused by the +-- CLI (@docs/plans/12-cli-and-developer-experience.md@) and by golden tests. +-- +-- 'EvalConfig' carries the run knobs (bounded 'concurrency', the 'FailurePolicy' +-- for per-example errors, and 'numSamples' per example for multi-sample metrics); +-- 'defaultEvalConfig' scores failures @0@ and keeps going. +module Shikumi.Eval.Report + ( -- * Per-example outcome + ExampleResult (..), + FailureReason (..), + FailurePolicy (..), + + -- * Run configuration + EvalConfig (..), + defaultEvalConfig, + + -- * Usage totals + UsageTotals (..), + emptyUsageTotals, + + -- * The report + Report (..), + mkReport, + renderReportText, + ) +where + +import Data.Maybe (isJust, mapMaybe) +import Data.Text (Text) +import Data.Text qualified as T +import Numeric (showFFloat) +import Numeric.Natural (Natural) +import Shikumi.Eval.Types (Score, scoreZero, unScore) + +-- | The reason an example did not complete normally. +data FailureReason + = -- | @runProgram@ threw a 'Shikumi.Error.ShikumiError' (rendered to text) + ProgramError !Text + | -- | the metric itself failed (an effectful metric raised an error) + MetricError !Text + | -- | the example exceeded its time budget + TimedOut + deriving stock (Eq, Show) + +-- | What to do when an example fails. +data FailurePolicy + = -- | record the failure, assign this score, continue + FailScore !Score + | -- | re-throw, ending the whole run + FailAbort + deriving stock (Eq, Show) + +-- | Per-example outcome retained in the report for failure analysis. +data ExampleResult = ExampleResult + { -- | position in the dataset (0-based) + index :: !Int, + score :: !Score, + failure :: !(Maybe FailureReason), + latencyMs :: !Integer + } + deriving stock (Eq, Show) + +-- | Summed token/cost usage across the run. +data UsageTotals = UsageTotals + { totalInputTokens :: !Natural, + totalOutputTokens :: !Natural, + totalTokens :: !Natural, + totalCostUsd :: !Rational + } + deriving stock (Eq, Show) + +-- | The zero of 'UsageTotals'. +emptyUsageTotals :: UsageTotals +emptyUsageTotals = UsageTotals 0 0 0 0 + +instance Semigroup UsageTotals where + a <> b = + UsageTotals + (totalInputTokens a + totalInputTokens b) + (totalOutputTokens a + totalOutputTokens b) + (totalTokens a + totalTokens b) + (totalCostUsd a + totalCostUsd b) + +instance Monoid UsageTotals where + mempty = emptyUsageTotals + +-- | Knobs for an evaluation run. +data EvalConfig = EvalConfig + { -- | max examples evaluated at once (forced to @>= 1@ by 'evaluate') + concurrency :: !Int, + failurePolicy :: !FailurePolicy, + -- | samples per example for multi-sample metrics (forced to @>= 1@) + numSamples :: !Int + } + deriving stock (Eq, Show) + +-- | The default: four-way concurrency, score failures @0@ and keep going, one +-- sample per example. +defaultEvalConfig :: EvalConfig +defaultEvalConfig = + EvalConfig + { concurrency = 4, + failurePolicy = FailScore scoreZero, + numSamples = 1 + } + +-- | The aggregate result of an evaluation run. +data Report = Report + { -- | mean of the per-example scores, in @[0, 1]@ (0 when there are no examples) + aggregateScore :: !Double, + -- | examples whose score is exactly @1.0@ + passCount :: !Int, + -- | examples carrying a 'FailureReason' + failCount :: !Int, + total :: !Int, + -- | per-example outcomes, retained in dataset order + results :: ![ExampleResult], + usage :: !UsageTotals, + totalLatencyMs :: !Integer + } + deriving stock (Eq, Show) + +-- | Build a report from per-example results and the run's usage totals. +-- 'aggregateScore' is the arithmetic mean of the per-example scores (0 if there +-- are no examples). 'passCount' counts perfect scores; 'failCount' counts +-- examples with a 'FailureReason'. +mkReport :: [ExampleResult] -> UsageTotals -> Report +mkReport rs u = + Report + { aggregateScore = + if null rs + then 0 + else sum (map (unScore . score) rs) / fromIntegral (length rs), + passCount = length (filter ((== 1.0) . unScore . score) rs), + failCount = length (filter (isJust . failure) rs), + total = length rs, + results = rs, + usage = u, + totalLatencyMs = sum (map latencyMs rs) + } + +-- | A deterministic, human-readable multi-line summary of a report. The format +-- is stable (fixed 4-decimal score and cost, examples in index order) so the CLI +-- and golden tests can rely on it. The exact shape, for a 3-example run with one +-- failure: +-- +-- > score=0.6667 pass=2/3 fail=1 +-- > tokens: in=120 out=45 total=165 +-- > cost: $0.0023 +-- > latency: 1234 ms +-- > failures: +-- > [2] ProgramError: decode failed +-- +-- When there are no failures the trailing @failures:@ block is omitted. +renderReportText :: Report -> Text +renderReportText r = + T.intercalate "\n" (header ++ failureLines) + where + header = + [ "score=" + <> fixed4 (aggregateScore r) + <> " pass=" + <> tshow (passCount r) + <> "/" + <> tshow (total r) + <> " fail=" + <> tshow (failCount r), + "tokens: in=" + <> tshowNat (totalInputTokens (usage r)) + <> " out=" + <> tshowNat (totalOutputTokens (usage r)) + <> " total=" + <> tshowNat (totalTokens (usage r)), + "cost: $" <> fixed4 (fromRational (totalCostUsd (usage r))), + "latency: " <> tshow (totalLatencyMs r) <> " ms" + ] + failingResults = mapMaybe asFailure (results r) + asFailure er = (\fr -> (index er, fr)) <$> failure er + failureLines + | null failingResults = [] + | otherwise = "failures:" : map renderFailure failingResults + renderFailure (i, fr) = " [" <> tshow i <> "] " <> renderReason fr + +-- | Render a failure reason for the report. +renderReason :: FailureReason -> Text +renderReason = \case + ProgramError t -> "ProgramError: " <> t + MetricError t -> "MetricError: " <> t + TimedOut -> "TimedOut" + +-- | A 'Double' shown with exactly four decimal places. +fixed4 :: Double -> Text +fixed4 d = T.pack (showFFloat (Just 4) d "") + +tshow :: (Show a) => a -> Text +tshow = T.pack . show + +tshowNat :: Natural -> Text +tshowNat = T.pack . show diff --git a/src/Shikumi/Eval/Types.hs b/src/Shikumi/Eval/Types.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Types.hs @@ -0,0 +1,111 @@ +-- | The evaluation data model — MasterPlan integration point #5. The optimizer +-- (@docs/plans/10-optimizer-framework.md@) and the CLI +-- (@docs/plans/12-cli-and-developer-experience.md@) /consume/ these types and +-- must not redefine them. +-- +-- A 'Score' is a 'Double' clamped to the closed interval @[0, 1]@ at +-- construction, so downstream ranking code can assume @0 <= s <= 1@ without +-- re-checking (a @Bool@ metric maps @True -> 1@, @False -> 0@). An 'Example' is +-- one labelled datum (input + expected output); a 'Dataset' is a list of them. A +-- 'Prediction' is a program's actual output for one example, carrying a primary +-- output plus the non-empty set of sampled outputs (a single-output program +-- yields a one-element 'Prediction'), so agreement-rewarding metrics +-- (majority-vote, ensemble) can inspect every sample. +module Shikumi.Eval.Types + ( -- * Scores + Score (..), + mkScore, + scoreZero, + scoreOne, + boolScore, + unScore, + + -- * Examples and datasets + Example (..), + example, + Dataset (..), + dataset, + datasetExamples, + datasetSize, + + -- * Predictions (program outputs) + Prediction (..), + prediction, + predictionPrimary, + predictionSamples, + ) +where + +import Data.Aeson (Value) +import Data.List.NonEmpty (NonEmpty (..)) + +-- | A metric score, always in the closed interval @[0, 1]@. +newtype Score = Score Double + deriving stock (Eq, Ord, Show) + +-- | Clamp any 'Double' into @[0, 1]@. +mkScore :: Double -> Score +mkScore d = Score (max 0 (min 1 d)) + +-- | The two interval endpoints as scores. +scoreZero, scoreOne :: Score +scoreZero = Score 0 +scoreOne = Score 1 + +-- | Map a boolean outcome to a score: @True -> 1@, @False -> 0@. +boolScore :: Bool -> Score +boolScore True = scoreOne +boolScore False = scoreZero + +-- | Project the underlying 'Double' (always in @[0, 1]@). +unScore :: Score -> Double +unScore (Score d) = d + +-- | One labelled datum: an input of type @i@ and its expected output of type @o@. +data Example i o = Example + { input :: !i, + expected :: !o + } + deriving stock (Eq, Show) + +-- | Build an 'Example' from an input and its expected output. +example :: i -> o -> Example i o +example = Example + +-- | A typed evaluation dataset. +newtype Dataset i o = Dataset [Example i o] + deriving stock (Eq, Show) + +-- | Build a 'Dataset' from a list of examples. +dataset :: [Example i o] -> Dataset i o +dataset = Dataset + +-- | The examples of a dataset, in order. +datasetExamples :: Dataset i o -> [Example i o] +datasetExamples (Dataset xs) = xs + +-- | The number of examples in a dataset. +datasetSize :: Dataset i o -> Int +datasetSize (Dataset xs) = length xs + +-- | A program's actual output for one example. Carries a primary output and the +-- non-empty set of sampled outputs (a single-output program yields one sample). +-- An optional raw JSON @detail@ lets adapters or judges attach structure. +data Prediction o = Prediction + { primary :: !o, + samples :: !(NonEmpty o), + detail :: !(Maybe Value) + } + deriving stock (Eq, Show) + +-- | Build a single-sample prediction (primary output, one sample, no detail). +prediction :: o -> Prediction o +prediction o = Prediction o (o :| []) Nothing + +-- | The designated primary output of a prediction. +predictionPrimary :: Prediction o -> o +predictionPrimary = primary + +-- | The non-empty set of sampled outputs. +predictionSamples :: Prediction o -> NonEmpty o +predictionSamples = samples diff --git a/src/Shikumi/Eval/Usage.hs b/src/Shikumi/Eval/Usage.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Eval/Usage.hs @@ -0,0 +1,51 @@ +-- | Usage accounting for an evaluation run. 'withUsageTotals' runs an 'Eff' +-- action while accumulating the token usage and cost of every @LLM@ call into a +-- 'UsageTotals', returning both the action's result and the totals. +-- +-- The accumulation is a local 'interpose' on EP-1's @LLM@ effect — the same seam +-- EP-6's @cachedLLM@ and EP-7's @tracedLLM@ use. Because @LLM.complete@ returns +-- the full 'Response' (which already carries 'Baikai.Usage.Usage' and +-- 'Baikai.Cost.Cost'), no substrate hook or writer effect is needed; the +-- interposed handler reads usage straight off each response. A shared 'IORef' +-- mutated with 'atomicModifyIORef'' makes the accumulation safe under the bounded +-- concurrency @evaluate@ uses (each pooled worker gets a cloned env that still +-- references the one ref). +module Shikumi.Eval.Usage + ( withUsageTotals, + ) +where + +import Control.Lens ((^.)) +import Data.Generics.Labels () +import Effectful (Eff, (:>)) +import Effectful.Dispatch.Dynamic (interpose) +import Effectful.Prim.IORef (Prim, atomicModifyIORef', newIORef, readIORef) +import Shikumi.Eval.Report (UsageTotals (..), emptyUsageTotals) +import Shikumi.LLM (LLM (..), Response, complete, stream) + +-- | Run @act@, accumulating every @LLM@ call's usage/cost into a 'UsageTotals'. +withUsageTotals :: (LLM :> es, Prim :> es) => Eff es a -> Eff es (a, UsageTotals) +withUsageTotals act = do + ref <- newIORef emptyUsageTotals + result <- + interpose + ( \_ -> \case + Complete m c o -> do + resp <- complete m c o + atomicModifyIORef' ref (\u -> (u <> usageOf resp, ())) + pure resp + Stream m c o -> stream m c o + ) + act + totals <- readIORef ref + pure (result, totals) + +-- | Project a response's token usage and cost into a 'UsageTotals'. +usageOf :: Response -> UsageTotals +usageOf resp = + UsageTotals + { totalInputTokens = resp ^. #message . #usage . #inputTokens, + totalOutputTokens = resp ^. #message . #usage . #outputTokens, + totalTokens = resp ^. #message . #usage . #totalTokens, + totalCostUsd = resp ^. #message . #usage . #cost . #usd + } diff --git a/test/DocSpec.hs b/test/DocSpec.hs new file mode 100644 --- /dev/null +++ b/test/DocSpec.hs @@ -0,0 +1,39 @@ +-- | Exercises the worked example from the 'Shikumi.Eval' module header end-to-end +-- under a mock LM, so the documented usage is guaranteed to compile and run. The +-- single import 'Shikumi.Eval' must expose everything the snippet needs. +module DocSpec (tests) where + +import Effectful (runEff) +import Effectful.Concurrent (runConcurrent) +import Effectful.Error.Static (runErrorNoCallStack) +import Effectful.Prim (runPrim) +import EvalFixtures (Answer (..), Question (..), answerResponse, qaProg, runConstLLM) +import Shikumi.Effect.Time (runTime) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval (aggregateScore, dataset, evaluatePure, exactMatch, example) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertBool, assertFailure, testCase) + +tests :: TestTree +tests = + testGroup + "Doc" + [ testCase "module-header example runs under a mock" $ do + -- The mock answers "4" for every call, so one of the two examples + -- matches its expected answer: an aggregate score of 0.5. + let qaData = + dataset + [ example (Question "What is 2+2?") (Answer "4"), + example (Question "Capital of France?") (Answer "Paris") + ] + result <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runConstLLM (answerResponse "4") $ + evaluatePure qaData exactMatch qaProg + case result of + Left e -> assertFailure ("unexpected error: " <> show e) + Right report -> + assertBool + ("expected ~0.5, got " <> show (aggregateScore report)) + (abs (aggregateScore report - 0.5) < 1e-9) + ] diff --git a/test/EmbeddingSpec.hs b/test/EmbeddingSpec.hs new file mode 100644 --- /dev/null +++ b/test/EmbeddingSpec.hs @@ -0,0 +1,64 @@ +{-# LANGUAGE TypeApplications #-} + +-- | End-to-end tests for the embeddings path (EP-15, M3). +-- +-- The hermetic test drives @semanticSimilarity@ through the /real/ interpreter +-- shape ('runEmbeddingBy') with a deterministic stub embedder — exercising the new +-- code path without a network — and asserts a semantically-close pair scores +-- strictly higher than a distant pair. The live test is gated on +-- @SHIKUMI_EMBEDDING_LIVE=1@ and asserts a real vector's dimensionality. +module EmbeddingSpec (tests) where + +import Data.Text (Text) +import Data.Vector (Vector) +import Data.Vector qualified as V +import Effectful (runEff) +import Effectful.Error.Static (runErrorNoCallStack) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Embedding (runEmbeddingBy, runEmbeddingLLM) +import Shikumi.Eval.Metric (embedText, semanticSimilarity) +import Shikumi.Eval.Types (Score, prediction, unScore) +import System.Environment (lookupEnv) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=)) + +-- | A deterministic stub embedder. The two cat sentences point in nearly the same +-- direction (cosine ≈ 0.995); the revenue sentence is near-orthogonal/opposite to +-- them, so the close pair must outscore the distant pair under @cosineScore@. +stubEmbed :: [Text] -> IO (Vector (Vector Double)) +stubEmbed = pure . V.fromList . map vecFor + where + vecFor "a cat sat on the mat" = V.fromList [1, 0.9] + vecFor "the cat is on the rug" = V.fromList [0.9, 1] + vecFor "quarterly revenue rose" = V.fromList [1, -1] + vecFor _ = V.fromList [0, 0] + +-- | Score the similarity of two texts through the real interpreter over the stub. +runSim :: Text -> Text -> IO (Either ShikumiError Score) +runSim expd predd = + runEff . runErrorNoCallStack @ShikumiError . runEmbeddingBy stubEmbed $ + semanticSimilarity id expd (prediction predd) + +tests :: TestTree +tests = + testGroup + "Embedding" + [ testCase "semanticSimilarity scores close pair above distant pair (hermetic)" $ do + close <- runSim "a cat sat on the mat" "the cat is on the rug" + distant <- runSim "a cat sat on the mat" "quarterly revenue rose" + case (close, distant) of + (Right c, Right d) -> + assertBool + ("close (" <> show (unScore c) <> ") must exceed distant (" <> show (unScore d) <> ")") + (unScore c > unScore d) + _ -> assertFailure "stub-backed semanticSimilarity errored", + testCase "live embedding returns expected dimensionality" $ do + live <- lookupEnv "SHIKUMI_EMBEDDING_LIVE" + case live of + Just "1" -> do + r <- runEff . runErrorNoCallStack @ShikumiError . runEmbeddingLLM $ embedText "hello" + case r of + Right v -> V.length v @?= 1536 + Left e -> assertFailure ("live embedding failed: " <> show e) + _ -> putStrLn "SHIKUMI_EMBEDDING_LIVE not set; skipping live test" + ] diff --git a/test/EvalFixtures.hs b/test/EvalFixtures.hs new file mode 100644 --- /dev/null +++ b/test/EvalFixtures.hs @@ -0,0 +1,146 @@ +-- | Shared, network-free fixtures for the @evaluate@ / golden specs: a +-- @Question -> Answer@ signature and program, canned fallback-adapter responses, +-- and two mock @LLM@ interpreters — a constant one that returns a fixed response +-- for every call (order-independent, used for the aggregate-score acceptance) and +-- a scripted one that pops replies in order and can inject a typed failure (used +-- for the failure-policy cases under @concurrency = 1@). +module EvalFixtures + ( -- * Records + Question (..), + Answer (..), + + -- * Signature and program + qaSig, + qaProg, + + -- * Canned responses + answerResponse, + markerResponse, + + -- * Mock LLM interpreters + MockReply (..), + runConstLLM, + runScriptedLLM, + ) +where + +import Baikai (AssistantContent (..), Response, _Response, _TextContent) +import Control.Lens ((&), (.~)) +import Data.Generics.Labels () +import Data.IORef (atomicModifyIORef', newIORef) +import Data.Text (Text) +import Data.Text qualified as T +import Data.Vector qualified as V +import Effectful (Eff, IOE, liftIO, type (:>)) +import Effectful.Dispatch.Dynamic (interpret) +import Effectful.Error.Static (Error, throwError) +import GHC.Generics (Generic) +import Shikumi.Adapter (ToPrompt) +import Shikumi.Error (ShikumiError (..)) +import Shikumi.LLM (LLM (..)) +import Shikumi.Module (predict) +import Shikumi.Program (Program) +import Shikumi.Schema (FromModel, ToSchema, Validatable) +import Shikumi.Signature (Signature, mkSignature) + +-- --------------------------------------------------------------------------- +-- Records +-- --------------------------------------------------------------------------- + +newtype Question = Question {question :: Text} + deriving stock (Generic, Show, Eq) + +instance ToSchema Question + +instance FromModel Question + +instance ToPrompt Question + +instance Validatable Question + +newtype Answer = Answer {answer :: Text} + deriving stock (Generic, Show, Eq) + +instance ToSchema Answer + +instance FromModel Answer + +instance ToPrompt Answer + +instance Validatable Answer + +-- --------------------------------------------------------------------------- +-- Signature and program +-- --------------------------------------------------------------------------- + +qaSig :: Signature Question Answer +qaSig = mkSignature "Answer the question." + +-- | The simplest program: a single predict node. Its behaviour is fully +-- determined by the mock LLM the test wires underneath it. +qaProg :: Program Question Answer +qaProg = predict qaSig + +-- --------------------------------------------------------------------------- +-- Canned responses (rendered as the fallback adapter's [[ ## field ## ]] form) +-- --------------------------------------------------------------------------- + +-- | An assistant 'Response' whose body decodes (via the fallback adapter) to +-- @Answer t@. +answerResponse :: Text -> Response +answerResponse t = mkResponse (markerBody [("answer", t)]) + +-- | An assistant 'Response' whose body decodes (via the fallback adapter) to a +-- record with the given @(field, value)@ sections. +markerResponse :: [(Text, Text)] -> Response +markerResponse = mkResponse . markerBody + +-- | Build a fallback-style response body from @(field, value)@ sections. +markerBody :: [(Text, Text)] -> Text +markerBody fields = T.unlines (concatMap sect fields ++ ["[[ ## completed ## ]]"]) + where + sect (k, v) = ["[[ ## " <> k <> " ## ]]", v] + +-- | An assistant 'Response' carrying @t@ as its single text block. +mkResponse :: Text -> Response +mkResponse t = + _Response & #message . #content .~ V.singleton (AssistantText (_TextContent & #text .~ t)) + +-- --------------------------------------------------------------------------- +-- Mock LLM interpreters +-- --------------------------------------------------------------------------- + +-- | One scripted reply: a canned success or a typed failure. +data MockReply + = MockOk Response + | MockFail ShikumiError + +-- | Return @resp@ for every completion, ignoring the request. Order-independent, +-- so it is safe under any concurrency. +runConstLLM :: Response -> Eff (LLM : es) a -> Eff es a +runConstLLM resp = interpret $ \_ -> \case + Complete {} -> pure resp + Stream {} -> pure [] + +-- | Pop scripted replies in order; a 'MockFail' is thrown through +-- @Error ShikumiError@. Deterministic only under @concurrency = 1@. +runScriptedLLM :: + (IOE :> es, Error ShikumiError :> es) => + [MockReply] -> + Eff (LLM : es) a -> + Eff es a +runScriptedLLM replies act = do + ref <- liftIO (newIORef replies) + interpret + ( \_ -> \case + Complete {} -> do + r <- liftIO (atomicModifyIORef' ref pop) + case r of + MockOk resp -> pure resp + MockFail e -> throwError e + Stream {} -> pure [] + ) + act + where + pop (x : xs) = (xs, x) + pop [] = ([], MockFail (ProviderFailure "mock: script exhausted")) diff --git a/test/EvaluateSpec.hs b/test/EvaluateSpec.hs new file mode 100644 --- /dev/null +++ b/test/EvaluateSpec.hs @@ -0,0 +1,106 @@ +-- | The plan's primary acceptance: @evaluate@ end-to-end against a mock LM, with +-- no network. Three cases — a four-of-five exact-match run scoring 0.8; a +-- program error scored 0 under the default 'FailScore' policy while the run +-- completes; and the same error aborting the run under 'FailAbort'. +module EvaluateSpec (tests) where + +import Effectful (runEff) +import Effectful.Concurrent (runConcurrent) +import Effectful.Error.Static (runErrorNoCallStack) +import Effectful.Prim (runPrim) +import EvalFixtures + ( Answer (..), + MockReply (..), + Question (..), + answerResponse, + qaProg, + runConstLLM, + runScriptedLLM, + ) +import Shikumi.Effect.Time (runTime) +import Shikumi.Error (ShikumiError (..)) +import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith) +import Shikumi.Eval.Metric (exactMatch, liftMetric) +import Shikumi.Eval.Report + ( EvalConfig (..), + ExampleResult (..), + FailurePolicy (..), + FailureReason (..), + Report (..), + defaultEvalConfig, + ) +import Shikumi.Eval.Types (Score, dataset, example, scoreZero) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=)) + +-- | Five questions; the program (driven by the constant mock) always answers +-- "yes", so the four examples whose expected answer is "yes" pass exactMatch and +-- the one whose expected answer is "no" fails — a 0.8 aggregate, independent of +-- evaluation order. +aggregateData :: [(Question, Answer)] +aggregateData = + [ (Question "q1", Answer "yes"), + (Question "q2", Answer "yes"), + (Question "q3", Answer "yes"), + (Question "q4", Answer "yes"), + (Question "q5", Answer "no") + ] + +tests :: TestTree +tests = + testGroup + "Evaluate" + [ testCase "four-of-five exact match -> aggregateScore 0.8" $ do + let ds = dataset [example q a | (q, a) <- aggregateData] + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runConstLLM (answerResponse "yes") $ + evaluatePure ds exactMatch qaProg + case report of + Left e -> assertFailure ("unexpected error: " <> show e) + Right r -> do + assertBool + ("expected ~0.8, got " <> show (aggregateScore r)) + (abs (aggregateScore r - 0.8) < 1e-9) + passCount r @?= 4 + failCount r @?= 0 + total r @?= 5 + length (results r) @?= 5 + map index (results r) @?= [0, 1, 2, 3, 4], + testCase "program error -> FailScore scoreZero, run completes" $ do + -- concurrency = 1 so the scripted replies pop in dataset order: the first + -- example fails, the rest succeed. + let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")] + replies = [MockFail providerErr, MockOk (answerResponse "ok")] + cfg = defaultEvalConfig {concurrency = 1} + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runScriptedLLM replies $ + evaluateWith cfg ds (liftMetric exactMatch) qaProg + case report of + Left e -> assertFailure ("run should not abort, but got: " <> show e) + Right r -> do + total r @?= 2 + failCount r @?= 1 + case results r of + (er0 : _) -> do + index er0 @?= 0 + score er0 @?= (scoreZero :: Score) + case failure er0 of + Just (ProgramError _) -> pure () + other -> assertFailure ("expected ProgramError, got " <> show other) + [] -> assertFailure "expected two results", + testCase "FailAbort surfaces the error" $ do + let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")] + replies = [MockFail providerErr, MockOk (answerResponse "ok")] + cfg = defaultEvalConfig {concurrency = 1, failurePolicy = FailAbort} + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runScriptedLLM replies $ + evaluateWith cfg ds (liftMetric exactMatch) qaProg + case report of + Left _ -> pure () + Right _ -> assertFailure "expected FailAbort to surface a Left" + ] + where + providerErr = ProviderFailure "boom" diff --git a/test/GoldenSpec.hs b/test/GoldenSpec.hs new file mode 100644 --- /dev/null +++ b/test/GoldenSpec.hs @@ -0,0 +1,44 @@ +{-# LANGUAGE RankNTypes #-} + +-- | A golden test for a stub program, run deterministically and offline under a +-- constant mock LM. The committed @test/golden/qa-program.golden@ pins the +-- transcript; the plan records the fail-before/pass-after demonstration. +module GoldenSpec (tests) where + +import Effectful (Eff, IOE, runEff) +import Effectful.Error.Static (Error, runErrorNoCallStack) +import EvalFixtures (Answer (..), Question (..), answerResponse, qaProg, runConstLLM) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Golden (goldenProgram) +import Shikumi.Eval.Types (Dataset, dataset, example) +import Shikumi.LLM (LLM) +import Test.Tasty (TestTree, testGroup) + +-- | Two questions; the constant mock always answers "yes", so the transcript is +-- deterministic regardless of the inputs. +goldenData :: Dataset Question Answer +goldenData = + dataset + [ example (Question "what is 2+2?") (Answer "ignored"), + example (Question "capital of France?") (Answer "ignored") + ] + +-- | Discharge @'[LLM, Error ShikumiError, IOE]@ to 'IO' under the constant mock, +-- failing on a 'ShikumiError'. +runMock :: forall a. Eff '[LLM, Error ShikumiError, IOE] a -> IO a +runMock act = do + r <- runEff . runErrorNoCallStack @ShikumiError . runConstLLM (answerResponse "yes") $ act + either (fail . show) pure r + +tests :: TestTree +tests = + testGroup + "Golden" + [ goldenProgram + "qa-program golden" + "test/golden/qa-program.golden" + runMock + goldenData + qaProg + answer + ] diff --git a/test/Main.hs b/test/Main.hs new file mode 100644 --- /dev/null +++ b/test/Main.hs @@ -0,0 +1,30 @@ +-- | EP-8 acceptance suite for @shikumi-eval@: the data model (@Types@), pure and +-- LM-backed metrics (@Metric@), report aggregation (@Report@), the end-to-end +-- @evaluate@ runner (@Evaluate@), golden program tests (@Golden@), and the +-- documented-usage doctest. Every group is hermetic — no network, no API key. +module Main (main) where + +import DocSpec qualified +import EmbeddingSpec qualified +import EvaluateSpec qualified +import GoldenSpec qualified +import MetricLMSpec qualified +import MetricSpec qualified +import ReportSpec qualified +import Test.Tasty (defaultMain, testGroup) +import TypesSpec qualified + +main :: IO () +main = + defaultMain $ + testGroup + "shikumi-eval" + [ TypesSpec.tests, + MetricSpec.tests, + ReportSpec.tests, + EvaluateSpec.tests, + MetricLMSpec.tests, + GoldenSpec.tests, + EmbeddingSpec.tests, + DocSpec.tests + ] diff --git a/test/MetricLMSpec.hs b/test/MetricLMSpec.hs new file mode 100644 --- /dev/null +++ b/test/MetricLMSpec.hs @@ -0,0 +1,65 @@ +-- | Unit tests for the LM-backed metrics, run against mock interpreters with no +-- network: 'semanticSimilarity' over a pure 'Embedding' interpreter (identical / +-- orthogonal / opposite vectors), and 'modelJudge' over a constant mock LLM that +-- returns a fixed grade. +module MetricLMSpec (tests) where + +import Data.Text (Text) +import Data.Vector (Vector) +import Data.Vector qualified as V +import Effectful (runEff, runPureEff) +import Effectful.Error.Static (runErrorNoCallStack) +import EvalFixtures (markerResponse, runConstLLM) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Metric + ( modelJudge, + runEmbedding, + semanticSimilarity, + ) +import Shikumi.Eval.Types + ( Prediction, + Score, + mkScore, + prediction, + scoreOne, + scoreZero, + ) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertFailure, testCase, (@?=)) + +-- | A fixed embedding table: each label maps to a 2-D vector. "a" and "b" are +-- orthogonal (cosine 0 -> score 0.5); "c" is the opposite of "a" (cosine -1 -> +-- score 0); identical labels share a vector (cosine 1 -> score 1); "z" is the +-- zero vector (no signal -> score 0). +embedTable :: Text -> Vector Double +embedTable "a" = V.fromList [1, 0] +embedTable "b" = V.fromList [0, 1] +embedTable "c" = V.fromList [-1, 0] +embedTable _ = V.fromList [0, 0] + +-- | Score the similarity of two text labels under the fixed embedding table. +sim :: Text -> Prediction Text -> Score +sim expd predd = + runPureEff . runEmbedding embedTable $ semanticSimilarity id expd predd + +tests :: TestTree +tests = + testGroup + "Metric (LM)" + [ testCase "semanticSimilarity identical -> 1" $ + sim "a" (prediction "a") @?= scoreOne, + testCase "semanticSimilarity orthogonal -> 0.5" $ + sim "a" (prediction "b") @?= mkScore 0.5, + testCase "semanticSimilarity opposite -> 0" $ + sim "a" (prediction "c") @?= scoreZero, + testCase "semanticSimilarity zero-norm -> 0" $ + sim "z" (prediction "z") @?= scoreZero, + testCase "modelJudge returns the model's grade" $ do + result <- + runEff . runErrorNoCallStack @ShikumiError $ + runConstLLM (markerResponse [("grade", "0.7")]) $ + modelJudge "Grade the answer." id ("ref" :: Text) (prediction "cand") + case result of + Left e -> assertFailure ("unexpected error: " <> show e) + Right s -> s @?= mkScore 0.7 + ] diff --git a/test/MetricSpec.hs b/test/MetricSpec.hs new file mode 100644 --- /dev/null +++ b/test/MetricSpec.hs @@ -0,0 +1,60 @@ +-- | Unit tests for the pure metrics and combinators: exact match, normalized +-- string similarity (identical / near / disjoint), and the @threshold@, +-- @weightedMean@, and @invert@ combinators. All offline and deterministic. +module MetricSpec (tests) where + +import Data.Text (Text) +import Shikumi.Eval.Metric + ( exactMatch, + invert, + normalizedStringSimilarity, + threshold, + weightedMean, + ) +import Shikumi.Eval.Types + ( Prediction, + Score, + mkScore, + prediction, + scoreOne, + scoreZero, + unScore, + ) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertBool, testCase, (@?=)) + +-- | A constant metric ignoring its inputs, for combinator tests. +constMetric :: Double -> (o -> Prediction o -> Score) +constMetric d _ _ = mkScore d + +tests :: TestTree +tests = + testGroup + "Metric" + [ testCase "exactMatch equal" $ + exactMatch ("a" :: Text) (prediction "a") @?= scoreOne, + testCase "exactMatch unequal" $ + exactMatch ("a" :: Text) (prediction "b") @?= scoreZero, + testCase "normalizedStringSimilarity identical" $ + normalizedStringSimilarity id ("the cat" :: Text) (prediction "the cat") @?= scoreOne, + testCase "normalizedStringSimilarity case/space-insensitive" $ + normalizedStringSimilarity id ("The Cat" :: Text) (prediction "the cat") @?= scoreOne, + testCase "normalizedStringSimilarity near is strictly between" $ + let s = unScore (normalizedStringSimilarity id ("the cat sat" :: Text) (prediction "the cat ran")) + in assertBool ("expected 0 < " <> show s <> " < 1") (s > 0 && s < 1), + testCase "normalizedStringSimilarity disjoint is low" $ + let s = unScore (normalizedStringSimilarity id ("alpha" :: Text) (prediction "zzzzz")) + in assertBool ("expected " <> show s <> " < 0.2") (s < 0.2), + testCase "threshold passes a high score" $ + threshold 0.5 (constMetric 0.7) () (prediction ()) @?= scoreOne, + testCase "threshold fails a low score" $ + threshold 0.5 (constMetric 0.3) () (prediction ()) @?= scoreZero, + testCase "weightedMean averages by weight" $ + -- (1*0.2 + 3*0.6) / (1+3) = 2.0/4 = 0.5 (within float tolerance) + let s = unScore (weightedMean [(1, constMetric 0.2), (3, constMetric 0.6)] () (prediction ())) + in assertBool ("expected ~0.5, got " <> show s) (abs (s - 0.5) < 1e-9), + testCase "weightedMean of empty is zero" $ + weightedMean [] () (prediction ()) @?= scoreZero, + testCase "invert flips" $ + invert (constMetric 0.25) () (prediction ()) @?= mkScore 0.75 + ] diff --git a/test/ReportSpec.hs b/test/ReportSpec.hs new file mode 100644 --- /dev/null +++ b/test/ReportSpec.hs @@ -0,0 +1,69 @@ +-- | Unit tests for report aggregation: 'mkReport' over synthetic per-example +-- results, the 'UsageTotals' 'Monoid', and the deterministic 'renderReportText' +-- format (an inline golden-by-equality check). +module ReportSpec (tests) where + +import Data.Ratio ((%)) +import Data.Text (Text) +import Shikumi.Eval.Report + ( ExampleResult (..), + FailureReason (..), + Report (..), + UsageTotals (..), + emptyUsageTotals, + mkReport, + renderReportText, + ) +import Shikumi.Eval.Types (mkScore, scoreOne, scoreZero) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertBool, testCase, (@?=)) + +-- | Three synthetic outcomes: a perfect pass, a partial, and a program failure. +fixtureResults :: [ExampleResult] +fixtureResults = + [ ExampleResult {index = 0, score = scoreOne, failure = Nothing, latencyMs = 100}, + ExampleResult {index = 1, score = mkScore 0.5, failure = Nothing, latencyMs = 200}, + ExampleResult + { index = 2, + score = scoreZero, + failure = Just (ProgramError "decode failed"), + latencyMs = 934 + } + ] + +fixtureUsage :: UsageTotals +fixtureUsage = UsageTotals {totalInputTokens = 120, totalOutputTokens = 45, totalTokens = 165, totalCostUsd = 23 % 10000} + +fixtureReport :: Report +fixtureReport = mkReport fixtureResults fixtureUsage + +expectedRender :: Text +expectedRender = + "score=0.5000 pass=1/3 fail=1\n\ + \tokens: in=120 out=45 total=165\n\ + \cost: $0.0023\n\ + \latency: 1234 ms\n\ + \failures:\n\ + \ [2] ProgramError: decode failed" + +tests :: TestTree +tests = + testGroup + "Report" + [ testCase "aggregateScore is the mean" $ + assertBool + ("expected ~0.5, got " <> show (aggregateScore fixtureReport)) + (abs (aggregateScore fixtureReport - 0.5) < 1e-9), + testCase "passCount counts perfect scores" $ passCount fixtureReport @?= 1, + testCase "failCount counts failures" $ failCount fixtureReport @?= 1, + testCase "total counts examples" $ total fixtureReport @?= 3, + testCase "totalLatencyMs sums" $ totalLatencyMs fixtureReport @?= 1234, + testCase "results retained in order" $ map index (results fixtureReport) @?= [0, 1, 2], + testCase "empty report is zero" $ aggregateScore (mkReport [] emptyUsageTotals) @?= 0, + testCase "UsageTotals Monoid sums" $ + UsageTotals 10 20 30 (1 % 100) <> UsageTotals 1 2 3 (2 % 100) + @?= UsageTotals 11 22 33 (3 % 100), + testCase "UsageTotals mempty is empty" $ (mempty :: UsageTotals) @?= emptyUsageTotals, + testCase "renderReportText matches the documented format" $ + renderReportText fixtureReport @?= expectedRender + ] diff --git a/test/TypesSpec.hs b/test/TypesSpec.hs new file mode 100644 --- /dev/null +++ b/test/TypesSpec.hs @@ -0,0 +1,31 @@ +-- | Unit tests for the owned data model: 'Score' clamping, dataset +-- construction, and single-sample 'Prediction' construction. +module TypesSpec (tests) where + +import Data.List.NonEmpty (NonEmpty (..)) +import Shikumi.Eval.Types + ( Score (..), + dataset, + datasetSize, + example, + mkScore, + prediction, + predictionSamples, + scoreOne, + scoreZero, + ) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, (@?=)) + +tests :: TestTree +tests = + testGroup + "Types" + [ testCase "mkScore clamps high" $ mkScore 1.5 @?= scoreOne, + testCase "mkScore clamps low" $ mkScore (-0.2) @?= scoreZero, + testCase "mkScore passes through in-range" $ mkScore 0.5 @?= Score 0.5, + testCase "datasetSize counts examples" $ + datasetSize (dataset [example (1 :: Int) 'a', example 2 'b']) @?= 2, + testCase "prediction is single-sample" $ + predictionSamples (prediction ("x" :: String)) @?= ("x" :| []) + ]