shikumi-eval 0.1.1.0 → 0.2.0.0

raw patch · 12 files changed

+286/−39 lines, 12 filesdep ~baikaidep ~shikumidep ~shikumi-evalPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: baikai, shikumi, shikumi-eval

API changes (from Hackage documentation)

Files

• CHANGELOG.md +19/−0
• shikumi-eval.cabal +7/−6
• src/Shikumi/Eval/Evaluate.hs +12/−5
• src/Shikumi/Eval/Report.hs +19/−11
• src/Shikumi/Eval/Types.hs +1/−1
• src/Shikumi/Eval/Usage.hs +33/−6
• test/EvalFixtures.hs +55/−2
• test/EvaluateSpec.hs +60/−4
• test/Main.hs +2/−0
• test/ReportSpec.hs +1/−1
• test/TypesSpec.hs +3/−3
• test/UsageSpec.hs +74/−0

@@ -2,6 +2,25 @@  ## Unreleased +## 0.2.0.0 - 2026-07-05++### Added++- Evaluation can enforce an optional per-example timeout via+  `EvalConfig.exampleTimeoutMs`.+- Usage accounting now includes streamed calls and exposes helpers for assistant+  messages and stream terminal events.++### Changed++- **BREAKING** `EvalConfig` record construction requires the new+  `exampleTimeoutMs` field.+- **BREAKING** `Score` no longer exports its constructor; use `mkScore`,+  `scoreZero`, `scoreOne`, and `unScore`.+- Reports render summed per-example latency as `latency-sum`, making concurrent+  evaluation accounting explicit.+- Refreshed the internal `shikumi` bound for the `0.3` series.+ ## 0.1.1.0 - 2026-06-28  ### Added

@@ -1,6 +1,6 @@ cabal-version:   3.4 name:            shikumi-eval-version:         0.1.1.0+version:         0.2.0.0 synopsis:        Typed evaluation framework for shikumi LM programs (EP-8) category:        AI description:@@ -46,14 +46,14 @@    build-depends:     , aeson-    , baikai        >=0.2      && <0.3+    , baikai        >=0.3      && <0.4     , base          >=4.20     && <5     , bytestring     , containers     , effectful     , generic-lens     , lens          ^>=5.3-    , shikumi       ^>=0.2.0.0+    , shikumi       ^>=0.3.0.0     , tasty     , tasty-golden     , text          ^>=2.1@@ -75,16 +75,17 @@     MetricSpec     ReportSpec     TypesSpec+    UsageSpec    build-depends:     , aeson-    , baikai        >=0.2      && <0.3+    , baikai        >=0.3      && <0.4     , base     , effectful     , generic-lens     , lens-    , shikumi       ^>=0.2.0.0-    , shikumi-eval  ^>=0.1.1.0+    , shikumi       ^>=0.3.0.0+    , shikumi-eval  ^>=0.2.0.0     , tasty     , tasty-golden     , tasty-hunit

@@ -22,12 +22,12 @@ import Data.List.NonEmpty (NonEmpty ((:|))) import Data.Text qualified as T import Effectful (Eff, (:>))-import Effectful.Concurrent (Concurrent)-import Effectful.Concurrent.Async (pooledForConcurrentlyN)+import Effectful.Concurrent (Concurrent, threadDelay)+import Effectful.Concurrent.Async (pooledForConcurrentlyN, race) import Effectful.Error.Static (Error, catchError, throwError) import Effectful.Prim (Prim) import Shikumi.Effect.Time (Time, getMonotonicTimeNSec)-import Shikumi.Error (ShikumiError)+import Shikumi.Error (ShikumiError (..)) import Shikumi.Eval.Metric (Metric, MetricM, liftMetric) import Shikumi.Eval.Report   ( EvalConfig (..),@@ -87,7 +87,7 @@ -- | Evaluate a single indexed example inside its error boundary, timing it with a -- monotonic clock. evalOne ::-  (LLM :> es, Error ShikumiError :> es, Time :> es) =>+  (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es) =>   EvalConfig ->   MetricM es o ->   Program i o ->@@ -95,7 +95,14 @@   Eff es ExampleResult evalOne cfg metric prog (ix, Example inp expd) = do   start <- getMonotonicTimeNSec-  (s, mFail) <- scoreExample cfg metric prog inp expd+  outcome <- case exampleTimeoutMs cfg of+    Nothing -> Right <$> scoreExample cfg metric prog inp expd+    Just ms -> race (threadDelay (max 0 ms * 1000)) (scoreExample cfg metric prog inp expd)+  (s, mFail) <- case outcome of+    Right r -> pure r+    Left () -> case failurePolicy cfg of+      FailAbort -> throwError (Timeout "evaluate: example timed out")+      FailScore sc -> pure (sc, Just TimedOut)   end <- getMonotonicTimeNSec   let ms = fromIntegral ((end - start) `div` 1_000_000)   pure ExampleResult {index = ix, score = s, failure = mFail, latencyMs = ms}

@@ -1,13 +1,13 @@ -- | The evaluation report and its aggregation. A 'Report' summarises a run over -- a dataset: the mean per-example score, pass/fail counts, summed token usage and--- cost, total latency, and a per-example breakdown ('ExampleResult', retained in--- dataset order for failure analysis). 'mkReport' is the pure aggregator;+-- cost, summed per-example latency, and a per-example breakdown ('ExampleResult',+-- retained in dataset order for failure analysis). 'mkReport' is the pure aggregator; -- 'renderReportText' is a deterministic human-readable rendering reused by the -- CLI (@docs/plans/12-cli-and-developer-experience.md@) and by golden tests. -- -- 'EvalConfig' carries the run knobs (bounded 'concurrency', the 'FailurePolicy'--- for per-example errors, and 'numSamples' per example for multi-sample metrics);--- 'defaultEvalConfig' scores failures @0@ and keeps going.+-- for per-example errors, optional per-example timeout, and 'numSamples' per example+-- for multi-sample metrics); 'defaultEvalConfig' scores failures @0@ and keeps going. module Shikumi.Eval.Report   ( -- * Per-example outcome     ExampleResult (..),@@ -94,18 +94,21 @@   { -- | max examples evaluated at once (forced to @>= 1@ by 'evaluate')     concurrency :: !Int,     failurePolicy :: !FailurePolicy,+    -- | wall-clock budget per example in milliseconds; 'Nothing' means no timeout+    exampleTimeoutMs :: !(Maybe Int),     -- | samples per example for multi-sample metrics (forced to @>= 1@)     numSamples :: !Int   }   deriving stock (Eq, Show, Generic) --- | The default: four-way concurrency, score failures @0@ and keep going, one--- sample per example.+-- | The default: four-way concurrency, no timeout, score failures @0@ and keep+-- going, one sample per example. defaultEvalConfig :: EvalConfig defaultEvalConfig =   EvalConfig     { concurrency = 4,       failurePolicy = FailScore scoreZero,+      exampleTimeoutMs = Nothing,       numSamples = 1     } @@ -113,7 +116,8 @@ data Report = Report   { -- | mean of the per-example scores, in @[0, 1]@ (0 when there are no examples)     aggregateScore :: !Double,-    -- | examples whose score is exactly @1.0@+    -- | examples whose score is exactly @1.0@; fractional metrics can have good+    -- aggregate scores while still reporting zero passes     passCount :: !Int,     -- | examples carrying a 'FailureReason'     failCount :: !Int,@@ -121,14 +125,18 @@     -- | per-example outcomes, retained in dataset order     results :: ![ExampleResult],     usage :: !UsageTotals,+    -- | sum of per-example latencies; under concurrent evaluation this can exceed+    -- wall-clock time because it is total compute latency, not elapsed time     totalLatencyMs :: !Integer   }   deriving stock (Eq, Show)  -- | Build a report from per-example results and the run's usage totals. -- 'aggregateScore' is the arithmetic mean of the per-example scores (0 if there--- are no examples). 'passCount' counts perfect scores; 'failCount' counts--- examples with a 'FailureReason'.+-- are no examples). 'passCount' counts only exact @1.0@ scores, so fractional+-- metrics such as similarity or partial-credit metrics should use+-- 'aggregateScore' to read run quality. 'failCount' counts examples with a+-- 'FailureReason'. mkReport :: [ExampleResult] -> UsageTotals -> Report mkReport rs u =   Report@@ -152,7 +160,7 @@ -- > score=0.6667  pass=2/3  fail=1 -- > tokens: in=120 out=45 total=165 -- > cost: $0.0023--- > latency: 1234 ms+-- > latency-sum: 1234 ms -- > failures: -- >   [2] ProgramError: decode failed --@@ -177,7 +185,7 @@           <> " total="           <> tshowNat (totalTokens (usage r)),         "cost: $" <> fixed4 (fromRational (totalCostUsd (usage r))),-        "latency: " <> tshow (totalLatencyMs r) <> " ms"+        "latency-sum: " <> tshow (totalLatencyMs r) <> " ms"       ]     failingResults = mapMaybe asFailure (results r)     asFailure er = (\fr -> (index er, fr)) <$> failure er

@@ -13,7 +13,7 @@ -- (majority-vote, ensemble) can inspect every sample. module Shikumi.Eval.Types   ( -- * Scores-    Score (..),+    Score,     mkScore,     scoreZero,     scoreOne,

@@ -15,6 +15,7 @@   ) where +import Baikai (AssistantMessageEvent (..), AssistantPayload, Message (..), TerminalPayload (..)) import Control.Lens ((^.)) import Data.Generics.Labels () import Effectful (Eff, (:>))@@ -34,7 +35,10 @@             resp <- complete m c o             atomicModifyIORef' ref (\u -> (u <> usageOf resp, ()))             pure resp-          Stream m c o -> stream m c o+          Stream m c o -> do+            events <- stream m c o+            atomicModifyIORef' ref (\u -> (u <> streamUsage events, ()))+            pure events       )       act   totals <- readIORef ref@@ -42,10 +46,33 @@  -- | Project a response's token usage and cost into a 'UsageTotals'. usageOf :: Response -> UsageTotals-usageOf resp =+usageOf resp = usageOfAssistant (resp ^. #message)++-- | Project an assistant message's token usage and cost into a 'UsageTotals'.+usageOfMessage :: Message -> UsageTotals+usageOfMessage (AssistantMessage msg) = usageOfAssistant msg+usageOfMessage _ = emptyUsageTotals++-- | Project an assistant payload's token usage and cost into a 'UsageTotals'.+usageOfAssistant :: AssistantPayload -> UsageTotals+usageOfAssistant msg =   UsageTotals-    { totalInputTokens = resp ^. #message . #usage . #inputTokens,-      totalOutputTokens = resp ^. #message . #usage . #outputTokens,-      totalTokens = resp ^. #message . #usage . #totalTokens,-      totalCostUsd = resp ^. #message . #usage . #cost . #usd+    { totalInputTokens = msg ^. #usage . #inputTokens,+      totalOutputTokens = msg ^. #usage . #outputTokens,+      totalTokens = msg ^. #usage . #totalTokens,+      totalCostUsd = msg ^. #usage . #cost . #usd     }++-- | The usage of one streamed call: read off terminal events. Baikai streams+-- emit exactly one 'EventDone' or 'EventError' carrying the assembled message+-- with final usage; deltas carry no usage.+streamUsage :: [AssistantMessageEvent] -> UsageTotals+streamUsage evs =+  mconcat+    [ usageOfMessage msg+    | ev <- evs,+      msg <- case ev of+        EventDone TerminalPayload {message = m} -> [m]+        EventError TerminalPayload {message = m} -> [m]+        _ -> []+    ]

@@ -16,27 +16,44 @@     -- * Canned responses     answerResponse,     markerResponse,+    usageResponse,+    usageTotalsPerCall,+    usageTerminalEvents,      -- * Mock LLM interpreters     MockReply (..),     runConstLLM,     runScriptedLLM,+    runStreamLLM,+    runSlowLLM,   ) where -import Baikai (AssistantContent (..), Response, _Response, _TextContent)-import Control.Lens ((&), (.~))+import Baikai+  ( AssistantContent (..),+    AssistantMessageEvent (..),+    Message (AssistantMessage),+    Response,+    StopReason (..),+    doneTerminal,+    _Response,+    _TextContent,+  )+import Control.Lens ((&), (.~), (^.)) import Data.Generics.Labels () import Data.IORef (atomicModifyIORef', newIORef)+import Data.Ratio ((%)) import Data.Text (Text) import Data.Text qualified as T import Data.Vector qualified as V import Effectful (Eff, IOE, liftIO, type (:>))+import Effectful.Concurrent (Concurrent, threadDelay) import Effectful.Dispatch.Dynamic (interpret) import Effectful.Error.Static (Error, throwError) import GHC.Generics (Generic) import Shikumi.Adapter (ToPrompt) import Shikumi.Error (ShikumiError (..))+import Shikumi.Eval.Report (UsageTotals (..)) import Shikumi.LLM (LLM (..)) import Shikumi.Module (predict) import Shikumi.Program (Program)@@ -90,6 +107,30 @@ answerResponse :: Text -> Response answerResponse t = mkResponse (markerBody [("answer", t)]) +-- | One response carrying known non-zero usage for accounting tests.+usageResponse :: Text -> Response+usageResponse t =+  answerResponse t+    & #message . #usage . #inputTokens .~ 100+    & #message . #usage . #outputTokens .~ 20+    & #message . #usage . #totalTokens .~ 120+    & #message . #usage . #cost . #usd .~ (1 % 1000)++usageTotalsPerCall :: UsageTotals+usageTotalsPerCall =+  UsageTotals+    { totalInputTokens = 100,+      totalOutputTokens = 20,+      totalTokens = 120,+      totalCostUsd = 1 % 1000+    }++-- | A successful terminal stream whose assembled message carries the same known+-- non-zero usage as 'usageResponse'.+usageTerminalEvents :: Text -> [AssistantMessageEvent]+usageTerminalEvents t =+  [EventDone (doneTerminal Nothing Stop (AssistantMessage (usageResponse t ^. #message)))]+ -- | An assistant 'Response' whose body decodes (via the fallback adapter) to a -- record with the given @(field, value)@ sections. markerResponse :: [(Text, Text)] -> Response@@ -121,6 +162,18 @@ runConstLLM resp = interpret $ \_ -> \case   Complete {} -> pure resp   Stream {} -> pure []++-- | Return a fixed event list for every streaming call.+runStreamLLM :: [AssistantMessageEvent] -> Eff (LLM : es) a -> Eff es a+runStreamLLM events = interpret $ \_ -> \case+  Complete {} -> pure (answerResponse "stream fixture complete path")+  Stream {} -> pure events++-- | Delay every completion before returning @resp@. Used for timeout tests.+runSlowLLM :: (Concurrent :> es) => Int -> Response -> Eff (LLM : es) a -> Eff es a+runSlowLLM micros resp = interpret $ \_ -> \case+  Complete {} -> threadDelay micros >> pure resp+  Stream {} -> threadDelay micros >> pure []  -- | Pop scripted replies in order; a 'MockFail' is thrown through -- @Error ShikumiError@. Deterministic only under @concurrency = 1@.

@@ -6,6 +6,7 @@  import Control.Lens ((&), (.~)) import Data.Generics.Labels ()+import Data.List.NonEmpty qualified as NE import Effectful (runEff) import Effectful.Concurrent (runConcurrent) import Effectful.Error.Static (runErrorNoCallStack)@@ -18,20 +19,20 @@     qaProg,     runConstLLM,     runScriptedLLM,+    runSlowLLM,   ) import Shikumi.Effect.Time (runTime) import Shikumi.Error (ShikumiError (..)) import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith) import Shikumi.Eval.Metric (exactMatch, liftMetric) import Shikumi.Eval.Report-  ( EvalConfig (..),-    ExampleResult (..),+  ( ExampleResult (..),     FailurePolicy (..),     FailureReason (..),     Report (..),     defaultEvalConfig,   )-import Shikumi.Eval.Types (Score, dataset, example, scoreZero)+import Shikumi.Eval.Types (Score, boolScore, dataset, example, mkScore, predictionSamples, scoreZero, unScore) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=)) @@ -102,7 +103,62 @@               evaluateWith cfg ds (liftMetric exactMatch) qaProg         case report of           Left _ -> pure ()-          Right _ -> assertFailure "expected FailAbort to surface a Left"+          Right _ -> assertFailure "expected FailAbort to surface a Left",+      testCase "example timeout records TimedOut under FailScore" $ do+        let ds = dataset [example (Question "slow") (Answer "yes")]+            cfg = defaultEvalConfig & #exampleTimeoutMs .~ Just 20+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runSlowLLM 500_000 (answerResponse "yes") $+              evaluateWith cfg ds (liftMetric exactMatch) qaProg+        case report of+          Left e -> assertFailure ("run should not abort, but got: " <> show e)+          Right r -> case results r of+            [er] -> do+              failure er @?= Just TimedOut+              score er @?= (scoreZero :: Score)+            other -> assertFailure ("expected one result, got " <> show other),+      testCase "example timeout aborts with Timeout under FailAbort" $ do+        let ds = dataset [example (Question "slow") (Answer "yes")]+            cfg = defaultEvalConfig & #exampleTimeoutMs .~ Just 20 & #failurePolicy .~ FailAbort+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runSlowLLM 500_000 (answerResponse "yes") $+              evaluateWith cfg ds (liftMetric exactMatch) qaProg+        case report of+          Left (Timeout _) -> pure ()+          Left e -> assertFailure ("expected Timeout, got: " <> show e)+          Right _ -> assertFailure "expected timeout to abort",+      testCase "numSamples > 1 produces a multi-sample prediction" $ do+        let ds = dataset [example (Question "q") (Answer "yes")]+            replies = [MockOk (answerResponse "yes"), MockOk (answerResponse "no"), MockOk (answerResponse "yes")]+            cfg = defaultEvalConfig & #concurrency .~ 1 & #numSamples .~ 3+            majorityMetric _ pr =+              boolScore+                ( length [() | Answer "yes" <- NE.toList (predictionSamples pr)]+                    >= 2+                )+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runScriptedLLM replies $+              evaluateWith cfg ds (liftMetric majorityMetric) qaProg+        case report of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right r -> aggregateScore r @?= 1.0,+      testCase "custom FailScore is honored" $ do+        let ds = dataset [example (Question "q") (Answer "yes")]+            cfg = defaultEvalConfig & #concurrency .~ 1 & #failurePolicy .~ FailScore (mkScore 0.25)+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runScriptedLLM [MockFail providerErr] $+              evaluateWith cfg ds (liftMetric exactMatch) qaProg+        case report of+          Left e -> assertFailure ("run should not abort, but got: " <> show e)+          Right r -> do+            aggregateScore r @?= 0.25+            case results r of+              [er] -> unScore (score er) @?= 0.25+              other -> assertFailure ("expected one result, got " <> show other)     ]   where     providerErr = ProviderFailure "boom"

@@ -13,6 +13,7 @@ import ReportSpec qualified import Test.Tasty (defaultMain, testGroup) import TypesSpec qualified+import UsageSpec qualified  main :: IO () main =@@ -22,6 +23,7 @@       [ TypesSpec.tests,         MetricSpec.tests,         ReportSpec.tests,+        UsageSpec.tests,         EvaluateSpec.tests,         MetricLMSpec.tests,         GoldenSpec.tests,

@@ -42,7 +42,7 @@   "score=0.5000  pass=1/3  fail=1\n\   \tokens: in=120 out=45 total=165\n\   \cost: $0.0023\n\-  \latency: 1234 ms\n\+  \latency-sum: 1234 ms\n\   \failures:\n\   \  [2] ProgramError: decode failed" 

@@ -4,8 +4,7 @@  import Data.List.NonEmpty (NonEmpty (..)) import Shikumi.Eval.Types-  ( Score (..),-    dataset,+  ( dataset,     datasetSize,     example,     mkScore,@@ -13,6 +12,7 @@     predictionSamples,     scoreOne,     scoreZero,+    unScore,   ) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (testCase, (@?=))@@ -23,7 +23,7 @@     "Types"     [ testCase "mkScore clamps high" $ mkScore 1.5 @?= scoreOne,       testCase "mkScore clamps low" $ mkScore (-0.2) @?= scoreZero,-      testCase "mkScore passes through in-range" $ mkScore 0.5 @?= Score 0.5,+      testCase "mkScore passes through in-range" $ unScore (mkScore 0.5) @?= 0.5,       testCase "datasetSize counts examples" $         datasetSize (dataset [example (1 :: Int) 'a', example 2 'b']) @?= 2,       testCase "prediction is single-sample" $

@@ -0,0 +1,74 @@+-- | Usage-accounting tests for both blocking and streamed LM calls.+module UsageSpec (tests) where++import Baikai (_Context, _Model, _Options)+import Effectful (runEff)+import Effectful.Concurrent (runConcurrent)+import Effectful.Error.Static (runErrorNoCallStack)+import Effectful.Prim (runPrim)+import EvalFixtures+  ( Answer (..),+    Question (..),+    qaProg,+    runConstLLM,+    runStreamLLM,+    usageResponse,+    usageTerminalEvents,+    usageTotalsPerCall,+  )+import Shikumi.Effect.Time (runTime)+import Shikumi.Error (ShikumiError)+import Shikumi.Eval.Evaluate (evaluatePure)+import Shikumi.Eval.Metric (exactMatch)+import Shikumi.Eval.Report (Report (..), UsageTotals (..))+import Shikumi.Eval.Types (dataset, example)+import Shikumi.Eval.Usage (withUsageTotals)+import Shikumi.LLM (stream)+import Shikumi.Program (runProgram)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.HUnit (assertFailure, testCase, (@?=))++tests :: TestTree+tests =+  testGroup+    "Usage"+    [ testCase "complete calls accumulate non-zero usage" $ do+        result <-+          runEff . runPrim . runErrorNoCallStack @ShikumiError $+            runConstLLM (usageResponse "yes") $+              withUsageTotals $ do+                _ <- runProgram qaProg (Question "q1")+                _ <- runProgram qaProg (Question "q2")+                pure ()+        case result of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right (_, totals) -> totals @?= usageTotalsPerCall <> usageTotalsPerCall,+      testCase "stream calls accumulate usage" $ do+        let events = usageTerminalEvents "yes"+        result <-+          runEff . runPrim $+            runStreamLLM events $+              withUsageTotals (stream _Model _Context _Options)+        snd result @?= usageTotalsPerCall,+      testCase "evaluate reports non-zero usage end-to-end" $ do+        let ds =+              dataset+                [ example (Question "q1") (Answer "yes"),+                  example (Question "q2") (Answer "yes"),+                  example (Question "q3") (Answer "yes")+                ]+        report <-+          runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $+            runConstLLM (usageResponse "yes") $+              evaluatePure ds exactMatch qaProg+        case report of+          Left e -> assertFailure ("unexpected error: " <> show e)+          Right r ->+            usage r+              @?= UsageTotals+                { totalInputTokens = 300,+                  totalOutputTokens = 60,+                  totalTokens = 360,+                  totalCostUsd = 3 / 1000+                }+    ]