diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +## 0.2.0.0 - 2026-07-05 + +### Added + +- Evaluation can enforce an optional per-example timeout via + `EvalConfig.exampleTimeoutMs`. +- Usage accounting now includes streamed calls and exposes helpers for assistant + messages and stream terminal events. + +### Changed + +- **BREAKING** `EvalConfig` record construction requires the new + `exampleTimeoutMs` field. +- **BREAKING** `Score` no longer exports its constructor; use `mkScore`, + `scoreZero`, `scoreOne`, and `unScore`. +- Reports render summed per-example latency as `latency-sum`, making concurrent + evaluation accounting explicit. +- Refreshed the internal `shikumi` bound for the `0.3` series. + ## 0.1.1.0 - 2026-06-28 ### Added diff --git a/shikumi-eval.cabal b/shikumi-eval.cabal --- a/shikumi-eval.cabal +++ b/shikumi-eval.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: shikumi-eval -version: 0.1.1.0 +version: 0.2.0.0 synopsis: Typed evaluation framework for shikumi LM programs (EP-8) category: AI description: @@ -46,14 +46,14 @@ build-depends: , aeson - , baikai >=0.2 && <0.3 + , baikai >=0.3 && <0.4 , base >=4.20 && <5 , bytestring , containers , effectful , generic-lens , lens ^>=5.3 - , shikumi ^>=0.2.0.0 + , shikumi ^>=0.3.0.0 , tasty , tasty-golden , text ^>=2.1 @@ -75,16 +75,17 @@ MetricSpec ReportSpec TypesSpec + UsageSpec build-depends: , aeson - , baikai >=0.2 && <0.3 + , baikai >=0.3 && <0.4 , base , effectful , generic-lens , lens - , shikumi ^>=0.2.0.0 - , shikumi-eval ^>=0.1.1.0 + , shikumi ^>=0.3.0.0 + , shikumi-eval ^>=0.2.0.0 , tasty , tasty-golden , tasty-hunit diff --git a/src/Shikumi/Eval/Evaluate.hs b/src/Shikumi/Eval/Evaluate.hs --- a/src/Shikumi/Eval/Evaluate.hs +++ b/src/Shikumi/Eval/Evaluate.hs @@ -22,12 +22,12 @@ import Data.List.NonEmpty (NonEmpty ((:|))) import Data.Text qualified as T import Effectful (Eff, (:>)) -import Effectful.Concurrent (Concurrent) -import Effectful.Concurrent.Async (pooledForConcurrentlyN) +import Effectful.Concurrent (Concurrent, threadDelay) +import Effectful.Concurrent.Async (pooledForConcurrentlyN, race) import Effectful.Error.Static (Error, catchError, throwError) import Effectful.Prim (Prim) import Shikumi.Effect.Time (Time, getMonotonicTimeNSec) -import Shikumi.Error (ShikumiError) +import Shikumi.Error (ShikumiError (..)) import Shikumi.Eval.Metric (Metric, MetricM, liftMetric) import Shikumi.Eval.Report ( EvalConfig (..), @@ -87,7 +87,7 @@ -- | Evaluate a single indexed example inside its error boundary, timing it with a -- monotonic clock. evalOne :: - (LLM :> es, Error ShikumiError :> es, Time :> es) => + (LLM :> es, Concurrent :> es, Error ShikumiError :> es, Time :> es) => EvalConfig -> MetricM es o -> Program i o -> @@ -95,7 +95,14 @@ Eff es ExampleResult evalOne cfg metric prog (ix, Example inp expd) = do start <- getMonotonicTimeNSec - (s, mFail) <- scoreExample cfg metric prog inp expd + outcome <- case exampleTimeoutMs cfg of + Nothing -> Right <$> scoreExample cfg metric prog inp expd + Just ms -> race (threadDelay (max 0 ms * 1000)) (scoreExample cfg metric prog inp expd) + (s, mFail) <- case outcome of + Right r -> pure r + Left () -> case failurePolicy cfg of + FailAbort -> throwError (Timeout "evaluate: example timed out") + FailScore sc -> pure (sc, Just TimedOut) end <- getMonotonicTimeNSec let ms = fromIntegral ((end - start) `div` 1_000_000) pure ExampleResult {index = ix, score = s, failure = mFail, latencyMs = ms} diff --git a/src/Shikumi/Eval/Report.hs b/src/Shikumi/Eval/Report.hs --- a/src/Shikumi/Eval/Report.hs +++ b/src/Shikumi/Eval/Report.hs @@ -1,13 +1,13 @@ -- | The evaluation report and its aggregation. A 'Report' summarises a run over -- a dataset: the mean per-example score, pass/fail counts, summed token usage and --- cost, total latency, and a per-example breakdown ('ExampleResult', retained in --- dataset order for failure analysis). 'mkReport' is the pure aggregator; +-- cost, summed per-example latency, and a per-example breakdown ('ExampleResult', +-- retained in dataset order for failure analysis). 'mkReport' is the pure aggregator; -- 'renderReportText' is a deterministic human-readable rendering reused by the -- CLI (@docs/plans/12-cli-and-developer-experience.md@) and by golden tests. -- -- 'EvalConfig' carries the run knobs (bounded 'concurrency', the 'FailurePolicy' --- for per-example errors, and 'numSamples' per example for multi-sample metrics); --- 'defaultEvalConfig' scores failures @0@ and keeps going. +-- for per-example errors, optional per-example timeout, and 'numSamples' per example +-- for multi-sample metrics); 'defaultEvalConfig' scores failures @0@ and keeps going. module Shikumi.Eval.Report ( -- * Per-example outcome ExampleResult (..), @@ -94,18 +94,21 @@ { -- | max examples evaluated at once (forced to @>= 1@ by 'evaluate') concurrency :: !Int, failurePolicy :: !FailurePolicy, + -- | wall-clock budget per example in milliseconds; 'Nothing' means no timeout + exampleTimeoutMs :: !(Maybe Int), -- | samples per example for multi-sample metrics (forced to @>= 1@) numSamples :: !Int } deriving stock (Eq, Show, Generic) --- | The default: four-way concurrency, score failures @0@ and keep going, one --- sample per example. +-- | The default: four-way concurrency, no timeout, score failures @0@ and keep +-- going, one sample per example. defaultEvalConfig :: EvalConfig defaultEvalConfig = EvalConfig { concurrency = 4, failurePolicy = FailScore scoreZero, + exampleTimeoutMs = Nothing, numSamples = 1 } @@ -113,7 +116,8 @@ data Report = Report { -- | mean of the per-example scores, in @[0, 1]@ (0 when there are no examples) aggregateScore :: !Double, - -- | examples whose score is exactly @1.0@ + -- | examples whose score is exactly @1.0@; fractional metrics can have good + -- aggregate scores while still reporting zero passes passCount :: !Int, -- | examples carrying a 'FailureReason' failCount :: !Int, @@ -121,14 +125,18 @@ -- | per-example outcomes, retained in dataset order results :: ![ExampleResult], usage :: !UsageTotals, + -- | sum of per-example latencies; under concurrent evaluation this can exceed + -- wall-clock time because it is total compute latency, not elapsed time totalLatencyMs :: !Integer } deriving stock (Eq, Show) -- | Build a report from per-example results and the run's usage totals. -- 'aggregateScore' is the arithmetic mean of the per-example scores (0 if there --- are no examples). 'passCount' counts perfect scores; 'failCount' counts --- examples with a 'FailureReason'. +-- are no examples). 'passCount' counts only exact @1.0@ scores, so fractional +-- metrics such as similarity or partial-credit metrics should use +-- 'aggregateScore' to read run quality. 'failCount' counts examples with a +-- 'FailureReason'. mkReport :: [ExampleResult] -> UsageTotals -> Report mkReport rs u = Report @@ -152,7 +160,7 @@ -- > score=0.6667 pass=2/3 fail=1 -- > tokens: in=120 out=45 total=165 -- > cost: $0.0023 --- > latency: 1234 ms +-- > latency-sum: 1234 ms -- > failures: -- > [2] ProgramError: decode failed -- @@ -177,7 +185,7 @@ <> " total=" <> tshowNat (totalTokens (usage r)), "cost: $" <> fixed4 (fromRational (totalCostUsd (usage r))), - "latency: " <> tshow (totalLatencyMs r) <> " ms" + "latency-sum: " <> tshow (totalLatencyMs r) <> " ms" ] failingResults = mapMaybe asFailure (results r) asFailure er = (\fr -> (index er, fr)) <$> failure er diff --git a/src/Shikumi/Eval/Types.hs b/src/Shikumi/Eval/Types.hs --- a/src/Shikumi/Eval/Types.hs +++ b/src/Shikumi/Eval/Types.hs @@ -13,7 +13,7 @@ -- (majority-vote, ensemble) can inspect every sample. module Shikumi.Eval.Types ( -- * Scores - Score (..), + Score, mkScore, scoreZero, scoreOne, diff --git a/src/Shikumi/Eval/Usage.hs b/src/Shikumi/Eval/Usage.hs --- a/src/Shikumi/Eval/Usage.hs +++ b/src/Shikumi/Eval/Usage.hs @@ -15,6 +15,7 @@ ) where +import Baikai (AssistantMessageEvent (..), AssistantPayload, Message (..), TerminalPayload (..)) import Control.Lens ((^.)) import Data.Generics.Labels () import Effectful (Eff, (:>)) @@ -34,7 +35,10 @@ resp <- complete m c o atomicModifyIORef' ref (\u -> (u <> usageOf resp, ())) pure resp - Stream m c o -> stream m c o + Stream m c o -> do + events <- stream m c o + atomicModifyIORef' ref (\u -> (u <> streamUsage events, ())) + pure events ) act totals <- readIORef ref @@ -42,10 +46,33 @@ -- | Project a response's token usage and cost into a 'UsageTotals'. usageOf :: Response -> UsageTotals -usageOf resp = +usageOf resp = usageOfAssistant (resp ^. #message) + +-- | Project an assistant message's token usage and cost into a 'UsageTotals'. +usageOfMessage :: Message -> UsageTotals +usageOfMessage (AssistantMessage msg) = usageOfAssistant msg +usageOfMessage _ = emptyUsageTotals + +-- | Project an assistant payload's token usage and cost into a 'UsageTotals'. +usageOfAssistant :: AssistantPayload -> UsageTotals +usageOfAssistant msg = UsageTotals - { totalInputTokens = resp ^. #message . #usage . #inputTokens, - totalOutputTokens = resp ^. #message . #usage . #outputTokens, - totalTokens = resp ^. #message . #usage . #totalTokens, - totalCostUsd = resp ^. #message . #usage . #cost . #usd + { totalInputTokens = msg ^. #usage . #inputTokens, + totalOutputTokens = msg ^. #usage . #outputTokens, + totalTokens = msg ^. #usage . #totalTokens, + totalCostUsd = msg ^. #usage . #cost . #usd } + +-- | The usage of one streamed call: read off terminal events. Baikai streams +-- emit exactly one 'EventDone' or 'EventError' carrying the assembled message +-- with final usage; deltas carry no usage. +streamUsage :: [AssistantMessageEvent] -> UsageTotals +streamUsage evs = + mconcat + [ usageOfMessage msg + | ev <- evs, + msg <- case ev of + EventDone TerminalPayload {message = m} -> [m] + EventError TerminalPayload {message = m} -> [m] + _ -> [] + ] diff --git a/test/EvalFixtures.hs b/test/EvalFixtures.hs --- a/test/EvalFixtures.hs +++ b/test/EvalFixtures.hs @@ -16,27 +16,44 @@ -- * Canned responses answerResponse, markerResponse, + usageResponse, + usageTotalsPerCall, + usageTerminalEvents, -- * Mock LLM interpreters MockReply (..), runConstLLM, runScriptedLLM, + runStreamLLM, + runSlowLLM, ) where -import Baikai (AssistantContent (..), Response, _Response, _TextContent) -import Control.Lens ((&), (.~)) +import Baikai + ( AssistantContent (..), + AssistantMessageEvent (..), + Message (AssistantMessage), + Response, + StopReason (..), + doneTerminal, + _Response, + _TextContent, + ) +import Control.Lens ((&), (.~), (^.)) import Data.Generics.Labels () import Data.IORef (atomicModifyIORef', newIORef) +import Data.Ratio ((%)) import Data.Text (Text) import Data.Text qualified as T import Data.Vector qualified as V import Effectful (Eff, IOE, liftIO, type (:>)) +import Effectful.Concurrent (Concurrent, threadDelay) import Effectful.Dispatch.Dynamic (interpret) import Effectful.Error.Static (Error, throwError) import GHC.Generics (Generic) import Shikumi.Adapter (ToPrompt) import Shikumi.Error (ShikumiError (..)) +import Shikumi.Eval.Report (UsageTotals (..)) import Shikumi.LLM (LLM (..)) import Shikumi.Module (predict) import Shikumi.Program (Program) @@ -90,6 +107,30 @@ answerResponse :: Text -> Response answerResponse t = mkResponse (markerBody [("answer", t)]) +-- | One response carrying known non-zero usage for accounting tests. +usageResponse :: Text -> Response +usageResponse t = + answerResponse t + & #message . #usage . #inputTokens .~ 100 + & #message . #usage . #outputTokens .~ 20 + & #message . #usage . #totalTokens .~ 120 + & #message . #usage . #cost . #usd .~ (1 % 1000) + +usageTotalsPerCall :: UsageTotals +usageTotalsPerCall = + UsageTotals + { totalInputTokens = 100, + totalOutputTokens = 20, + totalTokens = 120, + totalCostUsd = 1 % 1000 + } + +-- | A successful terminal stream whose assembled message carries the same known +-- non-zero usage as 'usageResponse'. +usageTerminalEvents :: Text -> [AssistantMessageEvent] +usageTerminalEvents t = + [EventDone (doneTerminal Nothing Stop (AssistantMessage (usageResponse t ^. #message)))] + -- | An assistant 'Response' whose body decodes (via the fallback adapter) to a -- record with the given @(field, value)@ sections. markerResponse :: [(Text, Text)] -> Response @@ -121,6 +162,18 @@ runConstLLM resp = interpret $ \_ -> \case Complete {} -> pure resp Stream {} -> pure [] + +-- | Return a fixed event list for every streaming call. +runStreamLLM :: [AssistantMessageEvent] -> Eff (LLM : es) a -> Eff es a +runStreamLLM events = interpret $ \_ -> \case + Complete {} -> pure (answerResponse "stream fixture complete path") + Stream {} -> pure events + +-- | Delay every completion before returning @resp@. Used for timeout tests. +runSlowLLM :: (Concurrent :> es) => Int -> Response -> Eff (LLM : es) a -> Eff es a +runSlowLLM micros resp = interpret $ \_ -> \case + Complete {} -> threadDelay micros >> pure resp + Stream {} -> threadDelay micros >> pure [] -- | Pop scripted replies in order; a 'MockFail' is thrown through -- @Error ShikumiError@. Deterministic only under @concurrency = 1@. diff --git a/test/EvaluateSpec.hs b/test/EvaluateSpec.hs --- a/test/EvaluateSpec.hs +++ b/test/EvaluateSpec.hs @@ -6,6 +6,7 @@ import Control.Lens ((&), (.~)) import Data.Generics.Labels () +import Data.List.NonEmpty qualified as NE import Effectful (runEff) import Effectful.Concurrent (runConcurrent) import Effectful.Error.Static (runErrorNoCallStack) @@ -18,20 +19,20 @@ qaProg, runConstLLM, runScriptedLLM, + runSlowLLM, ) import Shikumi.Effect.Time (runTime) import Shikumi.Error (ShikumiError (..)) import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith) import Shikumi.Eval.Metric (exactMatch, liftMetric) import Shikumi.Eval.Report - ( EvalConfig (..), - ExampleResult (..), + ( ExampleResult (..), FailurePolicy (..), FailureReason (..), Report (..), defaultEvalConfig, ) -import Shikumi.Eval.Types (Score, dataset, example, scoreZero) +import Shikumi.Eval.Types (Score, boolScore, dataset, example, mkScore, predictionSamples, scoreZero, unScore) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=)) @@ -102,7 +103,62 @@ evaluateWith cfg ds (liftMetric exactMatch) qaProg case report of Left _ -> pure () - Right _ -> assertFailure "expected FailAbort to surface a Left" + Right _ -> assertFailure "expected FailAbort to surface a Left", + testCase "example timeout records TimedOut under FailScore" $ do + let ds = dataset [example (Question "slow") (Answer "yes")] + cfg = defaultEvalConfig & #exampleTimeoutMs .~ Just 20 + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runSlowLLM 500_000 (answerResponse "yes") $ + evaluateWith cfg ds (liftMetric exactMatch) qaProg + case report of + Left e -> assertFailure ("run should not abort, but got: " <> show e) + Right r -> case results r of + [er] -> do + failure er @?= Just TimedOut + score er @?= (scoreZero :: Score) + other -> assertFailure ("expected one result, got " <> show other), + testCase "example timeout aborts with Timeout under FailAbort" $ do + let ds = dataset [example (Question "slow") (Answer "yes")] + cfg = defaultEvalConfig & #exampleTimeoutMs .~ Just 20 & #failurePolicy .~ FailAbort + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runSlowLLM 500_000 (answerResponse "yes") $ + evaluateWith cfg ds (liftMetric exactMatch) qaProg + case report of + Left (Timeout _) -> pure () + Left e -> assertFailure ("expected Timeout, got: " <> show e) + Right _ -> assertFailure "expected timeout to abort", + testCase "numSamples > 1 produces a multi-sample prediction" $ do + let ds = dataset [example (Question "q") (Answer "yes")] + replies = [MockOk (answerResponse "yes"), MockOk (answerResponse "no"), MockOk (answerResponse "yes")] + cfg = defaultEvalConfig & #concurrency .~ 1 & #numSamples .~ 3 + majorityMetric _ pr = + boolScore + ( length [() | Answer "yes" <- NE.toList (predictionSamples pr)] + >= 2 + ) + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runScriptedLLM replies $ + evaluateWith cfg ds (liftMetric majorityMetric) qaProg + case report of + Left e -> assertFailure ("unexpected error: " <> show e) + Right r -> aggregateScore r @?= 1.0, + testCase "custom FailScore is honored" $ do + let ds = dataset [example (Question "q") (Answer "yes")] + cfg = defaultEvalConfig & #concurrency .~ 1 & #failurePolicy .~ FailScore (mkScore 0.25) + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runScriptedLLM [MockFail providerErr] $ + evaluateWith cfg ds (liftMetric exactMatch) qaProg + case report of + Left e -> assertFailure ("run should not abort, but got: " <> show e) + Right r -> do + aggregateScore r @?= 0.25 + case results r of + [er] -> unScore (score er) @?= 0.25 + other -> assertFailure ("expected one result, got " <> show other) ] where providerErr = ProviderFailure "boom" diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -13,6 +13,7 @@ import ReportSpec qualified import Test.Tasty (defaultMain, testGroup) import TypesSpec qualified +import UsageSpec qualified main :: IO () main = @@ -22,6 +23,7 @@ [ TypesSpec.tests, MetricSpec.tests, ReportSpec.tests, + UsageSpec.tests, EvaluateSpec.tests, MetricLMSpec.tests, GoldenSpec.tests, diff --git a/test/ReportSpec.hs b/test/ReportSpec.hs --- a/test/ReportSpec.hs +++ b/test/ReportSpec.hs @@ -42,7 +42,7 @@ "score=0.5000 pass=1/3 fail=1\n\ \tokens: in=120 out=45 total=165\n\ \cost: $0.0023\n\ - \latency: 1234 ms\n\ + \latency-sum: 1234 ms\n\ \failures:\n\ \ [2] ProgramError: decode failed" diff --git a/test/TypesSpec.hs b/test/TypesSpec.hs --- a/test/TypesSpec.hs +++ b/test/TypesSpec.hs @@ -4,8 +4,7 @@ import Data.List.NonEmpty (NonEmpty (..)) import Shikumi.Eval.Types - ( Score (..), - dataset, + ( dataset, datasetSize, example, mkScore, @@ -13,6 +12,7 @@ predictionSamples, scoreOne, scoreZero, + unScore, ) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (testCase, (@?=)) @@ -23,7 +23,7 @@ "Types" [ testCase "mkScore clamps high" $ mkScore 1.5 @?= scoreOne, testCase "mkScore clamps low" $ mkScore (-0.2) @?= scoreZero, - testCase "mkScore passes through in-range" $ mkScore 0.5 @?= Score 0.5, + testCase "mkScore passes through in-range" $ unScore (mkScore 0.5) @?= 0.5, testCase "datasetSize counts examples" $ datasetSize (dataset [example (1 :: Int) 'a', example 2 'b']) @?= 2, testCase "prediction is single-sample" $ diff --git a/test/UsageSpec.hs b/test/UsageSpec.hs new file mode 100644 --- /dev/null +++ b/test/UsageSpec.hs @@ -0,0 +1,74 @@ +-- | Usage-accounting tests for both blocking and streamed LM calls. +module UsageSpec (tests) where + +import Baikai (_Context, _Model, _Options) +import Effectful (runEff) +import Effectful.Concurrent (runConcurrent) +import Effectful.Error.Static (runErrorNoCallStack) +import Effectful.Prim (runPrim) +import EvalFixtures + ( Answer (..), + Question (..), + qaProg, + runConstLLM, + runStreamLLM, + usageResponse, + usageTerminalEvents, + usageTotalsPerCall, + ) +import Shikumi.Effect.Time (runTime) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Evaluate (evaluatePure) +import Shikumi.Eval.Metric (exactMatch) +import Shikumi.Eval.Report (Report (..), UsageTotals (..)) +import Shikumi.Eval.Types (dataset, example) +import Shikumi.Eval.Usage (withUsageTotals) +import Shikumi.LLM (stream) +import Shikumi.Program (runProgram) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (assertFailure, testCase, (@?=)) + +tests :: TestTree +tests = + testGroup + "Usage" + [ testCase "complete calls accumulate non-zero usage" $ do + result <- + runEff . runPrim . runErrorNoCallStack @ShikumiError $ + runConstLLM (usageResponse "yes") $ + withUsageTotals $ do + _ <- runProgram qaProg (Question "q1") + _ <- runProgram qaProg (Question "q2") + pure () + case result of + Left e -> assertFailure ("unexpected error: " <> show e) + Right (_, totals) -> totals @?= usageTotalsPerCall <> usageTotalsPerCall, + testCase "stream calls accumulate usage" $ do + let events = usageTerminalEvents "yes" + result <- + runEff . runPrim $ + runStreamLLM events $ + withUsageTotals (stream _Model _Context _Options) + snd result @?= usageTotalsPerCall, + testCase "evaluate reports non-zero usage end-to-end" $ do + let ds = + dataset + [ example (Question "q1") (Answer "yes"), + example (Question "q2") (Answer "yes"), + example (Question "q3") (Answer "yes") + ] + report <- + runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $ + runConstLLM (usageResponse "yes") $ + evaluatePure ds exactMatch qaProg + case report of + Left e -> assertFailure ("unexpected error: " <> show e) + Right r -> + usage r + @?= UsageTotals + { totalInputTokens = 300, + totalOutputTokens = 60, + totalTokens = 360, + totalCostUsd = 3 / 1000 + } + ]