shikumi-eval 0.2.0.3 → 0.3.0.0

raw patch · 9 files changed

+123/−48 lines, 9 filesdep ~baikaidep ~shikumidep ~shikumi-evalPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: baikai, shikumi, shikumi-eval

API changes (from Hackage documentation)

Files

• CHANGELOG.md +12/−0
• shikumi-eval.cabal +6/−6
• src/Shikumi/Eval/Evaluate.hs +31/−8
• src/Shikumi/Eval/Report.hs +42/−5
• src/Shikumi/Eval/Usage.hs +2/−8
• test/EvalFixtures.hs +2/−8
• test/EvaluateSpec.hs +23/−3
• test/ReportSpec.hs +3/−3
• test/UsageSpec.hs +2/−7

@@ -2,6 +2,18 @@  ## Unreleased +## 0.3.0.0 — 2026-09-08++- Raise the internal `shikumi` bound to `^>=0.4.0.0` for the breaking core release.++- Preserve logical usage quality and unknown-call counts, and explicitly attach/render whole-run transport billing separately. Public UsageTotals and Report fields require PVP major review; empty/default constructors retain zero semantics.++- Render structured provider failures through the shared readable renderer while retaining existing failure policy and legacy diagnostics.++- Upgrade the dependency on `mori://shinzui/baikai/packages/baikai` to `>=0.7.0.0 && <0.8`.++- Expose `scoreExecution` and `tryShikumi` for alternate typed runners retaining execution evidence. Existing evaluation failure, timing, concurrency, and usage behavior is unchanged.+ ## 0.2.0.3 — 2026-08-29  ### Changed

@@ -1,6 +1,6 @@ cabal-version:   3.4 name:            shikumi-eval-version:         0.2.0.3+version:         0.3.0.0 synopsis:        Typed evaluation framework for shikumi LM programs (EP-8) category:        AI description:@@ -46,14 +46,14 @@    build-depends:     , aeson         >=2.2      && <2.3-    , baikai        >=0.6      && <0.7+    , baikai        >=0.7.0.0  && <0.8     , base          >=4.20     && <5     , bytestring    >=0.11     && <0.13     , containers    >=0.6      && <0.9     , effectful     >=2.5      && <2.7     , generic-lens  >=2.2      && <2.4     , lens          ^>=5.3-    , shikumi       ^>=0.3.0.0+    , shikumi       ^>=0.4.0.0     , tasty         >=1.4      && <1.6     , tasty-golden  >=2.3      && <2.4     , text          ^>=2.1@@ -79,13 +79,13 @@    build-depends:     , aeson-    , baikai        >=0.6      && <0.7+    , baikai        >=0.7.0.0  && <0.8     , base     , effectful     , generic-lens     , lens-    , shikumi       ^>=0.3.0.0-    , shikumi-eval  ^>=0.2.0.0+    , shikumi       ^>=0.4.0.0+    , shikumi-eval  ^>=0.3.0.0     , tasty     , tasty-golden     , tasty-hunit

@@ -15,6 +15,8 @@   ( evaluate,     evaluatePure,     evaluateWith,+    scoreExecution,+    tryShikumi,   ) where @@ -27,7 +29,7 @@ import Effectful.Error.Static (Error, catchError, throwError) import Effectful.Prim (Prim) import Shikumi.Effect.Time (Time, getMonotonicTimeNSec)-import Shikumi.Error (ShikumiError (..))+import Shikumi.Error (ShikumiError (..), renderShikumiError) import Shikumi.Eval.Metric (Metric, MetricM, liftMetric) import Shikumi.Eval.Report   ( EvalConfig (..),@@ -119,16 +121,36 @@   o ->   Eff es (Score, Maybe FailureReason) scoreExample cfg metric prog inp expd = do-  predOrErr <- tryShikumi (buildPrediction cfg prog inp)-  case predOrErr of+  (_, result) <-+    scoreExecution+      (const (failurePolicy cfg))+      id+      (tryShikumi (buildPrediction cfg prog inp))+      (metric expd)+  pure result++-- | Shared typed execution boundary. The runner retains arbitrary evidence even+-- on failure; the projection identifies its root result. Only checked errors are+-- caught. Callers choose their error classification without losing evidence.+scoreExecution ::+  (Error ShikumiError :> es) =>+  (ShikumiError -> FailurePolicy) ->+  (a -> Either ShikumiError b) ->+  Eff es a ->+  (b -> Eff es Score) ->+  Eff es (a, (Score, Maybe FailureReason))+scoreExecution policy project runner metric = do+  evidence <- runner+  result <- case project evidence of     Left e -> boundary (ProgramError (renderErr e)) e-    Right pr -> do-      scoreOrErr <- tryShikumi (metric expd pr)-      case scoreOrErr of+    Right value -> do+      scored <- tryShikumi (metric value)+      case scored of         Left e -> boundary (MetricError (renderErr e)) e         Right s -> pure (s, Nothing)+  pure (evidence, result)   where-    boundary reason e = case failurePolicy cfg of+    boundary reason e = case policy e of       FailAbort -> throwError e       FailScore s -> pure (s, Just reason) @@ -157,4 +179,5 @@  -- | Render a shikumi error for a 'FailureReason'. renderErr :: ShikumiError -> T.Text-renderErr = T.pack . show+renderErr e@ProviderError {} = renderShikumiError e+renderErr e = T.pack (show e)

@@ -26,16 +26,24 @@     Report (..),     mkReport,     renderReportText,+    attachBillingSummary,+    usageTotalsFromUsage,   ) where +import Baikai.Cost qualified as C+import Baikai.Usage qualified as U+import Data.Aeson (encode)+import Data.ByteString.Lazy qualified as BL import Data.Maybe (isJust, mapMaybe) import Data.Text (Text) import Data.Text qualified as T+import Data.Text.Encoding qualified as TE import GHC.Generics (Generic) import Numeric (showFFloat) import Numeric.Natural (Natural) import Shikumi.Eval.Types (Score, scoreZero, unScore)+import Shikumi.LLM.Observation (BillingSummary, UsageRecord (..), renderBillingSummary, usageUnknown)  -- | The reason an example did not complete normally. data FailureReason@@ -70,13 +78,15 @@   { totalInputTokens :: !Natural,     totalOutputTokens :: !Natural,     totalTokens :: !Natural,-    totalCostUsd :: !Rational+    totalCostUsd :: !Rational,+    usageQuality :: !(Maybe UsageRecord),+    unknownUsageCalls :: !Int   }   deriving stock (Eq, Show)  -- | The zero of 'UsageTotals'. emptyUsageTotals :: UsageTotals-emptyUsageTotals = UsageTotals 0 0 0 0+emptyUsageTotals = UsageTotals 0 0 0 0 Nothing 0  instance Semigroup UsageTotals where   a <> b =@@ -85,7 +95,24 @@       (totalOutputTokens a + totalOutputTokens b)       (totalTokens a + totalTokens b)       (totalCostUsd a + totalCostUsd b)+      (combineQuality (usageQuality a) (usageQuality b))+      (unknownUsageCalls a + unknownUsageCalls b) +combineQuality :: Maybe UsageRecord -> Maybe UsageRecord -> Maybe UsageRecord+combineQuality Nothing b = b+combineQuality a Nothing = a+combineQuality (Just (UsageRecord a)) (Just (UsageRecord b)) = Just (UsageRecord (a <> b))++usageTotalsFromUsage :: U.Usage -> UsageTotals+usageTotalsFromUsage u =+  UsageTotals+    (U.inputTokens u)+    (U.outputTokens u)+    (U.totalTokens u)+    (C.usd (U.cost u))+    (Just (UsageRecord u))+    (if usageUnknown (Just (UsageRecord u)) then 1 else 0)+ instance Monoid UsageTotals where   mempty = emptyUsageTotals @@ -127,7 +154,8 @@     usage :: !UsageTotals,     -- | sum of per-example latencies; under concurrent evaluation this can exceed     -- wall-clock time because it is total compute latency, not elapsed time-    totalLatencyMs :: !Integer+    totalLatencyMs :: !Integer,+    transportBilling :: !(Maybe BillingSummary)   }   deriving stock (Eq, Show) @@ -149,9 +177,14 @@       total = length rs,       results = rs,       usage = u,-      totalLatencyMs = sum (map latencyMs rs)+      totalLatencyMs = sum (map latencyMs rs),+      transportBilling = Nothing     } +-- | Attach whole-run transport totals, without adding them to logical usage.+attachBillingSummary :: BillingSummary -> Report -> Report+attachBillingSummary b r = r {transportBilling = Just b}+ -- | A deterministic, human-readable multi-line summary of a report. The format -- is stable (fixed 4-decimal score and cost, examples in index order) so the CLI -- and golden tests can rely on it. The exact shape, for a 3-example run with one@@ -167,7 +200,7 @@ -- When there are no failures the trailing @failures:@ block is omitted. renderReportText :: Report -> Text renderReportText r =-  T.intercalate "\n" (header ++ failureLines)+  T.intercalate "\n" (header ++ qualityLines ++ billingLines ++ failureLines)   where     header =       [ "score="@@ -187,6 +220,10 @@         "cost: $" <> fixed4 (fromRational (totalCostUsd (usage r))),         "latency-sum: " <> tshow (totalLatencyMs r) <> " ms"       ]+    qualityLines = case usageQuality (usage r) of+      Nothing -> []+      Just u -> ["logical usage quality: unknown-calls=" <> tshow (unknownUsageCalls (usage r)) <> " " <> TE.decodeUtf8 (BL.toStrict (encode u))]+    billingLines = maybe [] (pure . renderBillingSummary) (transportBilling r)     failingResults = mapMaybe asFailure (results r)     asFailure er = (\fr -> (index er, fr)) <$> failure er     failureLines

@@ -21,7 +21,7 @@ import Effectful (Eff, (:>)) import Effectful.Dispatch.Dynamic (interpose) import Effectful.Prim.IORef (Prim, atomicModifyIORef', newIORef, readIORef)-import Shikumi.Eval.Report (UsageTotals (..), emptyUsageTotals)+import Shikumi.Eval.Report (UsageTotals, emptyUsageTotals, usageTotalsFromUsage) import Shikumi.LLM (LLM (..), Response, complete, stream)  -- | Run @act@, accumulating every @LLM@ call's usage/cost into a 'UsageTotals'.@@ -55,13 +55,7 @@  -- | Project an assistant payload's token usage and cost into a 'UsageTotals'. usageOfAssistant :: AssistantPayload -> UsageTotals-usageOfAssistant msg =-  UsageTotals-    { totalInputTokens = msg ^. #usage . #inputTokens,-      totalOutputTokens = msg ^. #usage . #outputTokens,-      totalTokens = msg ^. #usage . #totalTokens,-      totalCostUsd = msg ^. #usage . #cost . #usd-    }+usageOfAssistant msg = usageTotalsFromUsage (msg ^. #usage)  -- | The usage of one streamed call: read off terminal events. Baikai streams -- emit exactly one 'EventDone' or 'EventError' carrying the assembled message

@@ -53,7 +53,7 @@ import GHC.Generics (Generic) import Shikumi.Adapter (ToPrompt) import Shikumi.Error (ShikumiError (..))-import Shikumi.Eval.Report (UsageTotals (..))+import Shikumi.Eval.Report (UsageTotals, usageTotalsFromUsage) import Shikumi.LLM (LLM (..)) import Shikumi.Module (predict) import Shikumi.Program (Program)@@ -117,13 +117,7 @@     & #message . #usage . #cost . #usd .~ (1 % 1000)  usageTotalsPerCall :: UsageTotals-usageTotalsPerCall =-  UsageTotals-    { totalInputTokens = 100,-      totalOutputTokens = 20,-      totalTokens = 120,-      totalCostUsd = 1 % 1000-    }+usageTotalsPerCall = usageTotalsFromUsage (usageResponse "yes" ^. #message . #usage)  -- | A successful terminal stream whose assembled message carries the same known -- non-zero usage as 'usageResponse'.

@@ -9,7 +9,7 @@ import Data.List.NonEmpty qualified as NE import Effectful (runEff) import Effectful.Concurrent (runConcurrent)-import Effectful.Error.Static (runErrorNoCallStack)+import Effectful.Error.Static (runErrorNoCallStack, throwError) import Effectful.Prim (runPrim) import EvalFixtures   ( Answer (..),@@ -23,7 +23,7 @@   ) import Shikumi.Effect.Time (runTime) import Shikumi.Error (ShikumiError (..))-import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith)+import Shikumi.Eval.Evaluate (evaluatePure, evaluateWith, scoreExecution) import Shikumi.Eval.Metric (exactMatch, liftMetric) import Shikumi.Eval.Report   ( ExampleResult (..),@@ -53,7 +53,27 @@ tests =   testGroup     "Evaluate"-    [ testCase "four-of-five exact match -> aggregateScore 0.8" $ do+    [ testCase "alternate runner retains typed failure evidence" $ do+        let err = InvalidJSON "original"+            envelope = (Left err :: Either ShikumiError Answer, ["attempt"] :: [String])+        result <-+          runEff . runErrorNoCallStack @ShikumiError $+            scoreExecution (const (FailScore scoreZero)) fst (pure envelope) (const (pure (boolScore True)))+        result @?= Right (envelope, (scoreZero, Just (ProgramError "InvalidJSON \"original\"")))+        aborted <-+          runEff . runErrorNoCallStack @ShikumiError $+            scoreExecution (const FailAbort) fst (pure envelope) (const (pure scoreZero))+        aborted @?= Left err,+      testCase "alternate runner labels metric failures separately" $ do+        result <-+          runEff . runErrorNoCallStack @ShikumiError $+            scoreExecution+              (const (FailScore scoreZero))+              id+              (pure (Right ()))+              (const (throwError (ValidationFailure "metric")))+        result @?= Right (Right (), (scoreZero, Just (MetricError "ValidationFailure \"metric\""))),+      testCase "four-of-five exact match -> aggregateScore 0.8" $ do         let ds = dataset [example q a | (q, a) <- aggregateData]         report <-           runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $

@@ -32,7 +32,7 @@   ]  fixtureUsage :: UsageTotals-fixtureUsage = UsageTotals {totalInputTokens = 120, totalOutputTokens = 45, totalTokens = 165, totalCostUsd = 23 % 10000}+fixtureUsage = UsageTotals {totalInputTokens = 120, totalOutputTokens = 45, totalTokens = 165, totalCostUsd = 23 % 10000, usageQuality = Nothing, unknownUsageCalls = 0}  fixtureReport :: Report fixtureReport = mkReport fixtureResults fixtureUsage@@ -61,8 +61,8 @@       testCase "results retained in order" $ map index (results fixtureReport) @?= [0, 1, 2],       testCase "empty report is zero" $ aggregateScore (mkReport [] emptyUsageTotals) @?= 0,       testCase "UsageTotals Monoid sums" $-        UsageTotals 10 20 30 (1 % 100) <> UsageTotals 1 2 3 (2 % 100)-          @?= UsageTotals 11 22 33 (3 % 100),+        UsageTotals 10 20 30 (1 % 100) Nothing 0 <> UsageTotals 1 2 3 (2 % 100) Nothing 0+          @?= UsageTotals 11 22 33 (3 % 100) Nothing 0,       testCase "UsageTotals mempty is empty" $ (mempty :: UsageTotals) @?= emptyUsageTotals,       testCase "renderReportText matches the documented format" $         renderReportText fixtureReport @?= expectedRender

@@ -20,7 +20,7 @@ import Shikumi.Error (ShikumiError) import Shikumi.Eval.Evaluate (evaluatePure) import Shikumi.Eval.Metric (exactMatch)-import Shikumi.Eval.Report (Report (..), UsageTotals (..))+import Shikumi.Eval.Report (Report (..)) import Shikumi.Eval.Types (dataset, example) import Shikumi.Eval.Usage (withUsageTotals) import Shikumi.LLM (stream)@@ -65,10 +65,5 @@           Left e -> assertFailure ("unexpected error: " <> show e)           Right r ->             usage r-              @?= UsageTotals-                { totalInputTokens = 300,-                  totalOutputTokens = 60,-                  totalTokens = 360,-                  totalCostUsd = 3 / 1000-                }+              @?= mconcat (replicate 3 usageTotalsPerCall)     ]