shikumi-eval 0.1.0.1 → 0.1.1.0

raw patch · 5 files changed

+23/−10 lines, 5 filesdep ~shikumidep ~shikumi-evalPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: shikumi, shikumi-eval

API changes (from Hackage documentation)

Files

• CHANGELOG.md +10/−0
• shikumi-eval.cabal +4/−4
• src/Shikumi/Eval/Metric.hs +3/−3
• src/Shikumi/Eval/Report.hs +2/−1
• test/EvaluateSpec.hs +4/−2

@@ -2,6 +2,16 @@  ## Unreleased +## 0.1.1.0 - 2026-06-28++### Added++- `EvalConfig` now derives `Generic`.++### Changed++- Refreshed the internal `shikumi` bound for the `0.2` series.+ ## 0.1.0.1 - 2026-06-21  ### Changed

@@ -1,6 +1,6 @@ cabal-version:   3.4 name:            shikumi-eval-version:         0.1.0.1+version:         0.1.1.0 synopsis:        Typed evaluation framework for shikumi LM programs (EP-8) category:        AI description:@@ -53,7 +53,7 @@     , effectful     , generic-lens     , lens          ^>=5.3-    , shikumi       ^>=0.1.0.1+    , shikumi       ^>=0.2.0.0     , tasty     , tasty-golden     , text          ^>=2.1@@ -83,8 +83,8 @@     , effectful     , generic-lens     , lens-    , shikumi       ^>=0.1.0.1-    , shikumi-eval  ^>=0.1.0.1+    , shikumi       ^>=0.2.0.0+    , shikumi-eval  ^>=0.1.1.0     , tasty     , tasty-golden     , tasty-hunit

@@ -208,9 +208,9 @@ -- | The judge's input: the grading rubric plus the reference and candidate texts -- (plain fields so the rendered prompt stays clean). data JudgeInput = JudgeInput-  { rubric :: Text,-    reference :: Text,-    candidate :: Text+  { rubric :: !Text,+    reference :: !Text,+    candidate :: !Text   }   deriving stock (Generic, Show) 

@@ -32,6 +32,7 @@ import Data.Maybe (isJust, mapMaybe) import Data.Text (Text) import Data.Text qualified as T+import GHC.Generics (Generic) import Numeric (showFFloat) import Numeric.Natural (Natural) import Shikumi.Eval.Types (Score, scoreZero, unScore)@@ -96,7 +97,7 @@     -- | samples per example for multi-sample metrics (forced to @>= 1@)     numSamples :: !Int   }-  deriving stock (Eq, Show)+  deriving stock (Eq, Show, Generic)  -- | The default: four-way concurrency, score failures @0@ and keep going, one -- sample per example.

@@ -4,6 +4,8 @@ -- completes; and the same error aborting the run under 'FailAbort'. module EvaluateSpec (tests) where +import Control.Lens ((&), (.~))+import Data.Generics.Labels () import Effectful (runEff) import Effectful.Concurrent (runConcurrent) import Effectful.Error.Static (runErrorNoCallStack)@@ -72,7 +74,7 @@         -- example fails, the rest succeed.         let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")]             replies = [MockFail providerErr, MockOk (answerResponse "ok")]-            cfg = defaultEvalConfig {concurrency = 1}+            cfg = defaultEvalConfig & #concurrency .~ 1         report <-           runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $             runScriptedLLM replies $@@ -93,7 +95,7 @@       testCase "FailAbort surfaces the error" $ do         let ds = dataset [example (Question "q0") (Answer "a"), example (Question "q1") (Answer "ok")]             replies = [MockFail providerErr, MockOk (answerResponse "ok")]-            cfg = defaultEvalConfig {concurrency = 1, failurePolicy = FailAbort}+            cfg = defaultEvalConfig & #concurrency .~ 1 & #failurePolicy .~ FailAbort         report <-           runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $             runScriptedLLM replies $