shikumi-optimize 0.1.0.1 → 0.2.0.0

raw patch · 16 files changed

+90/−59 lines, 16 filesdep ~generic-lensdep ~lensdep ~shikumiPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: generic-lens, lens, shikumi, shikumi-compile, shikumi-eval, shikumi-optimize, shikumi-trace

API changes (from Hackage documentation)

Files

• CHANGELOG.md +15/−0
• shikumi-optimize.cabal +12/−10
• src/Shikumi/Optimize/Bootstrap.hs +2/−1
• src/Shikumi/Optimize/COPRO.hs +2/−1
• src/Shikumi/Optimize/GEPA.hs +10/−8
• src/Shikumi/Optimize/LabeledFewShot.hs +4/−2
• src/Shikumi/Optimize/MIPRO.hs +2/−1
• src/Shikumi/Optimize/Pareto.hs +8/−8
• src/Shikumi/Optimize/Propose/Grounded.hs +9/−9
• src/Shikumi/Optimize/Propose/Summarize.hs +5/−5
• src/Shikumi/Optimize/Propose/Types.hs +2/−2
• src/Shikumi/Optimize/RandomSearch.hs +4/−2
• src/Shikumi/Optimize/Search.hs +3/−1
• src/Shikumi/Optimize/Types.hs +1/−1
• test/CoproSpec.hs +6/−5
• test/Miprov2Spec.hs +5/−3

@@ -2,6 +2,21 @@  ## Unreleased +## 0.2.0.0 - 2026-06-28++### Added++- `Generic` instances for `BootstrapConfig`, `CoproConfig`, and `Miprov2Config`.++### Changed++- Renamed exported `Candidate` record selectors to `params`, `perExample`, and+  `aggregate`.+- Renamed exported `PastInstruction` record selectors to `instruction` and+  `score`.+- Refreshed internal `shikumi`, `shikumi-compile`, `shikumi-eval`, and+  `shikumi-trace` bounds for the current package set.+ ## 0.1.0.1 - 2026-06-21  ### Changed

@@ -1,6 +1,6 @@ cabal-version:   3.4 name:            shikumi-optimize-version:         0.1.0.1+version:         0.2.0.0 synopsis:        The optimizer framework for shikumi LM programs (EP-10) category:        AI description:@@ -64,10 +64,12 @@     , base             >=4.20     && <5     , containers     , effectful-    , shikumi          ^>=0.1.0.1-    , shikumi-compile  ^>=0.1.0.1-    , shikumi-eval     ^>=0.1.0.1-    , shikumi-trace    ^>=0.1.0.1+    , generic-lens     ^>=2.2+    , lens             ^>=5.3+    , shikumi          ^>=0.2.0.0+    , shikumi-compile  ^>=0.1.1.0+    , shikumi-eval     ^>=0.1.1.0+    , shikumi-trace    ^>=0.1.1.0     , text             ^>=2.1     , vector @@ -100,11 +102,11 @@     , effectful     , generic-lens     , lens-    , shikumi           ^>=0.1.0.1-    , shikumi-compile   ^>=0.1.0.1-    , shikumi-eval      ^>=0.1.0.1-    , shikumi-optimize  ^>=0.1.0.1-    , shikumi-trace     ^>=0.1.0.1+    , shikumi           ^>=0.2.0.0+    , shikumi-compile   ^>=0.1.1.0+    , shikumi-eval      ^>=0.1.1.0+    , shikumi-optimize  ^>=0.2.0.0+    , shikumi-trace     ^>=0.1.1.0     , tasty     , tasty-hunit     , text

@@ -29,6 +29,7 @@  import Data.Aeson (ToJSON, toJSON) import Effectful.Error.Static (catchError)+import GHC.Generics (Generic) import Shikumi.Error (ShikumiError) import Shikumi.Eval (Example (..), datasetExamples, prediction, unScore) import Shikumi.Optimize.LabeledFewShot (withDemos)@@ -44,7 +45,7 @@     -- | cap on the demos attached, so prompts stay bounded (default @4@)     maxBootstrappedDemos :: !Int   }-  deriving stock (Eq, Show)+  deriving stock (Eq, Show, Generic)  -- | Keep only perfectly-correct teacher runs; attach at most four demos. defaultBootstrapConfig :: BootstrapConfig

@@ -27,6 +27,7 @@ import Effectful.Concurrent (Concurrent) import Effectful.Error.Static (Error) import Effectful.Prim (Prim)+import GHC.Generics (Generic) import Shikumi.Effect.Time (Time) import Shikumi.Error (ShikumiError) import Shikumi.Eval (Dataset, Metric, datasetSize)@@ -50,7 +51,7 @@     -- | LM-call / candidate ceilings     budget :: !Budget   }-  deriving stock (Eq, Show)+  deriving stock (Eq, Show, Generic)  -- | Breadth 4, depth 3, the default budget. defaultCoproConfig :: CoproConfig

@@ -33,7 +33,9 @@   ) where +import Control.Lens ((&), (?~)) import Control.Monad (forM, forM_, when)+import Data.Generics.Labels () import Data.Maybe (fromMaybe) import Data.Text (Text) import Data.Text qualified as T@@ -87,11 +89,11 @@ -- | The reflective proposer's input: the node's current instruction, its accumulated -- critiques, program/dataset summaries, and the node's field names. data ReflectIn = ReflectIn-  { currentInstruction :: Text,-    feedback :: Text,-    programSummary :: Text,-    datasetSummary :: Text,-    fieldSummary :: Text+  { currentInstruction :: !Text,+    feedback :: !Text,+    programSummary :: !Text,+    datasetSummary :: !Text,+    fieldSummary :: !Text   }   deriving stock (Generic, Show) @@ -179,7 +181,7 @@                   fb = T.intercalate "\n" crits               ReflectOut newInstr <-                 runProgram proposer (ReflectIn cur fb progSummary dataSummary fldSummary)-              pure (mapParamsAt idx (\ps -> ps {instructionOverride = Just newInstr}) prog)+              pure (mapParamsAt idx (\ps -> ps & #instructionOverride ?~ newInstr) prog)  -- | The 'Params' at a node index (empty if out of range). paramsAt :: Int -> Program i o -> Params@@ -216,7 +218,7 @@       dataSummary = fallbackDatasetSummary (datasetSize train)       maxCalls = maxLmCalls budget       maxCands = maxCandidates budget-      rebuild cand = either (const student) id (setProgramParams (candParams cand) student)+      rebuild cand = either (const student) id (setProgramParams (params cand) student)    seedRpt <- evaluatePure train metric student   let seedCand = Candidate (foldParams student) (perEx seedRpt) (aggregateScore seedRpt)@@ -259,7 +261,7 @@ bestOf :: Candidate -> [Candidate] -> Candidate bestOf seedCand cs = case cs of   [] -> seedCand-  (x : xs) -> foldl' (\b c -> if candAggregate c > candAggregate b then c else b) x xs+  (x : xs) -> foldl' (\b c -> if aggregate c > aggregate b then c else b) x xs  -- | A minimal program summary (EP-19's program describer is the richer source). fallbackProgramSummary :: Int -> Text

@@ -14,11 +14,13 @@   ) where +import Control.Lens ((&), (.~)) import Data.Aeson (ToJSON, toJSON)+import Data.Generics.Labels () import Shikumi.Eval (Dataset, Example (..), datasetExamples) import Shikumi.Optimize.Search (freezeProgram, scoreOn, selectBest) import Shikumi.Optimize.Types (Optimizer (..), Scored (..), defaultBudget)-import Shikumi.Program (Demo (..), Params (..), Program, mapParams)+import Shikumi.Program (Demo (..), Program, mapParams)  -- | Select the best size-@k@ set of labelled demonstrations from the training set. -- Multi-node programs receive the same demo set at every node (the DSPy default).@@ -44,7 +46,7 @@  -- | Attach a demo set to every node, leaving each node's instruction untouched. withDemos :: [Demo] -> Program i o -> Program i o-withDemos ds = mapParams (\ps -> ps {demos = ds})+withDemos ds = mapParams (\ps -> ps & #demos .~ ds)  -- | All size-@k@ sub-lists of a list, preserving relative order -- (@combinations 2 [a,b,c] = [[a,b],[a,c],[b,c]]@).

@@ -51,6 +51,7 @@ import Effectful.Concurrent (Concurrent) import Effectful.Error.Static (Error, catchError) import Effectful.Prim (Prim)+import GHC.Generics (Generic) import Shikumi.Effect.Time (Time) import Shikumi.Error (ShikumiError) import Shikumi.Eval@@ -111,7 +112,7 @@     -- | hard LM-call / candidate ceiling (V1's 'Budget')     budget :: !Budget   }-  deriving stock (Eq, Show)+  deriving stock (Eq, Show, Generic)  -- | Turn a preset into a concrete config. Values are small enough for hermetic tests -- yet keep DSPy's monotonic ordering (heavier => more candidates, more trials).

@@ -17,9 +17,9 @@ -- | A candidate program, identified by its @foldParams@-order node parameters, with -- its per-example scores (dataset order) and their mean. data Candidate = Candidate-  { candParams :: ![Params],-    candPerExample :: ![Double],-    candAggregate :: !Double+  { params :: ![Params],+    perExample :: ![Double],+    aggregate :: !Double   }   deriving stock (Eq, Show) @@ -27,8 +27,8 @@ -- Equal vectors are NOT domination, so equal candidates coexist on the frontier. dominates :: Candidate -> Candidate -> Bool dominates a b =-  let pa = candPerExample a-      pb = candPerExample b+  let pa = perExample a+      pb = perExample b    in length pa == length pb && and (zipWith (>=) pa pb) && or (zipWith (>) pa pb)  -- | The non-dominated subset, preserving input order (so the frontier is@@ -44,9 +44,9 @@ sampleParent _ [] = Nothing sampleParent seed cands@(c0 : _) = Just (go r (zip cands weights), seed')   where-    cols = length (candPerExample c0)-    colMax j = maximum [candPerExample c !! j | c <- cands]-    weightOf c = max 1 (length [() | j <- [0 .. cols - 1], (candPerExample c !! j) >= colMax j])+    cols = length (perExample c0)+    colMax j = maximum [perExample c !! j | c <- cands]+    weightOf c = max 1 (length [() | j <- [0 .. cols - 1], (perExample c !! j) >= colMax j])     weights = map weightOf cands     total = max 1 (sum weights)     seed' = lcg seed

@@ -49,15 +49,15 @@ -- | The grounded proposer's input: every signal about the optimization target, -- rendered into the prompt under its field name by the generic 'ToPrompt'. data GenerateInstructionIn = GenerateInstructionIn-  { datasetDescription :: Text,-    programCode :: Text,-    programDescription :: Text,-    moduleSignature :: Text,-    moduleDescription :: Text,-    taskDemos :: Text,-    previousInstructions :: Text,-    basicInstruction :: Text,-    tip :: Text+  { datasetDescription :: !Text,+    programCode :: !Text,+    programDescription :: !Text,+    moduleSignature :: !Text,+    moduleDescription :: !Text,+    taskDemos :: !Text,+    previousInstructions :: !Text,+    basicInstruction :: !Text,+    tip :: !Text   }   deriving stock (Generic, Show) 

@@ -108,8 +108,8 @@ -- ---------------------------------------------------------------------------  data ProgramDescribeIn = ProgramDescribeIn-  { programCode :: Text,-    programExample :: Text+  { programCode :: !Text,+    programExample :: !Text   }   deriving stock (Generic, Show) @@ -215,9 +215,9 @@ -- ---------------------------------------------------------------------------  data ModuleDescribeIn = ModuleDescribeIn-  { programCode :: Text,-    programDescription :: Text,-    moduleSignature :: Text+  { programCode :: !Text,+    programDescription :: !Text,+    moduleSignature :: !Text   }   deriving stock (Generic, Show) 

@@ -57,8 +57,8 @@  -- | One past attempt for a node: the instruction text and the score it earned. data PastInstruction = PastInstruction-  { pastInstruction :: !Text,-    pastScore :: !Double+  { instruction :: !Text,+    score :: !Double   }   deriving stock (Eq, Show, Generic) 

@@ -15,12 +15,14 @@   ) where +import Control.Lens ((&), (.~)) import Data.Aeson (ToJSON)+import Data.Generics.Labels () import Data.List (sortBy) import Data.Ord (comparing) import Shikumi.Compile.Types (compiledProgram) import Shikumi.Eval (Dataset, dataset, datasetExamples)-import Shikumi.Optimize.Bootstrap (BootstrapConfig (..), bootstrapFewShotWith, defaultBootstrapConfig)+import Shikumi.Optimize.Bootstrap (bootstrapFewShotWith, defaultBootstrapConfig) import Shikumi.Optimize.Search (freezeProgram, scoreOn, selectBest) import Shikumi.Optimize.Types (Budget (..), Optimizer (..), Scored (..)) import Shikumi.Program (Program)@@ -77,7 +79,7 @@ bootstrapRandomSearchWith cfg teacher numCandidates budget = Optimizer $ \train metric student -> do   let seeds = [1 .. max 1 numCandidates]       candidateFor seed = do-        let cfg' = defaultBootstrapConfig {maxBootstrappedDemos = sizeFor cfg seed}+        let cfg' = defaultBootstrapConfig & #maxBootstrappedDemos .~ sizeFor cfg seed             opt = bootstrapFewShotWith cfg' teacher budget         compiledProgram <$> runOptimizer opt (shuffle seed train) metric student   seeded <- mapM candidateFor seeds

@@ -16,6 +16,8 @@   ) where +import Control.Lens ((&), (?~))+import Data.Generics.Labels () import Data.Text (Text) import Effectful (Eff, (:>)) import Effectful.Concurrent (Concurrent)@@ -73,7 +75,7 @@ -- | Set node @idx@'s instruction override (in @foldParams@ order). Shared by -- @instructionSearch@ and @copro@. setNodeInstr :: Int -> Text -> Program i o -> Program i o-setNodeInstr idx instr = mapParamsAt idx (\ps -> ps {instructionOverride = Just instr})+setNodeInstr idx instr = mapParamsAt idx (\ps -> ps & #instructionOverride ?~ instr)  -- | The instruction override stored at node @idx@ (in @foldParams@ order), if any. instructionAt :: Int -> Program i o -> Maybe Text

@@ -86,7 +86,7 @@ -- | A candidate paired with its score. Threaded as a plain value through the pure -- selection fold; there is no mutable search state. data Scored a = Scored-  { candidate :: a,+  { candidate :: !a,     score :: !Double   }   deriving stock (Eq, Show)

@@ -5,6 +5,8 @@ -- and the serialization round-trip — the breadth/depth control the MasterPlan tracks. module CoproSpec (tests) where +import Control.Lens ((&), (.~))+import Data.Generics.Labels () import Data.IORef (newIORef, readIORef) import Effectful (Eff, IOE, runEff) import Effectful.Concurrent (Concurrent, runConcurrent)@@ -18,7 +20,6 @@ import Shikumi.LLM (LLM) import Shikumi.Optimize   ( Budget (..),-    CoproConfig (..),     copro,     defaultCoproConfig,     optimize,@@ -67,7 +68,7 @@ breadthSurfacesMagic :: TestTree breadthSurfacesMagic =   testCase "breadth surfaces and selects the magic RULE instruction" $ do-    res <- runStub (optimize (copro defaultCoproConfig {depth = 1}) trainset exactMatch sentimentProg)+    res <- runStub (optimize (copro (defaultCoproConfig & #depth .~ 1)) trainset exactMatch sentimentProg)     case res of       Left e -> assertFailure ("unexpected error: " <> show e)       Right cp -> case programParams (compiledProgram cp) of@@ -79,9 +80,9 @@   testCase "deeper depth never scores worse than shallower on held-out" $ do     res <-       runStub $ do-        cp1 <- optimize (copro defaultCoproConfig {depth = 1}) trainset exactMatch sentimentProg+        cp1 <- optimize (copro (defaultCoproConfig & #depth .~ 1)) trainset exactMatch sentimentProg         s1 <- scoreOn heldout exactMatch (compiledProgram cp1)-        cp3 <- optimize (copro defaultCoproConfig {depth = 3}) trainset exactMatch sentimentProg+        cp3 <- optimize (copro (defaultCoproConfig & #depth .~ 3)) trainset exactMatch sentimentProg         s3 <- scoreOn heldout exactMatch (compiledProgram cp3)         pure (s1, s3)     case res of@@ -92,7 +93,7 @@ budgetRespected =   testCase "respects the LM-call budget" $ do     ref <- newIORef (0 :: Int)-    let cfg = defaultCoproConfig {budget = Budget {maxLmCalls = 8, maxCandidates = 32}}+    let cfg = defaultCoproConfig & #budget .~ Budget {maxLmCalls = 8, maxCandidates = 32}     res <-       runEff . runPrim . runTime . runConcurrent . runErrorNoCallStack @ShikumiError $         runStubLMCounting ref (optimize (copro cfg) trainset exactMatch sentimentProg)

@@ -8,6 +8,8 @@ -- joint selection), and M3 the lift plus the serialization round-trip. module Miprov2Spec (tests) where +import Control.Lens ((&), (.~), (?~))+import Data.Generics.Labels () import Data.IORef (newIORef, readIORef) import Effectful (Eff, IOE, runEff) import Effectful.Concurrent (Concurrent, runConcurrent)@@ -33,7 +35,7 @@     scoreOn,     searchJoint,   )-import Shikumi.Program (Params (..), mapParamsAt)+import Shikumi.Program (mapParamsAt) import StubLM   ( Label (..),     Sentence (..),@@ -129,7 +131,7 @@               scoreOn                 jointHeldout                 exactMatch-                (mapParamsAt 0 (\ps -> ps {instructionOverride = Just ruleInstruction}) sentimentProg)+                (mapParamsAt 0 (\ps -> ps & #instructionOverride ?~ ruleInstruction) sentimentProg)             pure (joint, ruleOnly)         case res of           Left e -> assertFailure ("unexpected error: " <> show e)@@ -139,7 +141,7 @@             joint @?= 1.0,       testCase "searchJoint honors the LM-call budget" $ do         ref <- newIORef (0 :: Int)-        let cfg = cfgLight {numTrials = 5, minibatchSize = 2, budget = Budget {maxLmCalls = 8, maxCandidates = 32}}+        let cfg = cfgLight & #numTrials .~ 5 & #minibatchSize .~ 2 & #budget .~ Budget {maxLmCalls = 8, maxCandidates = 32}             instrCands = [["", ruleInstruction]]             demoCands = [[[], []]]         res <-