shikumi-compile 0.2.0.3 → 0.2.1.0

raw patch · 9 files changed

+337/−9 lines, 9 filesdep ~baikaidep ~shikumidep ~shikumi-compilePVP ok

version bump matches the API change (PVP)

Dependency ranges changed: baikai, shikumi, shikumi-compile

API changes (from Hackage documentation)

Files

• CHANGELOG.md +10/−0
• shikumi-compile.cabal +8/−5
• src/Shikumi/Compile.hs +6/−1
• src/Shikumi/Compile/ChainOfThought.hs +22/−2
• src/Shikumi/Compile/RAG.hs +4/−0
• src/Shikumi/Compile/Structure.hs +121/−0
• src/Shikumi/Compile/Structure/Serialize.hs +88/−0
• test/Main.hs +3/−1
• test/StructureSpec.hs +75/−0

@@ -2,6 +2,16 @@  ## Unreleased +## 0.2.1.0 — 2026-09-08++- Raise the internal `shikumi` bound to `^>=0.4.0.0` for the breaking core release.++- Upgrade the dependency on `mori://shinzui/baikai/packages/baikai` to `>=0.7.0.0 && <0.8`.++- Add validated finite typed recipe registries and distinct version-1 experimental structure artifacts with exact registry/revision/schema/shape checks; existing compiled state is unchanged.++- Preserve capture codecs through parameter restoration and RAG; adapt codecs to reasoning/value outputs in the chain-of-thought rewrite. Saved parameter shapes remain unchanged.+ ## 0.2.0.3 — 2026-08-29  ### Changed

@@ -1,6 +1,6 @@ cabal-version:   3.4 name:            shikumi-compile-version:         0.2.0.3+version:         0.2.1.0 synopsis:        The compiler layer for shikumi LM programs (EP-9) category:        AI description:@@ -44,6 +44,8 @@     Shikumi.Compile.RAG     Shikumi.Compile.Retriever     Shikumi.Compile.Serialize+    Shikumi.Compile.Structure+    Shikumi.Compile.Structure.Serialize     Shikumi.Compile.Types     Shikumi.Compile.ZeroShot @@ -54,7 +56,7 @@     , effectful     >=2.5      && <2.7     , generic-lens  >=2.2      && <2.4     , lens          ^>=5.3-    , shikumi       ^>=0.3.0.0+    , shikumi       ^>=0.4.0.0     , text          ^>=2.1  test-suite shikumi-compile-test@@ -64,19 +66,20 @@   main-is:        Main.hs   ghc-options:    -threaded -with-rtsopts=-N   other-modules:+    StructureSpec     Test.Capture     Test.Fixtures    build-depends:     , aeson-    , baikai           >=0.6      && <0.7+    , baikai           >=0.7.0.0  && <0.8     , base     , bytestring     , effectful     , generic-lens     , lens-    , shikumi          ^>=0.3.0.0-    , shikumi-compile  ^>=0.2.0.0+    , shikumi          ^>=0.4.0.0+    , shikumi-compile  ^>=0.2.1.0     , tasty     , tasty-hunit     , text

@@ -19,7 +19,10 @@ -- This module re-exports the full public surface; see the per-strategy modules for -- detail and caveats. module Shikumi.Compile-  ( -- * Types+  ( module Shikumi.Compile.Structure.Serialize,+    module Shikumi.Compile.Structure,++    -- * Types     Compiler (..),     CompiledProgram (..),     compile,@@ -51,5 +54,7 @@ import Shikumi.Compile.RAG (formatPassages, rag) import Shikumi.Compile.Retriever (Passage (..), Retriever (..), inMemoryRetriever) import Shikumi.Compile.Serialize (decodeCompiledOnto, encodeCompiled)+import Shikumi.Compile.Structure+import Shikumi.Compile.Structure.Serialize import Shikumi.Compile.Types (CompiledProgram (..), Compiler (..), compile, identity, runCompiled) import Shikumi.Compile.ZeroShot (zeroShot, zeroShotClear)

@@ -41,10 +41,12 @@   ) where +import Data.Aeson (object, (.=)) import Shikumi.Compile.Types (Compiler (..))-import Shikumi.Module (chainOfThoughtRaw, value)+import Shikumi.Module (WithReasoning (..), chainOfThoughtRaw) import Shikumi.Program-  ( Program+  ( CaptureCodec (..),+    Program       ( Compose,         Embed,         Ensemble,@@ -53,12 +55,14 @@         Map,         Parallel,         Predict,+        PredictCaptured,         Retry,         RetryWhen,         Validate       ),     mapParams,   )+import Shikumi.Schema.Types (objectSchema, stringSchema)  -- | Rewrite every 'Predict' node into its chain-of-thought form, recursing through -- every composite/combinator node so nodes nested arbitrarily deep are reached.@@ -72,6 +76,22 @@ -- ('Shikumi.Schema.FromModel' @i@/@o@, 'Shikumi.Schema.ToSchema' @o@, the -- 'Shikumi.Adapter.ToPrompt's) are exactly what 'chainOfThoughtRaw' needs. cot :: Program i o -> Program i o+cot (PredictCaptured codec sig ps) =+  case chainOfThoughtRaw sig of+    Predict augmented _ ->+      FMap+        value+        ( PredictCaptured+            ( CaptureCodec+                (encodeCaptureInput codec)+                (\o -> object ["reasoning" .= reasoning o, "value" .= encodeCaptureOutput codec (value o)])+                (captureInputSchema codec)+                (objectSchema [("reasoning", stringSchema), ("value", captureOutputSchema codec)] ["reasoning", "value"])+            )+            augmented+            ps+        )+    other -> FMap value other cot (Predict sig ps) = FMap value (mapParams (const ps) (chainOfThoughtRaw sig)) cot (Compose a b) = Compose (cot a) (cot b) cot (FMap k p) = FMap k (cot p)

@@ -48,6 +48,7 @@         Map,         Parallel,         Predict,+        PredictCaptured,         Retry,         RetryWhen,         Validate@@ -80,6 +81,9 @@   | otherwise = go   where     go :: forall i o. Program i o -> Program i o+    go (PredictCaptured codec sig ps) =+      let base = fromMaybe (getInstruction sig) (instructionOverride ps)+       in PredictCaptured codec sig ps {instructionOverride = Just (base <> "\n\n" <> ctx)}     go (Predict sig ps) =       let base = fromMaybe (getInstruction sig) (instructionOverride ps)        in Predict sig ps {instructionOverride = Just (base <> "\n\n" <> ctx)}

@@ -0,0 +1,121 @@+-- | Finite, trusted typed implementations. Identity is a caller-owned contract:+-- advance revisions whenever signatures, reducers or opaque code change.+module Shikumi.Compile.Structure+  ( RecipeId,+    recipeIdText,+    RecipeRevision,+    recipeRevisionNumber,+    RegistryId,+    registryIdText,+    StructureRecipe,+    recipeId,+    recipeRevision,+    recipeDescription,+    recipeProgram,+    StructureRegistry,+    registryId,+    registryRecipes,+    registryInputSchema,+    registryOutputSchema,+    StructureRegistryError (..),+    structureRecipe,+    structureRegistry,+    lookupRecipe,+    directCotRegistry,+  )+where++import Control.Monad (unless)+import Data.Aeson (Value)+import Data.List (find, nub)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NE+import Data.Proxy (Proxy (..))+import Data.Text (Text)+import Data.Text qualified as T+import Shikumi.Compile.ChainOfThought (chainOfThoughtCompiler)+import Shikumi.Compile.Types (Compiler (..))+import Shikumi.Program (Program, emptyParams, programParams)+import Shikumi.Schema (ToSchema (..))++newtype RecipeId = RecipeId Text deriving stock (Eq, Ord, Show)++newtype RecipeRevision = RecipeRevision Int deriving stock (Eq, Ord, Show)++newtype RegistryId = RegistryId Text deriving stock (Eq, Ord, Show)++recipeIdText :: RecipeId -> Text+recipeIdText (RecipeId ident) = ident++recipeRevisionNumber :: RecipeRevision -> Int+recipeRevisionNumber (RecipeRevision revision) = revision++registryIdText :: RegistryId -> Text+registryIdText (RegistryId ident) = ident++-- Constructors and record labels are private, so record updates cannot bypass+-- registry validation or replace its typed schema evidence.+data StructureRecipe i o = StructureRecipe RecipeId RecipeRevision Text (Program i o)++data StructureRegistry i o = StructureRegistry RegistryId Value Value (NonEmpty (StructureRecipe i o))++recipeId :: StructureRecipe i o -> RecipeId+recipeId (StructureRecipe x _ _ _) = x++recipeRevision :: StructureRecipe i o -> RecipeRevision+recipeRevision (StructureRecipe _ x _ _) = x++recipeDescription :: StructureRecipe i o -> Text+recipeDescription (StructureRecipe _ _ x _) = x++recipeProgram :: StructureRecipe i o -> Program i o+recipeProgram (StructureRecipe _ _ _ x) = x++registryId :: StructureRegistry i o -> RegistryId+registryId (StructureRegistry x _ _ _) = x++registryInputSchema :: StructureRegistry i o -> Value+registryInputSchema (StructureRegistry _ x _ _) = x++registryOutputSchema :: StructureRegistry i o -> Value+registryOutputSchema (StructureRegistry _ _ x _) = x++registryRecipes :: StructureRegistry i o -> NonEmpty (StructureRecipe i o)+registryRecipes (StructureRegistry _ _ _ x) = x++data StructureRegistryError+  = EmptyRecipeId+  | InvalidRecipeRevision Text Int+  | EmptyRegistryId+  | EmptyRegistry+  | DuplicateRecipeId Text+  | PopulatedBase+  deriving stock (Eq, Show)++structureRecipe :: Text -> Int -> Text -> Program i o -> Either StructureRegistryError (StructureRecipe i o)+structureRecipe ident revision description program = do+  unless (not (T.null (T.strip ident))) (Left EmptyRecipeId)+  unless (revision > 0) (Left (InvalidRecipeRevision ident revision))+  pure (StructureRecipe (RecipeId ident) (RecipeRevision revision) description program)++structureRegistry :: forall i o. (ToSchema i, ToSchema o) => Text -> [StructureRecipe i o] -> Either StructureRegistryError (StructureRegistry i o)+structureRegistry ident recipes = do+  unless (not (T.null (T.strip ident))) (Left EmptyRegistryId)+  rs <- maybe (Left EmptyRegistry) Right (NE.nonEmpty recipes)+  let ids = map recipeId recipes+  case find (\x -> length (filter (== x) ids) > 1) (nub ids) of+    Just duplicate -> Left (DuplicateRecipeId (recipeIdText duplicate))+    Nothing -> pure (StructureRegistry (RegistryId ident) (toSchema (Proxy @i)) (toSchema (Proxy @o)) rs)++lookupRecipe :: RecipeId -> StructureRegistry i o -> Maybe (StructureRecipe i o)+lookupRecipe ident = find ((== ident) . recipeId) . registryRecipes++-- | Register @direct@ then @cot@, both revision 1. Reject any populated+-- parameters; explicitly use @mapParams (const emptyParams)@ first if desired.+-- Independently optimized variants should use 'structureRecipe' instead.+directCotRegistry :: (ToSchema i, ToSchema o) => Text -> Program i o -> Either StructureRegistryError (StructureRegistry i o)+directCotRegistry ident base = do+  unless (all (== emptyParams) (programParams base)) (Left PopulatedBase)+  direct <- structureRecipe "direct" 1 "Direct base program" base+  cot <- structureRecipe "cot" 1 "Reasoning then projection" (runCompiler chainOfThoughtCompiler base)+  structureRegistry ident [direct, cot]

@@ -0,0 +1,88 @@+-- | Versioned experimental artifacts, restored only through trusted caller code.+-- Schema/shape equality detects declared incompatibility; it does not attest to+-- opaque function identity. Owners must advance revisions for behavioral changes.+module Shikumi.Compile.Structure.Serialize+  ( StructureArtifact (..),+    StructureArtifactError (..),+    encodeStructureArtifact,+    decodeStructureArtifact,+  )+where++import Control.Monad (unless)+import Data.Aeson (FromJSON, ToJSON, Value, eitherDecode, encode)+import Data.ByteString.Lazy (ByteString)+import Data.List (find)+import Data.Text (Text)+import GHC.Generics (Generic)+import Shikumi.Compile.Structure+import Shikumi.Compile.Types (CompiledProgram (..))+import Shikumi.Program (Params, ProgramShape, programParams, programShape, setProgramParams)++data StructureArtifact = StructureArtifact+  { artifactKind :: !Text,+    formatVersion :: !Int,+    artifactRegistryId :: !Text,+    artifactRecipeId :: !Text,+    artifactRecipeRevision :: !Int,+    inputSchema :: !Value,+    outputSchema :: !Value,+    selectedShape :: !ProgramShape,+    orderedParams :: ![Params]+  }+  deriving stock (Eq, Show, Generic)+  deriving anyclass (ToJSON, FromJSON)++data StructureArtifactError+  = MalformedStructureArtifact String+  | WrongArtifactKind Text+  | UnsupportedStructureVersion Int+  | RegistryMismatch Text Text+  | UnknownRecipe Text+  | RevisionMismatch Text Int Int+  | InputSchemaMismatch Text+  | OutputSchemaMismatch Text+  | StructureShapeMismatch Text+  | StructureParameterCountMismatch Text Int Int+  deriving stock (Eq, Show)++encodeStructureArtifact :: StructureRegistry i o -> RecipeId -> CompiledProgram i o -> Either StructureArtifactError ByteString+encodeStructureArtifact registry ident (CompiledProgram program) = do+  recipe <- maybe (Left (UnknownRecipe (recipeIdText ident))) Right (lookupRecipe ident registry)+  let artifact =+        StructureArtifact+          "shikumi.experimental.structure"+          1+          (registryIdText (registryId registry))+          (recipeIdText ident)+          (recipeRevisionNumber (recipeRevision recipe))+          (registryInputSchema registry)+          (registryOutputSchema registry)+          (programShape program)+          (programParams program)+  _ <- restore registry artifact+  pure (encode artifact)++decodeStructureArtifact :: StructureRegistry i o -> ByteString -> Either StructureArtifactError (CompiledProgram i o)+decodeStructureArtifact registry bytes = do+  artifact <- either (Left . MalformedStructureArtifact) Right (eitherDecode bytes)+  restore registry artifact++restore :: StructureRegistry i o -> StructureArtifact -> Either StructureArtifactError (CompiledProgram i o)+restore registry artifact = do+  unless (artifactKind artifact == "shikumi.experimental.structure") (Left (WrongArtifactKind (artifactKind artifact)))+  unless (formatVersion artifact == 1) (Left (UnsupportedStructureVersion (formatVersion artifact)))+  let expectedRegistry = registryIdText (registryId registry)+      ident = artifactRecipeId artifact+  unless (artifactRegistryId artifact == expectedRegistry) (Left (RegistryMismatch expectedRegistry (artifactRegistryId artifact)))+  recipe <- maybe (Left (UnknownRecipe ident)) Right (find ((== ident) . recipeIdText . recipeId) (registryRecipes registry))+  let revision = recipeRevisionNumber (recipeRevision recipe)+      template = recipeProgram recipe+      count = length (programParams template)+      actual = length (orderedParams artifact)+  unless (artifactRecipeRevision artifact == revision) (Left (RevisionMismatch ident revision (artifactRecipeRevision artifact)))+  unless (inputSchema artifact == registryInputSchema registry) (Left (InputSchemaMismatch ident))+  unless (outputSchema artifact == registryOutputSchema registry) (Left (OutputSchemaMismatch ident))+  unless (selectedShape artifact == programShape template) (Left (StructureShapeMismatch ident))+  unless (actual == count) (Left (StructureParameterCountMismatch ident count actual))+  either (const (Left (StructureParameterCountMismatch ident count actual))) (Right . CompiledProgram) (setProgramParams (orderedParams artifact) template)

@@ -22,6 +22,7 @@   ) import Shikumi.Compile.Retriever (Passage (..)) import Shikumi.Program (Params (..), foldParams)+import StructureSpec qualified import Test.Capture (runWithCapture) import Test.Fixtures   ( Question (..),@@ -43,7 +44,8 @@ tests =   testGroup     "shikumi-compile (EP-9)"-    [ m0_identity,+    [ StructureSpec.tests,+      m0_identity,       m1_baseline,       m2_zeroShot,       m3_fewShot,

@@ -0,0 +1,75 @@+module StructureSpec (tests) where++import Data.Aeson (Value (Null), eitherDecode, encode)+import Data.Either (isLeft)+import Data.List.NonEmpty qualified as NE+import Shikumi.Compile+import Shikumi.Program (programShape)+import Test.Capture (runWithCapture)+import Test.Fixtures+import Test.Tasty+import Test.Tasty.HUnit++right :: (Show e) => Either e a -> IO a+right = either (fail . show) pure++tests :: TestTree+tests =+  testGroup+    "typed structures"+    [ testCase "invalid metadata rejected" $ do+        assertBool "empty ID" (isLeft (structureRecipe " " 1 "" qaBase))+        assertBool "revision" (isLeft (structureRecipe "a" 0 "" qaBase))+        a <- right (structureRecipe "a" 1 "" qaBase)+        assertBool "empty registry" (isLeft (structureRegistry "r" ([] :: [StructureRecipe Question Answer])))+        assertBool "registry ID" (isLeft (structureRegistry "" [a]))+        assertBool "duplicate" (isLeft (structureRegistry "r" [a, a])),+      testCase "direct/CoT order and distinct shapes" $ do+        r <- right (directCotRegistry "r" qaBase)+        let rs = NE.toList (registryRecipes r)+        map (recipeIdText . recipeId) rs @?= ["direct", "cot"]+        case rs of+          [direct, cot] -> assertBool "different shapes" (programShape (recipeProgram direct) /= programShape (recipeProgram cot))+          _ -> assertFailure "expected direct and cot",+      testCase "populated helper base rejected; explicit pipelines accepted" $ do+        assertBool "no silent deletion" (isLeft (directCotRegistry "r" (compiledProgram (compile (fewShotTyped demoPairs) qaBase))))+        a <- right (structureRecipe "direct" 1 "" qaBase)+        b <- right (structureRecipe "pipeline" 1 "" qaPipeline)+        r <- right (structureRegistry "r" [a, b])+        length (registryRecipes r) @?= 2,+      testCase "artifact restores compatible demos and requests" $ do+        r <- right (directCotRegistry "r" qaBase)+        let ident = recipeId (NE.head (registryRecipes r))+            original = compile (fewShotTyped demoPairs) qaBase+        bytes <- right (encodeStructureArtifact r ident original)+        restored <- right (decodeStructureArtifact r bytes)+        before <- runWithCapture [answerResponse] (compiledProgram original) (Question "test")+        restoredCapture <- runWithCapture [answerResponse] (compiledProgram restored) (Question "test")+        restoredCapture @?= before,+      testCase "artifact incompatibilities are pure categorized failures" $ do+        r <- right (directCotRegistry "r" qaBase)+        let direct = NE.head (registryRecipes r)+            ident = recipeId direct+        bytes <- right (encodeStructureArtifact r ident (compile identity qaBase))+        artifact <- right (eitherDecode bytes)+        let check change expected = case decodeStructureArtifact r (encode (change artifact)) of+              Left err -> err @?= expected+              Right _ -> assertFailure "unexpected compatible artifact"+        check (\a -> a {formatVersion = 2}) (UnsupportedStructureVersion 2)+        check (\a -> a {artifactKind = "production"}) (WrongArtifactKind "production")+        check (\a -> a {artifactRegistryId = "other"}) (RegistryMismatch "r" "other")+        check (\a -> a {artifactRecipeId = "missing"}) (UnknownRecipe "missing")+        check (\a -> a {artifactRecipeRevision = 2}) (RevisionMismatch "direct" 1 2)+        check (\a -> a {inputSchema = Null}) (InputSchemaMismatch "direct")+        check (\a -> a {outputSchema = Null}) (OutputSchemaMismatch "direct")+        check (\a -> a {selectedShape = programShape qaPipeline}) (StructureShapeMismatch "direct")+        check (\a -> a {orderedParams = []}) (StructureParameterCountMismatch "direct" 1 0)+        check (\a -> a {orderedParams = orderedParams a ++ orderedParams a}) (StructureParameterCountMismatch "direct" 1 2)+        assertBool "malformed" (isLeft (decodeStructureArtifact r "{"))+        assertBool "compiled state is a different envelope" (isLeft (decodeStructureArtifact r (encodeCompiled (compile identity qaBase))))+        assertBool "structure envelope is not compiled state" (isLeft (decodeCompiledOnto qaBase bytes))+        assertBool "encode mismatch" (isLeft (encodeStructureArtifact r ident (compile identity qaPipeline)))+        changed <- right (structureRecipe "direct" 2 "" qaBase)+        changedRegistry <- right (structureRegistry "r" [changed])+        assertBool "registry revision changed" (isLeft (decodeStructureArtifact changedRegistry bytes))+    ]