diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +## 0.4.2.0 — 2026-11-02 + +### Changed + +- Upgrade `mori://shinzui/baikai/packages/baikai` to `>=0.10.0.0 && <0.11`. + +- Update internal dependency bounds for the breaking `shikumi 0.5.0.0` release. +- Omit provider structured data from agent summaries while retaining it in history. + ## 0.4.1.1 — 2026-10-16 ### Changed diff --git a/shikumi-tools.cabal b/shikumi-tools.cabal --- a/shikumi-tools.cabal +++ b/shikumi-tools.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: shikumi-tools -version: 0.4.1.1 +version: 0.4.2.0 synopsis: Typed tools and ReAct agents for shikumi LM programs (EP-11) @@ -68,7 +68,7 @@ build-depends: aeson >=2.2 && <2.3, - baikai >=0.8.0.0 && <0.9, + baikai >=0.10.0.0 && <0.11, base >=4.20 && <5, bytestring >=0.11 && <0.13, containers >=0.6 && <0.9, @@ -82,7 +82,7 @@ lens ^>=5.3, process >=1.6 && <1.7, regex-tdfa >=1.3 && <1.4, - shikumi ^>=0.4.1.1, + shikumi ^>=0.5.0.0, text ^>=2.1, vector >=0.13 && <0.14, @@ -116,13 +116,14 @@ SchemaSpec SessionSpec ShellSpec + SystemOneIntegrationSpec ToolOutputSpec ToolSpec WebSpec build-depends: aeson, - baikai >=0.8.0.0 && <0.9, + baikai >=0.10.0.0 && <0.11, base, bytestring, containers, @@ -131,10 +132,10 @@ filepath, generic-lens, lens, - shikumi ^>=0.4.1.1, - shikumi-cache ^>=0.2.0.2, + shikumi ^>=0.5.0.0, + shikumi-cache ^>=0.2.1.0, shikumi-testing ^>=0.1.0.0, - shikumi-tools ^>=0.4.1.1, + shikumi-tools ^>=0.4.2.0, tasty, tasty-hunit, text, diff --git a/src/Shikumi/Agent/History.hs b/src/Shikumi/Agent/History.hs --- a/src/Shikumi/Agent/History.hs +++ b/src/Shikumi/Agent/History.hs @@ -204,13 +204,14 @@ -- | Structural projection: provider-owned reasoning and unknown rich extensions -- never enter a summarizer request. The full audit retains all original bytes. renderSummaryEntry :: HistoryEntry -> Text -renderSummaryEntry (UserTurn p) = T.intercalate "\n" [case b of B.UserText (B.TextContent t) -> t; B.UserImage _ -> "[image omitted]" | b <- V.toList (p ^. #content)] +renderSummaryEntry (UserTurn p) = T.intercalate "\n" [case b of B.UserText (B.TextContent t) -> t; B.UserImage _ -> "[image omitted]"; B.UserData _ -> "[structured data omitted]" | b <- V.toList (p ^. #content)] renderSummaryEntry (Exchange p outputs correction) = T.intercalate "\n" (map block (V.toList (p ^. #content)) <> map output outputs <> maybe [] (\t -> ["Correction: " <> t]) correction) where block (B.AssistantText (B.TextContent t)) = t block (B.AssistantThinking t) | t ^. #redacted = "[redacted reasoning omitted]" | otherwise = t ^. #thinking <> "\n[reasoning continuation omitted]" + block (B.AssistantData _) = "[structured data omitted]" block (B.AssistantToolCall c) = c ^. #name <> " " <> jsonText (c ^. #arguments) output (c, o) = c ^. #name diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -19,6 +19,7 @@ import SchemaSpec qualified import SessionSpec qualified import ShellSpec qualified +import SystemOneIntegrationSpec qualified import Test.Tasty (defaultMain, testGroup) import ToolOutputSpec qualified import ToolSpec qualified @@ -29,7 +30,8 @@ defaultMain $ testGroup "shikumi-tools" - [ ResponsesSchemaSpec.tests, + [ SystemOneIntegrationSpec.tests, + ResponsesSchemaSpec.tests, ResponsesIntegrationSpec.tests, SessionSpec.tests, RLMSpec.tests, diff --git a/test/SystemOneIntegrationSpec.hs b/test/SystemOneIntegrationSpec.hs new file mode 100644 --- /dev/null +++ b/test/SystemOneIntegrationSpec.hs @@ -0,0 +1,82 @@ +module SystemOneIntegrationSpec (tests) where + +import Baikai qualified as B +import Data.Aeson +import Data.Aeson.KeyMap qualified as KM +import Data.Map.Strict qualified as Map +import Data.Text qualified as T +import Effectful (runEff) +import Effectful.Dispatch.Dynamic (interpose) +import Effectful.Error.Static (runErrorNoCallStack) +import Shikumi.Adapter (stampTemperature) +import Shikumi.Decision +import Shikumi.Error +import Shikumi.LLM qualified as L +import Shikumi.LLM.Defaults +import Shikumi.Program +import Shikumi.Routing +import Shikumi.Schema (field, unField) +import Shikumi.Testing.DecisionFixtures +import Shikumi.Testing.Responses (fixtureOptions, withTransportOptions) +import Shikumi.Testing.SystemOne +import Test.Tasty +import Test.Tasty.HUnit + +reply :: Value +reply = + object + [ "model" .= String "jev-latest", + "usage" .= object ["input_tokens" .= (10 :: Int), "output_tokens" .= (3 :: Int)], + "answers" + .= object + [ "urgent" .= object ["type" .= String "noul", "noul" .= (0.83 :: Double)], + "severity" .= object ["type" .= String "score", "probabilities" .= object ["0" .= (0.1 :: Double), "1" .= (0.3 :: Double), "2" .= (0.6 :: Double)], "score" .= (1.5 :: Double), "confidence" .= (0.55 :: Double), "legend" .= object ["0" .= String "Minor", "1" .= String "Disruptive", "2" .= String "Blocking"]], + "category" .= object ["type" .= String "choice", "probabilities" .= object ["billing" .= (0.2 :: Double), "technical" .= (0.8 :: Double)], "choice" .= String "technical", "confidence" .= (0.65 :: Double)] + ] + ] + +tests :: TestTree +tests = + testGroup + "SystemOne" + [ testCase "released provider sends state and criteria; defaults exempt" $ + withSystemOneFixture reply $ \model registry requests -> do + let defaults = emptyRequestDefaults {defaultThinking = Just B.ThinkingHigh, defaultMaxTokens = Just 100} + result <- + runEff . runErrorNoCallStack @ShikumiError . runRouting model . L.runLLMWith registry . withTransportOptions fixtureOptions . withRequestDefaults defaults . routeLLM $ + runProgram (Predict assessmentSig emptyParams) (Ticket (field "Checkout is unavailable.")) + case result of + Right (Assessment u s c) -> do + case unField u of Noul value p _ -> do value @?= True; p @?= 0.83 + case unField s of Score value level _ confidence -> do value @?= 1.5; level @?= 2; confidence @?= Just 0.55 + case unField c of Choice value _ confidence -> do value @?= Technical; confidence @?= Just 0.65 + Left err -> assertFailure (show err) + bodies <- requests + bodies @?= [object ["model" .= String "jev-latest", "state" .= renderDecisionState assessmentSig (Ticket (field "Checkout is unavailable.")), "questions" .= Map.fromList [(judgmentField j, questionJSON j) | j <- judgmentsOf @Assessment]]], + testCase "explicit temperature is terminal with zero HTTP requests" $ + withSystemOneFixture reply $ \model registry requests -> do + result <- + runEff . runErrorNoCallStack @ShikumiError . runRouting model . L.runLLMWith registry . withTransportOptions fixtureOptions . routeLLM . interpose (\_ -> \case L.Complete m c o -> L.complete m c (stampTemperature 0.7 o); L.Stream m c o -> L.stream m c o) $ + runProgram (Predict assessmentSig emptyParams) (Ticket (field "x")) + assertBool "terminal refusal" (case result of Left (SchemaMismatch msg) -> "temperature" `T.isInfixOf` msg; _ -> False) + requests >>= (@?= []), + testCase "missing wire evidence or undeclared answers fail after one HTTP request" $ do + let modifyAnswer name f = modifyObject "answers" (adjust (\case Object o -> Object (f o); v -> v) name) reply + malformed = [modifyAnswer "category" (KM.delete "confidence"), modifyAnswer "severity" (KM.delete "score"), modifyAnswer "severity" (KM.delete "legend"), modifyObject "answers" (KM.insert "unexpected" (object ["type" .= String "noul", "noul" .= (0.5 :: Double)])) reply] + mapM_ + ( \bad -> withSystemOneFixture bad $ \model registry requests -> do + result <- + runEff . runErrorNoCallStack @ShikumiError . runRouting model . L.runLLMWith registry . withTransportOptions fixtureOptions . routeLLM $ + runProgram (Predict assessmentSig emptyParams) (Ticket (field "x")) + assertBool "terminal InvalidJSON" (case result of Left (InvalidJSON _) -> True; _ -> False) + requests >>= (\xs -> length xs @?= 1) + ) + malformed + ] + +adjust :: (a -> a) -> Key -> KM.KeyMap a -> KM.KeyMap a +adjust f k o = maybe o (\v -> KM.insert k (f v) o) (KM.lookup k o) + +modifyObject :: Key -> (KM.KeyMap Value -> KM.KeyMap Value) -> Value -> Value +modifyObject key f (Object fields) = Object (adjust (\case Object o -> Object (f o); v -> v) key fields) +modifyObject _ _ v = v