packages feed

baikai-openai-0.7.3.1: test/StructuredCliSpec.hs

-- | Structured-output passthrough for the @codex exec --json@
-- subprocess provider.
--
-- The process cases run a real child process: a few lines of @sh@
-- written into a temporary directory that record the argument vector,
-- copy the @--output-schema@ file before the provider can delete it,
-- print a canned event stream and stderr, and exit with a chosen code.
module StructuredCliSpec (tests) where

import Baikai
import Baikai.Provider.OpenAI.Cli qualified as CodexCli
import Control.Lens ((&), (.~), (^.))
import Data.Aeson (Value (..), (.=))
import Data.Aeson qualified as Aeson
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as BS
import Data.ByteString.Lazy qualified as LBS
import Data.Generics.Labels ()
import Data.List (isPrefixOf)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as Text
import Data.Text.IO qualified as TextIO
import Data.Vector qualified as Vector
import Streamly.Data.Stream qualified as Stream
import System.Directory (doesFileExist, getPermissions, setOwnerExecutable, setPermissions)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))

tests :: TestTree
tests =
  testGroup
    "StructuredCliSpec: codex exec --output-schema passthrough"
    [ renderingTests,
      processTests,
      finalAnswerTests,
      finalAnswerIntegrationTests,
      testCase "the provider declares NativeJsonSchema without spawning anything" $
        CodexCli.codexCliProvider CodexCli.defaultCodexCliConfig ^. #structuredOutput
          @?= NativeJsonSchema
    ]

-- ============================================================
-- Pure argument rendering
-- ============================================================

renderingTests :: TestTree
renderingTests =
  testGroup
    "argument rendering"
    [ testCase "JsonObject renders the same vector as no response format" $
        CodexCli.codexCliCommand cfg testModel testContext (emptyOptions & #responseFormat .~ Just JsonObject)
          @?= CodexCli.codexCliCommand cfg testModel testContext emptyOptions,
      testCase "codexCliCommand renders no --output-schema even for a JsonSchema request" $
        assertBool
          "no schema flag"
          ("--output-schema" `notElem` snd (CodexCli.codexCliCommand cfg testModel testContext schemaOptions)),
      testCase "codexCliCommandWith Nothing is codexCliCommand" $
        CodexCli.codexCliCommandWith cfg Nothing testModel testContext schemaOptions
          @?= CodexCli.codexCliCommand cfg testModel testContext schemaOptions,
      testCase "codexCliCommandWith places --output-schema <file> before the extra args" $ do
        let (_, args) = CodexCli.codexCliCommandWith cfg (Just "/tmp/s.json") testModel testContext schemaOptions
        case break (== "--output-schema") args of
          (_, "--output-schema" : "/tmp/s.json" : rest) ->
            assertBool ("extra args follow the schema: " <> show rest) (["--color", "never"] `isPrefixOf` rest)
          _ -> assertFailure ("no --output-schema in " <> show args)
    ]
  where
    cfg = CodexCli.defaultCodexCliConfig {CodexCli.extraArgs = ["--color", "never"]}

-- ============================================================
-- A real child process
-- ============================================================

processTests :: TestTree
processTests =
  testGroup
    "fake codex process"
    [ testCase "two complete schema-conforming objects return only the final object" $
        withFakeCodex $ \run -> do
          (resp, captured) <- run briefingOptions (Fake (map agentMessage [briefingText "Intermediate", briefingText "Final"]) "" 0)
          responseError resp @?= Nothing
          let body = flattenAssistantText (resp ^. #message . #content)
          body @?= briefingText "Final"
          Aeson.eitherDecodeStrict' (Text.encodeUtf8 body) @?= Right (briefing "Final")
          resp ^. #evidence @?= Nothing
          case captured of
            Nothing -> assertFailure "the tool received no --output-schema"
            Just (path, bytes) -> do
              Aeson.decodeStrict bytes @?= Just briefingSchema
              doesFileExist (Text.unpack path) >>= (@?= False),
      testCase "the schema file carries the schema, the reply is returned unchanged, the file is removed" $
        withFakeCodex $ \run -> do
          (resp, captured) <- run schemaOptions (Fake conformingEvents "" 0)
          responseError resp @?= Nothing
          flattenAssistantText (resp ^. #message . #content) @?= conformingText
          case captured of
            Nothing -> assertFailure "the tool received no --output-schema"
            Just (path, bytes) -> do
              Aeson.decodeStrict bytes @?= Just itemsSchema
              doesFileExist (Text.unpack path) >>= (@?= False),
      testCase "the schema file is removed after a failed run too" $
        withFakeCodex $ \run -> do
          (resp, captured) <- run schemaOptions (Fake [] "the tool refused\n" 3)
          fmap (^. #category) (responseError resp) @?= Just ProcessFailure
          case captured of
            Nothing -> assertFailure "the tool received no --output-schema"
            Just (path, _) -> doesFileExist (Text.unpack path) >>= (@?= False),
      testCase "no response format sends no --output-schema" $
        withFakeCodex $ \run -> do
          (resp, captured) <- run emptyOptions (Fake conformingEvents "" 0)
          responseError resp @?= Nothing
          fmap fst captured @?= Nothing,
      testCase "codex rejecting --output-schema is InvalidRequest with the exit code" $
        withFakeCodex $ \run -> do
          (resp, _) <- run schemaOptions (Fake [] "error: unexpected argument '--output-schema' found\n" 2)
          case responseError resp of
            Nothing -> assertFailure "expected an error-shaped response"
            Just e -> do
              e ^. #category @?= InvalidRequest
              e ^. #exitCode @?= Just 2,
      testCase "the same stderr without a schema request stays a ProcessFailure" $
        withFakeCodex $ \run -> do
          (resp, _) <- run emptyOptions (Fake [] "error: unexpected argument '--output-schema' found\n" 2)
          fmap (^. #category) (responseError resp) @?= Just ProcessFailure,
      -- The spawned vector names a fresh random path each call; the
      -- evidence envelope names the schema instead, so the commitment
      -- is reproducible and still changes with the schema.
      testCase "the request commitment is reproducible and commits to the schema" $
        withFakeCodex $ \run -> do
          (first, _) <- run (schemaOptions & #evidence .~ Just (evidenceRequest "run-87")) (Fake conformingEvents "" 0)
          (second, _) <- run (schemaOptions & #evidence .~ Just (evidenceRequest "run-87")) (Fake conformingEvents "" 0)
          (plain, _) <- run (emptyOptions & #evidence .~ Just (evidenceRequest "run-87")) (Fake conformingEvents "" 0)
          let c1 = encodedCommitment first
          assertBool ("a commitment was recorded: " <> show c1) (c1 /= Nothing)
          c1 @?= encodedCommitment second
          assertBool "the schema is part of the commitment" (c1 /= encodedCommitment plain)
    ]

finalAnswerTests :: TestTree
finalAnswerTests =
  testGroup
    "final answer selection"
    [ testCase name $
        withFakeCodex $ \run -> do
          (resp, _) <- run opts (Fake (map agentMessage bodies) "" 0)
          responseError resp @?= Nothing
          flattenAssistantText (resp ^. #message . #content) @?= expected
    | (name, opts, bodies, expected) <-
        [ ("single briefing is unchanged", briefingOptions, [briefingText "Final"], briefingText "Final"),
          ("three objects select the last", briefingOptions, map briefingText ["First", "Second", "Final"], briefingText "Final"),
          ("prose commentary precedes a JSON answer", briefingOptions, ["Working on it.", briefingText "Final"], briefingText "Final"),
          ("plain-text calls select the final message and trim outer whitespace", emptyOptions, ["commentary", "  final answer\n "], "final answer"),
          ("an empty final answer replaces commentary", briefingOptions, [briefingText "Earlier", ""], ""),
          ("missing schema fields do not recover an earlier object", briefingOptions, [briefingText "Earlier", "{}"], "{}")
        ]
    ]

finalAnswerIntegrationTests :: TestTree
finalAnswerIntegrationTests =
  testGroup
    "final answer integration"
    [ testCase "malformed final JSON is returned faithfully and still fails decoding" $
        withFakeCodex $ \run -> do
          (resp, _) <- run briefingOptions (Fake (map agentMessage [briefingText "Earlier", " \n{\"why\":  "]) "" 0)
          responseError resp @?= Nothing
          let body = flattenAssistantText (resp ^. #message . #content)
          body @?= "{\"why\":"
          assertBool "inner JSON remains malformed" (case Aeson.eitherDecodeStrict' @Value (Text.encodeUtf8 body) of Left _ -> True; Right _ -> False),
      testCase "complete and synthetic stream expose one identical final text block" $
        withFakeCodexAndStream $ \run -> do
          let fake = Fake (briefingEvents "commentary" (briefingText "Final")) "" 0
              opts = briefingOptions & #evidence .~ Just (evidenceRequest "stream-103")
          (direct, _, _) <- run False opts fake
          (collected, events, _) <- run True opts fake
          responseError direct @?= Nothing
          responseError collected @?= Nothing
          let directBody = flattenAssistantText (direct ^. #message . #content)
          flattenAssistantText (collected ^. #message . #content) @?= directBody
          Aeson.eitherDecodeStrict' (Text.encodeUtf8 directBody) @?= Right (briefing "Final")
          Vector.toList (direct ^. #message . #content) @?= [AssistantText (TextContent (briefingText "Final"))]
          [body | TextEnd BlockEndPayload {content = body} <- events] @?= [briefingText "Final"]
          length [() | EventDone _ <- events] @?= 1
          length [() | EventError _ <- events] @?= 0
          case reverse events of
            EventDone TerminalPayload {reason = reason} : _ -> reason @?= Stop
            _ -> assertFailure ("expected final EventDone: " <> show events)
          direct ^. #responseId @?= Just "briefing-thread"
          collected ^. #responseId @?= direct ^. #responseId
          direct ^. #message . #usage . #inputTokens @?= 8
          direct ^. #message . #usage . #cacheReadTokens @?= 2
          direct ^. #message . #usage . #outputTokens @?= 3
          collected ^. #message . #usage @?= direct ^. #message . #usage
          evidenceField "response_commitment" collected @?= evidenceField "response_commitment" direct
          evidenceField "usage" collected @?= evidenceField "usage" direct,
      testCase "response evidence commits to the final answer independently of commentary" $
        withFakeCodex $ \run -> do
          let opts = briefingOptions & #evidence .~ Just (evidenceRequest "run-103")
              fake intermediate final = Fake (briefingEvents intermediate final) "" 0
          (first, _) <- run opts (fake "first commentary" (briefingText "Final"))
          (second, _) <- run opts (fake "different commentary" (briefingText "Final"))
          (changed, _) <- run opts (fake "first commentary" (briefingText "Changed"))
          let commitment = evidenceField "response_commitment" first
          assertBool "a successful response is observed" (case commitment of Just (Object o) -> KeyMap.member "observed" o; _ -> False)
          commitment @?= evidenceField "response_commitment" second
          assertBool "changing the final answer changes the commitment" (commitment /= evidenceField "response_commitment" changed)
          evidenceField "response_id" first @?= Just (Aeson.object ["observed" .= ("briefing-thread" :: Text)])
          evidenceField "observed_model" first @?= Just (String "unobserved"),
      testCase "a valid-looking answer cannot turn a nonzero exit into success" $
        withFakeCodex $ \run -> do
          (resp, captured) <- run briefingOptions (Fake (briefingEvents "commentary" (briefingText "Final")) "failed after answering\n" 3)
          fmap (^. #category) (responseError resp) @?= Just ProcessFailure
          fmap (^. #exitCode) (responseError resp) @?= Just (Just 3)
          resp ^. #responseId @?= Just "briefing-thread"
          case captured of
            Just (path, _) -> doesFileExist (Text.unpack path) >>= (@?= False)
            Nothing -> assertFailure "the tool received no schema"
    ]

-- | What the fake prints and how it exits.
data Fake = Fake
  { eventLines :: [Text],
    stderrBody :: Text,
    code :: Int
  }

-- | Write one fake @codex@ and hand back a function that runs a call
-- against it and returns the response with the schema path and bytes
-- the tool received.
--
-- Calls whose commitments are compared must share one fake, because
-- the argument vector the commitment digests begins with the
-- executable's path.
withFakeCodex :: ((Options -> Fake -> IO (Response, Maybe (Text, BS.ByteString))) -> IO a) -> IO a
withFakeCodex k =
  withFakeCodexAndStream $ \run -> k $ \opts fake -> do
    (resp, _, captured) <- run False opts fake
    pure (resp, captured)

withFakeCodexAndStream :: ((Bool -> Options -> Fake -> IO (Response, [AssistantMessageEvent], Maybe (Text, BS.ByteString))) -> IO a) -> IO a
withFakeCodexAndStream k =
  withSystemTempDirectory "baikai-codex-structured" $ \dir -> do
    let outPath = dir </> "stdout"
        errPath = dir </> "stderr"
        codePath = dir </> "code"
        pathPath = dir </> "schema-path"
        copyPath = dir </> "schema-copy"
    exe <-
      writeFakeExecutable dir "codex" $
        unlines
          [ "#!/bin/sh",
            "if [ \"$1\" = \"--version\" ]; then echo 'codex-cli 9.9.9'; exit 0; fi",
            "rm -f '" <> pathPath <> "' '" <> copyPath <> "'",
            "prev=",
            "for a in \"$@\"; do",
            "  if [ \"$prev\" = \"--output-schema\" ]; then",
            "    printf '%s' \"$a\" > '" <> pathPath <> "'",
            "    cp \"$a\" '" <> copyPath <> "'",
            "  fi",
            "  prev=\"$a\"",
            "done",
            "cat '" <> outPath <> "'",
            "cat '" <> errPath <> "' >&2",
            "exit $(cat '" <> codePath <> "')"
          ]
    reg <- newProviderRegistry
    registerApiProviderWith
      reg
      (CodexCli.codexCliProvider CodexCli.defaultCodexCliConfig {CodexCli.executable = exe})
    k $ \streaming opts fake -> do
      TextIO.writeFile outPath (Text.unlines (eventLines fake))
      TextIO.writeFile errPath (stderrBody fake)
      writeFile codePath (show (code fake))
      (resp, events) <-
        if streaming
          then do
            events <- Stream.toList (streamRequestWith reg testModel testContext opts)
            resp <- Stream.fold (reassembleResponse testModel) (Stream.fromList events)
            pure (resp, events)
          else do
            resp <- completeRequestWith reg testModel testContext opts
            pure (resp, [])
      sent <- doesFileExist pathPath
      captured <-
        if sent
          then do
            path <- TextIO.readFile pathPath
            bytes <- BS.readFile copyPath
            pure (Just (path, bytes))
          else pure Nothing
      pure (resp, events, captured)

writeFakeExecutable :: FilePath -> String -> String -> IO FilePath
writeFakeExecutable dir name body = do
  let path = dir </> name
  writeFile path body
  perms <- getPermissions path
  setPermissions path (setOwnerExecutable True perms)
  pure path

-- | The encoded @request_commitment@ of a response's evidence record.
encodedCommitment :: Response -> Maybe Value
encodedCommitment resp = case Aeson.toJSON <$> (resp ^. #evidence) of
  Just (Object o) -> KeyMap.lookup "request_commitment" o
  _ -> Nothing

evidenceField :: Text -> Response -> Maybe Value
evidenceField key resp = case Aeson.toJSON <$> (resp ^. #evidence) of
  Just (Object o) -> KeyMap.lookup (Key.fromText key) o
  _ -> Nothing

-- ============================================================
-- Fixtures
-- ============================================================

testModel :: Model
testModel =
  emptyModel
    & #modelId .~ "gpt-5.6"
    & #api .~ OpenAICompletionsCli
    & #provider .~ "openai"

testContext :: Context
testContext = emptyContext & #messages .~ Vector.singleton (user "List two fruits.")

schemaOptions :: Options
schemaOptions =
  emptyOptions & #responseFormat .~ Just (JsonSchema (jsonSchemaFormat "fruits" itemsSchema))

-- | An object holding an array of objects, one field an enum, in the
-- strict shape codex requires.
itemsSchema :: Value
itemsSchema =
  Aeson.object
    [ "type" .= ("object" :: Text),
      "properties"
        .= Aeson.object
          [ "items"
              .= Aeson.object
                [ "type" .= ("array" :: Text),
                  "items"
                    .= Aeson.object
                      [ "type" .= ("object" :: Text),
                        "properties"
                          .= Aeson.object
                            [ "label" .= Aeson.object ["type" .= ("string" :: Text)],
                              "level"
                                .= Aeson.object
                                  [ "type" .= ("string" :: Text),
                                    "enum" .= (["low", "high"] :: [Text])
                                  ]
                            ],
                        "required" .= (["label", "level"] :: [Text]),
                        "additionalProperties" .= False
                      ]
                ]
          ],
      "required" .= (["items"] :: [Text]),
      "additionalProperties" .= False
    ]

conformingText :: Text
conformingText =
  "{\"items\":[{\"label\":\"Strawberry\",\"level\":\"low\"},{\"label\":\"Mango\",\"level\":\"high\"}]}"

briefing :: Text -> Value
briefing why = Aeson.object ["why" .= why, "what" .= ("X" :: Text), "blocking" .= ("B" :: Text), "status" .= ("S" :: Text)]

briefingText :: Text -> Text
briefingText why = "{\"why\":\"" <> why <> "\",\"what\":\"X\",\"blocking\":\"B\",\"status\":\"S\"}"

briefingSchema :: Value
briefingSchema =
  Aeson.object
    [ "type" .= ("object" :: Text),
      "properties" .= Aeson.object ["why" .= string, "what" .= string, "blocking" .= string, "status" .= string],
      "required" .= (["why", "what", "blocking", "status"] :: [Text]),
      "additionalProperties" .= False
    ]
  where
    string = Aeson.object ["type" .= ("string" :: Text)]

briefingOptions :: Options
briefingOptions = emptyOptions & #responseFormat .~ Just (JsonSchema (jsonSchemaFormat "briefing" briefingSchema))

agentMessage :: Text -> Text
agentMessage body = Text.decodeUtf8 . LBS.toStrict . Aeson.encode $ Aeson.object ["type" .= ("item.completed" :: Text), "item" .= Aeson.object ["type" .= ("agent_message" :: Text), "text" .= body]]

briefingEvents :: Text -> Text -> [Text]
briefingEvents intermediate final =
  [ "{\"type\":\"thread.started\",\"thread_id\":\"briefing-thread\"}",
    agentMessage intermediate,
    "{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}",
    "{\"type\":\"thread.started\",\"thread_id\":\"later-thread\"}",
    agentMessage final,
    "{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"cached_input_tokens\":2,\"output_tokens\":3}}"
  ]

-- | The event stream @codex-cli 0.159.0@ emits for an @--output-schema@
-- run: the structured reply is the text of the ordinary final
-- @agent_message@.
conformingEvents :: [Text]
conformingEvents =
  [ "{\"type\":\"thread.started\",\"thread_id\":\"019fd471-4a48-7c83-be67-6b7c49646e87\"}",
    "{\"type\":\"turn.started\"}",
    Text.decodeUtf8 . LBS.toStrict . Aeson.encode $
      Aeson.object
        [ "type" .= ("item.completed" :: Text),
          "item"
            .= Aeson.object
              [ "id" .= ("item_0" :: Text),
                "type" .= ("agent_message" :: Text),
                "text" .= conformingText
              ]
        ],
    "{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"cached_input_tokens\":0,\"output_tokens\":20}}"
  ]