packages feed

claude-hs-0.1.0.0: test/StreamingSpec.hs

module StreamingSpec (run) where

import Claude.Client
import Claude.Messages
import Claude.Streaming
import Control.Concurrent (forkFinally, killThread, newEmptyMVar, putMVar, takeMVar, threadDelay)
import Control.Exception (bracket, try)
import Control.Lens ((&), (.~), (^.))
import Control.Monad (forM_, void)
import Data.Aeson (Key, Value (..), eitherDecode, eitherDecodeStrict, encode, object, toJSON, (.=))
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString (ByteString)
import Data.ByteString qualified as BS
import Data.ByteString.Builder (byteString)
import Data.ByteString.Char8 qualified as BSC
import Data.ByteString.Lazy qualified as LBS
import Data.Generics.Labels ()
import Data.IORef (modifyIORef', newIORef, readIORef)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import FeaturesSpec (thinkingBlock, webCitation)
import MessagesSpec (baseConversation, messageFixture, sampleRequest)
import Network.HTTP.Types (status200)
import Network.Wai qualified as Wai
import Numeric (showHex)
import Support.Server

run :: IO ()
run = do
  gatedRelease
  fragmentedEvents
  streamToolContinuation
  advancedEvidence
  streamFailures
  legacyBridge
  putStrLn "Streaming: delivery, cancellation, framing, continuation and failure fixtures passed"

event :: Text -> [(Key, Value)] -> Value
event tag fields = object (("type" .= tag) : fields)

frame :: Value -> ByteString
frame payload = "data: " <> LBS.toStrict (encode payload) <> "\n\n"

messageStart :: Value
messageStart = event "message_start" ["message" .= messageFixture []]

textDelta :: Int -> Text -> Value
textDelta index value = event "content_block_delta" ["index" .= index, "delta" .= event "text_delta" ["text" .= value]]

blockStart :: Int -> Value -> Value
blockStart index value = event "content_block_start" ["index" .= index, "content_block" .= value]

blockDelta :: Int -> Value -> Value
blockDelta index value = event "content_block_delta" ["index" .= index, "delta" .= value]

blockStop :: Int -> Value
blockStop index = event "content_block_stop" ["index" .= index]

messageStop :: Value
messageStop = event "message_stop" []

sse :: ByteString -> Wai.Response
sse body = Wai.responseLBS status200 [("content-type", "text/event-stream"), ("request-id", "req_stream")] (LBS.fromStrict body)

gatedRelease :: IO ()
gatedRelease = forM_ (["stop", "exception", "cancel", "finish"] :: [String]) $ \mode -> do
  entered <- newEmptyMVar
  released <- newEmptyMVar
  allowFinish <- newEmptyMVar
  never <- newEmptyMVar
  request <- sampleRequest baseConversation
  let app req send = do
        sent <- requestJson req
        assertEqual "stream request" (addStreamFlag (toJSON request)) sent
        assertEqual "stream accept" (Just "text/event-stream") (lookup "accept" (Wai.requestHeaders req))
        send $
          Wai.responseRaw
            ( \recv write -> do
                write "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nRequest-Id: req_gated\r\n\r\n"
                write (httpChunk (frame messageStart))
                if mode == "finish" then takeMVar allowFinish >> write (httpChunk (frame messageStop)) else pure ()
                recv >>= putMVar released
            )
            (sse "")
      checkStart meta streamEvent = do
        assertEqual "stream correlation" (Just "req_gated") (meta ^. #requestId)
        case streamEvent ^. #eventPayload of
          MessageStart _ -> putMVar entered ()
          MessageStop -> pure ()
          _ -> fail "Unexpected event"
  withClient app $ \client -> do
    let streaming = streamMessage client defaultRequestOptions request
    case mode of
      "stop" -> do
        Response _ terminal <- streaming (\meta ev -> checkStart meta ev >> pure Stop) >>= expectRight
        assertEqual "caller stopped" Stopped terminal
        takeMVar entered
      "exception" -> do
        result <- try (streaming (\meta ev -> checkStart meta ev >> ioError (userError "callback failure")))
        case result of
          Left (_ :: IOError) -> takeMVar entered
          Right _ -> fail "Callback exception did not propagate"
      "cancel" -> do
        finished <- newEmptyMVar
        bracket
          (forkFinally (void (streaming (\meta ev -> checkStart meta ev >> takeMVar never))) (putMVar finished))
          killThread
          ( \tid -> do
              takeMVar entered
              killThread tid
              takeMVar finished >>= \case
                Left _ -> pure ()
                Right () -> fail "Expected cancellation"
          )
      _ -> do
        Response _ terminal <-
          streaming
            ( \meta ev -> do
                checkStart meta ev
                case ev ^. #eventPayload of
                  MessageStart _ -> putMVar allowFinish ()
                  _ -> pure ()
                pure Continue
            )
            >>= expectRight
        assertEqual "provider finished" Finished terminal
    takeMVar released >>= assertEqual ("socket released after " <> mode) BS.empty
  where
    httpChunk bytes = BSC.pack (showHex (BS.length bytes) "") <> "\r\n" <> bytes <> "\r\n"
    addStreamFlag (Object fields) = Object (KeyMap.insert "stream" (Bool True) fields)
    addStreamFlag raw = raw

fragmentedEvents :: IO ()
fragmentedEvents = do
  request <- sampleRequest baseConversation
  let futureDelta = event "future_delta" ["unknown" .= True]
      transcript =
        ": comment\r\nevent: message_start\r\ndata: "
          <> LBS.toStrict (encode messageStart)
          <> "\r\n\r\nevent: ping\ndata: {\ndata: \"type\": \"ping\"}\n\n"
          <> BS.concat
            ( map
                frame
                [ blockStart 0 (event "text" ["text" .= ("" :: Text)]),
                  textDelta 0 "café",
                  blockDelta 0 (event "citations_delta" ["citation" .= webCitation]),
                  blockStop 0,
                  blockStart 1 (thinkingBlock "" ""),
                  blockDelta 1 (event "thinking_delta" ["thinking" .= ("summary" :: Text)]),
                  blockDelta 1 (event "signature_delta" ["signature" .= ("opaque-signature" :: Text)]),
                  blockDelta 1 futureDelta,
                  blockStop 1,
                  event "future_event" ["v" .= True],
                  event
                    "message_delta"
                    [ "delta" .= object ["stop_reason" .= ("end_turn" :: Text), "stop_sequence" .= Null],
                      "usage" .= object ["input_tokens" .= (20 :: Int), "output_tokens" .= (9 :: Int), "cache_read_input_tokens" .= (5 :: Int)]
                    ],
                  messageStop
                ]
            )
      check split = do
        seen <- newIORef []
        let app req send = do
              _ <- Wai.strictRequestBody req
              if split
                then send $ Wai.responseStream status200 [] $ \write flush ->
                  BS.foldr (\byte next -> write (byteString (BS.singleton byte)) >> flush >> threadDelay 100 >> next) (pure ()) transcript
                else send (sse transcript)
        withClient app $ \client -> do
          Response _ terminal <-
            streamMessage
              client
              defaultRequestOptions
              request
              (\_ item -> modifyIORef' seen (item :) >> pure Continue)
              >>= expectRight
          assertEqual "finished fragmented stream" Finished terminal
          reverse <$> readIORef seen
  whole <- check False
  split <- check True
  assertEqual "single byte fragments" whole split
  case map (^. #eventPayload) split of
    [ MessageStart _,
      Ping,
      ContentBlockStart 0 _,
      ContentBlockDelta 0 (TextDelta "café"),
      ContentBlockDelta 0 (CitationDelta (Citation (WebLocation _ _ _ "encrypted-index") _)),
      ContentBlockStop 0,
      ContentBlockStart 1 _,
      ContentBlockDelta 1 (ThinkingDelta "summary"),
      ContentBlockDelta 1 (SignatureDelta "opaque-signature"),
      ContentBlockDelta 1 (UnknownDelta "future_delta" _),
      ContentBlockStop 1,
      UnknownEvent "future_event",
      MessageDelta (MessageUpdate (Just EndTurn) Nothing Nothing Nothing) (UsageUpdate (Just 20) (Just 9) Nothing (Just 5) Nothing Nothing (UsageEvidence Nothing Nothing Nothing Nothing Nothing Nothing)),
      MessageStop
      ] -> pure ()
    _ -> fail "unexpected event or delta sequence"

streamToolContinuation :: IO ()
streamToolContinuation = do
  request <- sampleRequest baseConversation
  let unsigned = thinkingBlock "" ""
      signed = thinkingBlock "" "signature-stream"
      toolId = "tool_stream" :: Text
      tool input = event "tool_use" ["id" .= toolId, "name" .= ("lookup" :: Text), "input" .= input]
      finalInput = object ["value" .= ("green" :: Text)]
      transcript =
        BS.concat
          ( map
              frame
              [ messageStart,
                blockStart 0 unsigned,
                blockDelta 0 (event "signature_delta" ["signature" .= ("signature-stream" :: Text)]),
                blockStop 0,
                blockStart 1 (tool (object [])),
                blockDelta 1 (event "input_json_delta" ["partial_json" .= ("{\"value\":" :: Text)]),
                blockDelta 1 (event "input_json_delta" ["partial_json" .= ("\"green\"}" :: Text)]),
                blockStop 1,
                messageStop
              ]
          )
      app req send = do
        body <- requestJson req
        if lookup "accept" (Wai.requestHeaders req) == Just "text/event-stream"
          then send (sse transcript)
          else do
            let expectedMessages =
                  toJSON
                    [ toJSON (Message User (InputText "Hello" :| [])),
                      object ["role" .= ("assistant" :: Text), "content" .= [signed, tool finalInput]],
                      object ["role" .= ("user" :: Text), "content" .= [event "tool_result" ["tool_use_id" .= toolId, "content" .= ("done" :: Text)]]]
                    ]
            assertEqual
              "HTTP replay of signed stream tool turn"
              (Just expectedMessages)
              ( case body of
                  Object fields -> KeyMap.lookup "messages" fields
                  _ -> Nothing
              )
            send (json status200 [] (messageFixture []))
  withClient app $ \client -> do
    seen <- newIORef []
    _ <-
      streamMessage
        client
        defaultRequestOptions
        request
        (\_ ev -> modifyIORef' seen ((ev ^. #eventPayload) :) >> pure Continue)
        >>= expectRight
    events <- reverse <$> readIORef seen
    case events of
      [ MessageStart starting,
        ContentBlockStart 0 first,
        ContentBlockDelta 0 (SignatureDelta signature),
        ContentBlockStop 0,
        ContentBlockStart 1 second,
        ContentBlockDelta 1 (InputJsonDelta prefix),
        ContentBlockDelta 1 (InputJsonDelta suffix),
        ContentBlockStop 1,
        MessageStop
        ] -> do
          input <- expectRight (eitherDecodeStrict (encodeUtf8 (prefix <> suffix)))
          thinking <- parseBlock "signature" (String signature) first
          called <- parseBlock "input" input second
          let updated = starting & #content .~ [thinking, called]
          reply <- maybe (fail "missing stream continuation") pure (assistantContinuation updated)
          let toolResult = Message User (InputToolResult (ToolResult toolId (ToolResultText "done") Nothing Nothing) :| [])
              continued = request {conversation = withConversationHistory (conversationHistory baseConversation <> [reply, toolResult]) baseConversation}
          void (createMessage client defaultRequestOptions continued >>= expectRight)
      _ -> fail "Unexpected continuation transcript"
  where
    parseBlock :: Key -> Value -> OutputBlock -> IO OutputBlock
    parseBlock key item block = case blockJSON block of
      Object fields -> expectRight (eitherDecode (encode (Object (KeyMap.insert key item fields))))
      _ -> fail "Expected a block object"

streamFailures :: IO ()
streamFailures = do
  request <- sampleRequest baseConversation
  let malformed = event "content_block_delta" ["index" .= (0 :: Int), "delta" .= event "signature_delta" []]
      apiError = event "error" ["error" .= object ["type" .= ("overloaded_error" :: Text), "message" .= ("Retry later loopback-secret" :: Text)]]
      exercise payload verify = withClient (\_ send -> send (sse (frame messageStart <> payload))) $ \client ->
        streamMessage client defaultRequestOptions request (\_ _ -> pure Continue) >>= verify
      decodeFailure = \case
        Left (DecodeError meta _) -> assertEqual "decode failure correlation" (Just "req_stream") (meta ^. #requestId)
        _ -> fail "Expected stream decode failure"
  exercise "" decodeFailure
  exercise (frame malformed) decodeFailure
  exercise (frame apiError) $ \case
    Left (StreamApiError meta (ApiError "overloaded_error" detail)) -> do
      assertEqual "stream error correlation" (Just "req_stream") (meta ^. #requestId)
      assertEqual "stream error redaction" "Retry later [redacted]" detail
    _ -> fail "Expected a typed in-stream API error"
  withClient (\_ _ -> fail "Incompatible request dispatched") $ \client -> do
    let legacy = case toJSON request of
          Object fields -> Object (KeyMap.insert "temperature" (toJSON (0.5 :: Double)) fields)
          raw -> raw
    streamMessageValue client defaultRequestOptions legacy (\_ _ -> pure Continue) >>= \case
      Left (InvalidRequest _) -> pure ()
      _ -> fail "Expected local failure for removed sampling"

legacyBridge :: IO ()
legacyBridge = do
  let config = (defaultClientConfig "loopback-secret") {apiVersion = Just "2023-06-01"}
      request = object ["model" .= ("legacy-model" :: Text), "max_tokens" .= (10 :: Int), "messages" .= ([] :: [Value]), "temperature" .= (0.5 :: Double)]
      app req send = do
        assertEqual "configured legacy API version" (Just "2023-06-01") (lookup "anthropic-version" (Wai.requestHeaders req))
        requestJson req >>= \case
          Object fields -> do
            assertEqual "legacy sampling value" (Just (toJSON (0.5 :: Double))) (KeyMap.lookup "temperature" fields)
            assertEqual "legacy empty history" (Just (toJSON ([] :: [Value]))) (KeyMap.lookup "messages" fields)
          _ -> fail "Expected legacy request object"
        send (sse (frame messageStart <> frame messageStop))
  withClientConfig config app $ \client -> do
    Response _ terminal <- streamMessageValue client defaultRequestOptions request (\_ _ -> pure Continue) >>= expectRight
    assertEqual "legacy stream completed" Finished terminal

advancedEvidence :: IO ()
advancedEvidence = do
  request <- sampleRequest baseConversation
  let refusal = event "refusal" ["category" .= ("other" :: Text), "explanation" .= ("declined" :: Text)]
      initialUsage =
        object
          [ "input_tokens" .= (17 :: Int),
            "output_tokens" .= (1 :: Int),
            "speed" .= ("fast" :: Text),
            "service_tier" .= ("priority" :: Text),
            "inference_geo" .= ("us" :: Text),
            "cache_creation_input_tokens" .= (4 :: Int),
            "cache_read_input_tokens" .= (5 :: Int),
            "server_tool_use" .= object ["web_search_requests" .= (3 :: Int), "tool_search_requests" .= (2 :: Int), "extension" .= True]
          ]
      iteration = event "fallback_message" ["model" .= ("served-model" :: Text), "output_tokens" .= (19 :: Int), "extension" .= True]
      patch = object ["output_tokens" .= (19 :: Int), "cache_read_input_tokens" .= (0 :: Int), "output_tokens_details" .= object ["thinking_tokens" .= (12 :: Int)], "iterations" .= [iteration], "server_tool_use" .= object ["tool_search_requests" .= (0 :: Int)]]
      start = event "message_start" ["message" .= replace "usage" initialUsage (messageFixture [])]
      delta reason details counters = event "message_delta" ["delta" .= object ["stop_reason" .= (reason :: Text), "stop_details" .= details], "usage" .= counters]
      exhausted = delta "model_context_window_exceeded" Null patch
      refused = delta "refusal" refusal (object ["unrecognised" .= True])
      failedCode = event "code_execution_tool_result" ["tool_use_id" .= ("srvtoolu_1" :: Text), "content" .= event "code_execution_tool_result_error" ["error_code" .= ("invalid_tool_input" :: Text)]]
      app _ send = send (sse (foldMap frame [start, blockStart 0 failedCode, blockStop 0, exhausted, refused, messageStop]))
  withClient app $ \client -> do
    seen <- newIORef []
    _ <- streamMessage client defaultRequestOptions request (\_ entry -> modifyIORef' seen (entry :) >> pure Continue) >>= expectRight
    reverse <$> readIORef seen >>= \case
      [ StreamEvent (MessageStart first) _,
        StreamEvent (ContentBlockStart 0 block) _,
        StreamEvent (ContentBlockStop 0) _,
        StreamEvent (MessageDelta context update) _,
        StreamEvent (MessageDelta final missing) raw,
        StreamEvent MessageStop _
        ] -> do
          assertEqual "typed code error" (OutputCodeExecutionResult "srvtoolu_1" (CodeExecutionFailed "invalid_tool_input")) (blockContent block)
          assertEqual "context length distinct from refusal" (Just ModelContextWindowExceeded) (context ^. #stopReason)
          assertEqual "refusal category, explanation and reason" (Just Refusal, Just (RefusalDetails (Just "other") (Just "declined") Nothing)) (final ^. #stopReason, final ^. #stopDetails)
          assertEqual "raw delta" refused raw
          let totals = applyUsageUpdate (applyUsageUpdate (first ^. #usage) update) missing
              evidence = totals ^. #usageEvidence
          assertEqual "cumulative usage and explicit zero" (17, 19, Just 4, Just 0) (totals ^. #inputTokens, totals ^. #outputTokens, totals ^. #cacheCreationInputTokens, totals ^. #cacheReadInputTokens)
          assertEqual "pricing observations" (Just Fast, Just "priority", Just "us", Just (OutputTokenDetails (Just 12))) (evidence ^. #observedSpeed, evidence ^. #serviceTier, evidence ^. #inferenceGeography, evidence ^. #outputTokensDetails)
          assertEqual "incremental server tool counts" (Just (ServerToolCounts (Just 3) Nothing (Just 0) Nothing)) (evidence ^. #serverToolCounts)
          assertEqual "retained tool usage extensions" (Just (Bool True)) (totals ^. #serverToolUse >>= KeyMap.lookup "extension")
          assertEqual "final fallback iterations" (Just [("fallback_message", Just "served-model", Just (Bool True))]) ((map (\i -> (iterationType i, iterationModel i, KeyMap.lookup "extension" (iterationJSON i)))) <$> (evidence ^. #iterations))
      _ -> fail "expected advanced evidence transcript"
  where
    replace name value (Object fields) = Object (KeyMap.insert name value fields)
    replace _ _ other = other