claude-hs-0.1.0.0: test/StreamingSpec.hs
module StreamingSpec (run) where
import Claude.Client
import Claude.Messages
import Claude.Streaming
import Control.Concurrent (forkFinally, killThread, newEmptyMVar, putMVar, takeMVar, threadDelay)
import Control.Exception (bracket, try)
import Control.Lens ((&), (.~), (^.))
import Control.Monad (forM_, void)
import Data.Aeson (Key, Value (..), eitherDecode, eitherDecodeStrict, encode, object, toJSON, (.=))
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString (ByteString)
import Data.ByteString qualified as BS
import Data.ByteString.Builder (byteString)
import Data.ByteString.Char8 qualified as BSC
import Data.ByteString.Lazy qualified as LBS
import Data.Generics.Labels ()
import Data.IORef (modifyIORef', newIORef, readIORef)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)
import FeaturesSpec (thinkingBlock, webCitation)
import MessagesSpec (baseConversation, messageFixture, sampleRequest)
import Network.HTTP.Types (status200)
import Network.Wai qualified as Wai
import Numeric (showHex)
import Support.Server
run :: IO ()
run = do
gatedRelease
fragmentedEvents
streamToolContinuation
advancedEvidence
streamFailures
legacyBridge
putStrLn "Streaming: delivery, cancellation, framing, continuation and failure fixtures passed"
event :: Text -> [(Key, Value)] -> Value
event tag fields = object (("type" .= tag) : fields)
frame :: Value -> ByteString
frame payload = "data: " <> LBS.toStrict (encode payload) <> "\n\n"
messageStart :: Value
messageStart = event "message_start" ["message" .= messageFixture []]
textDelta :: Int -> Text -> Value
textDelta index value = event "content_block_delta" ["index" .= index, "delta" .= event "text_delta" ["text" .= value]]
blockStart :: Int -> Value -> Value
blockStart index value = event "content_block_start" ["index" .= index, "content_block" .= value]
blockDelta :: Int -> Value -> Value
blockDelta index value = event "content_block_delta" ["index" .= index, "delta" .= value]
blockStop :: Int -> Value
blockStop index = event "content_block_stop" ["index" .= index]
messageStop :: Value
messageStop = event "message_stop" []
sse :: ByteString -> Wai.Response
sse body = Wai.responseLBS status200 [("content-type", "text/event-stream"), ("request-id", "req_stream")] (LBS.fromStrict body)
gatedRelease :: IO ()
gatedRelease = forM_ (["stop", "exception", "cancel", "finish"] :: [String]) $ \mode -> do
entered <- newEmptyMVar
released <- newEmptyMVar
allowFinish <- newEmptyMVar
never <- newEmptyMVar
request <- sampleRequest baseConversation
let app req send = do
sent <- requestJson req
assertEqual "stream request" (addStreamFlag (toJSON request)) sent
assertEqual "stream accept" (Just "text/event-stream") (lookup "accept" (Wai.requestHeaders req))
send $
Wai.responseRaw
( \recv write -> do
write "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nRequest-Id: req_gated\r\n\r\n"
write (httpChunk (frame messageStart))
if mode == "finish" then takeMVar allowFinish >> write (httpChunk (frame messageStop)) else pure ()
recv >>= putMVar released
)
(sse "")
checkStart meta streamEvent = do
assertEqual "stream correlation" (Just "req_gated") (meta ^. #requestId)
case streamEvent ^. #eventPayload of
MessageStart _ -> putMVar entered ()
MessageStop -> pure ()
_ -> fail "Unexpected event"
withClient app $ \client -> do
let streaming = streamMessage client defaultRequestOptions request
case mode of
"stop" -> do
Response _ terminal <- streaming (\meta ev -> checkStart meta ev >> pure Stop) >>= expectRight
assertEqual "caller stopped" Stopped terminal
takeMVar entered
"exception" -> do
result <- try (streaming (\meta ev -> checkStart meta ev >> ioError (userError "callback failure")))
case result of
Left (_ :: IOError) -> takeMVar entered
Right _ -> fail "Callback exception did not propagate"
"cancel" -> do
finished <- newEmptyMVar
bracket
(forkFinally (void (streaming (\meta ev -> checkStart meta ev >> takeMVar never))) (putMVar finished))
killThread
( \tid -> do
takeMVar entered
killThread tid
takeMVar finished >>= \case
Left _ -> pure ()
Right () -> fail "Expected cancellation"
)
_ -> do
Response _ terminal <-
streaming
( \meta ev -> do
checkStart meta ev
case ev ^. #eventPayload of
MessageStart _ -> putMVar allowFinish ()
_ -> pure ()
pure Continue
)
>>= expectRight
assertEqual "provider finished" Finished terminal
takeMVar released >>= assertEqual ("socket released after " <> mode) BS.empty
where
httpChunk bytes = BSC.pack (showHex (BS.length bytes) "") <> "\r\n" <> bytes <> "\r\n"
addStreamFlag (Object fields) = Object (KeyMap.insert "stream" (Bool True) fields)
addStreamFlag raw = raw
fragmentedEvents :: IO ()
fragmentedEvents = do
request <- sampleRequest baseConversation
let futureDelta = event "future_delta" ["unknown" .= True]
transcript =
": comment\r\nevent: message_start\r\ndata: "
<> LBS.toStrict (encode messageStart)
<> "\r\n\r\nevent: ping\ndata: {\ndata: \"type\": \"ping\"}\n\n"
<> BS.concat
( map
frame
[ blockStart 0 (event "text" ["text" .= ("" :: Text)]),
textDelta 0 "café",
blockDelta 0 (event "citations_delta" ["citation" .= webCitation]),
blockStop 0,
blockStart 1 (thinkingBlock "" ""),
blockDelta 1 (event "thinking_delta" ["thinking" .= ("summary" :: Text)]),
blockDelta 1 (event "signature_delta" ["signature" .= ("opaque-signature" :: Text)]),
blockDelta 1 futureDelta,
blockStop 1,
event "future_event" ["v" .= True],
event
"message_delta"
[ "delta" .= object ["stop_reason" .= ("end_turn" :: Text), "stop_sequence" .= Null],
"usage" .= object ["input_tokens" .= (20 :: Int), "output_tokens" .= (9 :: Int), "cache_read_input_tokens" .= (5 :: Int)]
],
messageStop
]
)
check split = do
seen <- newIORef []
let app req send = do
_ <- Wai.strictRequestBody req
if split
then send $ Wai.responseStream status200 [] $ \write flush ->
BS.foldr (\byte next -> write (byteString (BS.singleton byte)) >> flush >> threadDelay 100 >> next) (pure ()) transcript
else send (sse transcript)
withClient app $ \client -> do
Response _ terminal <-
streamMessage
client
defaultRequestOptions
request
(\_ item -> modifyIORef' seen (item :) >> pure Continue)
>>= expectRight
assertEqual "finished fragmented stream" Finished terminal
reverse <$> readIORef seen
whole <- check False
split <- check True
assertEqual "single byte fragments" whole split
case map (^. #eventPayload) split of
[ MessageStart _,
Ping,
ContentBlockStart 0 _,
ContentBlockDelta 0 (TextDelta "café"),
ContentBlockDelta 0 (CitationDelta (Citation (WebLocation _ _ _ "encrypted-index") _)),
ContentBlockStop 0,
ContentBlockStart 1 _,
ContentBlockDelta 1 (ThinkingDelta "summary"),
ContentBlockDelta 1 (SignatureDelta "opaque-signature"),
ContentBlockDelta 1 (UnknownDelta "future_delta" _),
ContentBlockStop 1,
UnknownEvent "future_event",
MessageDelta (MessageUpdate (Just EndTurn) Nothing Nothing Nothing) (UsageUpdate (Just 20) (Just 9) Nothing (Just 5) Nothing Nothing (UsageEvidence Nothing Nothing Nothing Nothing Nothing Nothing)),
MessageStop
] -> pure ()
_ -> fail "unexpected event or delta sequence"
streamToolContinuation :: IO ()
streamToolContinuation = do
request <- sampleRequest baseConversation
let unsigned = thinkingBlock "" ""
signed = thinkingBlock "" "signature-stream"
toolId = "tool_stream" :: Text
tool input = event "tool_use" ["id" .= toolId, "name" .= ("lookup" :: Text), "input" .= input]
finalInput = object ["value" .= ("green" :: Text)]
transcript =
BS.concat
( map
frame
[ messageStart,
blockStart 0 unsigned,
blockDelta 0 (event "signature_delta" ["signature" .= ("signature-stream" :: Text)]),
blockStop 0,
blockStart 1 (tool (object [])),
blockDelta 1 (event "input_json_delta" ["partial_json" .= ("{\"value\":" :: Text)]),
blockDelta 1 (event "input_json_delta" ["partial_json" .= ("\"green\"}" :: Text)]),
blockStop 1,
messageStop
]
)
app req send = do
body <- requestJson req
if lookup "accept" (Wai.requestHeaders req) == Just "text/event-stream"
then send (sse transcript)
else do
let expectedMessages =
toJSON
[ toJSON (Message User (InputText "Hello" :| [])),
object ["role" .= ("assistant" :: Text), "content" .= [signed, tool finalInput]],
object ["role" .= ("user" :: Text), "content" .= [event "tool_result" ["tool_use_id" .= toolId, "content" .= ("done" :: Text)]]]
]
assertEqual
"HTTP replay of signed stream tool turn"
(Just expectedMessages)
( case body of
Object fields -> KeyMap.lookup "messages" fields
_ -> Nothing
)
send (json status200 [] (messageFixture []))
withClient app $ \client -> do
seen <- newIORef []
_ <-
streamMessage
client
defaultRequestOptions
request
(\_ ev -> modifyIORef' seen ((ev ^. #eventPayload) :) >> pure Continue)
>>= expectRight
events <- reverse <$> readIORef seen
case events of
[ MessageStart starting,
ContentBlockStart 0 first,
ContentBlockDelta 0 (SignatureDelta signature),
ContentBlockStop 0,
ContentBlockStart 1 second,
ContentBlockDelta 1 (InputJsonDelta prefix),
ContentBlockDelta 1 (InputJsonDelta suffix),
ContentBlockStop 1,
MessageStop
] -> do
input <- expectRight (eitherDecodeStrict (encodeUtf8 (prefix <> suffix)))
thinking <- parseBlock "signature" (String signature) first
called <- parseBlock "input" input second
let updated = starting & #content .~ [thinking, called]
reply <- maybe (fail "missing stream continuation") pure (assistantContinuation updated)
let toolResult = Message User (InputToolResult (ToolResult toolId (ToolResultText "done") Nothing Nothing) :| [])
continued = request {conversation = withConversationHistory (conversationHistory baseConversation <> [reply, toolResult]) baseConversation}
void (createMessage client defaultRequestOptions continued >>= expectRight)
_ -> fail "Unexpected continuation transcript"
where
parseBlock :: Key -> Value -> OutputBlock -> IO OutputBlock
parseBlock key item block = case blockJSON block of
Object fields -> expectRight (eitherDecode (encode (Object (KeyMap.insert key item fields))))
_ -> fail "Expected a block object"
streamFailures :: IO ()
streamFailures = do
request <- sampleRequest baseConversation
let malformed = event "content_block_delta" ["index" .= (0 :: Int), "delta" .= event "signature_delta" []]
apiError = event "error" ["error" .= object ["type" .= ("overloaded_error" :: Text), "message" .= ("Retry later loopback-secret" :: Text)]]
exercise payload verify = withClient (\_ send -> send (sse (frame messageStart <> payload))) $ \client ->
streamMessage client defaultRequestOptions request (\_ _ -> pure Continue) >>= verify
decodeFailure = \case
Left (DecodeError meta _) -> assertEqual "decode failure correlation" (Just "req_stream") (meta ^. #requestId)
_ -> fail "Expected stream decode failure"
exercise "" decodeFailure
exercise (frame malformed) decodeFailure
exercise (frame apiError) $ \case
Left (StreamApiError meta (ApiError "overloaded_error" detail)) -> do
assertEqual "stream error correlation" (Just "req_stream") (meta ^. #requestId)
assertEqual "stream error redaction" "Retry later [redacted]" detail
_ -> fail "Expected a typed in-stream API error"
withClient (\_ _ -> fail "Incompatible request dispatched") $ \client -> do
let legacy = case toJSON request of
Object fields -> Object (KeyMap.insert "temperature" (toJSON (0.5 :: Double)) fields)
raw -> raw
streamMessageValue client defaultRequestOptions legacy (\_ _ -> pure Continue) >>= \case
Left (InvalidRequest _) -> pure ()
_ -> fail "Expected local failure for removed sampling"
legacyBridge :: IO ()
legacyBridge = do
let config = (defaultClientConfig "loopback-secret") {apiVersion = Just "2023-06-01"}
request = object ["model" .= ("legacy-model" :: Text), "max_tokens" .= (10 :: Int), "messages" .= ([] :: [Value]), "temperature" .= (0.5 :: Double)]
app req send = do
assertEqual "configured legacy API version" (Just "2023-06-01") (lookup "anthropic-version" (Wai.requestHeaders req))
requestJson req >>= \case
Object fields -> do
assertEqual "legacy sampling value" (Just (toJSON (0.5 :: Double))) (KeyMap.lookup "temperature" fields)
assertEqual "legacy empty history" (Just (toJSON ([] :: [Value]))) (KeyMap.lookup "messages" fields)
_ -> fail "Expected legacy request object"
send (sse (frame messageStart <> frame messageStop))
withClientConfig config app $ \client -> do
Response _ terminal <- streamMessageValue client defaultRequestOptions request (\_ _ -> pure Continue) >>= expectRight
assertEqual "legacy stream completed" Finished terminal
advancedEvidence :: IO ()
advancedEvidence = do
request <- sampleRequest baseConversation
let refusal = event "refusal" ["category" .= ("other" :: Text), "explanation" .= ("declined" :: Text)]
initialUsage =
object
[ "input_tokens" .= (17 :: Int),
"output_tokens" .= (1 :: Int),
"speed" .= ("fast" :: Text),
"service_tier" .= ("priority" :: Text),
"inference_geo" .= ("us" :: Text),
"cache_creation_input_tokens" .= (4 :: Int),
"cache_read_input_tokens" .= (5 :: Int),
"server_tool_use" .= object ["web_search_requests" .= (3 :: Int), "tool_search_requests" .= (2 :: Int), "extension" .= True]
]
iteration = event "fallback_message" ["model" .= ("served-model" :: Text), "output_tokens" .= (19 :: Int), "extension" .= True]
patch = object ["output_tokens" .= (19 :: Int), "cache_read_input_tokens" .= (0 :: Int), "output_tokens_details" .= object ["thinking_tokens" .= (12 :: Int)], "iterations" .= [iteration], "server_tool_use" .= object ["tool_search_requests" .= (0 :: Int)]]
start = event "message_start" ["message" .= replace "usage" initialUsage (messageFixture [])]
delta reason details counters = event "message_delta" ["delta" .= object ["stop_reason" .= (reason :: Text), "stop_details" .= details], "usage" .= counters]
exhausted = delta "model_context_window_exceeded" Null patch
refused = delta "refusal" refusal (object ["unrecognised" .= True])
failedCode = event "code_execution_tool_result" ["tool_use_id" .= ("srvtoolu_1" :: Text), "content" .= event "code_execution_tool_result_error" ["error_code" .= ("invalid_tool_input" :: Text)]]
app _ send = send (sse (foldMap frame [start, blockStart 0 failedCode, blockStop 0, exhausted, refused, messageStop]))
withClient app $ \client -> do
seen <- newIORef []
_ <- streamMessage client defaultRequestOptions request (\_ entry -> modifyIORef' seen (entry :) >> pure Continue) >>= expectRight
reverse <$> readIORef seen >>= \case
[ StreamEvent (MessageStart first) _,
StreamEvent (ContentBlockStart 0 block) _,
StreamEvent (ContentBlockStop 0) _,
StreamEvent (MessageDelta context update) _,
StreamEvent (MessageDelta final missing) raw,
StreamEvent MessageStop _
] -> do
assertEqual "typed code error" (OutputCodeExecutionResult "srvtoolu_1" (CodeExecutionFailed "invalid_tool_input")) (blockContent block)
assertEqual "context length distinct from refusal" (Just ModelContextWindowExceeded) (context ^. #stopReason)
assertEqual "refusal category, explanation and reason" (Just Refusal, Just (RefusalDetails (Just "other") (Just "declined") Nothing)) (final ^. #stopReason, final ^. #stopDetails)
assertEqual "raw delta" refused raw
let totals = applyUsageUpdate (applyUsageUpdate (first ^. #usage) update) missing
evidence = totals ^. #usageEvidence
assertEqual "cumulative usage and explicit zero" (17, 19, Just 4, Just 0) (totals ^. #inputTokens, totals ^. #outputTokens, totals ^. #cacheCreationInputTokens, totals ^. #cacheReadInputTokens)
assertEqual "pricing observations" (Just Fast, Just "priority", Just "us", Just (OutputTokenDetails (Just 12))) (evidence ^. #observedSpeed, evidence ^. #serviceTier, evidence ^. #inferenceGeography, evidence ^. #outputTokensDetails)
assertEqual "incremental server tool counts" (Just (ServerToolCounts (Just 3) Nothing (Just 0) Nothing)) (evidence ^. #serverToolCounts)
assertEqual "retained tool usage extensions" (Just (Bool True)) (totals ^. #serverToolUse >>= KeyMap.lookup "extension")
assertEqual "final fallback iterations" (Just [("fallback_message", Just "served-model", Just (Bool True))]) ((map (\i -> (iterationType i, iterationModel i, KeyMap.lookup "extension" (iterationJSON i)))) <$> (evidence ^. #iterations))
_ -> fail "expected advanced evidence transcript"
where
replace name value (Object fields) = Object (KeyMap.insert name value fields)
replace _ _ other = other