packages feed

launchdarkly-server-sdk 4.6.1 → 4.6.2

raw patch · 7 files changed

+513/−45 lines, 7 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

CHANGELOG.md view
@@ -2,6 +2,18 @@  All notable changes to the LaunchDarkly Haskell Server-side SDK will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org). +## [4.6.2](https://github.com/launchdarkly/haskell-server-sdk/compare/4.6.1...4.6.2) (2026-11-16)+++### Bug Fixes++* Log an error when the stream stops because of a permanent error ([8ed7e35](https://github.com/launchdarkly/haskell-server-sdk/commit/8ed7e35735110edab2631903545dc5b30fc25168))+* Log the correct cause when the stream times out or the server closes it ([#130](https://github.com/launchdarkly/haskell-server-sdk/issues/130)) ([3cfeeef](https://github.com/launchdarkly/haskell-server-sdk/commit/3cfeeefea51a1966a3606d9ccfb5607e11fadd6d))+* Log the error and retry delay when the streaming connection reconnects ([#129](https://github.com/launchdarkly/haskell-server-sdk/issues/129)) ([8ed7e35](https://github.com/launchdarkly/haskell-server-sdk/commit/8ed7e35735110edab2631903545dc5b30fc25168))+* Make TestData valueForAll return its value when targeting is off ([#126](https://github.com/launchdarkly/haskell-server-sdk/issues/126)) ([6f1f61b](https://github.com/launchdarkly/haskell-server-sdk/commit/6f1f61b4d24a42ea325160feae9d7b7921437124))+* Reconnect the stream after non-HTTP read errors ([c92c118](https://github.com/launchdarkly/haskell-server-sdk/commit/c92c1187ed181d1530da6a83decd10d577d899b9))+* Reset the stream backoff only after 60 s of healthy connection ([c92c118](https://github.com/launchdarkly/haskell-server-sdk/commit/c92c1187ed181d1530da6a83decd10d577d899b9))+ ## [4.6.1](https://github.com/launchdarkly/haskell-server-sdk/compare/4.6.0...4.6.1) (2026-11-05)  
launchdarkly-server-sdk.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           launchdarkly-server-sdk-version:        4.6.1+version:        4.6.2 synopsis:       Server-side SDK for integrating with LaunchDarkly description:    Please see the README on GitHub at <https://github.com/launchdarkly/haskell-server-sdk#readme> category:       Web
src/LaunchDarkly/Server/Client/Internal.hs view
@@ -31,7 +31,7 @@  -- | The version string for this library. clientVersion :: Text-clientVersion = "4.6.1" -- x-release-please-version+clientVersion = "4.6.2" -- x-release-please-version  -- | -- Client is the LaunchDarkly client. Client instances are thread-safe.
src/LaunchDarkly/Server/Integrations/TestData/FlagBuilder.hs view
@@ -203,6 +203,7 @@     fb         & variations [Aeson.toJSON val]         & variationForAll (0 :: VariationIndex)+        & offVariation (0 :: VariationIndex)  {-# DEPRECATED valueForAllUsers "Use valueForAll instead" #-} 
src/LaunchDarkly/Server/Network/Streaming.hs view
@@ -1,13 +1,13 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE NumericUnderscores #-} -module LaunchDarkly.Server.Network.Streaming (streamingThread) where+module LaunchDarkly.Server.Network.Streaming (streamingThread, backoffCeiling, retryAttempt, readStream, streamReadTimeout) where  import Control.Applicative (many, (<|>)) import Control.Concurrent (threadDelay)-import Control.Exception (throwIO)+import Control.Exception (SomeAsyncException, SomeException, displayException, fromException, throwIO) import Control.Monad (mzero, void)-import Control.Monad.Catch (Exception, MonadCatch, MonadMask, try)+import Control.Monad.Catch (Exception, MonadCatch, MonadMask, MonadThrow, catchIf, throwM, try) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Logger (MonadLogger, logDebug, logError, logWarn) import Control.Monad.Loops (iterateUntilM)@@ -16,9 +16,11 @@ import Data.ByteString (ByteString) import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as L+import Data.IORef (IORef, newIORef, readIORef, writeIORef)+import Data.Maybe (isJust) import Data.Text (Text) import qualified Data.Text as T-import Data.Text.Encoding (decodeUtf8, encodeUtf8)+import Data.Text.Encoding (Decoding (Some), encodeUtf8, streamDecodeUtf8) import GHC.Generics (Generic) import GHC.Natural (Natural) import Network.HTTP.Client (HttpException (..), HttpExceptionContent (..), Manager, Request, Response (..), brRead)@@ -26,6 +28,7 @@ import System.Clock (Clock (Monotonic), TimeSpec (TimeSpec), getTime) import System.Random (Random (randomR), newStdGen) import System.Timeout (timeout)+import Text.Printf (printf)  import LaunchDarkly.AesonCompat (KeyMap, emptyObject) import LaunchDarkly.Server.Config.ClientContext (ClientContext (..))@@ -187,28 +190,42 @@ tryReadE :: MonadCatch m => m a -> m (Either ReadE a) tryReadE = try --- heartbeat expected every 120 seconds-readWithException :: IO ByteString -> IO Text-readWithException body =-    timeout (1_000_000 * 300) (brRead body) >>= \case+-- The time in microseconds to wait for data on the stream before it reconnects.+-- The server sends a heartbeat every 120 seconds.+streamReadTimeout :: Int+streamReadTimeout = 1_000_000 * 300++-- A read can end partway through a multi-byte character, so the decoder keeps those bytes for the next read.+-- `parseWith` treats empty text as end of input, so a read that decodes to nothing reads again.+readWithException :: Int -> IORef (ByteString -> Decoding) -> IO ByteString -> IO Text+readWithException readTimeout decoder body =+    timeout readTimeout (brRead body) >>= \case         Nothing -> throwIO ReadETimeout-        Just bytes -> if bytes == B.empty then throwIO ReadEClosed else pure (decodeUtf8 bytes)+        Just bytes ->+            if bytes == B.empty+                then throwIO ReadEClosed+                else do+                    decode <- readIORef decoder+                    let Some text _ next = decode bytes+                    writeIORef decoder next+                    if T.null text then readWithException readTimeout decoder body else pure text -readStream :: (MonadIO m, MonadLogger m, MonadMask m) => IO ByteString -> DataSourceUpdates -> m Bool-readStream body dataSourceUpdates = loop "" False+-- `readTimeout` is the time in microseconds to wait for each read. Tests use a shorter time than `streamReadTimeout`.+readStream :: (MonadIO m, MonadLogger m, MonadMask m) => Int -> IO ByteString -> DataSourceUpdates -> m Bool+readStream readTimeout body dataSourceUpdates = liftIO (newIORef streamDecodeUtf8) >>= \decoder -> loop decoder "" False   where-    loop initial processedEvent =-        tryReadE (parseWith (liftIO $ readWithException body) parseEvent initial) >>= \case+    loop decoder initial processedEvent =+        tryReadE (parseWith (liftIO $ readWithException readTimeout decoder body) parseEvent initial) >>= \case             (Left ReadETimeout) -> do-                $(logError) "streaming connection unexpectedly closed"+                $(logError) "timeout waiting for SSE event"                 pure processedEvent             (Left ReadEClosed) -> do-                $(logError) "timeout waiting for SSE event"+                $(logError) "streaming connection unexpectedly closed"                 pure processedEvent             (Right parsed) -> case parsed of                 Done remaining event -> do                     processed <- processEvent dataSourceUpdates (name event) (L.fromStrict $ encodeUtf8 $ buffer event)-                    if processed then loop remaining True else pure processedEvent+                    if processed then loop decoder remaining True else pure processedEvent                 Fail _ context err -> do                     $(logError) $ T.intercalate " " ["failed parsing SSE frame", T.pack $ show context, T.pack err]                     pure processedEvent@@ -216,32 +233,69 @@                     $(logError) "failed parsing SSE frame unexpected partial"                     pure processedEvent +-- `tryHTTP` only catches `HttpException`. Any other exception from reading the body or processing its events would end+-- the streaming thread with no log and no reconnect, such as a socket reset, a TLS error, invalid UTF-8, or a failed+-- store write. Async exceptions stay uncaught so `close` can stop the thread.+isRetryableReadError :: SomeException -> Bool+isRetryableReadError e = not (isJust (fromException @SomeAsyncException e) || isJust (fromException @HttpException e))++-- Rethrows the error as an `HttpException` so `tryHTTP` catches it and the stream reconnects with backoff.+-- `handleResponse` logs the error.+retryReadError :: MonadThrow m => Request -> SomeException -> m a+retryReadError request e = throwM $ HttpExceptionRequest request (InternalException e)++-- A short description of the error for the log. It leaves out the request, which `displayException` includes.+-- It also leaves out the responses of a redirect loop, because they can contain cookies and other server data.+-- It leaves out an invalid URI, because the URI can contain a password.+describeHttpException :: HttpException -> String+describeHttpException (HttpExceptionRequest _ (StatusCodeException response _)) = "HTTP error " ++ show (statusCode (responseStatus response))+describeHttpException (HttpExceptionRequest _ (TooManyRedirects _)) = "too many redirects"+describeHttpException (HttpExceptionRequest _ (InternalException e)) = displayException e+describeHttpException (HttpExceptionRequest _ content) = show content+describeHttpException (InvalidUrlException _ reason) = "invalid stream URI: " ++ reason++-- The reason to log when a connection ends. A connection that ends without an `HttpException` has already logged+-- the cause in `readStream`.+describeResult :: Either HttpException Bool -> String+describeResult (Left e) = describeHttpException e+describeResult (Right _) = "the stream ended"++logPermanentError :: MonadLogger m => String -> m ()+logPermanentError reason =+    $(logError) $ T.pack $ "A permanent error has been encountered with the LaunchDarkly streaming connection. It will not retry. Reason: " ++ reason+ -- This function is responsible for consuming a streaming connection. -- -- It is responsible for opening the connection, parsing the body for as long as it is able, and then handling shut down -- cleanly. While events are being processed, it is responsible for implementing a sane retry policy that prevents the -- stampeding herd problem in the unlikely event of upstream failures. startNewConnection :: (MonadIO m, MonadLogger m, MonadMask m) => Manager -> Request -> DataSourceUpdates -> StreamingState -> m StreamingState-startNewConnection manager request dataSourceUpdates state@(StreamingState {activeSince}) = do+startNewConnection manager request dataSourceUpdates state = do     $(logDebug) "starting new streaming connection"+    connectedAt <- liftIO $ newIORef Nothing     status <- tryHTTP $ withResponseGeneric request manager $ \response -> do         checkAuthorization response-            >> throwIfNot200 request response-            >> readStream (responseBody response) dataSourceUpdates+            >> catchIf isRetryableReadError (throwIfNot200 request response) (retryReadError request)+            >> liftIO (getTime Monotonic >>= writeIORef connectedAt . Just)+            >> catchIf isRetryableReadError (readStream streamReadTimeout (responseBody response) dataSourceUpdates) (retryReadError request)+    activeSince <- liftIO $ readIORef connectedAt     now <- liftIO $ getTime Monotonic-    handleResponse state now status+    handleResponse state now activeSince status   where     -- This function is responsible for parsing the final output from a now closed streaming connection.     --     -- Given the result of the stream run, this function can choose to either mark the state as cancelled, stopping all     -- further attempts, or it can update the state and wait some amount of time before starting another attempt.-    handleResponse :: (MonadIO m, MonadLogger m, MonadMask m) => StreamingState -> TimeSpec -> (Either HttpException Bool) -> m StreamingState-    handleResponse state now result =-        let state' = updateState state now result+    handleResponse :: (MonadIO m, MonadLogger m, MonadMask m) => StreamingState -> TimeSpec -> Maybe TimeSpec -> (Either HttpException Bool) -> m StreamingState+    handleResponse state now activeSince result =+        let state' = updateState state now activeSince result          in if cancel state'-                then pure state'+                then do+                    logPermanentError (describeResult result)+                    pure state'                 else do                     delay <- calculateDelay state'+                    $(logWarn) $ T.pack $ printf "LaunchDarkly streaming connection was interrupted and will retry in %.3f s. Reason: %s" (fromIntegral delay / 1_000_000 :: Double) (describeResult result)                     _ <- liftIO $ threadDelay delay                     pure state' @@ -249,19 +303,11 @@     --     -- Given the result of a stream run, this function can either choose to mark the state as cancelled, meaning halt all     -- further attempts, or it can be updated according to the rules of our streaming specification.-    updateState :: StreamingState -> TimeSpec -> (Either HttpException Bool) -> StreamingState-    updateState state now (Right _) = state {initialConnection = False, activeSince = Just now, attempt = 1}-    updateState state@(StreamingState {attempt = att}) now (Left (HttpExceptionRequest _ (StatusCodeException response _)))-        | isHttpUnrecoverable code = state {cancel = True}-        | otherwise = do-            case activeSince of-                (Just time)-                    | (now >= time + (TimeSpec 60 0)) -> state {attempt = 1, activeSince = Nothing}-                    | otherwise -> state {attempt = att + 1}-                Nothing -> state {attempt = att + 1}-      where-        code = statusCode (responseStatus response)-    updateState state@(StreamingState {attempt = att}) _ _ = state {attempt = att + 1}+    updateState :: StreamingState -> TimeSpec -> Maybe TimeSpec -> (Either HttpException Bool) -> StreamingState+    updateState state@(StreamingState {attempt = att}) now activeSince (Right _) = state {initialConnection = False, attempt = retryAttempt att activeSince now}+    updateState state _ _ (Left (HttpExceptionRequest _ (StatusCodeException response _)))+        | isHttpUnrecoverable (statusCode (responseStatus response)) = state {cancel = True}+    updateState state@(StreamingState {attempt = att}) now activeSince (Left _) = state {attempt = retryAttempt att activeSince now}      -- Calculate the next delay period following a backoff + jitter + max delay algorithm.     --@@ -270,14 +316,33 @@     calculateDelay StreamingState {initialRetryDelay, attempt = att} = do         liftIO $             newStdGen >>= \gen ->-                let timespan = min (30 * 1_000_000) ((initialRetryDelay * 1_000) * (2 ^ (att - 1)))+                let timespan = backoffCeiling initialRetryDelay att                     jitter = fst $ randomR (0, timespan `div` 2) gen                  in pure $ (timespan - jitter) +-- The retry delay in microseconds before jitter, given the initial delay in milliseconds and the attempt number.+--+-- The product uses `Integer` because in `Int` it overflows on 32-bit runtimes at the default 1 s initial delay, and+-- on any runtime for a large enough initial delay. Any initial delay of 1 ms or more reaches the 30 s cap by 2^15,+-- so the exponent cap keeps the `Integer` small without changing the result.+backoffCeiling :: Int -> Int -> Int+backoffCeiling initialRetryDelay att =+    fromInteger $ min (30 * 1_000_000) (toInteger initialRetryDelay * 1_000 * 2 ^ min 15 (att - 1))++-- The attempt number for the retry after a connection ends.+--+-- Every end of a connection counts as a failure, also a clean close by the server.+-- `activeSince` is the time the connection got a 200 response, or `Nothing` if it did not get one.+-- A connection that stayed open for 60 s or more resets the backoff to the first attempt.+-- Otherwise, the attempt number increases by one.+retryAttempt :: Int -> Maybe TimeSpec -> TimeSpec -> Int+retryAttempt att activeSince now = case activeSince of+    Just time | now >= time + TimeSpec 60 0 -> 1+    _ -> att + 1+ data StreamingState = StreamingState     { initialConnection :: Bool -- Marker used to determine the first time the streamer connects.     , initialRetryDelay :: Int -- The base duration used for the retry delay calculation-    , activeSince :: Maybe TimeSpec -- TimeSpec to denote the last time the SDK successfully connected to the stream     , attempt :: Int -- A number representing the attempt # of this connection     , cancel :: Bool -- A marker to shortcut logic and halt the streaming process     }@@ -287,9 +352,10 @@ streamingThread :: (MonadIO m, MonadLogger m, MonadMask m) => Text -> Int -> ClientContext -> DataSourceUpdates -> m () streamingThread streamURI initialRetryDelay clientContext dataSourceUpdates = do     let manager = tlsManager $ httpConfiguration clientContext-    req <- prepareRequest (httpConfiguration clientContext) (T.unpack streamURI ++ "/all")-    handleUnauthorized dataSourceUpdates (processStream manager req)+    tryHTTP (prepareRequest (httpConfiguration clientContext) (T.unpack streamURI ++ "/all")) >>= \case+        Left e -> logPermanentError (describeHttpException e)+        Right req -> handleUnauthorized dataSourceUpdates (processStream manager req)   where     processStream :: (MonadIO m, MonadLogger m, MonadMask m) => Manager -> Request -> m ()     processStream manager req = do-        void $ iterateUntilM (cancel) (\state -> startNewConnection manager req dataSourceUpdates state) StreamingState {initialConnection = True, initialRetryDelay, activeSince = Nothing, attempt = 0, cancel = False}+        void $ iterateUntilM (cancel) (\state -> startNewConnection manager req dataSourceUpdates state) StreamingState {initialConnection = True, initialRetryDelay, attempt = 0, cancel = False}
test/Spec/Integrations/TestData.hs view
@@ -24,6 +24,7 @@         , testTargeting         , testRules         , testValueForAll+        , testValueForAllTargetingOff         ]  testConfig :: DataSourceFactory -> Config@@ -225,5 +226,22 @@      assertEqual "ben receives CustomType1" "CustomType1" =<< stringVariation client "flag-key-1" ben "CustomType2"     assertEqual "todd receives CustomType1" "CustomType1" =<< stringVariation client "flag-key-1" todd "CustomType2"++    close client++testValueForAllTargetingOff :: Test+testValueForAllTargetingOff = TestCase $ do+    td <- TestData.newTestData+    TestData.update td+        =<< ( TestData.flag td "flag-key-1"+                <&> TestData.valueForAll ("value" :: Text)+                <&> TestData.on False+            )+    let config = testConfig (TestData.dataSourceFactory td)+        user = makeContext "user" "user"++    client <- makeClient config++    assertEqual "user receives value with targeting off" "value" =<< stringVariation client "flag-key-1" user "default"      close client
test/Spec/Streaming.hs view
@@ -1,6 +1,377 @@+{-# LANGUAGE NamedFieldPuns #-}+{-# LANGUAGE NumericUnderscores #-}+ module Spec.Streaming (allTests) where +import Control.Concurrent (forkFinally, killThread, threadDelay)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar, tryPutMVar, tryReadMVar)+import Control.Exception (AsyncException (ThreadKilled), SomeException, fromException, throwIO, toException)+import Control.Monad (forever, void)+import Control.Monad.Logger (LogLevel (..), fromLogStr, runLoggingT)+import Data.Aeson (encode, object, (.=))+import Data.ByteString (ByteString)+import qualified Data.ByteString as B+import qualified Data.ByteString.Char8 as B8+import qualified Data.ByteString.Lazy as L+import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef)+import Data.Maybe (isJust)+import Data.Text (Text)+import qualified Data.Text as T+import Data.Text.Encoding (decodeUtf8With)+import Data.Text.Encoding.Error (lenientDecode)+import Network.HTTP.Client (Manager, defaultManagerSettings, managerRawConnection, newManager, responseTimeoutNone)+import Network.HTTP.Client.Internal (makeConnection)+import Numeric (showHex)+import System.Clock (Clock (Monotonic), TimeSpec (TimeSpec), getTime, toNanoSecs)+import System.Timeout (timeout) import Test.HUnit +import LaunchDarkly.AesonCompat (KeyMap, lookupKey)+import LaunchDarkly.Server.Config.ClientContext (ClientContext (..))+import LaunchDarkly.Server.Config.HttpConfiguration (HttpConfiguration (..))+import LaunchDarkly.Server.DataSource.Internal (DataSourceUpdates (..))+import LaunchDarkly.Server.Features (Flag)+import LaunchDarkly.Server.Network.Streaming (backoffCeiling, readStream, retryAttempt, streamingThread)+import Util.Features (makeTestFlag)+ allTests :: Test-allTests = TestList []+allTests =+    TestList+        [ TestLabel "healthy stream initializes and killThread stops it" testKillThreadStopsStream+        , TestLabel "reconnects after an HttpException mid-body" testReconnectAfterHttpException+        , TestLabel "reconnects after a socket reset mid-body" testReconnectAfterSocketReset+        , TestLabel "logs a closed connection and reconnects" testLogsClosedConnection+        , TestLabel "logs a read timeout" testLogsReadTimeout+        , TestLabel "logs a retried HTTP error status" testLogsRetriedStatus+        , TestLabel "logs a connection failure before the response" testLogsConnectionFailure+        , TestLabel "logs a failure reading an error response body" testLogsErrorBodyFailure+        , TestLabel "logs the delay the stream waits before it retries" testLogsActualDelay+        , TestLabel "logs a retry after the server closes the stream" testLogsCleanClose+        , TestLabel "logs a redirect loop without the responses" testLogsRedirectLoop+        , TestLabel "logs a permanent HTTP error and stops" testLogsPermanentStatus+        , TestLabel "does not retry an unauthorized SDK key" testUnauthorizedDoesNotRetry+        , TestLabel "logs an invalid stream URI and stops" testLogsInvalidUri+        , TestLabel "decodes UTF-8 split across reads" testUtf8SplitAcrossReads+        , TestLabel "backoff stays at the cap after many attempts" testBackoffCeiling+        , TestLabel "a connection open for 60 s resets the backoff" testRetryAttempt+        ]++-- | One socket read. 'Fail' throws from inside the read, the way a reset socket does.+data ReadStep = Send ByteString | Fail SomeException++-- | The reads one TCP connection returns, in order. Once they run out, reads block.+type ConnectionScript = [ReadStep]++data StreamRun = StreamRun+    { connections :: Int+    -- ^ How many scripted connections the SDK opened+    , initialized :: Maybe (KeyMap Flag)+    -- ^ The flags passed to 'dataSourceUpdatesInit', if it was called+    , initializedAt :: Maybe TimeSpec+    -- ^ The monotonic time when 'dataSourceUpdatesInit' was called+    , logs :: [(LogLevel, Text)]+    , logTimes :: [TimeSpec]+    -- ^ The monotonic time of each entry in 'logs'+    , exited :: Maybe (Either SomeException ())+    -- ^ How the streaming thread ended. 'Nothing' when it was still running 1 s after 'killThread'.+    }++-- | Serves each new connection from the next script. Once the scripts run out, opening a connection blocks.+scriptedManager :: IORef [ConnectionScript] -> IO Manager+scriptedManager pending =+    newManager defaultManagerSettings {managerRawConnection = pure $ \_ _ _ -> openNext}+  where+    openNext =+        pop pending >>= \case+            Nothing -> blockForever+            Just script -> do+                steps <- newIORef script+                makeConnection (readStep steps) (const $ pure ()) (pure ())+    readStep steps =+        pop steps >>= \case+            Nothing -> blockForever+            Just (Send bytes) -> pure bytes+            Just (Fail e) -> throwIO e++pop :: IORef [a] -> IO (Maybe a)+pop ref = atomicModifyIORef' ref $ \case+    [] -> ([], Nothing)+    x : rest -> (rest, Just x)++blockForever :: IO a+blockForever = forever $ threadDelay 1_000_000_000++-- | Runs 'streamingThread' against the scripts until it initializes or exits, for at most 5 s, then kills it.+runStream :: [ConnectionScript] -> IO StreamRun+runStream = runStreamAt "http://localhost"++-- | Like 'runStream', with the stream URI given.+runStreamAt :: Text -> [ConnectionScript] -> IO StreamRun+runStreamAt streamURI = runStreamWith streamURI 1++-- | Like 'runStreamAt', with the initial retry delay in milliseconds given.+runStreamWith :: Text -> Int -> [ConnectionScript] -> IO StreamRun+runStreamWith streamURI initialRetryDelay scripts = do+    pending <- newIORef scripts+    logRef <- newIORef []+    initVar <- newEmptyMVar+    settled <- newEmptyMVar+    done <- newEmptyMVar+    manager <- scriptedManager pending+    let logTo _ _ level msg = do+            now <- getTime Monotonic+            atomicModifyIORef' logRef (\xs -> ((now, (level, decodeUtf8With lenientDecode $ fromLogStr msg)) : xs, ()))+        context =+            ClientContext+                { runLogger = (`runLoggingT` logTo)+                , httpConfiguration =+                    HttpConfiguration+                        { defaultRequestHeaders = []+                        , defaultRequestTimeout = responseTimeoutNone+                        , tlsManager = manager+                        }+                }+        updates =+            DataSourceUpdates+                { dataSourceUpdatesInit = \flags _ -> do+                    now <- getTime Monotonic+                    void $ tryPutMVar initVar (flags, now)+                    void $ tryPutMVar settled ()+                    pure (Right ())+                , dataSourceUpdatesInsertFlag = const $ pure (Right ())+                , dataSourceUpdatesInsertSegment = const $ pure (Right ())+                , dataSourceUpdatesDeleteFlag = \_ _ -> pure (Right ())+                , dataSourceUpdatesDeleteSegment = \_ _ -> pure (Right ())+                , dataSourceUpdatesSetStatus = const $ pure ()+                }+    thread <-+        forkFinally+            (runLogger context $ streamingThread streamURI initialRetryDelay context updates)+            (\result -> putMVar done result >> void (tryPutMVar settled ()))+    void $ timeout 5_000_000 $ readMVar settled+    killThread thread+    exited <- timeout 1_000_000 $ readMVar done+    initResult <- tryReadMVar initVar+    remaining <- readIORef pending+    timedLogs <- reverse <$> readIORef logRef+    pure+        StreamRun+            { connections = length scripts - length remaining+            , initialized = fst <$> initResult+            , initializedAt = snd <$> initResult+            , logs = map snd timedLogs+            , logTimes = map fst timedLogs+            , exited+            }++responseHead :: ReadStep+responseHead = Send "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n"++-- | Frames the bytes as one HTTP chunk, which the SDK receives from a single 'brRead'.+chunk :: ByteString -> ReadStep+chunk bytes = Send $ B8.pack (showHex (B.length bytes) "") <> "\r\n" <> bytes <> "\r\n"++-- | A put event whose only flag key, @café@, is the first non-ASCII text in the payload.+putEvent :: ByteString+putEvent = "event: put\ndata: " <> L.toStrict (encode payload) <> "\n\n"+  where+    payload =+        object+            [ "path" .= ("/" :: Text)+            , "data" .= object ["flags" .= object ["café" .= makeTestFlag "café" 1], "segments" .= object []]+            ]++-- | 'putEvent' split at the first byte of the two-byte UTF-8 encoding of @é@ (0xC3 0xA9).+beforeAccent, fromAccent :: ByteString+(beforeAccent, fromAccent) = B.break (== 0xC3) putEvent++healthyConnection :: ConnectionScript+healthyConnection = [responseHead, chunk putEvent]++assertInitialized :: StreamRun -> Assertion+assertInitialized run =+    assertBool+        ("stream never initialized; thread exit: " ++ show (exited run) ++ "; logs: " ++ show (logs run))+        (isJust $ initialized run >>= lookupKey "café")++testReconnectAfterHttpException :: Test+testReconnectAfterHttpException = TestCase $ do+    run <- runStream [[responseHead, Send "zz\r\n"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "InvalidChunkHeaders"++-- | The log lines that report a retry or a permanent error.+retryLogs :: StreamRun -> [(LogLevel, Text)]+retryLogs = filter (\(_, msg) -> T.isInfixOf "will retry" msg || T.isInfixOf "will not retry" msg) . logs++-- | Asserts one Warn retry log with the delay and the given reason.+-- The test delay is 1 ms minus up to 0.5 ms of jitter. It prints as 0.001, or as 0.000 at the full 0.5 ms of jitter.+assertOneRetryLog :: StreamRun -> Text -> Assertion+assertOneRetryLog run reason = case retryLogs run of+    [(level, msg)] -> do+        assertEqual "log level" LevelWarn level+        assertBool ("logs the delay and the reason, got: " ++ T.unpack msg) $+            msg `elem` [T.concat ["LaunchDarkly streaming connection was interrupted and will retry in ", delay, " s. Reason: ", reason] | delay <- ["0.000", "0.001"]]+    other -> assertFailure $ "expected one retry log, got: " ++ show other++-- | Asserts one Error log for a permanent error with the given reason, and no retry.+assertPermanentError :: StreamRun -> Text -> Assertion+assertPermanentError run reason = do+    assertEqual "connections" 1 (connections run)+    assertEqual+        "logs"+        [(LevelError, "A permanent error has been encountered with the LaunchDarkly streaming connection. It will not retry. Reason: " <> reason)]+        (retryLogs run)++-- | A response with no body that closes the connection, so the next request opens a new scripted connection.+emptyResponse :: ByteString -> [ByteString] -> ConnectionScript+emptyResponse status headers = [Send $ "HTTP/1.1 " <> status <> "\r\n" <> B.concat (map (<> "\r\n") headers) <> "Content-Length: 0\r\nConnection: close\r\n\r\n"]++testReconnectAfterSocketReset :: Test+testReconnectAfterSocketReset = TestCase $ do+    run <- runStream [[responseHead, chunk beforeAccent, Fail $ toException $ userError "Connection reset by peer"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "user error (Connection reset by peer)"++testLogsRetriedStatus :: Test+testLogsRetriedStatus = TestCase $ do+    run <- runStream [emptyResponse "503 Service Unavailable" [], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "HTTP error 503"++testLogsConnectionFailure :: Test+testLogsConnectionFailure = TestCase $ do+    run <- runStream [[Fail $ toException $ userError "Connection refused"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "user error (Connection refused)"++testLogsErrorBodyFailure :: Test+testLogsErrorBodyFailure = TestCase $ do+    run <- runStream [[Send "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 100\r\n\r\nab", Fail $ toException $ userError "reset in body"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "user error (reset in body)"++-- The logged delay must be the delay the stream waits, after jitter.+-- The 200 ms initial delay gives a delay of 100 ms to 200 ms. These bounds are whole milliseconds, so the logged delay+-- must be in that range with no allowance for rounding.+-- The stream waits at least the delay before it connects again, so the logged delay must also be no more than the time+-- from the log to initialization. This check allows for the rounding to 1 ms.+-- A log of the delay before jitter fails the second check, except when the jitter is less than about 0.6 ms.+testLogsActualDelay :: Test+testLogsActualDelay = TestCase $ do+    run <- runStreamWith "http://localhost" 200 [emptyResponse "503 Service Unavailable" [], healthyConnection]+    assertInitialized run+    let prefix = "LaunchDarkly streaming connection was interrupted and will retry in "+    case [(time, msg) | (time, (_, msg)) <- zip (logTimes run) (logs run), T.isPrefixOf prefix msg] of+        [(loggedAt, msg)] -> do+            let logged = read (T.unpack $ T.takeWhile (/= ' ') $ T.drop (T.length prefix) msg) :: Double+                waited = maybe 0 (\time -> fromIntegral (toNanoSecs (time - loggedAt)) / 1e9) (initializedAt run)+            assertBool ("logged a delay of " ++ show logged ++ " s, outside 0.1 s to 0.2 s") $ logged >= 0.1 && logged <= 0.2+            assertBool ("logged a delay of " ++ show logged ++ " s but waited " ++ show waited ++ " s") $ waited >= logged - 0.0005+        other -> assertFailure $ "expected one retry log, got: " ++ show other++testLogsCleanClose :: Test+testLogsCleanClose = TestCase $ do+    let closingHead = Send "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"+    run <- runStream [[closingHead, Send "0\r\n\r\n"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertOneRetryLog run "the stream ended"++testLogsRedirectLoop :: Test+testLogsRedirectLoop = TestCase $ do+    let redirect = emptyResponse "302 Found" ["Location: http://localhost/all", "Set-Cookie: session=secret"]+    run <- runStream (replicate 11 redirect ++ [healthyConnection])+    assertInitialized run+    assertEqual "connections" 12 (connections run)+    assertOneRetryLog run "too many redirects"++testLogsPermanentStatus :: Test+testLogsPermanentStatus = TestCase $ do+    run <- runStream [emptyResponse "404 Not Found" [], healthyConnection]+    assertPermanentError run "HTTP error 404"++testUnauthorizedDoesNotRetry :: Test+testUnauthorizedDoesNotRetry = TestCase $ do+    run <- runStream [emptyResponse "401 Unauthorized" [], healthyConnection]+    assertEqual "connections" 1 (connections run)+    assertEqual "retry logs" [] (retryLogs run)++testLogsInvalidUri :: Test+testLogsInvalidUri = TestCase $ do+    run <- runStreamAt "https://user:secret@relay:port" [healthyConnection]+    assertEqual "connections" 0 (connections run)+    assertEqual+        "logs"+        [(LevelError, "A permanent error has been encountered with the LaunchDarkly streaming connection. It will not retry. Reason: invalid stream URI: Invalid URL")]+        (retryLogs run)+    assertBool "no log line contains the password" $ not $ any (T.isInfixOf "secret" . snd) (logs run)++testUtf8SplitAcrossReads :: Test+testUtf8SplitAcrossReads = TestCase $ do+    assertBool "payload contains 0xC3" (not $ B.null fromAccent)+    run <- runStream [[responseHead, chunk beforeAccent, chunk (B.take 1 fromAccent), chunk (B.drop 1 fromAccent)]]+    assertInitialized run+    assertEqual "connections" 1 (connections run)++testKillThreadStopsStream :: Test+testKillThreadStopsStream = TestCase $ do+    run <- runStream [healthyConnection]+    assertInitialized run+    assertEqual "connections" 1 (connections run)+    assertBool ("thread stopped by ThreadKilled, got: " ++ show (exited run)) $+        case exited run of+            Just (Left e) -> fromException e == Just ThreadKilled+            _ -> False++testBackoffCeiling :: Test+testBackoffCeiling = TestCase $ do+    assertEqual "first attempt waits the initial delay" 1_000_000 (backoffCeiling 1000 1)+    assertEqual "attempt 46 stays at the 30 s cap" 30_000_000 (backoffCeiling 1000 46)+    assertEqual "an initial delay too large for Int arithmetic stays at the 30 s cap" 30_000_000 (backoffCeiling maxBound 1)++testRetryAttempt :: Test+testRetryAttempt = TestCase $ do+    assertEqual "a connection that did not open increments the attempt" 7 (retryAttempt 6 Nothing (TimeSpec 1000 0))+    assertEqual "a connection open for less than 60 s increments the attempt" 7 (retryAttempt 6 (Just (TimeSpec 100 0)) (TimeSpec 159 0))+    assertEqual "a connection open for 60 s resets to the first attempt" 1 (retryAttempt 6 (Just (TimeSpec 100 0)) (TimeSpec 160 0))+    assertEqual "a connection open for hours resets to the first attempt" 1 (retryAttempt 6 (Just (TimeSpec 100 0)) (TimeSpec 10_000 0))++hasError :: Text -> [(LogLevel, Text)] -> Bool+hasError message = elem (LevelError, message)++testLogsClosedConnection :: Test+testLogsClosedConnection = TestCase $ do+    -- The server ends the chunked body. 'Connection: close' stops http-client from reusing the socket for the retry.+    let closingHead = Send "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"+    run <- runStream [[closingHead, Send "0\r\n\r\n"], healthyConnection]+    assertInitialized run+    assertEqual "connections" 2 (connections run)+    assertBool ("logs the closed connection, got: " ++ show (logs run)) $+        hasError "streaming connection unexpectedly closed" (logs run)+    assertBool "does not log a timeout" $ not $ hasError "timeout waiting for SSE event" (logs run)++-- The streaming thread waits 300 s for a read. This test calls 'readStream' with a 10 ms wait instead.+testLogsReadTimeout :: Test+testLogsReadTimeout = TestCase $ do+    logRef <- newIORef []+    let logTo _ _ level msg = atomicModifyIORef' logRef (\xs -> ((level, decodeUtf8With lenientDecode $ fromLogStr msg) : xs, ()))+        updates =+            DataSourceUpdates+                { dataSourceUpdatesInit = \_ _ -> pure (Right ())+                , dataSourceUpdatesInsertFlag = const $ pure (Right ())+                , dataSourceUpdatesInsertSegment = const $ pure (Right ())+                , dataSourceUpdatesDeleteFlag = \_ _ -> pure (Right ())+                , dataSourceUpdatesDeleteSegment = \_ _ -> pure (Right ())+                , dataSourceUpdatesSetStatus = const $ pure ()+                }+    result <- timeout 5_000_000 $ runLoggingT (readStream 10_000 blockForever updates) logTo+    assertEqual "readStream returns without a processed event" (Just False) result+    logs <- reverse <$> readIORef logRef+    assertBool ("logs the timeout, got: " ++ show logs) $ hasError "timeout waiting for SSE event" logs+    assertBool "does not log a closed connection" $ not $ hasError "streaming connection unexpectedly closed" logs