launchdarkly-server-sdk-4.6.2: src/LaunchDarkly/Server/Network/Streaming.hs
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE NumericUnderscores #-}
module LaunchDarkly.Server.Network.Streaming (streamingThread, backoffCeiling, retryAttempt, readStream, streamReadTimeout) where
import Control.Applicative (many, (<|>))
import Control.Concurrent (threadDelay)
import Control.Exception (SomeAsyncException, SomeException, displayException, fromException, throwIO)
import Control.Monad (mzero, void)
import Control.Monad.Catch (Exception, MonadCatch, MonadMask, MonadThrow, catchIf, throwM, try)
import Control.Monad.IO.Class (MonadIO, liftIO)
import Control.Monad.Logger (MonadLogger, logDebug, logError, logWarn)
import Control.Monad.Loops (iterateUntilM)
import Data.Aeson (FromJSON, Result (..), eitherDecode, fromJSON, parseJSON, withObject, (.!=), (.:), (.:?))
import Data.Attoparsec.Text as P hiding (Result, try)
import Data.ByteString (ByteString)
import qualified Data.ByteString as B
import qualified Data.ByteString.Lazy as L
import Data.IORef (IORef, newIORef, readIORef, writeIORef)
import Data.Maybe (isJust)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding (Decoding (Some), encodeUtf8, streamDecodeUtf8)
import GHC.Generics (Generic)
import GHC.Natural (Natural)
import Network.HTTP.Client (HttpException (..), HttpExceptionContent (..), Manager, Request, Response (..), brRead)
import Network.HTTP.Types.Status (Status (statusCode))
import System.Clock (Clock (Monotonic), TimeSpec (TimeSpec), getTime)
import System.Random (Random (randomR), newStdGen)
import System.Timeout (timeout)
import Text.Printf (printf)
import LaunchDarkly.AesonCompat (KeyMap, emptyObject)
import LaunchDarkly.Server.Config.ClientContext (ClientContext (..))
import LaunchDarkly.Server.Config.HttpConfiguration (HttpConfiguration (..), prepareRequest)
import LaunchDarkly.Server.DataSource.Internal (DataSourceUpdates (..))
import LaunchDarkly.Server.Features (Flag, Segment)
import LaunchDarkly.Server.Network.Common (checkAuthorization, handleUnauthorized, isHttpUnrecoverable, throwIfNot200, tryHTTP, withResponseGeneric)
import LaunchDarkly.Server.Store.Internal (StoreResult)
data PutBody = PutBody
{ flags :: !(KeyMap Flag)
, segments :: !(KeyMap Segment)
}
deriving (Generic, Show)
data PathData d = PathData
{ path :: !Text
, pathData :: !d
}
deriving (Generic, Show)
data PathVersion = PathVersion
{ path :: !Text
, version :: !Natural
}
deriving (Generic, Show, FromJSON)
instance FromJSON PutBody where
parseJSON = withObject "PutBody" $ \o -> do
flags <- o .: "flags"
segments <- o .:? "segments" .!= emptyObject
pure $ PutBody {flags = flags, segments = segments}
instance FromJSON a => FromJSON (PathData a) where
parseJSON = withObject "Put" $ \o -> do
pathData <- o .: "data"
path <- o .:? "path" .!= "/"
pure $ PathData {path = path, pathData = pathData}
data SSE = SSE
{ name :: !Text
, buffer :: !Text
, lastEventId :: !(Maybe Text)
, retry :: !(Maybe Text)
}
deriving (Generic, Show, Eq)
nameCharPredicate :: Char -> Bool
nameCharPredicate x = x /= '\r' && x /= ':' && x /= '\n'
anyCharPredicate :: Char -> Bool
anyCharPredicate x = x /= '\r' && x /= '\n'
endOfLineSSE :: Parser ()
endOfLineSSE = choice [void $ string "\r\n", void $ string "\r", void $ string "\n", endOfInput]
comment :: Parser ()
comment = char ':' >> P.takeWhile anyCharPredicate >> endOfLineSSE >> pure ()
parseField :: Parser (Text, Text)
parseField = do
fieldName <- P.takeWhile1 nameCharPredicate
void $ option ' ' $ char ':'
void $ option ' ' $ char ' '
fieldValue <- P.takeWhile anyCharPredicate
endOfLineSSE
pure (fieldName, fieldValue)
processField :: (Text, Text) -> SSE -> SSE
processField (fieldName, fieldValue) event = case fieldName of
"event" -> event {name = fieldValue}
"id" -> event {lastEventId = Just fieldValue}
"retry" -> event {retry = Just fieldValue}
"data" -> event {buffer = T.concat [buffer event, if T.null (buffer event) then "" else "\n", fieldValue]}
_ -> event
parseEvent :: Parser SSE
parseEvent = do
fields <- concat <$> many ((comment >> pure []) <|> fmap (: []) parseField)
endOfLineSSE
let event = foldr processField (SSE "" "" mzero mzero) fields
if T.null (name event) || T.null (buffer event) then parseEvent else pure event
processPut :: (MonadIO m, MonadLogger m) => DataSourceUpdates -> L.ByteString -> m Bool
processPut dataSourceUpdates value = case eitherDecode value of
Right (PathData _ (PutBody flags segments)) -> do
$(logDebug) "initializing dataSourceUpdates with put"
liftIO (dataSourceUpdatesInit dataSourceUpdates flags segments) >>= \case
Left err -> do
$(logError) $ T.append "dataSourceUpdates failed put: " err
pure False
_ -> pure True
Left err -> do
$(logError) $ T.append "failed to parse put body" (T.pack err)
pure False
processPatch :: forall m. (MonadIO m, MonadLogger m) => DataSourceUpdates -> L.ByteString -> m Bool
processPatch dataSourceUpdates value = case eitherDecode value of
Right (PathData path body)
| T.isPrefixOf "/flags/" path -> insPatch "flag" path dataSourceUpdatesInsertFlag (fromJSON body)
| T.isPrefixOf "/segments/" path -> insPatch "segment" path dataSourceUpdatesInsertSegment (fromJSON body)
| otherwise -> do
$(logError) "unknown patch path"
pure True
Left err -> do
$(logError) $ T.append "failed to parse patch generic" (T.pack err)
pure False
where
insPatch :: Text -> Text -> (DataSourceUpdates -> a -> IO (Either Text ())) -> Result a -> m Bool
insPatch name _ _ (Error err) = do
$(logError) $ T.concat ["failed to parse patch ", name, ": ", T.pack err]
pure False
insPatch name path insert (Success item) = do
$(logDebug) $ T.concat ["patching ", name, " with path: ", path]
status <- liftIO $ insert dataSourceUpdates item
either
( \err -> do
$(logError) $ T.concat ["dataSourceUpdates failed ", name, " patch: ", err]
pure False
)
(const $ pure True)
status
processDelete :: forall m. (MonadIO m, MonadLogger m) => DataSourceUpdates -> L.ByteString -> m Bool
processDelete dataSourceUpdates value = case eitherDecode value :: Either String PathVersion of
Right (PathVersion path version)
| T.isPrefixOf "/flags/" path -> logDelete "flag" path (dataSourceUpdatesDeleteFlag dataSourceUpdates (T.drop 7 path) version)
| T.isPrefixOf "/segments/" path -> logDelete "segment" path (dataSourceUpdatesDeleteSegment dataSourceUpdates (T.drop 10 path) version)
| otherwise -> do
$(logError) "unknown delete path"
pure False
Left err -> do
$(logError) $ T.append "failed to parse delete" (T.pack err)
pure False
where
logDelete :: Text -> Text -> StoreResult () -> m Bool
logDelete name path action = do
$(logDebug) $ T.concat ["deleting ", name, " with path: ", path]
status <- liftIO action
either
( \err -> do
$(logError) $ T.concat ["dataSourceUpdates failed ", name, " delete: ", err]
pure False
)
(const $ pure True)
status
processEvent :: (MonadIO m, MonadLogger m) => DataSourceUpdates -> Text -> L.ByteString -> m Bool
processEvent dataSourceUpdates name value = case name of
"put" -> processPut dataSourceUpdates value
"patch" -> processPatch dataSourceUpdates value
"delete" -> processDelete dataSourceUpdates value
_ -> do
$(logWarn) "unknown event type"
pure True
data ReadE = ReadETimeout | ReadEClosed deriving (Show, Exception)
tryReadE :: MonadCatch m => m a -> m (Either ReadE a)
tryReadE = try
-- The time in microseconds to wait for data on the stream before it reconnects.
-- The server sends a heartbeat every 120 seconds.
streamReadTimeout :: Int
streamReadTimeout = 1_000_000 * 300
-- A read can end partway through a multi-byte character, so the decoder keeps those bytes for the next read.
-- `parseWith` treats empty text as end of input, so a read that decodes to nothing reads again.
readWithException :: Int -> IORef (ByteString -> Decoding) -> IO ByteString -> IO Text
readWithException readTimeout decoder body =
timeout readTimeout (brRead body) >>= \case
Nothing -> throwIO ReadETimeout
Just bytes ->
if bytes == B.empty
then throwIO ReadEClosed
else do
decode <- readIORef decoder
let Some text _ next = decode bytes
writeIORef decoder next
if T.null text then readWithException readTimeout decoder body else pure text
-- `readTimeout` is the time in microseconds to wait for each read. Tests use a shorter time than `streamReadTimeout`.
readStream :: (MonadIO m, MonadLogger m, MonadMask m) => Int -> IO ByteString -> DataSourceUpdates -> m Bool
readStream readTimeout body dataSourceUpdates = liftIO (newIORef streamDecodeUtf8) >>= \decoder -> loop decoder "" False
where
loop decoder initial processedEvent =
tryReadE (parseWith (liftIO $ readWithException readTimeout decoder body) parseEvent initial) >>= \case
(Left ReadETimeout) -> do
$(logError) "timeout waiting for SSE event"
pure processedEvent
(Left ReadEClosed) -> do
$(logError) "streaming connection unexpectedly closed"
pure processedEvent
(Right parsed) -> case parsed of
Done remaining event -> do
processed <- processEvent dataSourceUpdates (name event) (L.fromStrict $ encodeUtf8 $ buffer event)
if processed then loop decoder remaining True else pure processedEvent
Fail _ context err -> do
$(logError) $ T.intercalate " " ["failed parsing SSE frame", T.pack $ show context, T.pack err]
pure processedEvent
Partial _ -> do
$(logError) "failed parsing SSE frame unexpected partial"
pure processedEvent
-- `tryHTTP` only catches `HttpException`. Any other exception from reading the body or processing its events would end
-- the streaming thread with no log and no reconnect, such as a socket reset, a TLS error, invalid UTF-8, or a failed
-- store write. Async exceptions stay uncaught so `close` can stop the thread.
isRetryableReadError :: SomeException -> Bool
isRetryableReadError e = not (isJust (fromException @SomeAsyncException e) || isJust (fromException @HttpException e))
-- Rethrows the error as an `HttpException` so `tryHTTP` catches it and the stream reconnects with backoff.
-- `handleResponse` logs the error.
retryReadError :: MonadThrow m => Request -> SomeException -> m a
retryReadError request e = throwM $ HttpExceptionRequest request (InternalException e)
-- A short description of the error for the log. It leaves out the request, which `displayException` includes.
-- It also leaves out the responses of a redirect loop, because they can contain cookies and other server data.
-- It leaves out an invalid URI, because the URI can contain a password.
describeHttpException :: HttpException -> String
describeHttpException (HttpExceptionRequest _ (StatusCodeException response _)) = "HTTP error " ++ show (statusCode (responseStatus response))
describeHttpException (HttpExceptionRequest _ (TooManyRedirects _)) = "too many redirects"
describeHttpException (HttpExceptionRequest _ (InternalException e)) = displayException e
describeHttpException (HttpExceptionRequest _ content) = show content
describeHttpException (InvalidUrlException _ reason) = "invalid stream URI: " ++ reason
-- The reason to log when a connection ends. A connection that ends without an `HttpException` has already logged
-- the cause in `readStream`.
describeResult :: Either HttpException Bool -> String
describeResult (Left e) = describeHttpException e
describeResult (Right _) = "the stream ended"
logPermanentError :: MonadLogger m => String -> m ()
logPermanentError reason =
$(logError) $ T.pack $ "A permanent error has been encountered with the LaunchDarkly streaming connection. It will not retry. Reason: " ++ reason
-- This function is responsible for consuming a streaming connection.
--
-- It is responsible for opening the connection, parsing the body for as long as it is able, and then handling shut down
-- cleanly. While events are being processed, it is responsible for implementing a sane retry policy that prevents the
-- stampeding herd problem in the unlikely event of upstream failures.
startNewConnection :: (MonadIO m, MonadLogger m, MonadMask m) => Manager -> Request -> DataSourceUpdates -> StreamingState -> m StreamingState
startNewConnection manager request dataSourceUpdates state = do
$(logDebug) "starting new streaming connection"
connectedAt <- liftIO $ newIORef Nothing
status <- tryHTTP $ withResponseGeneric request manager $ \response -> do
checkAuthorization response
>> catchIf isRetryableReadError (throwIfNot200 request response) (retryReadError request)
>> liftIO (getTime Monotonic >>= writeIORef connectedAt . Just)
>> catchIf isRetryableReadError (readStream streamReadTimeout (responseBody response) dataSourceUpdates) (retryReadError request)
activeSince <- liftIO $ readIORef connectedAt
now <- liftIO $ getTime Monotonic
handleResponse state now activeSince status
where
-- This function is responsible for parsing the final output from a now closed streaming connection.
--
-- Given the result of the stream run, this function can choose to either mark the state as cancelled, stopping all
-- further attempts, or it can update the state and wait some amount of time before starting another attempt.
handleResponse :: (MonadIO m, MonadLogger m, MonadMask m) => StreamingState -> TimeSpec -> Maybe TimeSpec -> (Either HttpException Bool) -> m StreamingState
handleResponse state now activeSince result =
let state' = updateState state now activeSince result
in if cancel state'
then do
logPermanentError (describeResult result)
pure state'
else do
delay <- calculateDelay state'
$(logWarn) $ T.pack $ printf "LaunchDarkly streaming connection was interrupted and will retry in %.3f s. Reason: %s" (fromIntegral delay / 1_000_000 :: Double) (describeResult result)
_ <- liftIO $ threadDelay delay
pure state'
-- Once a streaming connection run has ended, we need to update the streaming state.
--
-- Given the result of a stream run, this function can either choose to mark the state as cancelled, meaning halt all
-- further attempts, or it can be updated according to the rules of our streaming specification.
updateState :: StreamingState -> TimeSpec -> Maybe TimeSpec -> (Either HttpException Bool) -> StreamingState
updateState state@(StreamingState {attempt = att}) now activeSince (Right _) = state {initialConnection = False, attempt = retryAttempt att activeSince now}
updateState state _ _ (Left (HttpExceptionRequest _ (StatusCodeException response _)))
| isHttpUnrecoverable (statusCode (responseStatus response)) = state {cancel = True}
updateState state@(StreamingState {attempt = att}) now activeSince (Left _) = state {attempt = retryAttempt att activeSince now}
-- Calculate the next delay period following a backoff + jitter + max delay algorithm.
--
-- See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
calculateDelay :: (MonadIO m, MonadLogger m, MonadMask m) => StreamingState -> m Int
calculateDelay StreamingState {initialRetryDelay, attempt = att} = do
liftIO $
newStdGen >>= \gen ->
let timespan = backoffCeiling initialRetryDelay att
jitter = fst $ randomR (0, timespan `div` 2) gen
in pure $ (timespan - jitter)
-- The retry delay in microseconds before jitter, given the initial delay in milliseconds and the attempt number.
--
-- The product uses `Integer` because in `Int` it overflows on 32-bit runtimes at the default 1 s initial delay, and
-- on any runtime for a large enough initial delay. Any initial delay of 1 ms or more reaches the 30 s cap by 2^15,
-- so the exponent cap keeps the `Integer` small without changing the result.
backoffCeiling :: Int -> Int -> Int
backoffCeiling initialRetryDelay att =
fromInteger $ min (30 * 1_000_000) (toInteger initialRetryDelay * 1_000 * 2 ^ min 15 (att - 1))
-- The attempt number for the retry after a connection ends.
--
-- Every end of a connection counts as a failure, also a clean close by the server.
-- `activeSince` is the time the connection got a 200 response, or `Nothing` if it did not get one.
-- A connection that stayed open for 60 s or more resets the backoff to the first attempt.
-- Otherwise, the attempt number increases by one.
retryAttempt :: Int -> Maybe TimeSpec -> TimeSpec -> Int
retryAttempt att activeSince now = case activeSince of
Just time | now >= time + TimeSpec 60 0 -> 1
_ -> att + 1
data StreamingState = StreamingState
{ initialConnection :: Bool -- Marker used to determine the first time the streamer connects.
, initialRetryDelay :: Int -- The base duration used for the retry delay calculation
, attempt :: Int -- A number representing the attempt # of this connection
, cancel :: Bool -- A marker to shortcut logic and halt the streaming process
}
deriving (Generic, Show)
-- Start a thread for streaming events back from LaunchDarkly services.
streamingThread :: (MonadIO m, MonadLogger m, MonadMask m) => Text -> Int -> ClientContext -> DataSourceUpdates -> m ()
streamingThread streamURI initialRetryDelay clientContext dataSourceUpdates = do
let manager = tlsManager $ httpConfiguration clientContext
tryHTTP (prepareRequest (httpConfiguration clientContext) (T.unpack streamURI ++ "/all")) >>= \case
Left e -> logPermanentError (describeHttpException e)
Right req -> handleUnauthorized dataSourceUpdates (processStream manager req)
where
processStream :: (MonadIO m, MonadLogger m, MonadMask m) => Manager -> Request -> m ()
processStream manager req = do
void $ iterateUntilM (cancel) (\state -> startNewConnection manager req dataSourceUpdates state) StreamingState {initialConnection = True, initialRetryDelay, attempt = 0, cancel = False}