kenshou-telemetry-0.1.0.0: src/Kenshou/Telemetry/Scrape.hs
module Kenshou.Telemetry.Scrape
( EndpointSummary (..),
ScraperHandle (..),
startScraperInProcess,
runScraperInProcess,
runScraperRole,
SlotLeakResult (..),
wsSlotLeakProbe,
)
where
import Control.Concurrent.Async (Async, async, cancel, mapConcurrently_, waitCatch)
import Control.Concurrent.MVar (MVar, newMVar, withMVar)
import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)
import Control.Monad (forM_, replicateM_, void)
import Data.Aeson (FromJSON (..), ToJSON (..), encode, object, withObject, (.:), (.=))
import Data.Aeson.Types (parseMaybe, (.!=), (.:?))
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (traverse_)
import Data.IORef
import Data.List (sort)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import GHC.Clock (getMonotonicTimeNSec)
import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))
import Kenshou.Measure.Clock (Origin (..), captureOrigin, sleepUntilNs)
import Kenshou.Telemetry.Endpoint
import Network.HTTP.Client (Manager, defaultManagerSettings, httpLbs, newManager, parseRequest, responseBody, responseStatus)
import Network.HTTP.Types.Status (statusCode)
import Network.WebSockets qualified as WebSockets
import System.Directory (createDirectoryIfMissing)
import System.FilePath ((</>))
import System.IO (BufferMode (LineBuffering), Handle, IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)
import Text.Read (readMaybe)
data ScrapeSample = ScrapeSample
{ status :: Int,
latencyNs :: Integer,
bodyBytes :: Integer,
skipped :: Int,
errorMessage :: Maybe Text
}
data EndpointSummary = EndpointSummary
{ endpoint :: Text,
kind :: EndpointKind,
scrapes :: Int,
failures :: Int,
skippedTicks :: Int,
latencyP50Ns :: Integer,
latencyP99Ns :: Integer,
latencyMaxNs :: Integer,
meanBodyBytes :: Double,
maxBodyBytes :: Integer
}
deriving stock (Eq, Show)
instance ToJSON EndpointSummary where
toJSON summary =
object
[ "endpoint" .= summary.endpoint,
"kind" .= summary.kind,
"scrapes" .= summary.scrapes,
"failures" .= summary.failures,
"skippedTicks" .= summary.skippedTicks,
"latencyNs" .= object ["p50" .= summary.latencyP50Ns, "p99" .= summary.latencyP99Ns, "max" .= summary.latencyMaxNs],
"bodyBytes" .= object ["mean" .= summary.meanBodyBytes, "max" .= summary.maxBodyBytes]
]
instance FromJSON EndpointSummary where
parseJSON = withObject "EndpointSummary" \value -> do
latency <- value .: "latencyNs"
body <- value .: "bodyBytes"
EndpointSummary
<$> value .: "endpoint"
<*> value .: "kind"
<*> value .: "scrapes"
<*> value .: "failures"
<*> value .: "skippedTicks"
<*> latency .: "p50"
<*> latency .: "p99"
<*> latency .: "max"
<*> body .: "mean"
<*> body .: "max"
data ScraperHandle = ScraperHandle
{ register :: Endpoint -> IO (),
finish :: IO [EndpointSummary]
}
data ScraperState = ScraperState
{ manager :: Manager,
outputDir :: FilePath,
intervalNs :: Integer,
wsSubscribers :: Int,
workers :: IORef [(Endpoint, Async ())],
endpoints :: IORef [Endpoint],
samples :: IORef (Map Text [ScrapeSample])
}
startScraperInProcess :: FilePath -> Int -> Int -> IO ScraperHandle
startScraperInProcess outputDir intervalMs wsSubscribers = do
createDirectoryIfMissing True (outputDir </> "series")
manager <- newManager defaultManagerSettings
workers <- newIORef []
endpoints <- newIORef []
samples <- newIORef Map.empty
let state = ScraperState manager outputDir (fromIntegral intervalMs * 1_000_000) wsSubscribers workers endpoints samples
pure (ScraperHandle (registerEndpoint state) (finishScraper state))
runScraperInProcess :: FilePath -> Int -> Int -> [Endpoint] -> IO (IO [EndpointSummary])
runScraperInProcess outputDir intervalMs wsSubscribers endpoints = do
scraper <- startScraperInProcess outputDir intervalMs wsSubscribers
traverse_ scraper.register endpoints
pure scraper.finish
runScraperRole :: RoleContext -> IO ()
runScraperRole context = do
let (intervalMs, subscribers) = maybe (15_000, 0) id (parseMaybe parseConfig context.init.args)
scraper <- startScraperInProcess context.init.outDir intervalMs subscribers
context.send WrkReady
loop scraper
where
loop scraper =
context.receive >>= \case
Just (CtlCustom "endpoint" payload) -> case parseMaybe parseJSON payload of
Nothing -> context.send (WrkError "invalid scraper endpoint") >> loop scraper
Just endpoint -> scraper.register endpoint >> loop scraper
Just (CtlCustom "finish" _) -> scraper.finish >>= context.send . WrkCustom "scrape-summary" . toJSON >> loop scraper
Just (CtlStop _) -> void scraper.finish
Just _ -> loop scraper
Nothing -> void scraper.finish
parseConfig = withObject "scraper config" (\value -> (,) <$> value .: "intervalMs" <*> value .:? "wsSubscribers" .!= 0)
registerEndpoint :: ScraperState -> Endpoint -> IO ()
registerEndpoint state endpoint = do
modifyIORef' state.endpoints (<> [endpoint])
if endpoint.kind == WebSocketPush
then do
modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [])
if state.wsSubscribers <= 0
then pure ()
else do
worker <- async (websocketSubscribers state endpoint)
modifyIORef' state.workers (<> [(endpoint, worker)])
else do
worker <- async (scrapeLoop state endpoint)
modifyIORef' state.workers (<> [(endpoint, worker)])
scrapeLoop :: ScraperState -> Endpoint -> IO ()
scrapeLoop state endpoint = do
request <- parseRequest (Text.unpack endpoint.url)
handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> ".csv")) WriteMode
hSetBuffering handle LineBuffering
hPutStrLn handle "t_wall_ns,endpoint,kind,status,latency_ns,body_bytes,skipped,error"
started <- getMonotonicTimeNSec
let loop tick = do
sleepUntilNs (started + fromIntegral tick * fromIntegral state.intervalNs)
origin <- captureOrigin
callStarted <- getMonotonicTimeNSec
result <- trySync (httpLbs request state.manager)
callEnded <- getMonotonicTimeNSec
let elapsed = fromIntegral (callEnded - callStarted)
behind = max 0 (fromIntegral (callEnded - started) `div` state.intervalNs - tick)
sample = case result of
Left exception -> ScrapeSample 0 elapsed 0 (fromIntegral behind) (Just (Text.pack (displayException exception)))
Right response -> ScrapeSample (statusCode (responseStatus response)) elapsed (fromIntegral (LazyByteString.length (responseBody response))) (fromIntegral behind) Nothing
modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [sample])
hPutStrLn handle (csvRow origin endpoint sample)
loop (tick + 1 + behind)
loop 1 `finally` hClose handle
websocketSubscribers :: ScraperState -> Endpoint -> IO ()
websocketSubscribers state endpoint = do
(host, port, path) <- either (ioError . userError . Text.unpack) pure (parseWebSocketUrl endpoint.url)
handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> "-ws.csv")) WriteMode
hSetBuffering handle LineBuffering
hPutStrLn handle "t_wall_ns,subscriber,event,frames,bytes,gap_ns,error"
lock <- newMVar ()
mapConcurrently_ (subscriber host port path handle lock) [1 .. state.wsSubscribers] `finally` hClose handle
where
subscriber host port path handle lock subscriberId = do
outcome <- trySync $ WebSockets.runClient (Text.unpack host) port (Text.unpack path) \connection -> do
traverse_ (WebSockets.sendTextData connection . encode) endpoint.wsHello
writeWsRow handle lock subscriberId "connect" 0 0 0 Nothing
previous <- newIORef Nothing
let loop frames = do
payload <- WebSockets.receiveData connection :: IO LazyByteString.ByteString
now <- getMonotonicTimeNSec
prior <- atomicModifyIORef' previous (\value -> (Just now, value))
writeWsRow handle lock subscriberId "frame" frames (fromIntegral (LazyByteString.length payload)) (maybe 0 (fromIntegral . (now -)) prior) Nothing
loop (frames + 1)
loop 1
case outcome of
Left exception -> writeWsRow handle lock subscriberId "error" 0 0 0 (Just (Text.pack (displayException exception)))
Right () -> writeWsRow handle lock subscriberId "close" 0 0 0 Nothing
writeWsRow :: Handle -> MVar () -> Int -> Text -> Int -> Integer -> Integer -> Maybe Text -> IO ()
writeWsRow handle lock subscriberId event frames bytes gap errorMessage = withMVar lock \_ -> do
origin <- captureOrigin
hPutStrLn handle . Text.unpack . Text.intercalate "," $
[ Text.pack (show origin.wallUnixNs),
Text.pack (show subscriberId),
event,
Text.pack (show frames),
Text.pack (show bytes),
Text.pack (show gap),
maybe "" (Text.replace "," ";" . Text.replace "\n" " ") errorMessage
]
finishScraper :: ScraperState -> IO [EndpointSummary]
finishScraper state = do
workers <- readIORef state.workers
forM_ workers (cancel . snd)
forM_ workers (void . waitCatch . snd)
captured <- readIORef state.samples
endpoints <- readIORef state.endpoints
pure [summarize endpoint (Map.findWithDefault [] endpoint.name captured) | endpoint <- endpoints]
summarize :: Endpoint -> [ScrapeSample] -> EndpointSummary
summarize endpoint samples =
EndpointSummary
{ endpoint = endpoint.name,
kind = endpoint.kind,
scrapes = length samples,
failures = length [() | sample <- samples, sample.status < 200 || sample.status >= 300 || sample.errorMessage /= Nothing],
skippedTicks = sum (fmap (.skipped) samples),
latencyP50Ns = quantile 0.50 latencies,
latencyP99Ns = quantile 0.99 latencies,
latencyMaxNs = maybe 0 last nonEmptyLatencies,
meanBodyBytes = if null samples then 0 else fromIntegral (sum (fmap (.bodyBytes) samples)) / fromIntegral (length samples),
maxBodyBytes = maybe 0 last nonEmptyBodies
}
where
latencies = sort (fmap (.latencyNs) samples)
bodies = sort (fmap (.bodyBytes) samples)
nonEmptyLatencies = if null latencies then Nothing else Just latencies
nonEmptyBodies = if null bodies then Nothing else Just bodies
quantile :: Double -> [Integer] -> Integer
quantile _ [] = 0
quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
data SlotLeakResult = SlotLeakResult
{ attempted :: Int,
freshAccepted :: Bool
}
deriving stock (Eq, Show)
instance ToJSON SlotLeakResult where
toJSON result = object ["attempted" .= result.attempted, "freshAccepted" .= result.freshAccepted]
wsSlotLeakProbe :: Endpoint -> Int -> IO SlotLeakResult
wsSlotLeakProbe endpoint attempts = case parseWebSocketUrl endpoint.url of
Left _ -> pure (SlotLeakResult attempts False)
Right (host, port, path) -> do
replicateM_ attempts (void (trySync (connect host port path)))
fresh <- trySync (connect host port path)
pure (SlotLeakResult attempts (either (const False) (const True) fresh))
where
connect host port path = WebSockets.runClient (Text.unpack host) port (Text.unpack path) (const (pure ()))
parseWebSocketUrl :: Text -> Either Text (Text, Int, Text)
parseWebSocketUrl url = do
remainder <- maybe (Left "WebSocket URL must start with ws://") Right (Text.stripPrefix "ws://" url)
let (authority, rawPath) = Text.breakOn "/" remainder
path = if Text.null rawPath then "/" else rawPath
pieces = Text.splitOn ":" authority
case pieces of
[host] -> Right (host, 80, path)
[host, rawPort] -> maybe (Left "WebSocket URL has an invalid port") (\port -> Right (host, port, path)) (readMaybe (Text.unpack rawPort))
_ -> Left "WebSocket URL has an invalid authority"
csvRow :: Origin -> Endpoint -> ScrapeSample -> String
csvRow origin endpoint sample =
Text.unpack . Text.intercalate "," $
[ Text.pack (show origin.wallUnixNs),
endpoint.name,
Text.pack (show endpoint.kind),
Text.pack (show sample.status),
Text.pack (show sample.latencyNs),
Text.pack (show sample.bodyBytes),
Text.pack (show sample.skipped),
maybe "" sanitize sample.errorMessage
]
where
sanitize = Text.replace "," ";" . Text.replace "\n" " "
tryAny :: IO value -> IO (Either SomeException value)
tryAny = try
trySync :: IO value -> IO (Either SomeException value)
trySync action = do
outcome <- tryAny action
case outcome of
Left exception -> case fromException exception :: Maybe SomeAsyncException of
Just _ -> throwIO exception
Nothing -> pure outcome
Right _ -> pure outcome