kenshou-telemetry-0.1.0.0: src/Kenshou/Telemetry/Tracing/Pipeline.hs
module Kenshou.Telemetry.Tracing.Pipeline
( PipelineStats,
PipelineSnapshot (..),
LatencySummary (..),
newPipelineStats,
countingProcessor,
instrumentProcessorSuccess,
instrumentExporter,
snapshotPipeline,
startPipelineSampler,
)
where
import Control.Concurrent (threadDelay)
import Control.Concurrent.Async (async, cancel, waitCatch)
import Control.Exception (displayException, finally, onException)
import Control.Monad (void)
import Data.Aeson (ToJSON (..), object, (.=))
import Data.HashMap.Strict qualified as HashMap
import Data.IORef
import Data.List (sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Vector qualified as Vector
import GHC.Clock (getMonotonicTimeNSec)
import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))
import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))
import OpenTelemetry.Processor.Span (SpanProcessor (..))
import System.Directory (createDirectoryIfMissing)
import System.FilePath ((</>))
import System.IO (BufferMode (LineBuffering), IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)
data PipelineStats = PipelineStats
{ started :: IORef Int,
ended :: IORef Int,
exportedOk :: IORef Int,
exportFailed :: IORef Int,
exportCalls :: IORef Int,
exportLatencies :: IORef [Integer],
maxQueueDepth :: IORef Int,
lastExportError :: IORef (Maybe Text)
}
data LatencySummary = LatencySummary {p50 :: Integer, p99 :: Integer, maximum :: Integer}
deriving stock (Eq, Show)
data PipelineSnapshot = PipelineSnapshot
{ spansStarted :: Int,
spansEnded :: Int,
spansExportedOk :: Int,
spansExportFailed :: Int,
spansDropped :: Int,
maxQueueDepth :: Int,
exportCalls :: Int,
exportLatencyNs :: LatencySummary,
lastExportError :: Maybe Text
}
deriving stock (Eq, Show)
instance ToJSON LatencySummary where
toJSON summary = object ["p50" .= summary.p50, "p99" .= summary.p99, "max" .= summary.maximum]
instance ToJSON PipelineSnapshot where
toJSON snapshot =
object
[ "spansStarted" .= snapshot.spansStarted,
"spansEnded" .= snapshot.spansEnded,
"spansExportedOk" .= snapshot.spansExportedOk,
"spansExportFailed" .= snapshot.spansExportFailed,
"spansDropped" .= snapshot.spansDropped,
"maxQueueDepth" .= snapshot.maxQueueDepth,
"exportCalls" .= snapshot.exportCalls,
"exportLatencyNs" .= snapshot.exportLatencyNs,
"lastExportError" .= snapshot.lastExportError
]
newPipelineStats :: IO PipelineStats
newPipelineStats = PipelineStats <$> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef [] <*> newIORef 0 <*> newIORef Nothing
countingProcessor :: PipelineStats -> SpanProcessor
countingProcessor stats =
SpanProcessor
{ spanProcessorOnStart = \_ _ -> increment stats.started 1,
spanProcessorOnEnd = \_ -> increment stats.ended 1 >> updateQueueHighWater stats,
spanProcessorShutdown = pure ShutdownSuccess,
spanProcessorForceFlush = pure FlushSuccess
}
instrumentProcessorSuccess :: PipelineStats -> SpanProcessor -> SpanProcessor
instrumentProcessorSuccess stats processor =
processor
{ spanProcessorOnEnd = \spanValue -> processor.spanProcessorOnEnd spanValue >> increment stats.exportedOk 1
}
instrumentExporter :: PipelineStats -> SpanExporter -> SpanExporter
instrumentExporter stats exporter =
exporter
{ spanExporterExport = \batch -> do
let spanCount = sum (fmap Vector.length (HashMap.elems batch))
startedAt <- getMonotonicTimeNSec
increment stats.exportCalls 1
result <-
exporter.spanExporterExport batch
`onException` do
increment stats.exportFailed spanCount
writeIORef stats.lastExportError (Just "exporter raised an exception")
endedAt <- getMonotonicTimeNSec
modifyIORef' stats.exportLatencies (fromIntegral (endedAt - startedAt) :)
case result of
Success -> increment stats.exportedOk spanCount
Failure exception -> do
increment stats.exportFailed spanCount
writeIORef stats.lastExportError (fmap (Text.pack . displayException) exception)
pure result
}
snapshotPipeline :: PipelineStats -> IO PipelineSnapshot
snapshotPipeline stats = do
spansStarted <- readIORef stats.started
spansEnded <- readIORef stats.ended
spansExportedOk <- readIORef stats.exportedOk
spansExportFailed <- readIORef stats.exportFailed
exportCalls <- readIORef stats.exportCalls
latencies <- readIORef stats.exportLatencies
maxQueueDepth <- readIORef stats.maxQueueDepth
lastExportError <- readIORef stats.lastExportError
let spansDropped = max 0 (spansEnded - spansExportedOk - spansExportFailed)
pure
PipelineSnapshot
{ spansStarted,
spansEnded,
spansExportedOk,
spansExportFailed,
spansDropped,
maxQueueDepth,
exportCalls,
exportLatencyNs = summarize latencies,
lastExportError
}
startPipelineSampler :: FilePath -> PipelineStats -> IO (IO [Int])
startPipelineSampler outputDir stats = do
createDirectoryIfMissing True (outputDir </> "series")
samples <- newIORef []
worker <- async do
handle <- openFile (outputDir </> "series" </> "otel-pipeline.csv") WriteMode
hSetBuffering handle LineBuffering
hPutStrLn handle "t_mono_ns,spans_started,spans_ended,exported_ok,export_failed,queue_depth,export_calls"
let loop = do
threadDelay 1_000_000
timestamp <- getMonotonicTimeNSec
started <- readIORef stats.started
ended <- readIORef stats.ended
exportedOk <- readIORef stats.exportedOk
exportFailed <- readIORef stats.exportFailed
exportCalls <- readIORef stats.exportCalls
let queueDepth = max 0 (ended - exportedOk - exportFailed)
modifyIORef' samples (queueDepth :)
hPutStrLn handle (comma [timestamp, fromIntegral started, fromIntegral ended, fromIntegral exportedOk, fromIntegral exportFailed, fromIntegral queueDepth, fromIntegral exportCalls])
loop
loop `finally` hClose handle
pure do
cancel worker
void (waitCatch worker)
reverse <$> readIORef samples
where
comma = foldr1 (\left right -> left <> "," <> right) . fmap show
increment :: IORef Int -> Int -> IO ()
increment ref amount = void (atomicModifyIORef' ref (\value -> let updated = value + amount in (updated, updated)))
updateQueueHighWater :: PipelineStats -> IO ()
updateQueueHighWater stats = do
ended <- readIORef stats.ended
exportedOk <- readIORef stats.exportedOk
exportFailed <- readIORef stats.exportFailed
let queued = max 0 (ended - exportedOk - exportFailed)
atomicModifyIORef' stats.maxQueueDepth (\high -> (max high queued, ()))
summarize :: [Integer] -> LatencySummary
summarize [] = LatencySummary 0 0 0
summarize values =
let ordered = sort values
in LatencySummary (quantile 0.50 ordered) (quantile 0.99 ordered) (last ordered)
quantile :: Double -> [Integer] -> Integer
quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))