kenshou-measure-0.1.0.0: src/Kenshou/Measure/Samples.hs
module Kenshou.Measure.Samples
( ClockKind (..),
SampleHeader (..),
SampleRecord (..),
SampleWriter,
SampleFileReport (..),
openSampleWriter,
writeSampleBlock,
closeSampleWriter,
sampleWriterBackpressure,
readSamples,
)
where
import Control.Concurrent.Async (Async, async, wait)
import Control.Concurrent.MVar
import Control.Concurrent.STM
import Data.Bits
import Data.ByteString (ByteString)
import Data.ByteString qualified as ByteString
import Data.ByteString.Builder
import Data.ByteString.Lazy qualified as LazyByteString
import Data.IORef
import Data.Int (Int64)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as Text
import Data.Word
import Kenshou.Measure.Clock (Origin (..))
import System.IO
data ClockKind = MonotonicClock | WallClock
deriving stock (Eq, Show)
data SampleHeader = SampleHeader
{ origin :: !Origin,
clockKind :: !ClockKind,
operation :: !Text,
processLabel :: !Text
}
deriving stock (Eq, Show)
data SampleRecord = SampleRecord
{ intendedStartNs :: !Word64,
actualStartNs :: !Word64,
endNs :: !Word64,
outcomeCode :: !Word8,
units :: !Word64
}
deriving stock (Eq, Show)
data SampleCommand = WriteBlock !Word16 ![SampleRecord] | Stop !(MVar ())
data SampleWriter = SampleWriter
{ queue :: !(TBQueue SampleCommand),
worker :: !(Async ()),
backpressure :: !(IORef Word64)
}
data SampleFileReport = SampleFileReport
{ header :: !SampleHeader,
recordsRead :: !Word64,
ignoredBytes :: !Word64
}
deriving stock (Eq, Show)
openSampleWriter :: FilePath -> SampleHeader -> IO SampleWriter
openSampleWriter path header = do
handle <- openBinaryFile path WriteMode
ByteString.hPut handle (builderBytes (headerBuilder header))
queue <- newTBQueueIO 256
backpressure <- newIORef 0
worker <- async (writerLoop handle queue)
pure SampleWriter {queue, worker, backpressure}
writeSampleBlock :: SampleWriter -> Word16 -> [SampleRecord] -> IO ()
writeSampleBlock _ _ [] = pure ()
writeSampleBlock writer workerId records = do
wasFull <- atomically do
full <- isFullTBQueue writer.queue
writeTBQueue writer.queue (WriteBlock workerId records)
pure full
whenFull wasFull do
modifyIORef' writer.backpressure (+ 1)
where
whenFull True action = action
whenFull False _ = pure ()
closeSampleWriter :: SampleWriter -> IO ()
closeSampleWriter writer = do
stopped <- newEmptyMVar
atomically (writeTBQueue writer.queue (Stop stopped))
takeMVar stopped
wait writer.worker
sampleWriterBackpressure :: SampleWriter -> IO Word64
sampleWriterBackpressure = readIORef . (.backpressure)
writerLoop :: Handle -> TBQueue SampleCommand -> IO ()
writerLoop handle queue = do
command <- atomically (readTBQueue queue)
case command of
WriteBlock workerId records -> ByteString.hPut handle (encodeBlock workerId records) >> writerLoop handle queue
Stop stopped -> hFlush handle >> hClose handle >> putMVar stopped ()
encodeBlock :: Word16 -> [SampleRecord] -> ByteString
encodeBlock _ [] = ByteString.empty
encodeBlock workerId records@(firstRecord : _) =
builderBytes $
word16LE workerId
<> word32LE (fromIntegral (length records))
<> word32LE (fromIntegral (ByteString.length payload))
<> word64LE first
<> byteString payload
where
first = firstRecord.intendedStartNs
payload = builderBytes (recordsBuilder first records)
recordsBuilder :: Word64 -> [SampleRecord] -> Builder
recordsBuilder _ [] = mempty
recordsBuilder previous (record : rest) =
putVarWord (zigZag (wordDelta record.intendedStartNs previous))
<> putVarWord (record.actualStartNs - min record.actualStartNs record.intendedStartNs)
<> putVarWord (record.endNs - min record.endNs record.actualStartNs)
<> word8 record.outcomeCode
<> putVarWord record.units
<> recordsBuilder record.intendedStartNs rest
readSamples :: FilePath -> (SampleRecord -> IO ()) -> IO SampleFileReport
readSamples path consume = do
bytes <- ByteString.readFile path
case parseHeader bytes of
Left message -> ioError (userError (Text.unpack message))
Right (header, bodyOffset) -> do
(count, ignored) <- readBlocks bytes bodyOffset consume
pure (SampleFileReport header count ignored)
readBlocks :: ByteString -> Int -> (SampleRecord -> IO ()) -> IO (Word64, Word64)
readBlocks bytes = go 0
where
totalLength = ByteString.length bytes
go count offset consume
| offset == totalLength = pure (count, 0)
| offset + 18 > totalLength = pure (count, fromIntegral (totalLength - offset))
| otherwise = case parseBlockHeader bytes offset of
Left _ -> pure (count, fromIntegral (totalLength - offset))
Right (recordCount, payloadBytes, first, payloadOffset)
| payloadOffset + payloadBytes > totalLength -> pure (count, fromIntegral (totalLength - offset))
| otherwise -> case parseRecords bytes payloadOffset payloadBytes recordCount first of
Left _ -> pure (count, fromIntegral (totalLength - offset))
Right records -> mapM_ consume records >> go (count + fromIntegral (length records)) (payloadOffset + payloadBytes) consume
parseBlockHeader :: ByteString -> Int -> Either Text (Int, Int, Word64, Int)
parseBlockHeader bytes offset = do
_workerId <- word16At bytes offset
count <- word32At bytes (offset + 2)
payloadBytes <- word32At bytes (offset + 6)
first <- word64At bytes (offset + 10)
pure (fromIntegral count, fromIntegral payloadBytes, first, offset + 18)
parseRecords :: ByteString -> Int -> Int -> Int -> Word64 -> Either Text [SampleRecord]
parseRecords bytes offset payloadLength count first = do
(records, finalOffset) <- go [] offset first count
ensure (finalOffset == offset + payloadLength) "sample block payload length mismatch"
pure (reverse records)
where
limit = offset + payloadLength
go records current _ 0 = Right (records, current)
go records current previous remaining = do
ensure (current < limit) "truncated sample block"
(delta, afterDelta) <- readVarWord bytes current limit
(lag, afterLag) <- readVarWord bytes afterDelta limit
(service, afterService) <- readVarWord bytes afterLag limit
outcome <- word8AtLimit bytes afterService limit
(units, afterUnits) <- readVarWord bytes (afterService + 1) limit
let intended = addSigned previous (unZigZag delta)
actual = intended + lag
record = SampleRecord intended actual (actual + service) outcome units
go (record : records) afterUnits intended (remaining - 1)
headerBuilder :: SampleHeader -> Builder
headerBuilder header =
byteString "KSMP"
<> word16LE 1
<> word16LE 0
<> word64LE header.origin.monoNs
<> int64LE header.origin.wallUnixNs
<> word8 (case header.clockKind of MonotonicClock -> 0; WallClock -> 1)
<> textBuilder header.operation
<> textBuilder header.processLabel
parseHeader :: ByteString -> Either Text (SampleHeader, Int)
parseHeader bytes = do
ensure (ByteString.take 4 bytes == "KSMP") "invalid sample magic"
version <- word16At bytes 4
ensure (version == 1) "unsupported sample version"
flags <- word16At bytes 6
ensure (flags == 0) "unsupported sample flags"
mono <- word64At bytes 8
wall <- fromIntegral <$> word64At bytes 16
clockByte <- word8At bytes 24
clock <- case clockByte of 0 -> Right MonotonicClock; 1 -> Right WallClock; _ -> Left "unknown sample clock"
(operation, afterOperation) <- textAt bytes 25
(processLabel, bodyOffset) <- textAt bytes afterOperation
pure (SampleHeader (Origin mono wall) clock operation processLabel, bodyOffset)
textBuilder :: Text -> Builder
textBuilder value = word16LE (fromIntegral (ByteString.length encoded)) <> byteString encoded
where
encoded = Text.encodeUtf8 value
textAt :: ByteString -> Int -> Either Text (Text, Int)
textAt bytes offset = do
lengthValue <- word16At bytes offset
let start = offset + 2
end = start + fromIntegral lengthValue
ensure (end <= ByteString.length bytes) "truncated sample text"
case Text.decodeUtf8' (ByteString.take (fromIntegral lengthValue) (ByteString.drop start bytes)) of
Left _ -> Left "invalid UTF-8 in sample header"
Right value -> Right (value, end)
builderBytes :: Builder -> ByteString
builderBytes = LazyByteString.toStrict . toLazyByteString
putVarWord :: Word64 -> Builder
putVarWord value
| value < 0x80 = word8 (fromIntegral value)
| otherwise = word8 (fromIntegral (value .&. 0x7f) .|. 0x80) <> putVarWord (value `shiftR` 7)
readVarWord :: ByteString -> Int -> Int -> Either Text (Word64, Int)
readVarWord bytes = go 0 0
where
go bitOffset accumulator offset limit
| bitOffset >= 64 = Left "varint is too long"
| offset >= limit = Left "truncated varint"
| otherwise =
let byte = ByteString.index bytes offset
value = accumulator .|. (fromIntegral (byte .&. 0x7f) `shiftL` bitOffset)
in if byte .&. 0x80 == 0 then Right (value, offset + 1) else go (bitOffset + 7) value (offset + 1) limit
wordDelta :: Word64 -> Word64 -> Int64
wordDelta current previous = fromIntegral current - fromIntegral previous
addSigned :: Word64 -> Int64 -> Word64
addSigned value delta = fromIntegral (fromIntegral value + delta :: Int64)
zigZag :: Int64 -> Word64
zigZag value = fromIntegral ((value `shiftL` 1) `xor` (value `shiftR` 63))
unZigZag :: Word64 -> Int64
unZigZag value = fromIntegral (value `shiftR` 1) `xor` negate (fromIntegral (value .&. 1))
word8At :: ByteString -> Int -> Either Text Word8
word8At bytes offset = word8AtLimit bytes offset (ByteString.length bytes)
word8AtLimit :: ByteString -> Int -> Int -> Either Text Word8
word8AtLimit bytes offset limit
| offset < limit = Right (ByteString.index bytes offset)
| otherwise = Left "truncated sample file"
word16At :: ByteString -> Int -> Either Text Word16
word16At bytes offset = fromIntegral <$> littleEndianAt bytes offset 2
word32At :: ByteString -> Int -> Either Text Word32
word32At bytes offset = fromIntegral <$> littleEndianAt bytes offset 4
word64At :: ByteString -> Int -> Either Text Word64
word64At bytes offset = littleEndianAt bytes offset 8
littleEndianAt :: ByteString -> Int -> Int -> Either Text Word64
littleEndianAt bytes offset width
| offset + width > ByteString.length bytes = Left "truncated sample file"
| otherwise = Right $ foldr (.|.) 0 [fromIntegral (ByteString.index bytes (offset + index)) `shiftL` (8 * index) | index <- [0 .. width - 1]]
ensure :: Bool -> Text -> Either Text ()
ensure condition message = if condition then Right () else Left message