hpgsql-0.4.0: src/Hpgsql/Internal/PinnedByteArray.hs
{-# LANGUAGE BinaryLiterals #-}
{-# LANGUAGE CPP #-}
{-# LANGUAGE MagicHash #-}
{-# LANGUAGE UnboxedTuples #-}
{-# LANGUAGE UnliftedFFITypes #-}
-- |
--
-- = Pinned byte arrays
--
-- This is an internal module so the interface can change very often.
-- The author still commits to following Hackage's PVP when changes are made
-- here (and in any other Internal module), but don't expect this to be a stable
-- interface.
--
-- = Why this module?
--
-- Why create our own `PinnedByteArray` type instead of just using `ByteString`?
-- `ByteString` uses a `ForeignPtr` under the hood, so peeking into its bytes for
-- decoding means our field decoders must use `withForeignPtr`, which runs in IO.
-- We don't want IO infecting our decoding code at all, so this required
-- `unsafePerformIO` (or rather its variant `unsafeDupablePerformIO`), which calls
-- `lazy` internally.
--
-- As per the documentation, "The 'lazy' function restrains strictness analysis a little",
-- and also "After strictness analysis has run, calls to 'lazy' are inlined to be the
-- identity function".
--
-- What this meant in practice is that when inspecting row decoders for a type,
-- I could see boxing of intermediary values that was completely unnecessary.
-- When decoding a postgres int4 into a Haskell Int, for example, the
-- GHC Core clearly showed a Word32 being boxed/allocated when all we needed
-- was a short-lived Word32#.
--
-- That's when pinned byte arrays, backed up by `ByteArray#`, come to the rescue:
-- those aren't moved by the GC so they can be decoded without being in IO.
--
-- After I wrote all of this, I realized _maybe_ I could've just moved `withForeignPtr`
-- higher immediately above the call to parsing buffer chunks into rows, then pass down the
-- `Ptr Word8` in a newtype instead of doing this.
-- But not only did this thought come late, `PinnedByteArray` feels right and easier to use
-- as there's no concerns about scoping decoding, and our buffers are short-lived so heap
-- fragmentation is not a concern.
module Hpgsql.Internal.PinnedByteArray
( PinnedByteArray,
LazyPinnedByteArray,
append,
copyStrictSlice,
createPinnedByteArray',
drop,
emptyLazyPBA,
emptyPBA,
fromByteString,
fromChunks,
fromStrict,
lazyLength,
length,
splitAt,
take,
takePgMessageIdentAndLen,
toByteString,
toStrict,
unsafeToUtf8Text,
-- * Binary (de)serializer
ByteStringIdx (..),
decodeInt16BE,
decodeInt32BE,
decodeInt64BE,
decodeWord8,
decodeWord32BE,
decodeWord64BE,
encodeInt32BE,
encodeDouble,
encodeFloat,
encodeInt64BE,
encodeInt16BE,
encodePgBoolean,
decodeDataRow,
)
where
import Control.Monad (when)
import Data.ByteString.Internal (ByteString (..))
import qualified Data.ByteString.Internal as InternalBS
import Data.Int (Int16, Int32, Int64)
import Foreign.C (CInt (..))
import Foreign.Marshal.Utils (copyBytes)
import Foreign.Ptr (plusPtr)
import GHC.Base (Addr#, ByteArray#, Char (..), IO (..), Int (..), MutableByteArray#, RealWorld, byteArrayContents#, compareByteArrays#, indexWord8ArrayAsChar#, indexWord8ArrayAsWord32#, mutableByteArrayContents#, newPinnedByteArray#, unIO, unsafeFreezeByteArray#, (+#))
import GHC.Exts (indexWord8Array#, indexWord8ArrayAsWord16#, indexWord8ArrayAsWord64#)
import GHC.Ptr (Ptr (..))
import System.IO.Unsafe (unsafeDupablePerformIO)
import Prelude hiding (drop, encodeFloat, length, null, splitAt, take)
#if WORDS_BIGENDIAN
import Data.Word (Word16, Word64)
#else
import Data.Word (Word16, Word64, byteSwap16, byteSwap64, Word8, byteSwap32)
#endif
#if MIN_VERSION_base(4,19,0)
import Data.Array.Byte (ByteArray (..))
#else
import Data.Text.Array (Array (..))
#endif
import Data.Bits (Bits (unsafeShiftR))
import Data.Coerce (coerce)
import Data.Text.Internal (Text (..))
import Foreign (Storable (..), withForeignPtr, (.&.))
import GHC.Float (castDoubleToWord64, castFloatToWord32)
import GHC.Word (Word16 (..), Word32 (..), Word64 (..), Word8 (..))
data PinnedByteArray = PinnedByteArray
{ _start :: !Int,
_len :: !Int,
_array :: !ByteArray#
}
instance Eq PinnedByteArray where
PinnedByteArray (I# s1) l1@(I# len) arr1# == PinnedByteArray (I# s2) l2 arr2# =
l1 == l2 && case compareByteArrays# arr1# s1 arr2# s2 len of
0# -> True
_ -> False
instance Show PinnedByteArray where
show _ = "<PinnedByteArray>"
data LazyPinnedByteArray
= -- | PinnedByteArrays are in reverse order for efficient appending
LazyPinnedByteArray !Int ![PinnedByteArray]
append :: LazyPinnedByteArray -> PinnedByteArray -> LazyPinnedByteArray
append (LazyPinnedByteArray l1 pbs1) pba = LazyPinnedByteArray (l1 + length pba) (pba : pbs1 {- Reverse order -})
fromChunks :: [PinnedByteArray] -> LazyPinnedByteArray
fromChunks pbas =
let len = sum $ map length pbas
in LazyPinnedByteArray len (reverse pbas)
{-# NOINLINE emptyPBA #-}
emptyPBA :: PinnedByteArray
emptyPBA = unsafeDupablePerformIO $ createPinnedByteArray 0 (\_ -> pure 0)
emptyLazyPBA :: LazyPinnedByteArray
emptyLazyPBA = fromStrict emptyPBA
createPinnedByteArray :: Int -> (Addr# -> IO CInt) -> IO PinnedByteArray
createPinnedByteArray size f = fst <$> createPinnedByteArray' size withUnit
where
withUnit addr = do
len <- f addr
pure (len, ())
createPinnedByteArray' :: Int -> (Addr# -> IO (CInt, a)) -> IO (PinnedByteArray, a)
createPinnedByteArray' (I# size#) f = IO $ \s0 ->
let !(# newRW, mutArr# :: MutableByteArray# RealWorld #) = newPinnedByteArray# size# s0
!(# newRW', (lenCopied, ret) #) = unIO (f (mutableByteArrayContents# mutArr#)) newRW
!(# finalRW, frozenArr# #) = unsafeFreezeByteArray# mutArr# newRW'
in (# finalRW, (PinnedByteArray 0 (fromIntegral lenCopied) frozenArr#, ret) #)
fromByteString :: ByteString -> PinnedByteArray
fromByteString (BS fptr len) = unsafeDupablePerformIO $ createPinnedByteArray len $ \dst -> withForeignPtr fptr $ \src -> do
copyBytes (Ptr dst) src len
pure $ fromIntegral len
toByteString :: PinnedByteArray -> ByteString
toByteString (PinnedByteArray start len src) = unsafeDupablePerformIO $ InternalBS.create len $ \dst ->
copyBytes dst (Ptr (byteArrayContents# src) `plusPtr` start) len
{-# INLINE unsafeToUtf8Text #-}
-- | Assuming the pinned byte array contains valid UTF8 text, creates
-- returns an instance of `Text` with the same contents (but does make a copy).
unsafeToUtf8Text :: ByteStringIdx -> Int -> PinnedByteArray -> Either String Text
unsafeToUtf8Text idx desiredLen pba@(PinnedByteArray {}) = let !(PinnedByteArray start arrLen arr#) = copyStrictSlice idx.idx desiredLen (fromStrict pba) in if arrLen /= desiredLen then Left "Insufficient bytes in buffer in unsafeToUtf8Text" else Right $ Text (ByteArray arr#) start arrLen
takePgMessageIdentAndLen :: LazyPinnedByteArray -> Maybe (Char, Int32)
takePgMessageIdentAndLen lpba@(LazyPinnedByteArray len _) =
if len >= 5
then
let !(PinnedByteArray (I# start) _ arr#) = copyStrictSlice 0 5 lpba
in Just (C# (indexWord8ArrayAsChar# arr# start), fromIntegral $ fromBigEndian32 $ W32# (indexWord8ArrayAsWord32# arr# (start +# 1#)))
else Nothing
-- | Drops the next `n` bytes.
drop :: Int -> PinnedByteArray -> PinnedByteArray
drop (max 0 -> n) (PinnedByteArray start len arr#) =
if n >= len
then emptyPBA
else
PinnedByteArray (start + n) (len - n) arr#
-- | Takes the first `n` bytes.
take :: Int -> PinnedByteArray -> PinnedByteArray
take (max 0 -> n) (PinnedByteArray start len arr#) =
PinnedByteArray start (min n len) arr#
fromStrict :: PinnedByteArray -> LazyPinnedByteArray
fromStrict pba@(PinnedByteArray _ len _) = LazyPinnedByteArray len [pba]
-- | Copies chunks into a single contiguous 'PinnedByteArray'. Avoids the copy
-- when there's already just a single chunk.
toStrict :: LazyPinnedByteArray -> PinnedByteArray
toStrict (LazyPinnedByteArray _ [pba]) = pba
toStrict lpba@(LazyPinnedByteArray totalLen _) = copyStrictSlice 0 totalLen lpba
-- | Creates a strict PBA from a Lazy one, but just with the first `n`
-- bytes after the first `skip` (or less if they're not all there).
-- The returned `PinnedByteArray` is _always_ pointing to a new copy of
-- the bytes in the supplied `LazyPinnedByteArray` - no reference is kept
-- to the supplied bytes.
copyStrictSlice :: Int -> Int -> LazyPinnedByteArray -> PinnedByteArray
copyStrictSlice (max 0 -> skip) (max 0 -> n') (LazyPinnedByteArray totalLen' (reverse -> chunks)) =
let n = min n' totalLen'
in unsafeDupablePerformIO $ createPinnedByteArray n $ \dst -> do
let go !copied !_ !_ [] = pure copied
go !copied !_ 0 !_ = pure copied
go !offset !skipLeft !nLeft (PinnedByteArray start l arr# : rest) = do
let toSkipSrc = min l skipLeft
toCopy = min nLeft (l - toSkipSrc)
when (toCopy > 0 && toSkipSrc < l) $ copyBytes (Ptr dst `plusPtr` offset) (Ptr (byteArrayContents# arr#) `plusPtr` (start + toSkipSrc)) toCopy
when (toCopy < 0) $ error "toCopy < 0 should be impossible"
go (offset + toCopy) (skipLeft - toSkipSrc) (nLeft - toCopy) rest
fromIntegral <$> go 0 skip n chunks
splitAt :: Int -> PinnedByteArray -> (PinnedByteArray, PinnedByteArray)
splitAt n pba = (take n pba, drop n pba)
length :: PinnedByteArray -> Int
length (PinnedByteArray _ len _) = len
lazyLength :: LazyPinnedByteArray -> Int
lazyLength (LazyPinnedByteArray len _) = len
-- * Binary (de)serializer
-- A replacement for libraries like cereal or binary.
-- In our tests, this is ~6.0% faster than cereal, and it also
-- (or by virtue of) allocates ~13% less memory in some of our benchmarks.
-- And it also means one fewer dependency.
-- The caveat is that this module makes unaligned memory access. For the target
-- CPU architectures of this library, this should be fine.
fromBigEndian32 :: Word32 -> Word32
#if WORDS_BIGENDIAN
fromBigEndian32 = Prelude.id
#else
fromBigEndian32 = byteSwap32
#endif
fromBigEndian64 :: Word64 -> Word64
#if WORDS_BIGENDIAN
fromBigEndian64 = Prelude.id
#else
fromBigEndian64 = byteSwap64
#endif
fromBigEndian16 :: Word16 -> Word16
#if WORDS_BIGENDIAN
fromBigEndian16 = Prelude.id
#else
fromBigEndian16 = byteSwap16
#endif
data WordDecodingAll a where
CWord8 :: WordDecodingAll Word8
CWord16 :: WordDecodingAll Word16
CWord32 :: WordDecodingAll Word32
CWord64 :: WordDecodingAll Word64
{-# INLINE decodeWord #-}
decodeWord :: WordDecodingAll a -> ByteStringIdx -> PinnedByteArray -> (a -> a) -> Either String a
decodeWord wdec (ByteStringIdx boxedIdx@(I# idx)) (PinnedByteArray (I# start) len byArrSharp) endianConvert =
case wdec of
CWord8 -> if len < 1 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W8# (indexWord8Array# byArrSharp (idx +# start))
CWord16 -> if len < 2 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W16# (indexWord8ArrayAsWord16# byArrSharp (idx +# start))
CWord32 -> if len < 4 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W32# (indexWord8ArrayAsWord32# byArrSharp (idx +# start))
CWord64 -> if len < 8 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W64# (indexWord8ArrayAsWord64# byArrSharp (idx +# start))
{-# INLINE unsafeEncodeWord #-}
unsafeEncodeWord :: (Storable a) => a -> (a -> a) -> Int -> ByteString
unsafeEncodeWord n endianConvert len =
InternalBS.unsafeCreate len $ \bufferPtr ->
poke (coerce bufferPtr) $ endianConvert n
newtype ByteStringIdx = ByteStringIdx {idx :: Int}
deriving newtype (Eq, Num, Show)
{-# INLINE decodeInt16BE #-}
decodeInt16BE :: ByteStringIdx -> PinnedByteArray -> Either String Int16
decodeInt16BE idx bs = fromIntegral <$> decodeWord CWord16 idx bs fromBigEndian16
{-# INLINE encodeInt16BE #-}
encodeInt16BE :: Int16 -> ByteString
encodeInt16BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian16 2
{-# INLINE decodeWord8 #-}
decodeWord8 :: ByteStringIdx -> PinnedByteArray -> Either String Word8
decodeWord8 idx bs = decodeWord CWord8 idx bs Prelude.id
{-# INLINE decodeWord32BE #-}
decodeWord32BE :: ByteStringIdx -> PinnedByteArray -> Either String Word32
decodeWord32BE idx bs = decodeWord CWord32 idx bs fromBigEndian32
{-# INLINE decodeWord64BE #-}
decodeWord64BE :: ByteStringIdx -> PinnedByteArray -> Either String Word64
decodeWord64BE idx bs = decodeWord CWord64 idx bs fromBigEndian64
{-# INLINE decodeInt32BE #-}
decodeInt32BE :: ByteStringIdx -> PinnedByteArray -> Either String Int32
decodeInt32BE idx bs = fromIntegral <$> decodeWord CWord32 idx bs fromBigEndian32
{-# INLINE encodeInt32BE #-}
encodeInt32BE :: Int32 -> ByteString
encodeInt32BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian32 4
{-# INLINE decodeInt64BE #-}
decodeInt64BE :: ByteStringIdx -> PinnedByteArray -> Either String Int64
decodeInt64BE idx bs = fromIntegral <$> decodeWord CWord64 idx bs fromBigEndian64
{-# INLINE encodeInt64BE #-}
encodeInt64BE :: Int64 -> ByteString
encodeInt64BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian64 8
{-# INLINE encodeFloat #-}
encodeFloat :: Float -> ByteString
encodeFloat n = unsafeEncodeWord (castFloatToWord32 n) fromBigEndian32 4
{-# INLINE encodeDouble #-}
encodeDouble :: Double -> ByteString
encodeDouble n = unsafeEncodeWord (castDoubleToWord64 n) fromBigEndian64 8
-- TODO: Encode field length together with value for small types.
-- This can also be a performance boost by having fewer bytestrings?
{-# INLINE encodePgBoolean #-}
encodePgBoolean :: Bool -> ByteString
encodePgBoolean v = if v then "\SOH" else "\NUL"
{-# INLINE decodeDataRow #-}
-- | A super specialized decoder to decode a postgres DataRow message
-- more quickly than a naive implementation.
-- Returns the index into the left-unparsed contents of the supplied bytestring.
decodeDataRow :: ByteStringIdx -> PinnedByteArray -> Either String ByteStringIdx
decodeDataRow idx sbs@(PinnedByteArray _ len _) =
-- We have a fast path when rows are at least 8 bytes long (should be the case
-- for all but 0-column query results or bytestring chunks "cut in the middle of the message")
-- by playing with bitwise operations.
-- Whether this is worth keeping is sort of questionable. It's complex
-- (even if I think it's safe and well tested) and reduces runtime of one of
-- our benchmarks by 2% compared to not having it.
case decodeWord CWord64 idx sbs fromBigEndian64 of
Right (w64 :: Word64) ->
-- After fromBigEndian64, the Word64 has bytes in big-endian order:
-- byte 0 (msg type) in MSB, bytes 1-4 (length) next, bytes 5-6 (col count), byte 7 in LSB.
let msgIdentByte64 = w64 .&. 0b11111111_00000000_00000000_00000000_00000000_00000000_00000000_00000000
lenFullMsg = flip unsafeShiftR 24 $ w64 .&. 0b00000000_11111111_11111111_11111111_11111111_00000000_00000000_00000000
letterD :: Word64 = 0b01000100_00000000_00000000_00000000_00000000_00000000_00000000_00000000
in if msgIdentByte64 == letterD
then
toResult (fromIntegral lenFullMsg)
else Left "Not a DataRow (Word64 bits decoding path)"
Left _ ->
-- It is possible the DataRow has length less than 8 bytes, so
-- we still have to try to parse that.
if len >= 5 + idx.idx
then do
msgIdentChar <- decodeWord8 idx sbs
lenFullMsg <- decodeInt32BE (1 + idx) sbs
if msgIdentChar == 68 -- Letter 'D'
then toResult (fromIntegral lenFullMsg)
else Left "Not a DataRow"
else Left "Less than enough bytes to decode a DataRow"
where
toResult lenFullMsg
| len >= 1 + lenFullMsg + idx.idx = Right $ ByteStringIdx $ 1 + lenFullMsg + idx.idx
| otherwise = Left "Less than enough bytes to decode a full DataRow"