packages feed

hpgsql-0.4.0: src/Hpgsql/Internal/PinnedByteArray.hs

{-# LANGUAGE BinaryLiterals #-}
{-# LANGUAGE CPP #-}
{-# LANGUAGE MagicHash #-}
{-# LANGUAGE UnboxedTuples #-}
{-# LANGUAGE UnliftedFFITypes #-}

-- |
--
-- = Pinned byte arrays
--
-- This is an internal module so the interface can change very often.
-- The author still commits to following Hackage's PVP when changes are made
-- here (and in any other Internal module), but don't expect this to be a stable
-- interface.
--
-- = Why this module?
--
-- Why create our own `PinnedByteArray` type instead of just using `ByteString`?
-- `ByteString` uses a `ForeignPtr` under the hood, so peeking into its bytes for
-- decoding means our field decoders must use `withForeignPtr`, which runs in IO.
-- We don't want IO infecting our decoding code at all, so this required
-- `unsafePerformIO` (or rather its variant `unsafeDupablePerformIO`), which calls
-- `lazy` internally.
--
-- As per the documentation, "The 'lazy' function restrains strictness analysis a little",
-- and also "After strictness analysis has run, calls to 'lazy' are inlined to be the
-- identity function".
--
-- What this meant in practice is that when inspecting row decoders for a type,
-- I could see boxing of intermediary values that was completely unnecessary.
-- When decoding a postgres int4 into a Haskell Int, for example, the
-- GHC Core clearly showed a Word32 being boxed/allocated when all we needed
-- was a short-lived Word32#.
--
-- That's when pinned byte arrays, backed up by `ByteArray#`, come to the rescue:
-- those aren't moved by the GC so they can be decoded without being in IO.
--
-- After I wrote all of this, I realized _maybe_ I could've just moved `withForeignPtr`
-- higher immediately above the call to parsing buffer chunks into rows, then pass down the
-- `Ptr Word8` in a newtype instead of doing this.
-- But not only did this thought come late, `PinnedByteArray` feels right and easier to use
-- as there's no concerns about scoping decoding, and our buffers are short-lived so heap
-- fragmentation is not a concern.
module Hpgsql.Internal.PinnedByteArray
  ( PinnedByteArray,
    LazyPinnedByteArray,
    append,
    copyStrictSlice,
    createPinnedByteArray',
    drop,
    emptyLazyPBA,
    emptyPBA,
    fromByteString,
    fromChunks,
    fromStrict,
    lazyLength,
    length,
    splitAt,
    take,
    takePgMessageIdentAndLen,
    toByteString,
    toStrict,
    unsafeToUtf8Text,

    -- * Binary (de)serializer
    ByteStringIdx (..),
    decodeInt16BE,
    decodeInt32BE,
    decodeInt64BE,
    decodeWord8,
    decodeWord32BE,
    decodeWord64BE,
    encodeInt32BE,
    encodeDouble,
    encodeFloat,
    encodeInt64BE,
    encodeInt16BE,
    encodePgBoolean,
    decodeDataRow,
  )
where

import Control.Monad (when)
import Data.ByteString.Internal (ByteString (..))
import qualified Data.ByteString.Internal as InternalBS
import Data.Int (Int16, Int32, Int64)
import Foreign.C (CInt (..))
import Foreign.Marshal.Utils (copyBytes)
import Foreign.Ptr (plusPtr)
import GHC.Base (Addr#, ByteArray#, Char (..), IO (..), Int (..), MutableByteArray#, RealWorld, byteArrayContents#, compareByteArrays#, indexWord8ArrayAsChar#, indexWord8ArrayAsWord32#, mutableByteArrayContents#, newPinnedByteArray#, unIO, unsafeFreezeByteArray#, (+#))
import GHC.Exts (indexWord8Array#, indexWord8ArrayAsWord16#, indexWord8ArrayAsWord64#)
import GHC.Ptr (Ptr (..))
import System.IO.Unsafe (unsafeDupablePerformIO)
import Prelude hiding (drop, encodeFloat, length, null, splitAt, take)
#if WORDS_BIGENDIAN
import Data.Word (Word16, Word64)
#else
import Data.Word (Word16, Word64, byteSwap16, byteSwap64, Word8, byteSwap32)
#endif
#if MIN_VERSION_base(4,19,0)
import Data.Array.Byte (ByteArray (..))
#else
import Data.Text.Array (Array (..))
#endif
import Data.Bits (Bits (unsafeShiftR))
import Data.Coerce (coerce)
import Data.Text.Internal (Text (..))
import Foreign (Storable (..), withForeignPtr, (.&.))
import GHC.Float (castDoubleToWord64, castFloatToWord32)
import GHC.Word (Word16 (..), Word32 (..), Word64 (..), Word8 (..))

data PinnedByteArray = PinnedByteArray
  { _start :: !Int,
    _len :: !Int,
    _array :: !ByteArray#
  }

instance Eq PinnedByteArray where
  PinnedByteArray (I# s1) l1@(I# len) arr1# == PinnedByteArray (I# s2) l2 arr2# =
    l1 == l2 && case compareByteArrays# arr1# s1 arr2# s2 len of
      0# -> True
      _ -> False

instance Show PinnedByteArray where
  show _ = "<PinnedByteArray>"

data LazyPinnedByteArray
  = -- | PinnedByteArrays are in reverse order for efficient appending
    LazyPinnedByteArray !Int ![PinnedByteArray]

append :: LazyPinnedByteArray -> PinnedByteArray -> LazyPinnedByteArray
append (LazyPinnedByteArray l1 pbs1) pba = LazyPinnedByteArray (l1 + length pba) (pba : pbs1 {- Reverse order -})

fromChunks :: [PinnedByteArray] -> LazyPinnedByteArray
fromChunks pbas =
  let len = sum $ map length pbas
   in LazyPinnedByteArray len (reverse pbas)

{-# NOINLINE emptyPBA #-}
emptyPBA :: PinnedByteArray
emptyPBA = unsafeDupablePerformIO $ createPinnedByteArray 0 (\_ -> pure 0)

emptyLazyPBA :: LazyPinnedByteArray
emptyLazyPBA = fromStrict emptyPBA

createPinnedByteArray :: Int -> (Addr# -> IO CInt) -> IO PinnedByteArray
createPinnedByteArray size f = fst <$> createPinnedByteArray' size withUnit
  where
    withUnit addr = do
      len <- f addr
      pure (len, ())

createPinnedByteArray' :: Int -> (Addr# -> IO (CInt, a)) -> IO (PinnedByteArray, a)
createPinnedByteArray' (I# size#) f = IO $ \s0 ->
  let !(# newRW, mutArr# :: MutableByteArray# RealWorld #) = newPinnedByteArray# size# s0
      !(# newRW', (lenCopied, ret) #) = unIO (f (mutableByteArrayContents# mutArr#)) newRW
      !(# finalRW, frozenArr# #) = unsafeFreezeByteArray# mutArr# newRW'
   in (# finalRW, (PinnedByteArray 0 (fromIntegral lenCopied) frozenArr#, ret) #)

fromByteString :: ByteString -> PinnedByteArray
fromByteString (BS fptr len) = unsafeDupablePerformIO $ createPinnedByteArray len $ \dst -> withForeignPtr fptr $ \src -> do
  copyBytes (Ptr dst) src len
  pure $ fromIntegral len

toByteString :: PinnedByteArray -> ByteString
toByteString (PinnedByteArray start len src) = unsafeDupablePerformIO $ InternalBS.create len $ \dst ->
  copyBytes dst (Ptr (byteArrayContents# src) `plusPtr` start) len

{-# INLINE unsafeToUtf8Text #-}

-- | Assuming the pinned byte array contains valid UTF8 text, creates
-- returns an instance of `Text` with the same contents (but does make a copy).
unsafeToUtf8Text :: ByteStringIdx -> Int -> PinnedByteArray -> Either String Text
unsafeToUtf8Text idx desiredLen pba@(PinnedByteArray {}) = let !(PinnedByteArray start arrLen arr#) = copyStrictSlice idx.idx desiredLen (fromStrict pba) in if arrLen /= desiredLen then Left "Insufficient bytes in buffer in unsafeToUtf8Text" else Right $ Text (ByteArray arr#) start arrLen

takePgMessageIdentAndLen :: LazyPinnedByteArray -> Maybe (Char, Int32)
takePgMessageIdentAndLen lpba@(LazyPinnedByteArray len _) =
  if len >= 5
    then
      let !(PinnedByteArray (I# start) _ arr#) = copyStrictSlice 0 5 lpba
       in Just (C# (indexWord8ArrayAsChar# arr# start), fromIntegral $ fromBigEndian32 $ W32# (indexWord8ArrayAsWord32# arr# (start +# 1#)))
    else Nothing

-- | Drops the next `n` bytes.
drop :: Int -> PinnedByteArray -> PinnedByteArray
drop (max 0 -> n) (PinnedByteArray start len arr#) =
  if n >= len
    then emptyPBA
    else
      PinnedByteArray (start + n) (len - n) arr#

-- | Takes the first `n` bytes.
take :: Int -> PinnedByteArray -> PinnedByteArray
take (max 0 -> n) (PinnedByteArray start len arr#) =
  PinnedByteArray start (min n len) arr#

fromStrict :: PinnedByteArray -> LazyPinnedByteArray
fromStrict pba@(PinnedByteArray _ len _) = LazyPinnedByteArray len [pba]

-- | Copies chunks into a single contiguous 'PinnedByteArray'. Avoids the copy
-- when there's already just a single chunk.
toStrict :: LazyPinnedByteArray -> PinnedByteArray
toStrict (LazyPinnedByteArray _ [pba]) = pba
toStrict lpba@(LazyPinnedByteArray totalLen _) = copyStrictSlice 0 totalLen lpba

-- | Creates a strict PBA from a Lazy one, but just with the first `n`
-- bytes after the first `skip` (or less if they're not all there).
-- The returned `PinnedByteArray` is _always_ pointing to a new copy of
-- the bytes in the supplied `LazyPinnedByteArray` - no reference is kept
-- to the supplied bytes.
copyStrictSlice :: Int -> Int -> LazyPinnedByteArray -> PinnedByteArray
copyStrictSlice (max 0 -> skip) (max 0 -> n') (LazyPinnedByteArray totalLen' (reverse -> chunks)) =
  let n = min n' totalLen'
   in unsafeDupablePerformIO $ createPinnedByteArray n $ \dst -> do
        let go !copied !_ !_ [] = pure copied
            go !copied !_ 0 !_ = pure copied
            go !offset !skipLeft !nLeft (PinnedByteArray start l arr# : rest) = do
              let toSkipSrc = min l skipLeft
                  toCopy = min nLeft (l - toSkipSrc)
              when (toCopy > 0 && toSkipSrc < l) $ copyBytes (Ptr dst `plusPtr` offset) (Ptr (byteArrayContents# arr#) `plusPtr` (start + toSkipSrc)) toCopy
              when (toCopy < 0) $ error "toCopy < 0 should be impossible"
              go (offset + toCopy) (skipLeft - toSkipSrc) (nLeft - toCopy) rest
        fromIntegral <$> go 0 skip n chunks

splitAt :: Int -> PinnedByteArray -> (PinnedByteArray, PinnedByteArray)
splitAt n pba = (take n pba, drop n pba)

length :: PinnedByteArray -> Int
length (PinnedByteArray _ len _) = len

lazyLength :: LazyPinnedByteArray -> Int
lazyLength (LazyPinnedByteArray len _) = len

-- * Binary (de)serializer

-- A replacement for libraries like cereal or binary.
-- In our tests, this is ~6.0% faster than cereal, and it also
-- (or by virtue of) allocates ~13% less memory in some of our benchmarks.
-- And it also means one fewer dependency.
-- The caveat is that this module makes unaligned memory access. For the target
-- CPU architectures of this library, this should be fine.

fromBigEndian32 :: Word32 -> Word32
#if WORDS_BIGENDIAN
fromBigEndian32 = Prelude.id
#else
fromBigEndian32 = byteSwap32
#endif

fromBigEndian64 :: Word64 -> Word64
#if WORDS_BIGENDIAN
fromBigEndian64 = Prelude.id
#else
fromBigEndian64 = byteSwap64
#endif

fromBigEndian16 :: Word16 -> Word16
#if WORDS_BIGENDIAN
fromBigEndian16 = Prelude.id
#else
fromBigEndian16 = byteSwap16
#endif

data WordDecodingAll a where
  CWord8 :: WordDecodingAll Word8
  CWord16 :: WordDecodingAll Word16
  CWord32 :: WordDecodingAll Word32
  CWord64 :: WordDecodingAll Word64

{-# INLINE decodeWord #-}
decodeWord :: WordDecodingAll a -> ByteStringIdx -> PinnedByteArray -> (a -> a) -> Either String a
decodeWord wdec (ByteStringIdx boxedIdx@(I# idx)) (PinnedByteArray (I# start) len byArrSharp) endianConvert =
  case wdec of
    CWord8 -> if len < 1 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W8# (indexWord8Array# byArrSharp (idx +# start))
    CWord16 -> if len < 2 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W16# (indexWord8ArrayAsWord16# byArrSharp (idx +# start))
    CWord32 -> if len < 4 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W32# (indexWord8ArrayAsWord32# byArrSharp (idx +# start))
    CWord64 -> if len < 8 + boxedIdx then Left "Less than enough bytes to decode" else Right $ endianConvert $ W64# (indexWord8ArrayAsWord64# byArrSharp (idx +# start))

{-# INLINE unsafeEncodeWord #-}
unsafeEncodeWord :: (Storable a) => a -> (a -> a) -> Int -> ByteString
unsafeEncodeWord n endianConvert len =
  InternalBS.unsafeCreate len $ \bufferPtr ->
    poke (coerce bufferPtr) $ endianConvert n

newtype ByteStringIdx = ByteStringIdx {idx :: Int}
  deriving newtype (Eq, Num, Show)

{-# INLINE decodeInt16BE #-}
decodeInt16BE :: ByteStringIdx -> PinnedByteArray -> Either String Int16
decodeInt16BE idx bs = fromIntegral <$> decodeWord CWord16 idx bs fromBigEndian16

{-# INLINE encodeInt16BE #-}
encodeInt16BE :: Int16 -> ByteString
encodeInt16BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian16 2

{-# INLINE decodeWord8 #-}
decodeWord8 :: ByteStringIdx -> PinnedByteArray -> Either String Word8
decodeWord8 idx bs = decodeWord CWord8 idx bs Prelude.id

{-# INLINE decodeWord32BE #-}
decodeWord32BE :: ByteStringIdx -> PinnedByteArray -> Either String Word32
decodeWord32BE idx bs = decodeWord CWord32 idx bs fromBigEndian32

{-# INLINE decodeWord64BE #-}
decodeWord64BE :: ByteStringIdx -> PinnedByteArray -> Either String Word64
decodeWord64BE idx bs = decodeWord CWord64 idx bs fromBigEndian64

{-# INLINE decodeInt32BE #-}
decodeInt32BE :: ByteStringIdx -> PinnedByteArray -> Either String Int32
decodeInt32BE idx bs = fromIntegral <$> decodeWord CWord32 idx bs fromBigEndian32

{-# INLINE encodeInt32BE #-}
encodeInt32BE :: Int32 -> ByteString
encodeInt32BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian32 4

{-# INLINE decodeInt64BE #-}
decodeInt64BE :: ByteStringIdx -> PinnedByteArray -> Either String Int64
decodeInt64BE idx bs = fromIntegral <$> decodeWord CWord64 idx bs fromBigEndian64

{-# INLINE encodeInt64BE #-}
encodeInt64BE :: Int64 -> ByteString
encodeInt64BE n = unsafeEncodeWord (fromIntegral n) fromBigEndian64 8

{-# INLINE encodeFloat #-}
encodeFloat :: Float -> ByteString
encodeFloat n = unsafeEncodeWord (castFloatToWord32 n) fromBigEndian32 4

{-# INLINE encodeDouble #-}
encodeDouble :: Double -> ByteString
encodeDouble n = unsafeEncodeWord (castDoubleToWord64 n) fromBigEndian64 8

-- TODO: Encode field length together with value for small types.
-- This can also be a performance boost by having fewer bytestrings?
{-# INLINE encodePgBoolean #-}
encodePgBoolean :: Bool -> ByteString
encodePgBoolean v = if v then "\SOH" else "\NUL"

{-# INLINE decodeDataRow #-}

-- | A super specialized decoder to decode a postgres DataRow message
-- more quickly than a naive implementation.
-- Returns the index into the left-unparsed contents of the supplied bytestring.
decodeDataRow :: ByteStringIdx -> PinnedByteArray -> Either String ByteStringIdx
decodeDataRow idx sbs@(PinnedByteArray _ len _) =
  -- We have a fast path when rows are at least 8 bytes long (should be the case
  -- for all but 0-column query results or bytestring chunks "cut in the middle of the message")
  -- by playing with bitwise operations.
  -- Whether this is worth keeping is sort of questionable. It's complex
  -- (even if I think it's safe and well tested) and reduces runtime of one of
  -- our benchmarks by 2% compared to not having it.
  case decodeWord CWord64 idx sbs fromBigEndian64 of
    Right (w64 :: Word64) ->
      -- After fromBigEndian64, the Word64 has bytes in big-endian order:
      -- byte 0 (msg type) in MSB, bytes 1-4 (length) next, bytes 5-6 (col count), byte 7 in LSB.
      let msgIdentByte64 = w64 .&. 0b11111111_00000000_00000000_00000000_00000000_00000000_00000000_00000000
          lenFullMsg = flip unsafeShiftR 24 $ w64 .&. 0b00000000_11111111_11111111_11111111_11111111_00000000_00000000_00000000
          letterD :: Word64 = 0b01000100_00000000_00000000_00000000_00000000_00000000_00000000_00000000
       in if msgIdentByte64 == letterD
            then
              toResult (fromIntegral lenFullMsg)
            else Left "Not a DataRow (Word64 bits decoding path)"
    Left _ ->
      -- It is possible the DataRow has length less than 8 bytes, so
      -- we still have to try to parse that.
      if len >= 5 + idx.idx
        then do
          msgIdentChar <- decodeWord8 idx sbs
          lenFullMsg <- decodeInt32BE (1 + idx) sbs
          if msgIdentChar == 68 -- Letter 'D'
            then toResult (fromIntegral lenFullMsg)
            else Left "Not a DataRow"
        else Left "Less than enough bytes to decode a DataRow"
  where
    toResult lenFullMsg
      | len >= 1 + lenFullMsg + idx.idx = Right $ ByteStringIdx $ 1 + lenFullMsg + idx.idx
      | otherwise = Left "Less than enough bytes to decode a full DataRow"