bytebuild 0.3.12.0 → 0.3.13.0

raw patch · 5 files changed

+142/−6 lines, 5 filesdep ~primitive-unliftedPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: primitive-unlifted

API changes (from Hackage documentation)

Files

• CHANGELOG.md +6/−0
• bytebuild.cabal +5/−4
• src/Data/Bytes/Builder.hs +62/−2
• src/Data/Bytes/Builder/Bounded.hs +50/−0
• test/Main.hs +19/−0

@@ -5,6 +5,12 @@ `small-bytearray-builder` is now just a compatibility shim to ease the migration process. +## 0.3.13.0 -- 2023-05-01++* Add VLQ builders for Word32 and Word64.+* Add chunks-to-builder function+* Add textUtf8 for copying text to builder with text-2.0 and newer+ ## 0.3.12.0 -- 2022-12-01  * Support GHC 9.4.

@@ -1,6 +1,6 @@ cabal-version: 2.2 name: bytebuild-version: 0.3.12.0+version: 0.3.13.0 synopsis: Build byte arrays description:   This is similar to the builder facilities provided by@@ -18,7 +18,7 @@   that runs out of space simply aborts and is rerun at the beginning   of the next chunk. This strategy for building is suitable for most   CSVs and several line protocols (carbon, InfluxDB, etc.).-  + homepage: https://github.com/byteverse/bytebuild bug-reports: https://github.com/byteverse/bytebuild/issues license: BSD-3-Clause@@ -59,6 +59,7 @@     , primitive-unlifted >=0.1.2 && <0.2     , run-st >=0.1 && <0.2     , template-haskell >=2.16+    , text >=1.2 && <2.2     , text-short >=0.1.3 && <0.2     , wide-word >=0.1.0.9 && <0.2     , zigzag@@ -68,10 +69,10 @@     if impl(ghc >= 8.10)       hs-source-dirs: src-9.0   if flag(checked)-    build-depends: primitive-checked >= 0.7 && <0.8+    build-depends: primitive-checked >= 0.7 && <0.9     hs-source-dirs: src-checked   else-    build-depends: primitive >= 0.7 && <0.8+    build-depends: primitive >= 0.7 && <0.9     hs-source-dirs: src-unchecked   ghc-options: -Wall -O2   hs-source-dirs: src

@@ -1,3 +1,4 @@+{-# language CPP #-} {-# language BangPatterns #-} {-# language DataKinds #-} {-# language DuplicateRecordFields #-}@@ -21,12 +22,16 @@   , putManyConsLength     -- * Materialized Byte Sequences   , bytes+  , chunks   , copy   , copyCons   , copy2   , insert   , byteArray   , shortByteString+#if MIN_VERSION_text(2,0,0)+  , textUtf8+#endif   , shortTextUtf8   , shortTextJsonString   , cstring@@ -100,6 +105,10 @@   , wordLEB128   , word32LEB128   , word64LEB128+    -- **** VLQ+  , wordVlq+  , word32Vlq+  , word64Vlq     -- *** Many   , word8Array     -- **** Big Endian@@ -149,7 +158,7 @@ import Data.Bytes.Builder.Unsafe (commitsOntoChunks) import Data.Bytes.Builder.Unsafe (reverseCommitsOntoChunks) import Data.Bytes.Builder.Unsafe (stringUtf8,cstring,fromEffect)-import Data.Bytes.Chunks (Chunks(ChunksNil))+import Data.Bytes.Chunks (Chunks(ChunksCons,ChunksNil)) import Data.Bytes.Types (Bytes(Bytes),MutableBytes(MutableBytes)) import Data.ByteString.Short.Internal (ShortByteString(SBS)) import Data.Char (ord)@@ -162,7 +171,7 @@ import Data.Word.Zigzag (toZigzagNative,toZigzag32,toZigzag64) import Foreign.C.String (CStringLen) import GHC.ByteOrder (ByteOrder(BigEndian,LittleEndian),targetByteOrder)-import GHC.Exts (Addr#,(*#),oneShot)+import GHC.Exts (MutableByteArray#,Addr#,(*#),oneShot) import GHC.Exts (Int(I#),Char(C#),Int#,State#,ByteArray#,(>=#)) import GHC.Exts (RealWorld,(+#),(-#),(<#)) import GHC.Integer.Logarithms.Compat (integerLog2#)@@ -184,6 +193,12 @@ import qualified GHC.Exts as Exts import qualified Op as Op +#if MIN_VERSION_text(2,0,0)+import Data.Text (Text)+import qualified Data.Text.Internal as I+import qualified Data.Text.Array as A+#endif+ -- | Run a builder. run ::      Int -- ^ Size of initial chunk (use 4080 if uncertain)@@ -391,6 +406,29 @@       (# s1, buf0, off0 +# slen#, len0 -# slen#, cs0 #)   ) +-- | Paste byte chunks into a builder.+chunks :: Chunks -> Builder+{-# noinline chunks #-}+chunks xs0 =+  -- Implementation note: It would probably be good to begin with a+  -- goCopying phase before switching to goInserting. If the total+  -- size of the chunks is small, we could end up just copying+  -- everything into the existing buffer, which would be nice.+  -- Note: This function needs a test in the test suite.+  Builder $ \buf0 off0 len0 cs0 s0 -> case xs0 of+    ChunksNil -> (# s0, buf0, off0, len0, cs0 #)+    ChunksCons{} -> goInserting xs0 (Mutable buf0 off0 cs0) s0+  where+  -- Notice that goNoncopying does not take a buffer as an argument. At the+  -- very end, we create a 128-byte buffer with nothing in it and present+  -- that as the new buffer. We *cannot* simply reuse the old buffer with+  -- the length set to zero because commitDistance1 would get confused.+  goInserting :: Chunks -> Commits s -> State# s -> (# State# s, MutableByteArray# s, Int#, Int#, Commits s #)+  goInserting ChunksNil !cs s0 = case Exts.newByteArray# 128# s0 of+    (# s1, buf1 #) -> (# s1, buf1, 0#, 128#, cs #)+  goInserting (ChunksCons (Bytes (ByteArray b) (I# off) (I# len)) ys) !cs s0 =+    goInserting ys (Immutable b off len cs) s0+ -- | Create a builder from a byte sequence. This always results in a -- call to @memcpy@. This is beneficial when the byte sequence is -- known to be small (less than 256 bytes).@@ -794,6 +832,13 @@   let ba = shortTextToByteArray a    in bytes (Bytes ba 0 (PM.sizeofByteArray ba)) +#if MIN_VERSION_text(2,0,0)+-- | Create a builder from text. The text will be UTF-8 encoded.+textUtf8 :: Text -> Builder+textUtf8 (I.Text (A.ByteArray b) off len) =+  bytes (Bytes (ByteArray b) off len)+#endif+ -- | Create a builder from text. The text will be UTF-8 encoded, -- and JSON special characters will be escaped. Additionally, the -- result is surrounded by double quotes. For example:@@ -1175,6 +1220,21 @@ word64LEB128 :: Word64 -> Builder {-# inline word64LEB128 #-} word64LEB128 w = fromBounded Nat.constant (Bounded.word64LEB128 w)++-- | Encode a machine-sized word with VLQ.+wordVlq :: Word -> Builder+{-# inline wordVlq #-}+wordVlq w = fromBounded Nat.constant (Bounded.wordVlq w)++-- | Encode a 32-bit word with VLQ.+word32Vlq :: Word32 -> Builder+{-# inline word32Vlq #-}+word32Vlq w = fromBounded Nat.constant (Bounded.word32Vlq w)++-- | Encode a 64-bit word with VLQ.+word64Vlq :: Word64 -> Builder+{-# inline word64Vlq #-}+word64Vlq w = fromBounded Nat.constant (Bounded.word64Vlq w)  -- | Encode a signed arbitrary-precision integer as decimal. -- This encoding never starts with a zero unless the argument was zero.

@@ -5,6 +5,7 @@ {-# language KindSignatures #-} {-# language LambdaCase #-} {-# language MagicHash #-}+{-# language NumericUnderscores #-} {-# language RankNTypes #-} {-# language ScopedTypeVariables #-} {-# language TypeApplications #-}@@ -97,9 +98,19 @@   , int32LE   , int16LE     -- **** LEB128+    -- | LEB128 encodes an integer in 7-bit units, least significant bits first,+    -- with the high bit of each output byte set to 1 in all bytes except for+    -- the final byte.   , wordLEB128   , word32LEB128   , word64LEB128+    -- **** VLQ+    -- | VLQ (also known as VByte, Varint, VInt) encodes an integer in 7-bit+    -- units, most significant bits first, with the high bit of each output byte+    -- set to 1 in all bytes except for the final byte.+  , wordVlq+  , word32Vlq+  , word64Vlq     -- * Encode Floating-Point Types   , doubleDec   ) where@@ -886,6 +897,27 @@   primitive_ (writeCharArray# arr (off +# 7# ) c7)   pure (I# (off +# 8# )) +-- | Encode a machine-sized word with VLQ (also known as VByte, Varint, VInt).+wordVlq :: Word -> Builder 10+{-# inline wordVlq #-}+wordVlq (W# w) = vlqCommon (W# w)++-- | Encode a 32-bit word with VLQ (also known as VByte, Varint, VInt).+word32Vlq :: Word32 -> Builder 5+{-# inline word32Vlq #-}+word32Vlq (W32# w) = vlqCommon (W# (C.word32ToWord# w))++-- | Encode a 64-bit word with VLQ (also known as VByte, Varint, VInt).+word64Vlq :: Word64 -> Builder 10+{-# inline word64Vlq #-}+word64Vlq (W64# w) = vlqCommon (W#+#if MIN_VERSION_base(4,17,0)+    (word64ToWord# w)+#else+    w+#endif+  )+ -- | Encode a machine-sized word with LEB-128. wordLEB128 :: Word -> Builder 10 {-# inline wordLEB128 #-}@@ -906,6 +938,24 @@     w #endif   )++vlqCommon :: Word -> Builder n+vlqCommon !w = case w of+  0 -> unsafeWord8 0+  _ ->+    let !startIx = 7 * quot (63 - countLeadingZeros w) 7+     in vlqStep startIx w++vlqStep ::+     Int -- start index, must be in range [0,63] and 7 must divide it evenly+  -> Word+  -> Builder n+vlqStep !ix !w+  | ix <= 0 =+      unsafeWord8 (unsafeWordToWord8 (unsafeShiftR w ix .&. 0b0111_1111))+  | otherwise = unsafeAppend+      (unsafeWord8 (unsafeWordToWord8 (unsafeShiftR w ix .|. 0b1000_0000)))+      (vlqStep (ix - 7) w)  lebCommon :: Word -> Builder n lebCommon !w = case quotRem w 128 of

@@ -242,6 +242,10 @@          in runConcat 1 (foldMap word256BE xs)             ===             runConcat 1 (word256ArrayBE ys 0 (Prelude.length xs))+    , TQC.testProperty "word64Vlq" $ \(x :: Word64) ->+        runConcat 1 (word64Vlq x)+        ===+        naiveVlq (fromIntegral x)     , TQC.testProperty "word64LEB128" $ \(x :: Word64) ->         runConcat 1 (word64LEB128 x)         ===@@ -415,4 +419,19 @@         xs' = w : xs      in if q == 0            then L.reverse xs'+          else go xs' q++naiveVlq :: Natural -> ByteArray+naiveVlq x =+  Bytes.toByteArray (Bytes.unsafeDrop 1 (Exts.fromList (0xFF : go [] x)))+  where+  go !xs !n =+    let (q,r) = quotRem n 128+        r' = fromIntegral @Natural @Word8 r+        w = case xs of+          [] -> r'+          _ -> Bits.setBit r' 7+        xs' = w : xs+     in if q == 0 +          then xs'           else go xs' q