lz4-bytes 0.1.0.2 → 0.1.1.0

raw patch · 6 files changed

+229/−32 lines, 6 filesdep +byte-orderdep ~primitivePVP ok

version bump matches the API change (PVP)

Dependencies added: byte-order

Dependency ranges changed: primitive

API changes (from Hackage documentation)

Files

• CHANGELOG.md +6/−0
• cbits/hs_lz4.c +42/−0
• lz4-bytes.cabal +7/−2
• src/Lz4/Block.hs +54/−30
• src/Lz4/Frame.hs +78/−0
• src/Lz4/Internal.hs +42/−0

@@ -1,5 +1,11 @@ # Revision history for lz4-bytes +## 0.1.1.0 -- 2023-07-27++* Add `Lz4.Frame` module for producing LZ4 frames. This is being added+  because the Apache Arrow format uses LZ4 frames as one of its compression+  methods.+ ## 0.1.0.2 -- 2020-03-09  * Add `lz4hc.h` to `extra-source-files`.

@@ -42,3 +42,45 @@     r = (HsInt)LZ4_decompress_safe(src + soff, dst + doff, slen, dstCapacity);     return r;   }++// Started on streaming compression and then realized it has some+// restrictions that make it pretty hard to wrap in Haskell. Read+// the docs for LZ4_compress_fast_continue in lz4.h before trying+// to do this. Basically, if any attempt to do this is ever made to+// do this, the only way forward is to use UnliftedArray ByteArray+// and do the whole thing in one big uninterruptible FFI call since+// nothing is allowed to move. But even then, decompression speed+// will be negatively impacted since you might end up with a bunch+// of small LZ4 chunks. And once you've got more than one LZ4 chunk,+// you have to start thinking about framing too.+//+// HsInt hs_compress_fast_chunks_start+//   ( LZ4_stream_t* ctx+//   , const char* src+//   , HsInt soff+//   , char* dst+//   , HsInt doff+//   , HsInt slen+//   , HsInt dstCapacity+//   , HsInt acceleration+//   ) {+//   int result;+//   result = LZ4_compress_fast_extState(ctx, src+soff, dest+doff, slen, dstCapacity, acceleration);+//   return result;+// }+// +// HsInt hs_compress_fast_chunks_continue+//   ( LZ4_stream_t* ctx+//   , const char* src+//   , HsInt soff+//   , char* dst+//   , HsInt doff+//   , HsInt slen+//   , HsInt dstCapacity+//   , HsInt acceleration+//   ) {+//   int result;+//   result = LZ4_compress_fast_continue(ctx, src+soff, dest+doff, slen, dstCapacity, acceleration);+//   return result;+// }+

@@ -1,10 +1,11 @@ cabal-version: 2.2 name: lz4-bytes-version: 0.1.0.2+version: 0.1.1.0 synopsis: Bindings to LZ4 description:   This library is similar to the @lz4@ library except that it works   with the @Bytes@ type from @byteslice@ rather than @ByteString@.+  Also, this library has some amount of support for LZ4 frames. homepage: https://github.com/byteverse/lz4-bytes bug-reports: https://github.com/byteverse/lz4-bytes/issues license: BSD-3-Clause@@ -20,13 +21,17 @@   cbits/lz4hc.h  library+  other-modules:+    Lz4.Internal   exposed-modules:     Lz4.Block+    Lz4.Frame   build-depends:     , base >=4.12 && <5     , byteslice >=0.1.4 && <0.3-    , primitive >=0.7 && <0.8+    , primitive >=0.7 && <0.10     , run-st >=0.1.1 && <0.2+    , byte-order >=0.1.3   hs-source-dirs: src   default-language: Haskell2010   include-dirs: cbits

@@ -1,18 +1,30 @@+{-# language BangPatterns #-} {-# language BlockArguments #-}-{-# language UnliftedFFITypes #-}+{-# language DeriveAnyClass #-}+{-# language DerivingStrategies #-} {-# language MagicHash #-}-{-# language BangPatterns #-} {-# language UnboxedTuples #-}+{-# language UnliftedFFITypes #-} +-- | Compress a contiguous sequence of bytes into a single LZ4 block.+-- These functions do not perform any framing. module Lz4.Block-  ( compress+  ( -- * Compression+    compress   , compressU   , compressHighly   , compressHighlyU+    -- * Decompression   , decompress   , decompressU+    -- * Unsafe Compression+  , compressInto+    -- * Computing buffer size+  , requiredBufferSize   ) where +import Lz4.Internal (requiredBufferSize,c_hs_compress_HC)+ import Control.Monad.ST (runST) import Control.Monad.ST.Run (runByteArrayST) import Data.Bytes.Types (Bytes(Bytes))@@ -21,6 +33,7 @@ import GHC.IO (unsafeIOToST) import GHC.ST (ST(ST)) +import qualified Control.Exception import qualified Data.Primitive as PM import qualified GHC.Exts as Exts @@ -34,10 +47,10 @@   -> Bytes -- ^ Bytes to compress   -> Bytes compressHighly !lvl (Bytes (ByteArray arr) off len) = runST do-  let maxSz = inlineCompressBound len+  let maxSz = requiredBufferSize len   dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz   actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 0 len maxSz lvl)-  shrinkMutableByteArray dst actualSz+  PM.shrinkMutableByteArray dst actualSz   result <- PM.unsafeFreezeByteArray dst   pure (Bytes result 0 actualSz) @@ -47,10 +60,10 @@   -> Bytes -- ^ Bytes to compress   -> ByteArray compressHighlyU !lvl (Bytes (ByteArray arr) off len) = runST do-  let maxSz = inlineCompressBound len+  let maxSz = requiredBufferSize len   dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz   actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 0 len maxSz lvl)-  shrinkMutableByteArray dst actualSz+  PM.shrinkMutableByteArray dst actualSz   PM.unsafeFreezeByteArray dst  -- | Compress bytes using LZ4.@@ -63,23 +76,45 @@   -> Bytes -- ^ Bytes to compress   -> Bytes compress !lvl (Bytes (ByteArray arr) off len) = runST do-  let maxSz = inlineCompressBound len+  let maxSz = requiredBufferSize len   dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz   actualSz <- unsafeIOToST (c_hs_compress_fast arr off dst# 0 len maxSz lvl)-  shrinkMutableByteArray dst actualSz+  PM.shrinkMutableByteArray dst actualSz   result <- PM.unsafeFreezeByteArray dst   pure (Bytes result 0 actualSz) +-- | Compress bytes using LZ4, pasting the compressed bytes into the+-- mutable byte array at the specified offset.+--+-- Precondition: There must be at least+-- @'requiredBufferSize' (Bytes.length src)@ bytes available starting+-- from the offset in the destination buffer. This is checked, and+-- this function will throw an exception if this invariant is violated.+compressInto ::+     Int -- ^ Acceleration Factor (Use 1 if uncertain)+  -> Bytes -- ^ Bytes to compress+  -> MutableByteArray s -- ^ Destination buffer+  -> Int -- ^ Offset into destination buffer+  -> Int -- ^ Bytes remaining in destination buffer+  -> ST s Int -- ^ Next available offset in destination buffer +compressInto !lvl (Bytes (ByteArray arr) off len) dst@(MutableByteArray dst# ) !doff !dlen = do+  let maxSz = requiredBufferSize len+  if dlen < maxSz+    then unsafeIOToST (Control.Exception.throwIO Lz4BufferTooSmall)+    else do+      actualSz <- unsafeIOToST (c_hs_compress_fast arr off dst# doff len maxSz lvl)+      pure (doff + actualSz)+ -- | Variant of 'compress' with an unsliced result. compressU ::       Int -- ^ Acceleration Factor (Use 1 if uncertain)   -> Bytes -- ^ Bytes to compress   -> ByteArray compressU !lvl (Bytes (ByteArray arr) off len) = runByteArrayST do-  let maxSz = inlineCompressBound len+  let maxSz = requiredBufferSize len   dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz   actualSz <- unsafeIOToST (c_hs_compress_fast arr off dst# 0 len maxSz lvl)-  shrinkMutableByteArray dst actualSz+  PM.shrinkMutableByteArray dst actualSz   PM.unsafeFreezeByteArray dst  -- | Decompress a byte sequence. Fails if the actual decompressed@@ -106,11 +141,6 @@       pure (Just result)     else pure Nothing --- Copied from a macro lz4.h to avoid using FFI for simple--- arithmetic. Make sure this stays in sync with the macro.-inlineCompressBound :: Int -> Int-inlineCompressBound s = s + (div s 255) + 16- foreign import ccall unsafe "hs_compress_fast"   c_hs_compress_fast ::        ByteArray# -- Source@@ -122,17 +152,6 @@     -> Int       -- Acceleration factor     -> IO Int    -- Result length -foreign import ccall unsafe "hs_compress_HC"-  c_hs_compress_HC ::-       ByteArray# -- Source-    -> Int       -- Source offset-    -> MutableByteArray# s -- Destination-    -> Int       -- Destination offset-    -> Int       -- Input size-    -> Int       -- Destination capacity-    -> Int       -- Compression level-    -> IO Int    -- Result length- foreign import ccall unsafe "hs_decompress_safe"   c_hs_decompress_safe ::        ByteArray# -- Source@@ -143,6 +162,11 @@     -> Int       -- Destination capacity     -> IO Int    -- Result length -shrinkMutableByteArray :: MutableByteArray s -> Int -> ST s ()-shrinkMutableByteArray (MutableByteArray x) (Exts.I# i) =-  ST (\s -> (# Exts.shrinkMutableByteArray# x i s, () #) )+data Lz4BufferTooSmall = Lz4BufferTooSmall+  deriving stock (Eq,Show)+  deriving anyclass (Control.Exception.Exception)++-- foreign import capi "lz4.h value sizeof(LZ4_stream_t)" lz4StreamSz :: Int+-- +-- allocateLz4StreamT :: ST s (MutableByteArray s)+-- allocateLz4StreamT = PM.newPinnedByteArray lz4StreamSz

@@ -0,0 +1,78 @@+{-# language BangPatterns #-}+{-# language BinaryLiterals #-}+{-# language BlockArguments #-}+{-# language DeriveAnyClass #-}+{-# language DerivingStrategies #-}+{-# language MagicHash #-}+{-# language MultiWayIf #-}+{-# language NumericUnderscores #-}+{-# language UnboxedTuples #-}+{-# language UnliftedFFITypes #-}++-- | Compress a contiguous sequence of bytes into an LZ4 frame+-- containing a single block.+module Lz4.Frame+  ( -- * Compression+    compressHighlyU+  ) where++import Lz4.Internal (requiredBufferSize,c_hs_compress_HC)++import Control.Monad.ST (runST)+import Control.Monad.ST.Run (runByteArrayST)+import Data.Bytes.Types (Bytes(Bytes))+import Data.Int (Int32)+import Data.Primitive (MutableByteArray(..),ByteArray(..))+import Data.Word (Word8)+import GHC.Exts (ByteArray#,MutableByteArray#)+import GHC.IO (unsafeIOToST)+import GHC.ST (ST(ST))++import qualified Control.Exception+import qualified Data.Primitive as PM+import qualified GHC.Exts as Exts+import qualified Data.Primitive.ByteArray.LittleEndian as LE++-- | Use HC compression to produce a frame with a single block.+-- All optional fields (checksums, content sizes, and dictionary IDs)+-- are omitted.+--+-- Note: Currently, this produces incorrect output when the size of+-- the input to be compressed is greater than 4MiB. The only way+-- to correct this function is to make it not compress large input.+-- This can be done by setting the high bit of the size. This needs+-- to be tested though since it is an uncommon code path.+compressHighlyU ::+     Int -- ^ Compression level (Use 9 if uncertain)+  -> Bytes -- ^ Bytes to compress+  -> ByteArray+compressHighlyU !lvl (Bytes (ByteArray arr) off len) = runST do+  let maxSz = requiredBufferSize len + 15+  dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+  -- -- First 4 bytes: magic identifier+  PM.writeByteArray dst 0 (0x04 :: Word8)+  PM.writeByteArray dst 1 (0x22 :: Word8)+  PM.writeByteArray dst 2 (0x4D :: Word8)+  PM.writeByteArray dst 3 (0x18 :: Word8)+  -- Next 3 bytes: frame descriptor+  PM.writeByteArray dst 4 (0b0110_0000 :: Word8)+  if | len <= 65536 -> do+         PM.writeByteArray dst 5 (0b0100_0000 :: Word8)+         PM.writeByteArray dst 6 (0x82 :: Word8)+     | len <= 262144 -> do+         PM.writeByteArray dst 5 (0b0101_0000 :: Word8)+         PM.writeByteArray dst 6 (0xFB :: Word8)+     | len <= 1048576 -> do+         PM.writeByteArray dst 5 (0b0110_0000 :: Word8)+         PM.writeByteArray dst 6 (0x51 :: Word8)+     | otherwise -> do+         PM.writeByteArray dst 5 (0b0111_0000 :: Word8)+         PM.writeByteArray dst 6 (0x73 :: Word8)+  actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 11 len maxSz lvl)+  LE.writeUnalignedByteArray dst 7 (fromIntegral actualSz :: Int32)+  PM.writeByteArray dst (actualSz + 11) (0x00 :: Word8)+  PM.writeByteArray dst (actualSz + 12) (0x00 :: Word8)+  PM.writeByteArray dst (actualSz + 13) (0x00 :: Word8)+  PM.writeByteArray dst (actualSz + 14) (0x00 :: Word8)+  PM.shrinkMutableByteArray dst (actualSz + 15)+  PM.unsafeFreezeByteArray dst

@@ -0,0 +1,42 @@+{-# language BangPatterns #-}+{-# language BlockArguments #-}+{-# language DeriveAnyClass #-}+{-# language DerivingStrategies #-}+{-# language MagicHash #-}+{-# language UnboxedTuples #-}+{-# language UnliftedFFITypes #-}++-- | Compress a contiguous sequence of bytes into a single LZ4 block.+-- These functions do not perform any framing.+module Lz4.Internal+  ( requiredBufferSize+  , c_hs_compress_HC+  ) where++import Control.Monad.ST (runST)+import Control.Monad.ST.Run (runByteArrayST)+import Data.Bytes.Types (Bytes(Bytes))+import Data.Primitive (MutableByteArray(..),ByteArray(..))+import GHC.Exts (ByteArray#,MutableByteArray#)+import GHC.IO (unsafeIOToST)+import GHC.ST (ST(ST))++import qualified Control.Exception+import qualified Data.Primitive as PM+import qualified GHC.Exts as Exts++-- | Copied from the @LZ4_COMPRESSBOUND@ macro lz4.h to avoid using+-- FFI for simple arithmetic. Make sure this stays in sync with the macro.+requiredBufferSize :: Int -> Int+requiredBufferSize s = s + (div s 255) + 16++foreign import ccall unsafe "hs_compress_HC"+  c_hs_compress_HC ::+       ByteArray# -- Source+    -> Int       -- Source offset+    -> MutableByteArray# s -- Destination+    -> Int       -- Destination offset+    -> Int       -- Input size+    -> Int       -- Destination capacity+    -> Int       -- Compression level+    -> IO Int    -- Result length