lz4-bytes 0.1.1.0 → 0.1.1.1
raw patch · 8 files changed
+238/−204 lines, 8 filesdep ~byte-ordersetup-changednew-uploaderPVP ok
version bump matches the API change (PVP)
Dependency ranges changed: byte-order
API changes (from Hackage documentation)
Files
- CHANGELOG.md +4/−0
- Setup.hs +0/−2
- cbits/hs_lz4.c +1/−2
- lz4-bytes.cabal +39/−29
- src/Lz4/Block.hs +107/−81
- src/Lz4/Frame.hs +46/−43
- src/Lz4/Internal.hs +22/−23
- test/Main.hs +19/−24
CHANGELOG.md view
@@ -1,5 +1,9 @@ # Revision history for lz4-bytes +## 0.1.1.1 -- 2024-01-31++* Update package metadata.+ ## 0.1.1.0 -- 2023-07-27 * Add `Lz4.Frame` module for producing LZ4 frames. This is being added
− Setup.hs
@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain
cbits/hs_lz4.c view
@@ -68,7 +68,7 @@ // result = LZ4_compress_fast_extState(ctx, src+soff, dest+doff, slen, dstCapacity, acceleration); // return result; // }-// +// // HsInt hs_compress_fast_chunks_continue // ( LZ4_stream_t* ctx // , const char* src@@ -83,4 +83,3 @@ // result = LZ4_compress_fast_continue(ctx, src+soff, dest+doff, slen, dstCapacity, acceleration); // return result; // }-
lz4-bytes.cabal view
@@ -1,52 +1,62 @@-cabal-version: 2.2-name: lz4-bytes-version: 0.1.1.0-synopsis: Bindings to LZ4+cabal-version: 2.2+name: lz4-bytes+version: 0.1.1.1+synopsis: Bindings to LZ4 description: This library is similar to the @lz4@ library except that it works with the @Bytes@ type from @byteslice@ rather than @ByteString@. Also, this library has some amount of support for LZ4 frames.-homepage: https://github.com/byteverse/lz4-bytes-bug-reports: https://github.com/byteverse/lz4-bytes/issues-license: BSD-3-Clause-license-file: LICENSE-author: Andrew Martin-maintainer: andrew.thaddeus@gmail.com-copyright: 2020 Andrew Martin-category: Data-build-type: Simple++homepage: https://github.com/byteverse/lz4-bytes+bug-reports: https://github.com/byteverse/lz4-bytes/issues+license: BSD-3-Clause+license-file: LICENSE+author: Andrew Martin+maintainer: amartin@layer3com.com+copyright: 2020 Andrew Martin+category: Data+build-type: Simple extra-source-files:- CHANGELOG.md cbits/lz4.h cbits/lz4hc.h +extra-doc-files: CHANGELOG.md+ library- other-modules:- Lz4.Internal+ other-modules: Lz4.Internal exposed-modules: Lz4.Block Lz4.Frame+ build-depends:- , base >=4.12 && <5- , byteslice >=0.1.4 && <0.3- , primitive >=0.7 && <0.10- , run-st >=0.1.1 && <0.2- , byte-order >=0.1.3- hs-source-dirs: src+ , base >=4.12 && <5+ , byte-order >=0.1.3 && <0.2+ , byteslice >=0.1.4 && <0.3+ , primitive >=0.7 && <0.10+ , run-st >=0.1.1 && <0.2++ hs-source-dirs: src default-language: Haskell2010- include-dirs: cbits- c-sources: cbits/lz4.c cbits/lz4hc.c cbits/hs_lz4.c+ include-dirs: cbits+ c-sources:+ cbits/hs_lz4.c+ cbits/lz4.c+ cbits/lz4hc.c test-suite test default-language: Haskell2010- type: exitcode-stdio-1.0- hs-source-dirs: test- main-is: Main.hs- ghc-options: -Wall -O2+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ ghc-options: -Wall -O2 build-depends:- , base >=4.11.1 && <5+ , base >=4.11.1 && <5 , byteslice , lz4-bytes , primitive , tasty , tasty-quickcheck++source-repository head+ type: git+ location: git://github.com/byteverse/lz4-bytes.git
src/Lz4/Block.hs view
@@ -1,54 +1,61 @@-{-# language BangPatterns #-}-{-# language BlockArguments #-}-{-# language DeriveAnyClass #-}-{-# language DerivingStrategies #-}-{-# language MagicHash #-}-{-# language UnboxedTuples #-}-{-# language UnliftedFFITypes #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DerivingStrategies #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnboxedTuples #-}+{-# LANGUAGE UnliftedFFITypes #-} --- | Compress a contiguous sequence of bytes into a single LZ4 block.--- These functions do not perform any framing.+{- | Compress a contiguous sequence of bytes into a single LZ4 block.+These functions do not perform any framing.+-} module Lz4.Block ( -- * Compression compress , compressU , compressHighly , compressHighlyU+ -- * Decompression , decompress , decompressU+ -- * Unsafe Compression , compressInto+ -- * Computing buffer size , requiredBufferSize ) where -import Lz4.Internal (requiredBufferSize,c_hs_compress_HC)+import Lz4.Internal (c_hs_compress_HC, requiredBufferSize) import Control.Monad.ST (runST) import Control.Monad.ST.Run (runByteArrayST)-import Data.Bytes.Types (Bytes(Bytes))-import Data.Primitive (MutableByteArray(..),ByteArray(..))-import GHC.Exts (ByteArray#,MutableByteArray#)+import Data.Bytes.Types (Bytes (Bytes))+import Data.Primitive (ByteArray (..), MutableByteArray (..))+import GHC.Exts (ByteArray#, MutableByteArray#) import GHC.IO (unsafeIOToST)-import GHC.ST (ST(ST))+import GHC.ST (ST (ST)) import qualified Control.Exception import qualified Data.Primitive as PM import qualified GHC.Exts as Exts --- | Compress bytes using LZ4's HC algorithm. This is slower--- than 'compress' but provides better compression. A higher--- compression level increases compression but decreases speed.--- This function has undefined behavior on byte sequences larger--- than 2,113,929,216 bytes. This calls @LZ4_compress_HC@.+{- | Compress bytes using LZ4's HC algorithm. This is slower+than 'compress' but provides better compression. A higher+compression level increases compression but decreases speed.+This function has undefined behavior on byte sequences larger+than 2,113,929,216 bytes. This calls @LZ4_compress_HC@.+-} compressHighly ::- Int -- ^ Compression level (Use 9 if uncertain)- -> Bytes -- ^ Bytes to compress- -> Bytes+ -- | Compression level (Use 9 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ Bytes compressHighly !lvl (Bytes (ByteArray arr) off len) = runST do let maxSz = requiredBufferSize len- dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+ dst@(MutableByteArray dst#) <- PM.newByteArray maxSz actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 0 len maxSz lvl) PM.shrinkMutableByteArray dst actualSz result <- PM.unsafeFreezeByteArray dst@@ -56,48 +63,60 @@ -- | Variant of 'compressHighly' with an unsliced result. compressHighlyU ::- Int -- ^ Compression level (Use 9 if uncertain)- -> Bytes -- ^ Bytes to compress- -> ByteArray+ -- | Compression level (Use 9 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ ByteArray compressHighlyU !lvl (Bytes (ByteArray arr) off len) = runST do let maxSz = requiredBufferSize len- dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+ dst@(MutableByteArray dst#) <- PM.newByteArray maxSz actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 0 len maxSz lvl) PM.shrinkMutableByteArray dst actualSz PM.unsafeFreezeByteArray dst --- | Compress bytes using LZ4.--- A higher acceleration factor increases speed but decreases--- compression. This function has undefined--- behavior on byte sequences larger than 2,113,929,216 bytes.--- This calls @LZ4_compress_default@.+{- | Compress bytes using LZ4.+A higher acceleration factor increases speed but decreases+compression. This function has undefined+behavior on byte sequences larger than 2,113,929,216 bytes.+This calls @LZ4_compress_default@.+-} compress ::- Int -- ^ Acceleration Factor (Use 1 if uncertain)- -> Bytes -- ^ Bytes to compress- -> Bytes+ -- | Acceleration Factor (Use 1 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ Bytes compress !lvl (Bytes (ByteArray arr) off len) = runST do let maxSz = requiredBufferSize len- dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+ dst@(MutableByteArray dst#) <- PM.newByteArray maxSz actualSz <- unsafeIOToST (c_hs_compress_fast arr off dst# 0 len maxSz lvl) PM.shrinkMutableByteArray dst actualSz result <- PM.unsafeFreezeByteArray dst pure (Bytes result 0 actualSz) --- | Compress bytes using LZ4, pasting the compressed bytes into the--- mutable byte array at the specified offset.------ Precondition: There must be at least--- @'requiredBufferSize' (Bytes.length src)@ bytes available starting--- from the offset in the destination buffer. This is checked, and--- this function will throw an exception if this invariant is violated.+{- | Compress bytes using LZ4, pasting the compressed bytes into the+mutable byte array at the specified offset.++Precondition: There must be at least+@'requiredBufferSize' (Bytes.length src)@ bytes available starting+from the offset in the destination buffer. This is checked, and+this function will throw an exception if this invariant is violated.+-} compressInto ::- Int -- ^ Acceleration Factor (Use 1 if uncertain)- -> Bytes -- ^ Bytes to compress- -> MutableByteArray s -- ^ Destination buffer- -> Int -- ^ Offset into destination buffer- -> Int -- ^ Bytes remaining in destination buffer- -> ST s Int -- ^ Next available offset in destination buffer -compressInto !lvl (Bytes (ByteArray arr) off len) dst@(MutableByteArray dst# ) !doff !dlen = do+ -- | Acceleration Factor (Use 1 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ -- | Destination buffer+ MutableByteArray s ->+ -- | Offset into destination buffer+ Int ->+ -- | Bytes remaining in destination buffer+ Int ->+ -- | Next available offset in destination buffer+ ST s Int+compressInto !lvl (Bytes (ByteArray arr) off len) dst@(MutableByteArray dst#) !doff !dlen = do let maxSz = requiredBufferSize len if dlen < maxSz then unsafeIOToST (Control.Exception.throwIO Lz4BufferTooSmall)@@ -106,34 +125,41 @@ pure (doff + actualSz) -- | Variant of 'compress' with an unsliced result.-compressU :: - Int -- ^ Acceleration Factor (Use 1 if uncertain)- -> Bytes -- ^ Bytes to compress- -> ByteArray+compressU ::+ -- | Acceleration Factor (Use 1 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ ByteArray compressU !lvl (Bytes (ByteArray arr) off len) = runByteArrayST do let maxSz = requiredBufferSize len- dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+ dst@(MutableByteArray dst#) <- PM.newByteArray maxSz actualSz <- unsafeIOToST (c_hs_compress_fast arr off dst# 0 len maxSz lvl) PM.shrinkMutableByteArray dst actualSz PM.unsafeFreezeByteArray dst --- | Decompress a byte sequence. Fails if the actual decompressed--- result does not match the given expected length.+{- | Decompress a byte sequence. Fails if the actual decompressed+result does not match the given expected length.+-} decompress ::- Int -- ^ Expected length of decompressed bytes- -> Bytes -- ^ Compressed bytes- -> Maybe Bytes+ -- | Expected length of decompressed bytes+ Int ->+ -- | Compressed bytes+ Bytes ->+ Maybe Bytes decompress !dstSz !b = case decompressU dstSz b of Nothing -> Nothing Just r -> Just (Bytes r 0 dstSz) -- | Variant of 'decompress' with an unsliced result. decompressU ::- Int -- ^ Expected length of decompressed bytes- -> Bytes -- ^ Compressed bytes- -> Maybe ByteArray+ -- | Expected length of decompressed bytes+ Int ->+ -- | Compressed bytes+ Bytes ->+ Maybe ByteArray decompressU dstSz (Bytes (ByteArray arr) off len) = runST do- dst@(MutableByteArray dst# ) <- PM.newByteArray dstSz+ dst@(MutableByteArray dst#) <- PM.newByteArray dstSz actualSz <- unsafeIOToST (c_hs_decompress_safe arr off dst# 0 len dstSz) if actualSz == dstSz then do@@ -143,30 +169,30 @@ foreign import ccall unsafe "hs_compress_fast" c_hs_compress_fast ::- ByteArray# -- Source- -> Int -- Source offset- -> MutableByteArray# s -- Destination- -> Int -- Destination offset- -> Int -- Input size- -> Int -- Destination capacity- -> Int -- Acceleration factor- -> IO Int -- Result length+ ByteArray# -> -- Source+ Int -> -- Source offset+ MutableByteArray# s -> -- Destination+ Int -> -- Destination offset+ Int -> -- Input size+ Int -> -- Destination capacity+ Int -> -- Acceleration factor+ IO Int -- Result length foreign import ccall unsafe "hs_decompress_safe" c_hs_decompress_safe ::- ByteArray# -- Source- -> Int -- Source offset- -> MutableByteArray# s -- Destination- -> Int -- Destination offset- -> Int -- Input size- -> Int -- Destination capacity- -> IO Int -- Result length+ ByteArray# -> -- Source+ Int -> -- Source offset+ MutableByteArray# s -> -- Destination+ Int -> -- Destination offset+ Int -> -- Input size+ Int -> -- Destination capacity+ IO Int -- Result length data Lz4BufferTooSmall = Lz4BufferTooSmall- deriving stock (Eq,Show)+ deriving stock (Eq, Show) deriving anyclass (Control.Exception.Exception) -- foreign import capi "lz4.h value sizeof(LZ4_stream_t)" lz4StreamSz :: Int--- +-- -- allocateLz4StreamT :: ST s (MutableByteArray s) -- allocateLz4StreamT = PM.newPinnedByteArray lz4StreamSz
src/Lz4/Frame.hs view
@@ -1,54 +1,56 @@-{-# language BangPatterns #-}-{-# language BinaryLiterals #-}-{-# language BlockArguments #-}-{-# language DeriveAnyClass #-}-{-# language DerivingStrategies #-}-{-# language MagicHash #-}-{-# language MultiWayIf #-}-{-# language NumericUnderscores #-}-{-# language UnboxedTuples #-}-{-# language UnliftedFFITypes #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE BinaryLiterals #-}+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DerivingStrategies #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE MultiWayIf #-}+{-# LANGUAGE NumericUnderscores #-}+{-# LANGUAGE UnliftedFFITypes #-} --- | Compress a contiguous sequence of bytes into an LZ4 frame--- containing a single block.+{- | Compress a contiguous sequence of bytes into an LZ4 frame+containing a single block.+-} module Lz4.Frame ( -- * Compression compressHighlyU ) where -import Lz4.Internal (requiredBufferSize,c_hs_compress_HC)+import Lz4.Internal (c_hs_compress_HC, requiredBufferSize) import Control.Monad.ST (runST) import Control.Monad.ST.Run (runByteArrayST)-import Data.Bytes.Types (Bytes(Bytes))+import Data.Bytes.Types (Bytes (Bytes)) import Data.Int (Int32)-import Data.Primitive (MutableByteArray(..),ByteArray(..))+import Data.Primitive (ByteArray (..), MutableByteArray (..)) import Data.Word (Word8)-import GHC.Exts (ByteArray#,MutableByteArray#)+import GHC.Exts (ByteArray#, MutableByteArray#) import GHC.IO (unsafeIOToST)-import GHC.ST (ST(ST))+import GHC.ST (ST (ST)) import qualified Control.Exception import qualified Data.Primitive as PM-import qualified GHC.Exts as Exts import qualified Data.Primitive.ByteArray.LittleEndian as LE+import qualified GHC.Exts as Exts --- | Use HC compression to produce a frame with a single block.--- All optional fields (checksums, content sizes, and dictionary IDs)--- are omitted.------ Note: Currently, this produces incorrect output when the size of--- the input to be compressed is greater than 4MiB. The only way--- to correct this function is to make it not compress large input.--- This can be done by setting the high bit of the size. This needs--- to be tested though since it is an uncommon code path.+{- | Use HC compression to produce a frame with a single block.+All optional fields (checksums, content sizes, and dictionary IDs)+are omitted.++Note: Currently, this produces incorrect output when the size of+the input to be compressed is greater than 4MiB. The only way+to correct this function is to make it not compress large input.+This can be done by setting the high bit of the size. This needs+to be tested though since it is an uncommon code path.+-} compressHighlyU ::- Int -- ^ Compression level (Use 9 if uncertain)- -> Bytes -- ^ Bytes to compress- -> ByteArray+ -- | Compression level (Use 9 if uncertain)+ Int ->+ -- | Bytes to compress+ Bytes ->+ ByteArray compressHighlyU !lvl (Bytes (ByteArray arr) off len) = runST do let maxSz = requiredBufferSize len + 15- dst@(MutableByteArray dst# ) <- PM.newByteArray maxSz+ dst@(MutableByteArray dst#) <- PM.newByteArray maxSz -- -- First 4 bytes: magic identifier PM.writeByteArray dst 0 (0x04 :: Word8) PM.writeByteArray dst 1 (0x22 :: Word8)@@ -56,18 +58,19 @@ PM.writeByteArray dst 3 (0x18 :: Word8) -- Next 3 bytes: frame descriptor PM.writeByteArray dst 4 (0b0110_0000 :: Word8)- if | len <= 65536 -> do- PM.writeByteArray dst 5 (0b0100_0000 :: Word8)- PM.writeByteArray dst 6 (0x82 :: Word8)- | len <= 262144 -> do- PM.writeByteArray dst 5 (0b0101_0000 :: Word8)- PM.writeByteArray dst 6 (0xFB :: Word8)- | len <= 1048576 -> do- PM.writeByteArray dst 5 (0b0110_0000 :: Word8)- PM.writeByteArray dst 6 (0x51 :: Word8)- | otherwise -> do- PM.writeByteArray dst 5 (0b0111_0000 :: Word8)- PM.writeByteArray dst 6 (0x73 :: Word8)+ if+ | len <= 65_536 -> do+ PM.writeByteArray dst 5 (0b0100_0000 :: Word8)+ PM.writeByteArray dst 6 (0x82 :: Word8)+ | len <= 262_144 -> do+ PM.writeByteArray dst 5 (0b0101_0000 :: Word8)+ PM.writeByteArray dst 6 (0xFB :: Word8)+ | len <= 1_048_576 -> do+ PM.writeByteArray dst 5 (0b0110_0000 :: Word8)+ PM.writeByteArray dst 6 (0x51 :: Word8)+ | otherwise -> do+ PM.writeByteArray dst 5 (0b0111_0000 :: Word8)+ PM.writeByteArray dst 6 (0x73 :: Word8) actualSz <- unsafeIOToST (c_hs_compress_HC arr off dst# 11 len maxSz lvl) LE.writeUnalignedByteArray dst 7 (fromIntegral actualSz :: Int32) PM.writeByteArray dst (actualSz + 11) (0x00 :: Word8)
src/Lz4/Internal.hs view
@@ -1,13 +1,11 @@-{-# language BangPatterns #-}-{-# language BlockArguments #-}-{-# language DeriveAnyClass #-}-{-# language DerivingStrategies #-}-{-# language MagicHash #-}-{-# language UnboxedTuples #-}-{-# language UnliftedFFITypes #-}+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DerivingStrategies #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-} --- | Compress a contiguous sequence of bytes into a single LZ4 block.--- These functions do not perform any framing.+{- | Compress a contiguous sequence of bytes into a single LZ4 block.+These functions do not perform any framing.+-} module Lz4.Internal ( requiredBufferSize , c_hs_compress_HC@@ -15,28 +13,29 @@ import Control.Monad.ST (runST) import Control.Monad.ST.Run (runByteArrayST)-import Data.Bytes.Types (Bytes(Bytes))-import Data.Primitive (MutableByteArray(..),ByteArray(..))-import GHC.Exts (ByteArray#,MutableByteArray#)+import Data.Bytes.Types (Bytes (Bytes))+import Data.Primitive (ByteArray (..), MutableByteArray (..))+import GHC.Exts (ByteArray#, MutableByteArray#) import GHC.IO (unsafeIOToST)-import GHC.ST (ST(ST))+import GHC.ST (ST (ST)) import qualified Control.Exception import qualified Data.Primitive as PM import qualified GHC.Exts as Exts --- | Copied from the @LZ4_COMPRESSBOUND@ macro lz4.h to avoid using--- FFI for simple arithmetic. Make sure this stays in sync with the macro.+{- | Copied from the @LZ4_COMPRESSBOUND@ macro lz4.h to avoid using+FFI for simple arithmetic. Make sure this stays in sync with the macro.+-} requiredBufferSize :: Int -> Int requiredBufferSize s = s + (div s 255) + 16 foreign import ccall unsafe "hs_compress_HC" c_hs_compress_HC ::- ByteArray# -- Source- -> Int -- Source offset- -> MutableByteArray# s -- Destination- -> Int -- Destination offset- -> Int -- Input size- -> Int -- Destination capacity- -> Int -- Compression level- -> IO Int -- Result length+ ByteArray# -> -- Source+ Int -> -- Source offset+ MutableByteArray# s -> -- Destination+ Int -> -- Destination offset+ Int -> -- Input size+ Int -> -- Destination capacity+ Int -> -- Compression level+ IO Int -- Result length
test/Main.hs view
@@ -1,16 +1,11 @@-{-# language BangPatterns #-}-{-# language MultiWayIf #-}-{-# language NumDecimals #-}-{-# language OverloadedStrings #-}-{-# language ScopedTypeVariables #-}-{-# language TypeApplications #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-} -import Lz4.Block (compress,compressHighly,decompress) import Data.Bytes (Bytes) import Data.Word (Word8)-import Test.Tasty (defaultMain,testGroup,TestTree)-import Test.Tasty.QuickCheck ((===),testProperty,Gen)-import Test.Tasty.QuickCheck (choose,vectorOf,forAll)+import Lz4.Block (compress, compressHighly, decompress)+import Test.Tasty (TestTree, defaultMain, testGroup)+import Test.Tasty.QuickCheck (Gen, choose, forAll, testProperty, vectorOf, (===)) import qualified Data.Bytes as Bytes import qualified GHC.Exts as Exts@@ -19,21 +14,21 @@ main = defaultMain tests tests :: TestTree-tests = testGroup "lz4"- [ testProperty "roundtrip" $ forAll genBytes $ \bs ->- let cs = compress 1 bs in- Just bs- ===- decompress (Bytes.length bs) cs- , testProperty "roundtrip-HC" $ forAll genBytes $ \bs ->- let cs = compressHighly 3 bs in- Just bs- ===- decompress (Bytes.length bs) cs- ]+tests =+ testGroup+ "lz4"+ [ testProperty "roundtrip" $ forAll genBytes $ \bs ->+ let cs = compress 1 bs+ in Just bs+ === decompress (Bytes.length bs) cs+ , testProperty "roundtrip-HC" $ forAll genBytes $ \bs ->+ let cs = compressHighly 3 bs+ in Just bs+ === decompress (Bytes.length bs) cs+ ] genBytes :: Gen Bytes genBytes = do- n <- choose (0,200)- bs <- vectorOf n (choose (0,2 :: Word8)) + n <- choose (0, 200)+ bs <- vectorOf n (choose (0, 2 :: Word8)) pure (Exts.fromList bs)