diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.1.0.2 + +* GHC 9.8.4 compatibility. + # 0.1.0.1 * Fix incorrect compression logic (compressing input Bytes repeatedly). diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Haskell Conduit implementing the official LZ4 frame streaming format. +## Building + +Use `git clone --recursive`, because `lz4/` is a git submodule. + ## Rationale and comparison to non-`lz4`-compatible libraries There exist two `lz4` formats: diff --git a/lz4-frame-conduit.cabal b/lz4-frame-conduit.cabal --- a/lz4-frame-conduit.cabal +++ b/lz4-frame-conduit.cabal @@ -1,5 +1,5 @@ name: lz4-frame-conduit -version: 0.1.0.1 +version: 0.1.0.2 synopsis: Conduit implementing the official LZ4 frame streaming format description: Conduit implementing the official LZ4 frame streaming format. @@ -14,6 +14,7 @@ category: Web build-type: Simple cabal-version: >=1.10 +tested-with: GHC == 8.6.5, GHC == 9.8.4 extra-source-files: README.md CHANGELOG.md diff --git a/src/Codec/Compression/LZ4/CTypes.hsc b/src/Codec/Compression/LZ4/CTypes.hsc --- a/src/Codec/Compression/LZ4/CTypes.hsc +++ b/src/Codec/Compression/LZ4/CTypes.hsc @@ -3,6 +3,17 @@ {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-} +{-| +Module : Codec.Compression.LZ4.CTypes +Description : C type definitions for the lz4 compression codec. +Copyright : (c) Niklas Hambüchen, 2020 +License : MIT +Maintainer : mail@nh2.me +Stability : stable + +-} + + module Codec.Compression.LZ4.CTypes ( Lz4FrameException(..) , BlockSizeID(..) diff --git a/src/Codec/Compression/LZ4/Conduit.hsc b/src/Codec/Compression/LZ4/Conduit.hsc --- a/src/Codec/Compression/LZ4/Conduit.hsc +++ b/src/Codec/Compression/LZ4/Conduit.hsc @@ -1,15 +1,60 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiWayIf #-} -{-# LANGUAGE PartialTypeSignatures #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-partial-type-signatures #-} --- | TODO: Implement: --- --- * Block checksumming --- * Dictionary support +{-| +Module : Codec.Compression.LZ4.Conduit +Description : Conduit for the lz4 compression codec. +Copyright : (c) Niklas Hambüchen, 2020 +License : MIT +Maintainer : mail@nh2.me +Stability : stable + + +Help Wanted / TODOs + +Please feel free to send me a pull request for any of the following items: + +* TODO Block checksumming + +* TODO Dictionary support + +* TODO Performance: + Write a version of `compress` that emits ByteStrings of known + constant length. That will allow us to do compression in a zero-copy + fashion, writing compressed bytes directly into a the ByteStrings + (e.g using `unsafePackMallocCString` or equivalent). + We currently don't do that (instead, use allocaBytes + copying packCStringLen) + to ensure that the ByteStrings generated are as compact as possible + (for the case that `written < size`), since the current `compress` + conduit directly yields the outputs of LZ4F_compressUpdate() + (unless they are of 0 length when they are buffered in the context + tmp buffer). + +* TODO Try enabling checksums, then corrupt a bit and see if lz4c detects it. + +* TODO Add `with*` style bracketed functions for creating the + LZ4F_createCompressionContext and Lz4FramePreferencesPtr + for prompt resource release, + in addition to the GC'd variants below. + This would replace our use of `finalizeForeignPtr` in the conduit. + `finalizeForeignPtr` seems almost as good, but note that it + doesn't guarantee prompt resource release on exceptions; + a `with*` style function that uses `bracket` does. + However, it isn't clear yet which one would be faster + (what the cost of `mask` is compared to foreign pointer finalizers). + Also note that prompt freeing has side benefits, + such as reduced malloc() fragmentation (the closer malloc() + and free() are to each other, the smaller is the chance to + have malloc()s on top of the our malloc() in the heap, + thus the smaller the chance that we cannot decrease the + heap pointer upon free() (because "mallocs on top" render + heap memory unreturnable to the OS; memory fragmentation). +-} + module Codec.Compression.LZ4.Conduit ( Lz4FrameException(..) , BlockSizeID(..) @@ -167,43 +212,6 @@ return () - --- TODO Performance: --- Write a version of `compress` that emits ByteStrings of known --- constant length. That will allow us to do compression in a zero-copy --- fashion, writing compressed bytes directly into a the ByteStrings --- (e.g using `unsafePackMallocCString` or equivalent). --- We currently don't do that (instead, use allocaBytes + copying packCStringLen) --- to ensure that the ByteStrings generated are as compact as possible --- (for the case that `written < size`), since the current `compress` --- conduit directly yields the outputs of LZ4F_compressUpdate() --- (unless they are of 0 length when they are buffered in the context --- tmp buffer). - --- TODO Try enabling checksums, then corrupt a bit and see if lz4c detects it. - --- TODO Add `with*` style bracketed functions for creating the --- LZ4F_createCompressionContext and Lz4FramePreferencesPtr --- for prompt resource release, --- in addition to the GC'd variants below. --- This would replace our use of `finalizeForeignPtr` in the conduit. --- `finalizeForeignPtr` seems almost as good, but note that it --- doesn't guarantee prompt resource release on exceptions; --- a `with*` style function that uses `bracket` does. --- However, it isn't clear yet which one would be faster --- (what the cost of `mask` is compared to foreign pointer finalizers). --- Also note that prompt freeing has side benefits, --- such as reduced malloc() fragmentation (the closer malloc() --- and free() are to each other, the smaller is the chance to --- have malloc()s on top of the our malloc() in the heap, --- thus the smaller the chance that we cannot decrease the --- heap pointer upon free() (because "mallocs on top" render --- heap memory unreturnable to the OS; memory fragmentation). - - --- TODO Turn the above TODO into documentation - - withScopedLz4fCompressionContext :: (HasCallStack) => (ScopedLz4FrameCompressionContext -> IO a) -> IO a withScopedLz4fCompressionContext f = bracket @@ -296,9 +304,9 @@ } |] +-- | TODO allow passing in cOptPtr instead of NULL. lz4fCompressUpdate :: (HasCallStack) => ScopedLz4FrameCompressionContext -> Ptr CChar -> CSize -> Ptr CChar -> CSize -> IO CSize lz4fCompressUpdate (ScopedLz4FrameCompressionContext ctx) destBuf destBufLen srcBuf srcBufLen = do - -- TODO allow passing in cOptPtr instead of NULL. written <- handleLz4Error [C.block| size_t { size_t err_or_written = LZ4F_compressUpdate($(LZ4F_cctx* ctx), $(char* destBuf), $(size_t destBufLen), $(char* srcBuf), $(size_t srcBufLen), NULL); return err_or_written; @@ -306,9 +314,9 @@ return written +-- | TODO allow passing in cOptPtr instead of NULL. lz4fCompressEnd :: (HasCallStack) => ScopedLz4FrameCompressionContext -> Ptr CChar -> CSize -> IO CSize lz4fCompressEnd (ScopedLz4FrameCompressionContext ctx) footerBuf footerBufLen = do - -- TODO allow passing in cOptPtr instead of NULL. footerWritten <- handleLz4Error [C.block| size_t { size_t err_or_footerWritten = LZ4F_compressEnd($(LZ4F_cctx* ctx), $(char* footerBuf), $(size_t footerBufLen), NULL); return err_or_footerWritten; @@ -316,7 +324,7 @@ return footerWritten --- Note [Single call to LZ4F_compressUpdate() can create multiple blocks] +-- | Note [Single call to LZ4F_compressUpdate() can create multiple blocks] -- A single call to LZ4F_compressUpdate() can create multiple blocks, -- and handles buffers > 32-bit sizes; see: -- https://github.com/lz4/lz4/blob/52cac9a97342641315c76cfb861206d6acd631a8/lib/lz4frame.c#L601 @@ -527,8 +535,7 @@ --- All notes that apply to `haskell_lz4_freeCompressionContext` apply --- here as well. +-- | All notes that apply to `haskell_lz4_freeCompressionContext` apply here as well. C.verbatim [r| void haskell_lz4_freeDecompressionContext(LZ4F_dctx** ctxPtr) { @@ -548,10 +555,9 @@ foreign import ccall "&haskell_lz4_freeDecompressionContext" haskell_lz4_freeDecompressionContext :: FunPtr (Ptr (Ptr LZ4F_dctx) -> IO ()) +-- | All notes that apply to `lz4fCreateCompressonContext` apply here as well. lz4fCreateDecompressionContext :: (HasCallStack) => IO Lz4FrameDecompressionContext lz4fCreateDecompressionContext = do - -- All notes that apply to `lz4fCreateCompressonContext` apply here - -- as well. ctxForeignPtr :: ForeignPtr (Ptr LZ4F_dctx) <- mallocForeignPtr withForeignPtr ctxForeignPtr $ \ptr -> poke ptr nullPtr @@ -576,9 +582,9 @@ return decompressSizeHint +-- TODO allow passing in dOptPtr instead of NULL. lz4fDecompress :: (HasCallStack) => Lz4FrameDecompressionContext -> Ptr CChar -> Ptr CSize -> Ptr CChar -> Ptr CSize -> IO CSize lz4fDecompress (Lz4FrameDecompressionContext ctxForeignPtr) dstBuffer dstSizePtr srcBuffer srcSizePtr = do - -- TODO allow passing in dOptPtr instead of NULL. decompressSizeHint <- handleLz4Error [C.block| size_t { LZ4F_dctx* ctxPtr = *$fptr-ptr:(LZ4F_dctx** ctxForeignPtr); @@ -588,7 +594,8 @@ return decompressSizeHint -decompress :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m () +-- | TODO check why decompressSizeHint is always 4 +decompress :: forall m . (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m () decompress = do ctx <- liftIO lz4fCreateDecompressionContext @@ -651,7 +658,7 @@ poke dstBufferSizePtr size peek dstBufferPtr - let loopSingleBs :: CSize -> ByteString -> _ + let loopSingleBs :: CSize -> ByteString -> ConduitT ByteString ByteString m CSize loopSingleBs decompressSizeHint bs = do (outBs, srcRead, newDecompressSizeHint) <- liftIO $ unsafeUseAsCStringLen bs $ \(srcBuffer, srcSize) -> do