conduit-extra 1.1.10 → 1.1.10.1
raw patch · 4 files changed
+99/−32 lines, 4 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- ChangeLog.md +4/−0
- Data/Conduit/Zlib.hs +67/−27
- conduit-extra.cabal +1/−1
- test/Data/Conduit/ZlibSpec.hs +27/−4
ChangeLog.md view
@@ -1,3 +1,7 @@+## 1.1.10.1++* Fix a leftovers bug in helperDecompress #254+ ## 1.1.10 * `multiple` combinator for `Data.Conduit.Zlib` [#254](https://github.com/snoyberg/conduit/issues/254)
Data/Conduit/Zlib.hs view
@@ -23,6 +23,7 @@ import Control.Monad.Primitive (PrimMonad, unsafePrimToPrim) import Control.Monad.Base (MonadBase, liftBase) import Control.Monad.Trans.Resource (MonadThrow, monadThrow)+import Data.Function (fix) -- | Gzip compression with default parameters. gzip :: (MonadThrow m, MonadBase base m, PrimMonad base) => Conduit ByteString m ByteString@@ -63,39 +64,78 @@ -> (ByteString -> t m ()) -> WindowBits -> t m ()-helperDecompress await' yield' leftover' config =- await' >>= maybe (return ()) start- where- start input = do- inf <- lift $ unsafeLiftIO $ initInflate config- push inf input+helperDecompress await' yield' leftover' config = do+ -- Initialize the stateful inflater, which will be used below+ -- This inflater is never exposed outside of this function+ inf <- lift $ unsafeLiftIO $ initInflate config - rem' <- lift $ unsafeLiftIO $ getUnusedInflate inf- unless (S.null rem') $ leftover' rem'+ -- Some helper functions used by the main feeder loop below - continue inf = await' >>= maybe (close inf) (push inf)+ let -- Flush any remaining inflated bytes downstream+ flush = do+ chunk <- lift $ unsafeLiftIO $ flushInflate inf+ unless (S.null chunk) $ yield' $ Chunk chunk - goPopper popper = do- mbs <- lift $ unsafeLiftIO popper- case mbs of- PRDone -> return ()- PRNext bs -> yield' (Chunk bs) >> goPopper popper- PRError e -> lift $ monadThrow e+ -- Get any input which is unused by the inflater+ getUnused = lift $ unsafeLiftIO $ getUnusedInflate inf - push inf (Chunk x) = do- popper <- lift $ unsafeLiftIO $ feedInflate inf x- goPopper popper- continue inf+ -- If there is any unused data, return it as leftovers to the stream+ unused = do+ rem' <- getUnused+ unless (S.null rem') $ leftover' rem' - push inf Flush = do- chunk <- lift $ unsafeLiftIO $ flushInflate inf- unless (S.null chunk) $ yield' $ Chunk chunk- yield' Flush- continue inf+ -- Main loop: feed data from upstream into the inflater+ fix $ \feeder -> do+ mnext <- await'+ case mnext of+ -- No more data is available from upstream+ Nothing -> do+ -- Flush any remaining uncompressed data+ flush+ -- Return the rest of the unconsumed data as leftovers+ unused+ -- Another chunk of compressed data arrived+ Just (Chunk x) -> do+ -- Feed the compressed data into the inflater, returning a+ -- "popper" which will return chunks of decompressed data+ popper <- lift $ unsafeLiftIO $ feedInflate inf x - close inf = do- chunk <- lift $ unsafeLiftIO $ finishInflate inf- unless (S.null chunk) $ yield' $ Chunk chunk+ -- Loop over the popper grabbing decompressed chunks and+ -- yielding them downstream+ fix $ \pop -> do+ mbs <- lift $ unsafeLiftIO popper+ case mbs of+ -- No more data from this popper+ PRDone -> do+ rem' <- getUnused+ if S.null rem'+ -- No data was unused by the inflater, so let's+ -- fill it up again and get more data out of it+ then feeder+ -- In this case, there is some unconsumed data,+ -- meaning the compressed stream is complete.+ -- At this point, we need to stop feeding,+ -- return the unconsumed data as leftovers, and+ -- flush any remaining content (which should be+ -- nothing)+ else do+ flush+ leftover' rem'+ -- Another chunk available, yield it downstream and+ -- loop again+ PRNext bs -> do+ yield' (Chunk bs)+ pop+ -- An error occurred inside zlib, throw it+ PRError e -> lift $ monadThrow e+ -- We've been asked to flush the stream+ Just Flush -> do+ -- Get any uncompressed data waiting for us+ flush+ -- Put a Flush in the stream+ yield' Flush+ -- Feed in more data+ feeder -- | -- Compress (deflate) a stream of 'ByteString's. The 'WindowBits' also control
conduit-extra.cabal view
@@ -1,5 +1,5 @@ Name: conduit-extra-Version: 1.1.10+Version: 1.1.10.1 Synopsis: Batteries included conduit: adapters for common libraries. Description: The conduit package itself maintains relative small dependencies. The purpose of this package is to collect commonly used utility functions wrapping other library dependencies, without depending on heavier-weight dependencies. The basic idea is that this package should only depend on haskell-platform packages and conduit.
test/Data/Conduit/ZlibSpec.hs view
@@ -20,6 +20,7 @@ import Control.Monad.Trans.Class import Control.Monad.Catch.Pure import Control.Monad.Base+import Control.Monad (replicateM_) instance MonadBase base m => MonadBase base (CatchT m) where liftBase = lift . liftBase@@ -32,15 +33,14 @@ src = mconcat $ map (CL.sourceList . return) bss outBss <- runExceptionT_ $ src C.$= CZ.gzip C.$= CZ.ungzip C.$$ CL.consume return $ lbs == L.fromChunks outBss- prop "flush" $ \bss' -> runST $ do+ prop "flush" $ \bss' -> do let bss = map S.pack $ filter (not . null) bss' bssC = concatMap (\bs -> [C.Chunk bs, C.Flush]) bss src = mconcat $ map (CL.sourceList . return) bssC- outBssC <- runExceptionT_- $ src C.$= CZ.compressFlush 5 (CZ.WindowBits 31)+ outBssC <- src C.$= CZ.compressFlush 5 (CZ.WindowBits 31) C.$= CZ.decompressFlush (CZ.WindowBits 31) C.$$ CL.consume- return $ bssC == outBssC+ outBssC `shouldBe` bssC it "compressFlush large data" $ do let content = L.pack $ map (fromIntegral . fromEnum) $ concat $ ["BEGIN"] ++ map show [1..100000 :: Int] ++ ["END"] src = CL.sourceList $ map C.Chunk $ L.toChunks content@@ -73,3 +73,26 @@ C.yield s2 C.$= CZ.gzip actual <- src C.$$ CZ.multiple CZ.ungzip C.=$ CL.consume S.concat actual `shouldBe` S.concat [s1, s2]++ it "single compressed, multiple uncompressed chunks" $ do+ let s1 = "hello"+ s2 = "there"+ s3 = "world"+ s1Z <- fmap S.concat $ C.yield s1 C.$= CZ.gzip C.$$ CL.consume+ let src = do+ C.yield $ S.append s1Z s2+ C.yield s3+ actual <- src C.$$ do+ x <- fmap S.concat $ CZ.ungzip C.=$ CL.consume+ y <- CL.consume+ return (x, y)+ actual `shouldBe` (s1, [s2, s3])++ it "multiple, over 32k" $ do+ let str = "One line"+ cnt = 30000+ src = replicateM_ cnt $ C.yield str C.$= CZ.gzip+ actual <- fmap S.concat $ src C.$$ CZ.multiple CZ.ungzip C.=$ CL.consume+ let expected = S.concat (replicate cnt str)+ S.length actual `shouldBe` S.length expected+ actual `shouldBe` expected