conduit-extra 1.1.0.4 → 1.1.1

raw patch · 4 files changed

+55/−20 lines, 4 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• Data/Conduit/Text.hs +43/−16
• conduit-extra.cabal +1/−1
• test/Data/Conduit/BinarySpec.hs +3/−3
• test/Data/Conduit/TextSpec.hs +8/−0

@@ -30,6 +30,7 @@     , foldLines     , withLine     , Data.Conduit.Text.decodeUtf8+    , decodeUtf8Lenient     , encodeUtf8     ) where @@ -136,14 +137,15 @@     let (bs, mexc) = codecEncode codec t     maybe (return bs) (monadThrow . fst) mexc ---- | Convert bytes into text, using the provided codec. If the codec is--- not capable of decoding an input byte sequence, an exception will be thrown.------ Since 0.3.0-decode :: MonadThrow m => Codec -> Conduit B.ByteString m T.Text-decode (NewCodec name _ start) =-    loop 0 start+decodeNew+    :: Monad m+    => (Int -> B.ByteString -> T.Text -> B.ByteString -> Conduit B.ByteString m T.Text)+    -> t+    -> Int+    -> (B.ByteString -> DecodeResult)+    -> Conduit B.ByteString m T.Text+decodeNew onFailure name =+    loop   where     loop consumed dec =         await >>= maybe finish go@@ -151,7 +153,7 @@         finish =             case dec B.empty of                 DecodeResultSuccess _ _ -> return ()-                DecodeResultFailure t rest -> onFailure B.empty t rest+                DecodeResultFailure t rest -> onFailure consumed B.empty t rest         {-# INLINE finish #-}          go bs | B.null bs = loop consumed dec@@ -163,14 +165,39 @@                             unless (T.null t) (yield t)                             loop consumed' dec'                      in consumed' `seq` next-                DecodeResultFailure t rest -> onFailure bs t rest+                DecodeResultFailure t rest -> onFailure consumed bs t rest -        onFailure bs t rest = do-            unless (T.null t) (yield t)-            leftover rest -- rest will never be null, no need to check-            let consumed' = consumed + B.length bs - B.length rest-            monadThrow $ NewDecodeException name consumed' (B.take 4 rest)-        {-# INLINE onFailure #-}+-- | Decode a stream of UTF8 data, and replace invalid bytes with the Unicode+-- replacement character.+--+-- Since 1.1.1+decodeUtf8Lenient :: Monad m => Conduit B.ByteString m T.Text+decodeUtf8Lenient =+    decodeNew onFailure "UTF8-lenient" 0 Data.Streaming.Text.decodeUtf8+  where+    onFailure _consumed _bs t rest = do+        unless (T.null t) (yield t)+        case B.uncons rest of+            Nothing -> return ()+            Just (_, rest') -> do+                unless (B.null rest') (leftover rest')+                yield $ T.singleton '\xFFFD'+        decodeUtf8Lenient++-- | Convert bytes into text, using the provided codec. If the codec is+-- not capable of decoding an input byte sequence, an exception will be thrown.+--+-- Since 0.3.0+decode :: MonadThrow m => Codec -> Conduit B.ByteString m T.Text+decode (NewCodec name _ start) =+    decodeNew onFailure name 0 start+  where+    onFailure consumed bs t rest = do+        unless (T.null t) (yield t)+        leftover rest -- rest will never be null, no need to check+        let consumed' = consumed + B.length bs - B.length rest+        monadThrow $ NewDecodeException name consumed' (B.take 4 rest)+    {-# INLINE onFailure #-} decode codec =     loop id   where

@@ -1,5 +1,5 @@ Name:                conduit-extra-Version:             1.1.0.4+Version:             1.1.1 Synopsis:            Batteries included conduit: adapters for common libraries. Description:     The conduit package itself maintains relative small dependencies. The purpose of this package is to collect commonly used utility functions wrapping other library dependencies, without depending on heavier-weight dependencies. The basic idea is that this package should only depend on haskell-platform packages and conduit.

@@ -100,9 +100,9 @@      describe "binary take" $ do       let go n l = CL.sourceList l C.$$ do-          a <- CB.take n-          b <- CL.consume-          return (a, b)+              a <- CB.take n+              b <- CL.consume+              return (a, b)        -- Taking nothing should result in an empty Bytestring       it "nothing" $ do

@@ -10,6 +10,8 @@ import Data.Monoid import Control.Monad.ST import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Encoding.Error as TEE import qualified Data.Text.Lazy.Encoding as TLE import Data.Functor.Identity import Control.Arrow@@ -149,6 +151,12 @@                   ]                 , ["\128\128\0that was bad"]                 )+        prop "lenient UTF8 decoding" $ \good1 good2 -> do+            let bss = [TE.encodeUtf8 $ T.pack good1, "\128\129\130", TE.encodeUtf8 $ T.pack good2]+                bs = S.concat bss+                expected = TE.decodeUtf8With TEE.lenientDecode bs+                actual = runIdentity $ mapM_ C.yield bss C.$$ CT.decodeUtf8Lenient C.=$ CL.consume+            T.concat actual `shouldBe` expected      describe "text lines" $ do         it "yields nothing given nothing" $