text 0.11.0.5 → 0.11.0.6

raw patch · 6 files changed

+59/−12 lines, 6 filesPVP: minor bump suggested

API additions: PVP suggests at least a minor version bump

API changes (from Hackage documentation)

Files

• Data/Text/Encoding.hs +17/−4
• Data/Text/Encoding/Error.hs +1/−1
• Data/Text/Fusion/Internal.hs +2/−2
• Data/Text/Lazy/Encoding.hs +28/−3
• tests/Properties.hs +10/−1
• text.cabal +1/−1

@@ -28,6 +28,9 @@     , decodeUtf32LE     , decodeUtf32BE +    -- ** Catchable failure+    , decodeUtf8'+     -- ** Controllable error handling     , decodeUtf8With     , decodeUtf16LEWith@@ -43,11 +46,12 @@     , encodeUtf32BE     ) where +import Control.Exception (evaluate, try) import Data.Bits ((.&.)) import Data.ByteString as B import Data.ByteString.Internal as B import Data.ByteString.Unsafe as B-import Data.Text.Encoding.Error (OnDecodeError, strictDecode)+import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) import Data.Text.Internal (Text(..), textP) import Data.Text.UnsafeChar (ord, unsafeWrite) import Data.Text.UnsafeShift (shiftL, shiftR)@@ -116,17 +120,26 @@   desc = "Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream" {-# INLINE[0] decodeUtf8With #-} --- | Decode a 'ByteString' containing UTF-8 encoded text.+-- | Decode a 'ByteString' containing UTF-8 encoded text that is known+-- to be valid. -- -- If the input contains any invalid UTF-8 data, an exception will be--- thrown.  For more control over the handling of invalid data, use+-- thrown that cannot be caught in pure code.  For more control over+-- the handling of invalid data, use 'decodeUtf8'' or -- 'decodeUtf8With'. decodeUtf8 :: ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-}- {-# RULES "STREAM stream/decodeUtf8 fusion" [1]     forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}++-- | Decode a 'ByteString' containing UTF-8 encoded text..+--+-- If the input contains any invalid UTF-8 data, the relevant+-- exception will be returned, otherwise the decoded text.+decodeUtf8' :: ByteString -> Either UnicodeException Text+decodeUtf8' = unsafePerformIO . try . evaluate . decodeUtf8With strictDecode+{-# INLINE decodeUtf8' #-}  -- | Encode text using UTF-8 encoding. encodeUtf8 :: Text -> ByteString

@@ -77,7 +77,7 @@   | EncodeError String (Maybe Char)     -- ^ Tried to encode a character that could not be represented     -- under the given encoding, or ran out of input in mid-encode.-    deriving (Typeable)+    deriving (Eq, Typeable)  showUnicodeException :: UnicodeException -> String showUnicodeException (DecodeError desc (Just w))

@@ -39,7 +39,7 @@ type M8 = M Word8  -- Restreaming state.-data S s = S !s {-# UNPACK #-} !M8 {-# UNPACK #-} !M8 {-# UNPACK #-} !M8+data S s = S !s !M8 !M8 !M8  infixl 2 :*: data PairS a b = !a :*: !b@@ -81,7 +81,7 @@     forall s. Stream     (s -> Step s a)             -- stepper function     !s                          -- current state-    {-# UNPACK #-} !Size        -- size hint+    !Size                       -- size hint  -- | /O(n)/ Determines if two streams are equal. eq :: (Eq a) => Stream a -> Stream a -> Bool

@@ -25,6 +25,10 @@     , decodeUtf16BE     , decodeUtf32LE     , decodeUtf32BE++    -- ** Catchable failure+    , decodeUtf8'+     -- ** Controllable error handling     , decodeUtf8With     , decodeUtf16LEWith@@ -40,9 +44,11 @@     , encodeUtf32BE     ) where +import Control.Exception (evaluate, try) import Data.Bits ((.&.))-import Data.Text.Encoding.Error (OnDecodeError, strictDecode)+import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldrChunks)+import System.IO.Unsafe (unsafePerformIO) import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as B import qualified Data.ByteString.Lazy.Internal as B@@ -108,10 +114,12 @@     desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream" {-# INLINE[0] decodeUtf8With #-} --- | Decode a 'ByteString' containing UTF-8 encoded text.+-- | Decode a 'ByteString' containing UTF-8 encoded text that is known+-- to be valid. -- -- If the input contains any invalid UTF-8 data, an exception will be--- thrown.  For more control over the handling of invalid data, use+-- thrown that cannot be caught in pure code.  For more control over+-- the handling of invalid data, use 'decodeUtf8'' or -- 'decodeUtf8With'. decodeUtf8 :: B.ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode@@ -120,6 +128,23 @@ -- This rule seems to cause performance loss. {- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1]    forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-}++-- | Decode a 'ByteString' containing UTF-8 encoded text..+--+-- If the input contains any invalid UTF-8 data, the relevant+-- exception will be returned, otherwise the decoded text.+--+-- /Note/: this function is /not/ lazy, as it must decode its entire+-- input before it can return a result.  If you need lazy (streaming)+-- decoding, use 'decodeUtf8With' in lenient mode.+decodeUtf8' :: B.ByteString -> Either UnicodeException Text+decodeUtf8' bs = unsafePerformIO $ do+                   let t = decodeUtf8 bs+                   try (evaluate (rnf t `seq` t))+  where+    rnf Empty        = ()+    rnf (Chunk _ ts) = rnf ts+{-# INLINE decodeUtf8' #-}  encodeUtf8 :: Text -> B.ByteString encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)

@@ -78,7 +78,9 @@ tl_ascii t   = EL.decodeASCII (EL.encodeUtf8 a) == a     where a  = TL.map (\c -> chr (ord c `mod` 128)) t t_utf8       = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id+t_utf8'      = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) tl_utf8      = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id+tl_utf8'     = forAll genUnicode $ (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right) t_utf16LE    = forAll genUnicode $ (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id tl_utf16LE   = forAll genUnicode $ (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id t_utf16BE    = forAll genUnicode $ (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id@@ -110,6 +112,10 @@     Left err -> assert $ length (show err) >= 0     Right n  -> assert $ n >= 0 +t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of+                                        Left err -> length (show err) >= 0+                                        Right t  -> T.length t >= 0+ class Stringy s where     packS    :: String -> s     unpackS  :: s -> String@@ -831,7 +837,9 @@     testProperty "t_ascii" t_ascii,     testProperty "tl_ascii" tl_ascii,     testProperty "t_utf8" t_utf8,+    testProperty "t_utf8'" t_utf8',     testProperty "tl_utf8" tl_utf8,+    testProperty "tl_utf8'" tl_utf8',     testProperty "t_utf16LE" t_utf16LE,     testProperty "tl_utf16LE" tl_utf16LE,     testProperty "t_utf16BE" t_utf16BE,@@ -841,7 +849,8 @@     testProperty "t_utf32BE" t_utf32BE,     testProperty "tl_utf32BE" tl_utf32BE,     testGroup "errors" [-      testProperty "t_utf8_err" t_utf8_err+      testProperty "t_utf8_err" t_utf8_err,+      testProperty "t_utf8_err'" t_utf8_err'     ]   ], 

@@ -1,5 +1,5 @@ name:           text-version:        0.11.0.5+version:        0.11.0.6 homepage:       http://bitbucket.org/bos/text bug-reports:    http://bitbucket.org/bos/text/issues synopsis:       An efficient packed Unicode text type.