text 1.2.2.1 → 1.2.2.2
raw patch · 22 files changed
+201/−125 lines, 22 filesdep +bytestring-builderdep ~basedep ~bytestringdep ~quickcheck-unicode
Dependencies added: bytestring-builder
Dependency ranges changed: base, bytestring, quickcheck-unicode
Files
- Data/Text.hs +11/−8
- Data/Text/Array.hs +1/−1
- Data/Text/Encoding.hs +7/−18
- Data/Text/Foreign.hs +1/−1
- Data/Text/Internal/Builder.hs +3/−1
- Data/Text/Internal/Builder/Int/Digits.hs +1/−1
- Data/Text/Internal/Fusion.hs +27/−17
- Data/Text/Internal/Fusion/Common.hs +13/−12
- Data/Text/Lazy.hs +2/−2
- Data/Text/Lazy/Builder.hs +1/−1
- Data/Text/Lazy/Builder/Int.hs +1/−1
- Data/Text/Lazy/Encoding.hs +5/−15
- Data/Text/Lazy/Read.hs +3/−3
- Data/Text/Read.hs +3/−3
- benchmarks/haskell/Benchmarks.hs +1/−1
- benchmarks/haskell/Benchmarks/Stream.hs +16/−5
- benchmarks/text-benchmarks.cabal +12/−2
- changelog.md +18/−1
- tests/Tests/Properties.hs +37/−20
- tests/Tests/QuickCheckUtils.hs +1/−1
- tests/text-tests.cabal +19/−4
- text.cabal +18/−7
Data/Text.hs view
@@ -238,6 +238,7 @@ import qualified Data.Text.Lazy as L import Data.Int (Int64) #endif+import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) #if __GLASGOW_HASKELL__ >= 708 import qualified GHC.Exts as Exts #endif@@ -571,7 +572,9 @@ -- Subject to fusion. length :: Text -> Int length t = S.length (stream t)-{-# INLINE length #-}+{-# INLINE [0] length #-}+-- length needs to be phased after the compareN/length rules otherwise+-- it may inline before the rules have an opportunity to fire. -- | /O(n)/ Compare the count of characters in a 'Text' to a number. -- Subject to fusion.@@ -590,32 +593,32 @@ {-# RULES "TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.- (==) (length t) n = compareLength t n == EQ+ eqInt (length t) n = compareLength t n == EQ #-} {-# RULES "TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.- (/=) (length t) n = compareLength t n /= EQ+ neInt (length t) n = compareLength t n /= EQ #-} {-# RULES "TEXT <N/length -> compareLength/==LT" [~1] forall t n.- (<) (length t) n = compareLength t n == LT+ ltInt (length t) n = compareLength t n == LT #-} {-# RULES "TEXT <=N/length -> compareLength//=GT" [~1] forall t n.- (<=) (length t) n = compareLength t n /= GT+ leInt (length t) n = compareLength t n /= GT #-} {-# RULES "TEXT >N/length -> compareLength/==GT" [~1] forall t n.- (>) (length t) n = compareLength t n == GT+ gtInt (length t) n = compareLength t n == GT #-} {-# RULES "TEXT >=N/length -> compareLength//=LT" [~1] forall t n.- (>=) (length t) n = compareLength t n /= LT+ geInt (length t) n = compareLength t n /= LT #-} -- -----------------------------------------------------------------------------@@ -1096,8 +1099,8 @@ iterNEnd :: Int -> Text -> Int iterNEnd n t@(Text _arr _off len) = loop (len-1) n where loop i !m+ | m <= 0 = i+1 | i <= 0 = 0- | m <= 1 = i | otherwise = loop (i+d) (m-1) where d = reverseIter_ t i
Data/Text/Array.hs view
@@ -20,7 +20,7 @@ -- > import qualified Data.Text.Array as A -- -- The names in this module resemble those in the 'Data.Array' family--- of modules, but are shorter due to the assumption of qualifid+-- of modules, but are shorter due to the assumption of qualified -- naming. module Data.Text.Array (
Data/Text/Encoding.hs view
@@ -55,13 +55,9 @@ , encodeUtf32LE , encodeUtf32BE -#if MIN_VERSION_bytestring(0,10,4) -- * Encoding Text using ByteString Builders- -- | /Note/ that these functions are only available if built against- -- @bytestring >= 0.10.4.0@. , encodeUtf8Builder , encodeUtf8BuilderEscaped-#endif ) where #if __GLASGOW_HASKELL__ >= 702@@ -70,24 +66,15 @@ import Control.Monad.ST (unsafeIOToST, unsafeSTToIO) #endif -#if MIN_VERSION_bytestring(0,10,4)-import Data.Bits ((.&.))-import Data.Text.Internal.Unsafe.Char (ord)-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Builder.Prim.Internal as BP-import qualified Data.Text.Internal.Encoding.Utf16 as U16-#endif- import Control.Exception (evaluate, try) import Control.Monad.ST (runST)+import Data.Bits ((.&.)) import Data.ByteString as B import Data.ByteString.Internal as B hiding (c2w) import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) import Data.Text.Internal (Text(..), safe, text) import Data.Text.Internal.Private (runText)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) import Data.Text.Internal.Unsafe.Shift (shiftR) import Data.Text.Show () import Data.Text.Unsafe (unsafeDupablePerformIO)@@ -98,8 +85,13 @@ import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr) import Foreign.Storable (Storable, peek, poke) import GHC.Base (ByteArray#, MutableByteArray#)+import qualified Data.ByteString.Builder as B+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)+import qualified Data.ByteString.Builder.Prim as BP+import qualified Data.ByteString.Builder.Prim.Internal as BP import qualified Data.Text.Array as A import qualified Data.Text.Internal.Encoding.Fusion as E+import qualified Data.Text.Internal.Encoding.Utf16 as U16 import qualified Data.Text.Internal.Fusion as F #include "text_cbits.h"@@ -315,8 +307,6 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode {-# INLINE decodeUtf8' #-} -#if MIN_VERSION_bytestring(0,10,4)- -- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. encodeUtf8Builder :: Text -> B.Builder encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)@@ -377,7 +367,6 @@ outerLoop i (B.BufferRange op ope) where poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)-#endif -- | Encode text using UTF-8 encoding. encodeUtf8 :: Text -> ByteString
Data/Text/Foreign.hs view
@@ -108,7 +108,7 @@ | n >= len || m >= len = t | otherwise = Text arr off m where- m | w < 0xDB00 || w > 0xD8FF = n+ m | w < 0xD800 || w > 0xDBFF = n | otherwise = n+1 w = A.unsafeIndex arr (off+n-1)
Data/Text/Internal/Builder.hs view
@@ -6,7 +6,7 @@ -- Module : Data.Text.Internal.Builder -- Copyright : (c) 2013 Bryan O'Sullivan -- (c) 2010 Johan Tibell--- License : BSD3-style (see LICENSE)+-- License : BSD-style (see LICENSE) -- -- Maintainer : Johan Tibell <johan.tibell@gmail.com> -- Stability : experimental@@ -242,6 +242,8 @@ !t = Text arr o u ts <- inlineInterleaveST (k b) return $! t : ts+{-# INLINE [1] flush #-}+-- defer inlining so that flush/flush rule may fire. ------------------------------------------------------------------------
Data/Text/Internal/Builder/Int/Digits.hs view
@@ -2,7 +2,7 @@ -- Module: Data.Text.Internal.Builder.Int.Digits -- Copyright: (c) 2013 Bryan O'Sullivan--- License: BSD3+-- License: BSD-style -- Maintainer: Bryan O'Sullivan <bos@serpentine.com> -- Stability: experimental -- Portability: portable
Data/Text/Internal/Fusion.hs view
@@ -98,25 +98,35 @@ -- | /O(n)/ Convert a 'Stream Char' into a 'Text'. unstream :: Stream Char -> Text unstream (Stream next0 s0 len) = runText $ \done -> do- let mlen = upperBound 4 len+ -- Before encoding each char we perform a buffer realloc check assuming+ -- worst case encoding size of two 16-bit units for the char. Just add an+ -- extra space to the buffer so that we do not end up reallocating even when+ -- all the chars are encoded as single unit.+ let mlen = upperBound 4 len + 1 arr0 <- A.new mlen- let outer arr top = loop+ let outer !arr !maxi = encode where- loop !s !i =- case next0 s of- Done -> done arr i- Skip s' -> loop s' i- Yield x s'- | j >= top -> {-# SCC "unstream/resize" #-} do- let top' = (top + 1) `shiftL` 1- arr' <- A.new top'- A.copyM arr' 0 arr 0 top- outer arr' top' s i- | otherwise -> do d <- unsafeWrite arr i x- loop s' (i+d)- where j | ord x < 0x10000 = i- | otherwise = i + 1- outer arr0 mlen s0 0+ -- keep the common case loop as small as possible+ encode !si !di =+ case next0 si of+ Done -> done arr di+ Skip si' -> encode si' di+ Yield c si'+ -- simply check for the worst case+ | maxi < di + 1 -> realloc si di+ | otherwise -> do+ n <- unsafeWrite arr di c+ encode si' (di + n)++ -- keep uncommon case separate from the common case code+ {-# NOINLINE realloc #-}+ realloc !si !di = do+ let newlen = (maxi + 1) * 2+ arr' <- A.new newlen+ A.copyM arr' 0 arr 0 di+ outer arr' (newlen - 1) si di++ outer arr0 (mlen - 1) s0 0 {-# INLINE [0] unstream #-} {-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
Data/Text/Internal/Fusion/Common.hs view
@@ -107,7 +107,7 @@ import qualified Data.List as L import qualified Prelude as P import Data.Bits (shiftL)-import Data.Char (isLetter)+import Data.Char (isLetter, isSpace) import Data.Int (Int64) import Data.Text.Internal.Fusion.Types import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,@@ -462,15 +462,16 @@ where next (CC (letter :*: s) '\0' _) = case next0 s of- Done -> Done- Skip s' -> Skip (CC (letter :*: s') '\0' '\0')+ Done -> Done+ Skip s' -> Skip (CC (letter :*: s') '\0' '\0') Yield c s'- | letter' -> if letter- then lowerMapping c (letter' :*: s')- else titleMapping c (letter' :*: s')- | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')- where letter' = isLetter c- next (CC s a b) = Yield a (CC s b '\0')+ | nonSpace -> if letter+ then lowerMapping c (nonSpace :*: s')+ else titleMapping c (letter' :*: s')+ | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0')+ where nonSpace = P.not (isSpace c)+ letter' = isLetter c+ next (CC s a b) = Yield a (CC s b '\0') {-# INLINE [0] toTitle #-} data Justify i s = Just1 !i !s@@ -669,12 +670,12 @@ -- ** Generating and unfolding streams replicateCharI :: Integral a => a -> Char -> Stream Char-replicateCharI n c+replicateCharI !n !c | n < 0 = empty | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low where- next i | i >= n = Done- | otherwise = Yield c (i + 1)+ next !i | i >= n = Done+ | otherwise = Yield c (i + 1) {-# INLINE [0] replicateCharI #-} data RI s = RI !s {-# UNPACK #-} !Int64
Data/Text/Lazy.hs view
@@ -545,7 +545,7 @@ unstream (S.tail (stream t)) = tail t #-} --- | /O(1)/ Returns all but the last character of a 'Text', which must+-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must -- be non-empty. Subject to fusion. init :: Text -> Text init (Chunk t0 ts0) = go t0 ts0@@ -581,7 +581,7 @@ isSingleton = S.isSingleton . stream {-# INLINE isSingleton #-} --- | /O(1)/ Returns the last character of a 'Text', which must be+-- | /O(n\/c)/ Returns the last character of a 'Text', which must be -- non-empty. Subject to fusion. last :: Text -> Char last Empty = emptyError "last"
Data/Text/Lazy/Builder.hs view
@@ -8,7 +8,7 @@ -- Module : Data.Text.Lazy.Builder -- Copyright : (c) 2013 Bryan O'Sullivan -- (c) 2010 Johan Tibell--- License : BSD3-style (see LICENSE)+-- License : BSD-style (see LICENSE) -- -- Maintainer : Johan Tibell <johan.tibell@gmail.com> -- Stability : experimental
Data/Text/Lazy/Builder/Int.hs view
@@ -7,7 +7,7 @@ -- Module: Data.Text.Lazy.Builder.Int -- Copyright: (c) 2013 Bryan O'Sullivan -- (c) 2011 MailRank, Inc.--- License: BSD3+-- License: BSD-style -- Maintainer: Bryan O'Sullivan <bos@serpentine.com> -- Stability: experimental -- Portability: portable
Data/Text/Lazy/Encoding.hs view
@@ -46,27 +46,23 @@ , encodeUtf32LE , encodeUtf32BE -#if MIN_VERSION_bytestring(0,10,4) -- * Encoding Text using ByteString Builders , encodeUtf8Builder , encodeUtf8BuilderEscaped-#endif ) where import Control.Exception (evaluate, try)+import Data.Monoid (Monoid(..)) import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)-import qualified Data.ByteString as S-import qualified Data.ByteString.Lazy as B-import qualified Data.ByteString.Lazy.Internal as B-import qualified Data.ByteString.Unsafe as B-#if MIN_VERSION_bytestring(0,10,4) import Data.Word (Word8)-import Data.Monoid (Monoid(..))+import qualified Data.ByteString as S import qualified Data.ByteString.Builder as B import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith) import qualified Data.ByteString.Builder.Prim as BP-#endif+import qualified Data.ByteString.Lazy as B+import qualified Data.ByteString.Lazy.Internal as B+import qualified Data.ByteString.Unsafe as B import qualified Data.Text as T import qualified Data.Text.Encoding as TE import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E@@ -146,7 +142,6 @@ {-# INLINE decodeUtf8' #-} encodeUtf8 :: Text -> B.ByteString-#if MIN_VERSION_bytestring(0,10,4) encodeUtf8 Empty = B.empty encodeUtf8 lt@(Chunk t _) = B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt@@ -167,11 +162,6 @@ encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder encodeUtf8BuilderEscaped prim = foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty--#else-encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)-encodeUtf8 Empty = B.Empty-#endif -- | Decode text from little endian UTF-16 encoding. decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text
Data/Text/Lazy/Read.hs view
@@ -68,9 +68,9 @@ go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading--- @\"0x\"@ followed by at least one decimal digit. Input is consumed--- until a non-hex-digit or end of string is reached. This function--- is case insensitive.+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is+-- consumed until a non-hex-digit or end of string is reached.+-- This function is case insensitive. -- -- This function does not handle leading sign characters. If you need -- to handle signed input, use @'signed' 'hexadecimal'@.
Data/Text/Read.hs view
@@ -67,9 +67,9 @@ go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading--- @\"0x\"@ followed by at least one decimal digit. Input is consumed--- until a non-hex-digit or end of string is reached. This function--- is case insensitive.+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is+-- consumed until a non-hex-digit or end of string is reached.+-- This function is case insensitive. -- -- This function does not handle leading sign characters. If you need -- to handle signed input, use @'signed' 'hexadecimal'@.
benchmarks/haskell/Benchmarks.hs view
@@ -53,7 +53,7 @@ , Mul.benchmark , Pure.benchmark "tiny" (tf "tiny.txt") , Pure.benchmark "ascii" (tf "ascii-small.txt")- , Pure.benchmark "france" (tf "france.html")+ -- , Pure.benchmark "france" (tf "france.html") , Pure.benchmark "russian" (tf "russian-small.txt") , Pure.benchmark "japanese" (tf "japanese.txt") , ReadNumbers.benchmark (tf "numbers.txt")
benchmarks/haskell/Benchmarks/Stream.hs view
@@ -18,11 +18,11 @@ import qualified Data.Text.Encoding.Error as E import qualified Data.Text.Internal.Encoding.Fusion as T import qualified Data.Text.Internal.Encoding.Fusion.Common as F-import qualified Data.Text.Internal.Fusion as T+import qualified Data.Text.Internal.Fusion as F import qualified Data.Text.IO as T import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL-import qualified Data.Text.Internal.Lazy.Fusion as TL+import qualified Data.Text.Internal.Lazy.Fusion as FL import qualified Data.Text.Lazy.IO as TL instance NFData a => NFData (Stream a) where@@ -53,14 +53,25 @@ !utf32beL = TL.encodeUtf32BE tl -- For the functions which operate on streams- let !s = T.stream t+ let !s = F.stream t return $ bgroup "Stream" -- Fusion [ bgroup "stream" $- [ bench "Text" $ nf T.stream t- , bench "LazyText" $ nf TL.stream tl+ [ bench "Text" $ nf F.stream t+ , bench "LazyText" $ nf FL.stream tl+ ]+ -- must perform exactly the same as stream above due to+ -- stream/unstream (i.e. stream after unstream) fusion+ , bgroup "stream-fusion" $+ [ bench "Text" $ nf (F.stream . F.unstream . F.stream) t+ , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl+ ]+ -- measure the overhead of unstream after stream+ , bgroup "stream-unstream" $+ [ bench "Text" $ nf (F.unstream . F.stream) t+ , bench "LazyText" $ nf (FL.unstream . FL.stream) tl ] -- Encoding.Fusion
benchmarks/text-benchmarks.cabal view
@@ -3,7 +3,7 @@ synopsis: Benchmarks for the text package description: Benchmarks for the text package homepage: https://bitbucket.org/bos/text-license: BSD3+license: BSD2 license-file: ../LICENSE author: Jasper Van der Jeugt <jaspervdj@gmail.com>, Bryan O'Sullivan <bos@serpentine.com>,@@ -15,6 +15,11 @@ cabal-version: >=1.2 +flag bytestring-builder+ description: Depend on the bytestring-builder package for backwards compatibility.+ default: False+ manual: False+ flag llvm description: use LLVM default: False@@ -33,7 +38,6 @@ build-depends: base == 4.*, binary, blaze-builder,- bytestring, bytestring-lexing >= 0.5.0, containers, criterion >= 0.10.0.0,@@ -45,6 +49,12 @@ stringsearch, utf8-string, vector++ if flag(bytestring-builder)+ build-depends: bytestring >= 0.9 && < 0.10.4,+ bytestring-builder >= 0.10.4+ else+ build-depends: bytestring >= 0.10.4 executable text-multilang hs-source-dirs: haskell
changelog.md view
@@ -1,10 +1,27 @@+1.2.2.2++* The `toTitle` function now correctly handles letters that+ immediately follow punctuation. Before, `"there's"` would turn into+ `"There'S"`. Now, it becomes `"There's"`.++* The implementation of unstreaming is faster, resulting in operations+ such as `map` and `intersperse` speeding up by up to 30%, with+ smaller code generated.++* The optimised length comparison function is now more likely to be+ used after some rewrite rule tweaking.++* Bug fix: an off-by-one bug in `takeEnd` is fixed.++* Bug fix: a logic error in `takeWord16` is fixed.+ 1.2.2.1 * The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. The build flag has been renamed accordingly. Your army of diligent maintainers apologizes for the churn. -* Spec compliance: toCaseFold now follows the Unicode 8.0 spec+* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec (updated from 7.0) * An STG lint error has been fixed
tests/Tests/Properties.hs view
@@ -11,7 +11,7 @@ import Control.Applicative ((<$>), (<*>)) import Control.Arrow ((***), second) import Data.Bits ((.&.))-import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)+import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord) import Data.Int (Int8, Int16, Int32, Int64) import Data.Monoid (Monoid(..)) import Data.String (IsString(fromString))@@ -23,6 +23,7 @@ import Data.Text.Lazy.Read as TL import Data.Text.Read as T import Data.Word (Word, Word8, Word16, Word32, Word64)+import Data.Maybe (mapMaybe) import Numeric (showEFloat, showFFloat, showGFloat, showHex) import Prelude hiding (replicate) import Test.Framework (Test, testGroup)@@ -37,6 +38,7 @@ import qualified Data.Bits as Bits (shiftL, shiftR) import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL+import qualified Data.Char as C import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Encoding as E@@ -60,10 +62,10 @@ tl_pack_unpack = (TL.unpack . TL.pack) `eq` id t_stream_unstream = (S.unstream . S.stream) `eq` id tl_stream_unstream = (SL.unstream . SL.stream) `eq` id-t_reverse_stream t = (S.reverse . S.reverseStream) t == t-t_singleton c = [c] == (T.unpack . T.singleton) c-tl_singleton c = [c] == (TL.unpack . TL.singleton) c-tl_unstreamChunks x = f 11 x == f 1000 x+t_reverse_stream t = (S.reverse . S.reverseStream) t === t+t_singleton c = [c] === (T.unpack . T.singleton) c+tl_singleton c = [c] === (TL.unpack . TL.singleton) c+tl_unstreamChunks x = f 11 x === f 1000 x where f n = SL.unstreamChunks n . S.streamList tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id@@ -74,13 +76,13 @@ encodeLazyL1 :: TL.Text -> BL.ByteString encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks -t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a+t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a where a = T.map (\c -> chr (ord c `mod` 128)) t-tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) == a+tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a where a = TL.map (\c -> chr (ord c `mod` 128)) t-t_latin1 t = E.decodeLatin1 (encodeL1 a) == a+t_latin1 t = E.decodeLatin1 (encodeL1 a) === a where a = T.map (\c -> chr (ord c `mod` 256)) t-tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) == a+tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a where a = TL.map (\c -> chr (ord c `mod` 256)) t t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)@@ -111,7 +113,7 @@ let b = E.encodeUtf8 t ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t leftover = (++ [B.empty]) . init . tail . B.inits- in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b == ls+ in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls data Badness = Solo | Leading | Trailing deriving (Eq, Show)@@ -201,8 +203,8 @@ mconcat `eq` (unpackS . mconcat . L.map T.pack) tl_mconcat = unsquare $ mconcat `eq` (unpackS . mconcat . L.map TL.pack)-t_mempty = mempty == (unpackS (mempty :: T.Text))-tl_mempty = mempty == (unpackS (mempty :: TL.Text))+t_mempty = mempty === (unpackS (mempty :: T.Text))+tl_mempty = mempty === (unpackS (mempty :: TL.Text)) t_IsString = fromString `eqP` (T.unpack . fromString) tl_IsString = fromString `eqP` (TL.unpack . fromString) @@ -326,6 +328,19 @@ where p = T.length . T.filter isUpper tl_toUpper_upper t = p (TL.toUpper t) >= p t where p = TL.length . TL.filter isUpper+t_toTitle_title t = all (<= 1) (caps w)+ where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle+ -- TIL: there exist uppercase-only letters+ w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t+t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable+ where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words+ -- Surprise! The Spanish/Portuguese ordinal indicators changed+ -- from category Ll (letter, lowercase) to Lo (letter, other)+ -- in Unicode 7.0+ -- Oh, and there exist lowercase-only letters (see previous test)+ stable c = if isLower c+ then C.toUpper c /= c+ else c /= '\170' && c /= '\186' justifyLeft k c xs = xs ++ L.replicate (k - length xs) c justifyRight m n xs = L.replicate (m - length xs) n ++ xs@@ -720,7 +735,7 @@ conc = T.concat . TL.toChunks t_indices_occurs = unsquare $ \(NotEmpty t) ts -> let s = T.intercalate t ts- in Slow.indices t s == indices t s+ in Slow.indices t s === indices t s -- Bit shifts. shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k@@ -812,18 +827,18 @@ -- Reading. t_decimal (n::Int) s =- T.signed T.decimal (T.pack (show n) `T.append` t) == Right (n,t)+ T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t) where t = T.dropWhile isDigit s tl_decimal (n::Int) s =- TL.signed TL.decimal (TL.pack (show n) `TL.append` t) == Right (n,t)+ TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t) where t = TL.dropWhile isDigit s t_hexadecimal m s ox =- T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) == Right (n,t)+ T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t) where t = T.dropWhile isHexDigit s p = if ox then "0x" else "" n = getPositive m :: Int tl_hexadecimal m s ox =- TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) == Right (n,t)+ TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t) where t = TL.dropWhile isHexDigit s p = if ox then "0x" else "" n = getPositive m :: Int@@ -867,11 +882,11 @@ t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t-t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) == t+t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t where n = small m t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr) -t_copy t = T.copy t == t+t_copy t = T.copy t === t -- Regression tests. s_filter_eq s = S.filter p t == S.streamList (filter p s)@@ -1030,7 +1045,9 @@ testProperty "tl_toLower_lower" tl_toLower_lower, testProperty "t_toUpper_length" t_toUpper_length, testProperty "t_toUpper_upper" t_toUpper_upper,- testProperty "tl_toUpper_upper" tl_toUpper_upper+ testProperty "tl_toUpper_upper" tl_toUpper_upper,+ testProperty "t_toTitle_title" t_toTitle_title,+ testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower ], testGroup "justification" [
tests/Tests/QuickCheckUtils.hs view
@@ -106,7 +106,7 @@ where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs instance Arbitrary T.Text where- arbitrary = T.pack `fmap` arbitrary+ arbitrary = T.pack `fmap` string shrink = map T.pack . shrink . T.unpack instance Arbitrary TL.Text where
tests/text-tests.cabal view
@@ -3,7 +3,7 @@ synopsis: Functional tests for the text package description: Functional tests for the text package homepage: https://github.com/bos/text-license: BSD3+license: BSD2 license-file: ../LICENSE author: Jasper Van der Jeugt <jaspervdj@gmail.com>, Bryan O'Sullivan <bos@serpentine.com>,@@ -20,6 +20,11 @@ default: False manual: True +flag bytestring-builder+ description: Depend on the bytestring-builder package for backwards compatibility.+ default: False+ manual: False+ executable text-tests main-is: Tests.hs @@ -48,16 +53,21 @@ HUnit >= 1.2, QuickCheck >= 2.7, base == 4.*,- bytestring, deepseq, directory,- quickcheck-unicode,+ quickcheck-unicode >= 1.0.1.0, random, test-framework >= 0.4, test-framework-hunit >= 0.2, test-framework-quickcheck2 >= 0.2, text-tests + if flag(bytestring-builder)+ build-depends: bytestring >= 0.9 && < 0.10.4,+ bytestring-builder >= 0.10.4+ else+ build-depends: bytestring >= 0.10.4+ executable text-tests-stdio main-is: Tests/IO.hs @@ -137,7 +147,12 @@ array, base == 4.*, binary,- bytestring, deepseq, ghc-prim, integer-gmp++ if flag(bytestring-builder)+ build-depends: bytestring >= 0.9 && < 0.10.4,+ bytestring-builder >= 0.10.4+ else+ build-depends: bytestring >= 0.10.4
text.cabal view
@@ -1,5 +1,5 @@ name: text-version: 1.2.2.1+version: 1.2.2.2 homepage: https://github.com/bos/text bug-reports: https://github.com/bos/text/issues synopsis: An efficient packed Unicode text type.@@ -31,7 +31,7 @@ the @text-icu@ package: <http://hackage.haskell.org/package/text-icu> -license: BSD3+license: BSD2 license-file: LICENSE author: Bryan O'Sullivan <bos@serpentine.com> maintainer: Bryan O'Sullivan <bos@serpentine.com>@@ -64,6 +64,11 @@ tests/scripts/*.sh tests/text-tests.cabal +flag bytestring-builder+ description: Depend on the bytestring-builder package for backwards compatibility.+ default: False+ manual: False+ flag developer description: operate in developer mode default: False@@ -133,10 +138,11 @@ deepseq >= 1.1.0.0, ghc-prim >= 0.2 - if impl(ghc >= 7.7)- build-depends: bytestring >= 0.10.4.0+ if flag(bytestring-builder)+ build-depends: bytestring >= 0.9 && < 0.10.4,+ bytestring-builder >= 0.10.4 else- build-depends: bytestring >= 0.9+ build-depends: bytestring >= 0.10.4 cpp-options: -DHAVE_DEEPSEQ ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2@@ -171,15 +177,20 @@ array, base, binary,- bytestring, deepseq, directory, ghc-prim,- quickcheck-unicode,+ quickcheck-unicode >= 1.0.1.0, random, test-framework >= 0.4, test-framework-hunit >= 0.2, test-framework-quickcheck2 >= 0.2++ if flag(bytestring-builder)+ build-depends: bytestring >= 0.9 && < 0.10.4,+ bytestring-builder >= 0.10.4+ else+ build-depends: bytestring >= 0.10.4 if flag(integer-simple) cpp-options: -DINTEGER_SIMPLE