text 0.10.0.0 → 0.10.0.1

raw patch · 14 files changed

+652/−380 lines, 14 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• Data/Text.hs +170/−104
• Data/Text/Encoding.hs +38/−2
• Data/Text/IO.hs +22/−0
• Data/Text/Internal.hs +24/−0
• Data/Text/Lazy.hs +213/−133
• Data/Text/Lazy/Builder.hs +37/−24
• Data/Text/Lazy/Encoding.hs +36/−2
• Data/Text/Lazy/IO.hs +21/−0
• Data/Text/Lazy/Read.hs +6/−4
• Data/Text/Read.hs +7/−5
• tests/Properties.hs +61/−64
• tests/QuickCheckUtils.hs +0/−30
• tests/SlowFunctions.hs +7/−7
• text.cabal +10/−5

@@ -13,20 +13,31 @@ -- Stability   : experimental -- Portability : GHC ----- A time and space-efficient implementation of Unicode text using--- packed Word16 arrays.  Suitable for performance critical use, both--- in terms of large data quantities and high speed.+-- A time and space-efficient implementation of Unicode text.+-- Suitable for performance critical use, both in terms of large data+-- quantities and high speed. --+-- /Note/: Read below the synopsis for important notes on the use of+-- this module.+-- -- This module is intended to be imported @qualified@, to avoid name -- clashes with "Prelude" functions, e.g. -- -- > import qualified Data.Text as T+--+-- To use an extended and very rich family of functions for working+-- with Unicode text (including normalization, regular expressions,+-- non-standard encodings, text breaking, and locales), see the+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>  module Data.Text     (     -- * Strict vs lazy types     -- $strict +    -- * Acceptable data+    -- $replacement+     -- * Fusion     -- $fusion @@ -117,10 +128,10 @@     , stripStart     , stripEnd     , splitAt-    , spanBy+    , breakOn+    , breakOnEnd     , break-    , breakEnd-    , breakBy+    , span     , group     , groupBy     , inits@@ -128,8 +139,8 @@      -- ** Breaking into many substrings     -- $split+    , splitOn     , split-    , splitBy     , chunksOf      -- ** Breaking into lines and words@@ -150,9 +161,9 @@      -- * Searching     , filter+    , breakOnAll     , find-    , findBy-    , partitionBy+    , partition      -- , findSubstring     @@ -196,7 +207,7 @@ import qualified Data.Text.Fusion as S import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion (stream, reverseStream, unstream)-import Data.Text.Internal (Text(..), empty, text, textP)+import Data.Text.Internal (Text(..), empty, firstf, safe, text, textP) import qualified Prelude as P import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,                          unsafeHead, unsafeTail)@@ -230,6 +241,33 @@ -- difference being that the strict module uses 'Int' values for -- lengths and counts, while the lazy module uses 'Int64' lengths. +-- $replacement+--+-- A 'Text' value is a sequence of Unicode scalar values, as defined+-- in &#xa7;3.9, definition D76 of the Unicode 5.2 standard:+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF+-- inclusive. Haskell implementations admit all Unicode code points+-- (&#xa7;3.4, definition D10) as 'Char' values, including code points+-- from this invalid range.  This means that there are some 'Char'+-- values that are not valid Unicode scalar values, and the functions+-- in this module must handle those cases.+--+-- Within this module, many functions construct a 'Text' from one or+-- more 'Char' values. Those functions will substitute 'Char' values+-- that are not valid Unicode scalar values with the replacement+-- character \"&#xfffd;\" (U+FFFD).  Functions that perform this+-- inspection and replacement are documented with the phrase+-- \"Performs replacement on invalid scalar values\".+--+-- (One reason for this policy of replacement is that internally, a+-- 'Text' value is represented as packed UTF-16 data. Values in the+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate+-- code points, and so cannot be represented. The functions replace+-- invalid scalar values, instead of dropping them, as a security+-- measure. For details, see Unicode Technical Report 36, &#xa7;3.5:+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters>)+ -- $fusion -- -- Most of the functions in this module are subject to /fusion/,@@ -240,6 +278,7 @@ -- -- > import Data.Text as T -- > import Data.Text.Encoding as E+-- > import Data.ByteString (ByteString) -- > -- > countChars :: ByteString -> Int -- > countChars = T.length . T.toUpper . E.decodeUtf8@@ -251,7 +290,7 @@ -- function will be compiled down to a single loop over the source -- 'ByteString'. ----- Functions that can be fused by the compiler are marked with the+-- Functions that can be fused by the compiler are documented with the -- phrase \"Subject to fusion\".  instance Eq Text where@@ -320,9 +359,10 @@ -- ----------------------------------------------------------------------------- -- * Conversion to/from 'Text' --- | /O(n)/ Convert a 'String' into a 'Text'.  Subject to fusion.+-- | /O(n)/ Convert a 'String' into a 'Text'.  Subject to+-- fusion.  Performs replacement on invalid scalar values. pack :: String -> Text-pack = unstream . S.streamList+pack = unstream . S.streamList . L.map safe {-# INLINE [1] pack #-}  -- | /O(n)/ Convert a Text into a String.  Subject to fusion.@@ -330,10 +370,10 @@ unpack = S.unstreamList . stream {-# INLINE [1] unpack #-} --- | /O(1)/ Convert a character into a Text.--- Subject to fusion.+-- | /O(1)/ Convert a character into a Text.  Subject to fusion.+-- Performs replacement on invalid scalar values. singleton :: Char -> Text-singleton = unstream . S.singleton+singleton = unstream . S.singleton . safe {-# INLINE [1] singleton #-}  -- -----------------------------------------------------------------------------@@ -341,15 +381,17 @@  -- | /O(n)/ Adds a character to the front of a 'Text'.  This function -- is more costly than its 'List' counterpart because it requires--- copying a new array.  Subject to fusion.+-- copying a new array.  Subject to fusion.  Performs replacement on+-- invalid scalar values. cons :: Char -> Text -> Text-cons c t = unstream (S.cons c (stream t))+cons c t = unstream (S.cons (safe c) (stream t)) {-# INLINE cons #-}  -- | /O(n)/ Adds a character to the end of a 'Text'.  This copies the -- entire array in the process, unless fused.  Subject to fusion.+-- Performs replacement on invalid scalar values. snoc :: Text -> Char -> Text-snoc t c = unstream (S.snoc (stream t) c)+snoc t c = unstream (S.snoc (stream t) (safe c)) {-# INLINE snoc #-}  -- | /O(n)/ Appends one 'Text' to the other by copying both of them@@ -523,9 +565,10 @@ -- ----------------------------------------------------------------------------- -- * Transformations -- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.  Subject to fusion.+-- each element of @t@.  Subject to fusion.  Performs replacement on+-- invalid scalar values. map :: (Char -> Char) -> Text -> Text-map f t = unstream (S.map f (stream t))+map f t = unstream (S.map (safe . f) (stream t)) {-# INLINE [1] map #-}  -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of@@ -536,9 +579,10 @@ {-# INLINE intercalate #-}  -- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.  Subject to fusion.+-- between the characters of a 'Text'.  Subject to fusion.  Performs+-- replacement on invalid scalar values. intersperse     :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse c (stream t))+intersperse c t = unstream (S.intersperse (safe c) (stream t)) {-# INLINE intersperse #-}  -- | /O(n)/ Reverse the characters of a string. Subject to fusion.@@ -554,7 +598,7 @@         -> Text                 -- ^ Replacement text         -> Text                 -- ^ Input text         -> Text-replace s d = intercalate d . split s+replace s d = intercalate d . splitOn s {-# INLINE replace #-}  -- ----------------------------------------------------------------------------@@ -575,7 +619,8 @@ -- context-dependent operation. The case conversion functions in this -- module are /not/ locale sensitive. Programs that require locale -- sensitivity should use appropriate versions of the case mapping--- functions from the @text-icu@ package.+-- functions from the @text-icu@ package:+-- <http://hackage.haskell.org/package/text-icu>  -- | /O(n)/ Convert a string to folded case.  This function is mainly -- useful for performing caseless (also known as case insensitive)@@ -614,8 +659,11 @@ {-# INLINE toUpper #-}  -- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Subject to fusion. Examples:+-- specified fill character on the right. Subject to fusion.+-- Performs replacement on invalid scalar values. --+-- Examples:+-- -- > justifyLeft 7 'x' "foo"    == "fooxxxx" -- > justifyLeft 3 'x' "foobar" == "foobar" justifyLeft :: Int -> Char -> Text -> Text@@ -633,8 +681,11 @@   #-}  -- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Examples:+-- specified fill character on the left.  Performs replacement on+-- invalid scalar values. --+-- Examples:+-- -- > justifyRight 7 'x' "bar"    == "xxxxbar" -- > justifyRight 3 'x' "foobar" == "foobar" justifyRight :: Int -> Char -> Text -> Text@@ -644,9 +695,12 @@   where len = length t {-# INLINE justifyRight #-} --- | /O(n)/ Center a string to the given length, using the--- specified fill character on either side. Examples:+-- | /O(n)/ Center a string to the given length, using the specified+-- fill character on either side.  Performs replacement on invalid+-- scalar values. --+-- Examples:+-- -- > center 8 'x' "HS" = "xxxHSxxx" center :: Int -> Char -> Text -> Text center k c t@@ -760,6 +814,7 @@  -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of -- successive reduced values from the left. Subject to fusion.+-- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --@@ -767,11 +822,13 @@ -- -- > last (scanl f z xs) == foldl f z xs. scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl f z (stream t))+scanl f z t = unstream (S.scanl g z (stream t))+    where g a b = safe (f a b) {-# INLINE scanl #-}  -- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument.  Subject to fusion.+-- value argument.  Subject to fusion.  Performs replacement on+-- invalid scalar values. -- -- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] scanl1 :: (Char -> Char -> Char) -> Text -> Text@@ -779,15 +836,18 @@            | otherwise = scanl f (unsafeHead t) (unsafeTail t) {-# INLINE scanl1 #-} --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs+-- replacement on invalid scalar values. -- -- > scanr f v == reverse . scanl (flip f) v . reverse scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f z = S.reverse . S.reverseScanr f z . reverseStream+scanr f z = S.reverse . S.reverseScanr g z . reverseStream+    where g a b = safe (f a b) {-# INLINE scanr #-}  -- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument.  Subject to fusion.+-- value argument.  Subject to fusion.  Performs replacement on+-- invalid scalar values. scanr1 :: (Char -> Char -> Char) -> Text -> Text scanr1 f t | null t    = empty            | otherwise = scanr f (last t) (init t)@@ -795,9 +855,11 @@  -- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a -- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'.+-- parameter from left to right, and returns a final 'Text'.  Performs+-- replacement on invalid scalar values. mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumL f z0 = S.mapAccumL f z0 . stream+mapAccumL f z0 = S.mapAccumL g z0 . stream+    where g a b = second safe (f a b) {-# INLINE mapAccumL #-}  -- | The 'mapAccumR' function behaves like a combination of 'map' and@@ -805,8 +867,10 @@ -- 'Text', passing an accumulating parameter from right to left, and -- returning a final value of this accumulator together with the new -- 'Text'.+-- Performs replacement on invalid scalar values. mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumR f z0 = second reverse . S.mapAccumL f z0 . reverseStream+mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream+    where g a b = second safe (f a b) {-# INLINE mapAccumR #-}  -- -----------------------------------------------------------------------------@@ -839,7 +903,7 @@ -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the -- value of every element. Subject to fusion. replicateChar :: Int -> Char -> Text-replicateChar n c = unstream (S.replicateCharI n c)+replicateChar n c = unstream (S.replicateCharI n (safe c)) {-# INLINE replicateChar #-}  -- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'@@ -848,9 +912,9 @@ -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the -- string, and @b@ is the seed value for further production. Subject--- to fusion.+-- to fusion.  Performs replacement on invalid scalar values. unfoldr     :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr f s)+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) {-# INLINE unfoldr #-}  -- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed@@ -858,9 +922,9 @@ -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and -- correct, otherwise its performance is similar to 'unfoldr'. Subject--- to fusion.+-- to fusion.  Performs replacement on invalid scalar values. unfoldrN     :: Int -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n f s)+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) {-# INLINE unfoldrN #-}  -- -----------------------------------------------------------------------------@@ -1009,23 +1073,23 @@             where d                = iter_ t i {-# INLINE splitAt #-} --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list.-spanBy :: (Char -> Bool) -> Text -> (Text, Text)-spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))+span :: (Char -> Bool) -> Text -> (Text, Text)+span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))   where k = loop 0         loop !i | i >= len || not (p c) = i                 | otherwise             = loop (i+d)             where Iter c d              = iter t i-{-# INLINE spanBy #-}+{-# INLINE span #-} --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is+-- | /O(n)/ 'break' is like 'span', but the prefix returned is -- over elements that fail the predicate @p@.-breakBy :: (Char -> Bool) -> Text -> (Text, Text)-breakBy p = spanBy (not . p)-{-# INLINE breakBy #-}+break :: (Char -> Bool) -> Text -> (Text, Text)+break p = span (not . p)+{-# INLINE break #-}  -- | /O(n)/ Group characters in a string according to a predicate. groupBy :: (Char -> Char -> Bool) -> Text -> [Text]@@ -1075,30 +1139,30 @@ -- -- Examples: ----- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > split "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]--- > split "x"    "x"                == ["",""]+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]+-- > splitOn "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]+-- > splitOn "x"    "x"                == ["",""] --  -- and ----- > intercalate s . split s         == id--- > split (singleton c)             == splitBy (==c)+-- > intercalate s . splitOn s         == id+-- > splitOn (singleton c)             == split (==c) -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-split :: Text -> Text -> [Text]-split pat@(Text _ _ l) src@(Text arr off len)-    | l <= 0          = emptyError "split"-    | isSingleton pat = splitBy (== unsafeHead pat) src+splitOn :: Text -> Text -> [Text]+splitOn pat@(Text _ _ l) src@(Text arr off len)+    | l <= 0          = emptyError "splitOn"+    | isSingleton pat = split (== unsafeHead pat) src     | otherwise       = go 0 (indices pat src)   where     go !s (x:xs) =  textP arr (s+off) (x-s) : go (x+l) xs     go  s _      = [textP arr (s+off) (len-s)]-{-# INLINE [1] split #-}+{-# INLINE [1] splitOn #-}  {-# RULES-"TEXT split/singleton -> splitBy/==" [~1] forall c t.-    split (singleton c) t = splitBy (==c) t+"TEXT splitOn/singleton -> split/==" [~1] forall c t.+    splitOn (singleton c) t = split (==c) t   #-}  -- | /O(n)/ Splits a 'Text' into components delimited by separators,@@ -1106,15 +1170,15 @@ -- resulting components do not contain the separators.  Two adjacent -- separators result in an empty component in the output.  eg. ----- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]--- > splitBy (=='a') ""        == [""]-splitBy :: (Char -> Bool) -> Text -> [Text]-splitBy _ t@(Text _off _arr 0) = [t]-splitBy p t = loop t+-- > split (=='a') "aabbaca" == ["","","bb","c",""]+-- > split (=='a') ""        == [""]+split :: (Char -> Bool) -> Text -> [Text]+split _ t@(Text _off _arr 0) = [t]+split p t = loop t     where loop s | null s'   = [l]                  | otherwise = l : loop (unsafeTail s')-              where (l, s') = breakBy p s-{-# INLINE splitBy #-}+              where (l, s') = break p s+{-# INLINE split #-}  -- | /O(n)/ Splits a 'Text' into components of length @k@.  The last -- element may be shorter than the other chunks, depending on the@@ -1136,21 +1200,21 @@ ------------------------------------------------------------------------------- -- ** Searching with a predicate --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and--- returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.-findBy :: (Char -> Bool) -> Text -> Maybe Char-findBy p t = S.findBy p (stream t)-{-# INLINE findBy #-}+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and+-- returns the first element matching the predicate, or 'Nothing' if+-- there is no such element.+find :: (Char -> Bool) -> Text -> Maybe Char+find p t = S.findBy p (stream t)+{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. ----- > partitionBy p t == (filter p t, filter (not . p) t)-partitionBy :: (Char -> Bool) -> Text -> (Text, Text)-partitionBy p t = (filter p t, filter (not . p) t)-{-# INLINE partitionBy #-}+-- > partition p t == (filter p t, filter (not . p) t)+partition :: (Char -> Bool) -> Text -> (Text, Text)+partition p t = (filter p t, filter (not . p) t)+{-# INLINE partition #-}  -- | /O(n)/ 'filter', applied to a predicate and a 'Text', -- returns a 'Text' containing those characters that satisfy the@@ -1166,39 +1230,40 @@ -- -- Examples: ----- > break "::" "a::b::c" ==> ("a", "::b::c")--- > break "/" "foobar"   ==> ("foobar", "")+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")+-- > breakOn "/" "foobar"   ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack--- >   where (prefix, match) = break needle haystack+-- >   where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'find'+-- want to break on every instance of a substring), use 'breakOnAll' -- instead, as it has lower startup overhead. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-break :: Text -> Text -> (Text, Text)-break pat src@(Text arr off len)-    | null pat  = emptyError "break"+breakOn :: Text -> Text -> (Text, Text)+breakOn pat src@(Text arr off len)+    | null pat  = emptyError "breakOn"     | otherwise = case indices pat src of                     []    -> (src, empty)                     (x:_) -> (textP arr off x, textP arr (off+x) (len-x))-{-# INLINE break #-}+{-# INLINE breakOn #-} --- | /O(n+m)/ Similar to 'break', but searches from the end of the string.+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the+-- string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@.  The second is the -- remainder of @haystack@, following the match. ----- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")-breakEnd :: Text -> Text -> (Text, Text)-breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)-                   in  (reverse b, reverse a)-{-# INLINE breakEnd #-}+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")+breakOnEnd :: Text -> Text -> (Text, Text)+breakOnEnd pat src = (reverse b, reverse a)+    where (a,b) = breakOn (reverse pat) (reverse src)+{-# INLINE breakOnEnd #-}  -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@.  Each element of the returned list consists of a pair:@@ -1218,16 +1283,16 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty.-find :: Text                    -- ^ @needle@ to search for-     -> Text                    -- ^ @haystack@ in which to search-     -> [(Text, Text)]-find pat src@(Text arr off slen)-    | null pat  = emptyError "find"+breakOnAll :: Text              -- ^ @needle@ to search for+           -> Text              -- ^ @haystack@ in which to search+           -> [(Text, Text)]+breakOnAll pat src@(Text arr off slen)+    | null pat  = emptyError "breakOnAll"     | otherwise = L.map step (indices pat src)   where     step       x = (chunk 0 x, chunk x (slen-x))     chunk !n !l  = textP arr (n+off) l-{-# INLINE find #-}+{-# INLINE breakOnAll #-}  ------------------------------------------------------------------------------- -- ** Indexing 'Text's@@ -1297,8 +1362,10 @@  -- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function -- given as the first argument, instead of a tupling function.+-- Performs replacement on invalid scalar values. zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))+    where g a b = safe (f a b) {-# INLINE [0] zipWith #-}  -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's@@ -1325,7 +1392,7 @@          | otherwise = h : if null t                            then []                            else lines (unsafeTail t)-    where (h,t) = spanBy (/= '\n') ps+    where (h,t) = span (/= '\n') ps {-# INLINE lines #-}  {-@@ -1362,8 +1429,7 @@ {-# INLINE unwords #-}  -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.  This function is--- subject to fusion.+-- 'True' iff the first is a prefix of the second.  Subject to fusion. isPrefixOf :: Text -> Text -> Bool isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) =     alen <= blen && S.isPrefixOf (stream a) (stream b)

@@ -14,18 +14,20 @@ -- Functions for converting 'Text' values to and from 'ByteString', -- using several standard encodings. ----- To make use of a much larger variety of encodings, use the @text-icu@--- package.+-- To gain access to a much larger family of encodings, use the+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>  module Data.Text.Encoding     (     -- * Decoding ByteStrings to Text+    -- $strict       decodeASCII     , decodeUtf8     , decodeUtf16LE     , decodeUtf16BE     , decodeUtf32LE     , decodeUtf32BE+     -- ** Controllable error handling     , decodeUtf8With     , decodeUtf16LEWith@@ -60,11 +62,25 @@ import qualified Data.Text.Encoding.Utf8 as U8 import qualified Data.Text.Fusion as F +-- $strict+--+-- All of the single-parameter functions for decoding bytestrings+-- encoded in one of the Unicode Transformation Formats (UTF) operate+-- in a /strict/ mode: each will throw an exception if given invalid+-- input.+--+-- Each function has a variant, whose name is suffixed with -'With',+-- that gives greater control over the handling of decoding errors.+-- For instance, 'decodeUtf8' will throw an exception, but+-- 'decodeUtf8With' allows the programmer to determine what to do on a+-- decoding error.+ -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: ByteString -> Text decodeASCII bs = F.unstream (E.streamASCII bs) {-# INLINE decodeASCII #-} +-- | Decode a 'ByteString' containing UTF-8 encoded text. decodeUtf8With :: OnDecodeError -> ByteString -> Text decodeUtf8With onErr bs = textP (fst a) 0 (snd a)  where@@ -101,6 +117,10 @@ {-# INLINE[0] decodeUtf8With #-}  -- | Decode a 'ByteString' containing UTF-8 encoded text.+--+-- If the input contains any invalid UTF-8 data, an exception will be+-- thrown.  For more control over the handling of invalid data, use+-- 'decodeUtf8With'. decodeUtf8 :: ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-}@@ -156,6 +176,10 @@ {-# INLINE decodeUtf16LEWith #-}  -- | Decode text from little endian UTF-16 encoding.+--+-- If the input contains any invalid little endian UTF-16 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-}@@ -166,6 +190,10 @@ {-# INLINE decodeUtf16BEWith #-}  -- | Decode text from big endian UTF-16 encoding.+--+-- If the input contains any invalid big endian UTF-16 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-}@@ -186,6 +214,10 @@ {-# INLINE decodeUtf32LEWith #-}  -- | Decode text from little endian UTF-32 encoding.+--+-- If the input contains any invalid little endian UTF-32 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-}@@ -196,6 +228,10 @@ {-# INLINE decodeUtf32BEWith #-}  -- | Decode text from big endian UTF-32 encoding.+--+-- If the input contains any invalid big endian UTF-32 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-}

@@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for text I\/O.+--+-- Skip past the synopsis for some important notes on performance and+-- portability across different versions of GHC.  module Data.Text.IO     (+    -- * Performance+    -- $performance +     -- * Locale support     -- $locale     -- * File-at-a-time operations@@ -59,6 +65,22 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) import System.IO.Error (isEOFError) #endif++-- $performance+-- #performance#+--+-- The functions in this module obey the runtime system's locale,+-- character set encoding, and line ending conversion settings.+--+-- If you know in advance that you will be working with data that has+-- a specific encoding (e.g. UTF-8), and your application is highly+-- performance sensitive, you may find that it is faster to perform+-- I\/O with bytestrings and to encode and decode yourself than to use+-- the functions in this module.+--+-- Whether this will hold depends on the version of GHC you are using,+-- the platform you are working on, the data you are working with, and+-- the encodings you are using, so be sure to test for yourself.  -- | The 'readFile' function reads a file and returns the contents of -- the file as a string.  The entire file is read strictly, as with

@@ -22,8 +22,12 @@     -- * Construction     , text     , textP+    -- * Safety+    , safe     -- * Code that must be here for accessibility     , empty+    -- * Utilities+    , firstf     -- * Debugging     , showText     ) where@@ -31,7 +35,9 @@ #if defined(ASSERTS) import Control.Exception (assert) #endif+import Data.Bits ((.&.)) import qualified Data.Text.Array as A+import Data.Text.UnsafeChar (ord) import Data.Typeable (Typeable)  -- | A space efficient, packed, unboxed Unicode text type.@@ -72,3 +78,21 @@ showText (Text arr off len) =     "Text " ++ show (A.toList arr off len) ++ ' ' :             show off ++ ' ' : show len++-- | Map a 'Char' to a 'Text'-safe value.+--+-- UTF-16 surrogate code points are not included in the set of Unicode+-- scalar values, but are unfortunately admitted as valid 'Char'+-- values by Haskell.  They cannot be represented in a 'Text'.  This+-- function remaps those code points to the Unicode replacement+-- character \"�\", and leaves other code points unchanged.+safe :: Char -> Char+safe c+    | ord c .&. 0x1ff800 /= 0xd800 = c+    | otherwise                    = '\xfffd'+{-# INLINE safe #-}++-- | Apply a function to the first element of an optional pair.+firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)+firstf f (Just (a, b)) = Just (f a, b)+firstf _  Nothing      = Nothing

@@ -11,18 +11,23 @@ -- Portability : GHC -- -- A time and space-efficient implementation of Unicode text using--- lists of packed arrays.  This representation is suitable for high+-- lists of packed arrays.+--+-- /Note/: Read below the synopsis for important notes on the use of+-- this module.+--+-- The representation used by this module is suitable for high -- performance use and for streaming large quantities of data.  It -- provides a means to manipulate a large body of text without -- requiring that the entire content be resident in memory. -- -- Some operations, such as 'concat', 'append', 'reverse' and 'cons',--- have better complexity than their "Data.Text" equivalents, due to--- optimisations resulting from the list spine structure. And for--- other operations lazy 'Text's are usually within a few percent of--- strict ones, but with better heap usage. For data larger than--- available memory, or if you have tight memory constraints, this--- module will be the only option.+-- have better time complexity than their "Data.Text" equivalents, due+-- to the underlying representation being a list of chunks. For other+-- operations, lazy 'Text's are usually within a few percent of strict+-- ones, but often with better heap usage if used in a streaming+-- fashion. For data larger than available memory, or if you have+-- tight memory constraints, this module will be the only option. -- -- This module is intended to be imported @qualified@, to avoid name -- clashes with "Prelude" functions.  eg.@@ -31,7 +36,15 @@  module Data.Text.Lazy     (+    -- * Fusion+    -- $fusion++    -- * Acceptable data+    -- $replacement++    -- * Types       Text+     -- * Creation and elimination     , pack     , unpack@@ -122,10 +135,10 @@     , stripStart     , stripEnd     , splitAt-    , spanBy+    , span+    , breakOn+    , breakOnEnd     , break-    , breakEnd-    , breakBy     , group     , groupBy     , inits@@ -133,8 +146,8 @@      -- ** Breaking into many substrings     -- $split+    , splitOn     , split-    , splitBy     , chunksOf     -- , breakSubstring @@ -156,8 +169,8 @@     -- * Searching     , filter     , find-    , findBy-    , partitionBy+    , breakOnAll+    , partition      -- , findSubstring     @@ -200,10 +213,62 @@ import Data.Text.Fusion.Internal (PairS(..)) import Data.Text.Lazy.Fusion (stream, unstream) import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks)-import Data.Text.Internal (textP)+import Data.Text.Internal (firstf, safe, textP) import qualified Data.Text.Util as U import Data.Text.Lazy.Search (indices) +-- $fusion+--+-- Most of the functions in this module are subject to /fusion/,+-- meaning that a pipeline of such functions will usually allocate at+-- most one 'Text' value.+--+-- As an example, consider the following pipeline:+--+-- > import Data.Text.Lazy as T+-- > import Data.Text.Lazy.Encoding as E+-- > import Data.ByteString.Lazy (ByteString)+-- >+-- > countChars :: ByteString -> Int+-- > countChars = T.length . T.toUpper . E.decodeUtf8+--+-- From the type signatures involved, this looks like it should+-- allocate one 'ByteString' value, and two 'Text' values. However,+-- when a module is compiled with optimisation enabled under GHC, the+-- two intermediate 'Text' values will be optimised away, and the+-- function will be compiled down to a single loop over the source+-- 'ByteString'.+--+-- Functions that can be fused by the compiler are documented with the+-- phrase \"Subject to fusion\".++-- $replacement+--+-- A 'Text' value is a sequence of Unicode scalar values, as defined+-- in &#xa7;3.9, definition D76 of the Unicode 5.2 standard:+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35>. As+-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF+-- inclusive. Haskell implementations admit all Unicode code points+-- (&#xa7;3.4, definition D10) as 'Char' values, including code points+-- from this invalid range.  This means that there are some 'Char'+-- values that are not valid Unicode scalar values, and the functions+-- in this module must handle those cases.+--+-- Within this module, many functions construct a 'Text' from one or+-- more 'Char' values. Those functions will substitute 'Char' values+-- that are not valid Unicode scalar values with the replacement+-- character \"&#xfffd;\" (U+FFFD).  Functions that perform this+-- inspection and replacement are documented with the phrase+-- \"Performs replacement on invalid scalar values\".+--+-- (One reason for this policy of replacement is that internally, a+-- 'Text' value is represented as packed UTF-16 data. Values in the+-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate+-- code points, and so cannot be represented. The functions replace+-- invalid scalar values, instead of dropping them, as a security+-- measure. For details, see Unicode Technical Report 36, &#xa7;3.5:+-- <http://unicode.org/reports/tr36#Deletion_of_Noncharacters>)+ equal :: Text -> Text -> Bool equal Empty Empty = True equal Empty _     = False@@ -274,19 +339,19 @@  -- | /O(n)/ Convert a 'String' into a 'Text'. ----- This function is subject to array fusion.+-- Subject to fusion.  Performs replacement on invalid scalar values. pack :: String -> Text-pack s = unstream (S.streamList s)+pack = unstream . S.streamList . L.map safe {-# INLINE [1] pack #-}  -- | /O(n)/ Convert a 'Text' into a 'String'.--- Subject to array fusion.+-- Subject to fusion. unpack :: Text -> String unpack t = S.unstreamList (stream t) {-# INLINE [1] unpack #-} --- | /O(1)/ Convert a character into a Text.--- Subject to fusion.+-- | /O(1)/ Convert a character into a Text.  Subject to fusion.+-- Performs replacement on invalid scalar values. singleton :: Char -> Text singleton c = Chunk (T.singleton c) Empty {-# INLINE [1] singleton #-}@@ -346,8 +411,7 @@     unstream (S.snoc (stream t) c) = snoc t c  #-} --- | /O(n\/c)/ Appends one 'Text' to another.  Subject to array--- fusion.+-- | /O(n\/c)/ Appends one 'Text' to another.  Subject to fusion. append :: Text -> Text -> Text append xs ys = foldrChunks Chunk ys xs {-# INLINE [1] append #-}@@ -360,7 +424,7 @@  #-}  -- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty. Subject to array fusion.+-- 'Nothing' if empty. Subject to fusion. uncons :: Text -> Maybe (Char, Text) uncons Empty        = Nothing uncons (Chunk t ts) = Just (T.unsafeHead t, ts')@@ -369,13 +433,13 @@ {-# INLINE uncons #-}  -- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty.  Subject to array fusion.+-- non-empty.  Subject to fusion. head :: Text -> Char head t = S.head (stream t) {-# INLINE head #-}  -- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty.  Subject to array fusion.+-- must be non-empty.  Subject to fusion. tail :: Text -> Text tail (Chunk t ts) = chunk (T.tail t) ts tail Empty        = emptyError "tail"@@ -389,7 +453,7 @@  #-}  -- | /O(1)/ Returns all but the last character of a 'Text', which must--- be non-empty.  Subject to array fusion.+-- be non-empty.  Subject to fusion. init :: Text -> Text init (Chunk t0 ts0) = go t0 ts0     where go t (Chunk t' ts) = Chunk t (go t' ts)@@ -404,7 +468,7 @@     unstream (S.init (stream t)) = init t  #-} --- | /O(1)/ Tests whether a 'Text' is empty or not.  Subject to array+-- | /O(1)/ Tests whether a 'Text' is empty or not.  Subject to -- fusion. null :: Text -> Bool null Empty = True@@ -425,7 +489,7 @@ {-# INLINE isSingleton #-}  -- | /O(1)/ Returns the last character of a 'Text', which must be--- non-empty.  Subject to array fusion.+-- non-empty.  Subject to fusion. last :: Text -> Char last Empty        = emptyError "last" last (Chunk t ts) = go t ts@@ -469,9 +533,10 @@ -- properties of code.  -- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.  Subject to array fusion.+-- each element of @t@.  Subject to fusion.  Performs replacement on+-- invalid scalar values. map :: (Char -> Char) -> Text -> Text-map f t = unstream (S.map f (stream t))+map f t = unstream (S.map (safe . f) (stream t)) {-# INLINE [1] map #-}  -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of@@ -482,14 +547,18 @@ {-# INLINE intercalate #-}  -- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.  Subject to array fusion.-intersperse     :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse c (stream t))+-- between the characters of a 'Text'.  Subject to fusion.  Performs+-- replacement on invalid scalar values.+intersperse :: Char -> Text -> Text+intersperse c t = unstream (S.intersperse (safe c) (stream t)) {-# INLINE intersperse #-}  -- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Subject to fusion. Examples:+-- specified fill character on the right. Subject to fusion.  Performs+-- replacement on invalid scalar values. --+-- Examples:+-- -- > justifyLeft 7 'x' "foo"    == "fooxxxx" -- > justifyLeft 3 'x' "foobar" == "foobar" justifyLeft :: Int64 -> Char -> Text -> Text@@ -507,8 +576,11 @@   #-}  -- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Examples:+-- specified fill character on the left.  Performs replacement on+-- invalid scalar values. --+-- Examples:+-- -- > justifyRight 7 'x' "bar"    == "xxxxbar" -- > justifyRight 3 'x' "foobar" == "foobar" justifyRight :: Int64 -> Char -> Text -> Text@@ -518,9 +590,12 @@   where len = length t {-# INLINE justifyRight #-} --- | /O(n)/ Center a string to the given length, using the--- specified fill character on either side. Examples:+-- | /O(n)/ Center a string to the given length, using the specified+-- fill character on either side.  Performs replacement on invalid+-- scalar values. --+-- Examples:+-- -- > center 8 'x' "HS" = "xxxHSxxx" center :: Int64 -> Char -> Text -> Text center k c t@@ -555,7 +630,7 @@         -> Text                 -- ^ Replacement text         -> Text                 -- ^ Input text         -> Text-replace s d = intercalate d . split s+replace s d = intercalate d . splitOn s {-# INLINE replace #-}  -- ----------------------------------------------------------------------------@@ -608,26 +683,24 @@ -- | /O(n)/ 'foldl', applied to a binary operator, a starting value -- (typically the left-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from left to right.--- Subject to array fusion.+-- Subject to fusion. foldl :: (a -> Char -> a) -> a -> Text -> a foldl f z t = S.foldl f z (stream t) {-# INLINE foldl #-}  -- | /O(n)/ A strict version of 'foldl'.--- Subject to array fusion.+-- Subject to fusion. foldl' :: (a -> Char -> a) -> a -> Text -> a foldl' f z t = S.foldl' f z (stream t) {-# INLINE foldl' #-}  -- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.  Subject to array--- fusion.+-- and thus must be applied to a non-empty 'Text'.  Subject to fusion. foldl1 :: (Char -> Char -> Char) -> Text -> Char foldl1 f t = S.foldl1 f (stream t) {-# INLINE foldl1 #-} --- | /O(n)/ A strict version of 'foldl1'.--- Subject to array fusion.+-- | /O(n)/ A strict version of 'foldl1'.  Subject to fusion. foldl1' :: (Char -> Char -> Char) -> Text -> Char foldl1' f t = S.foldl1' f (stream t) {-# INLINE foldl1' #-}@@ -635,13 +708,13 @@ -- | /O(n)/ 'foldr', applied to a binary operator, a starting value -- (typically the right-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from right to left.--- Subject to array fusion.+-- Subject to fusion. foldr :: (Char -> a -> a) -> a -> Text -> a foldr f z t = S.foldr f z (stream t) {-# INLINE foldr #-} --- | /O(n)/ A variant of 'foldr' that has no starting value argument, and--- thust must be applied to a non-empty 'Text'.  Subject to array+-- | /O(n)/ A variant of 'foldr' that has no starting value argument,+-- and thust must be applied to a non-empty 'Text'.  Subject to -- fusion. foldr1 :: (Char -> Char -> Char) -> Text -> Char foldr1 f t = S.foldr1 f (stream t)@@ -664,32 +737,32 @@ {-# INLINE concatMap #-}  -- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisifes the predicate @p@. Subject to array fusion.+-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion. any :: (Char -> Bool) -> Text -> Bool any p t = S.any p (stream t) {-# INLINE any #-}  -- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisify the predicate @p@. Subject to array fusion.+-- 'Text' @t@ satisify the predicate @p@. Subject to fusion. all :: (Char -> Bool) -> Text -> Bool all p t = S.all p (stream t) {-# INLINE all #-}  -- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty. Subject to array fusion.+-- must be non-empty. Subject to fusion. maximum :: Text -> Char maximum t = S.maximum (stream t) {-# INLINE maximum #-}  -- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty. Subject to array fusion.+-- must be non-empty. Subject to fusion. minimum :: Text -> Char minimum t = S.minimum (stream t) {-# INLINE minimum #-}  -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left. This function is subject--- to array fusion.+-- successive reduced values from the left. Subject to fusion.+-- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --@@ -697,11 +770,13 @@ -- -- > last (scanl f z xs) == foldl f z xs. scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl f z (stream t))+scanl f z t = unstream (S.scanl g z (stream t))+    where g a b = safe (f a b) {-# INLINE scanl #-}  -- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument.  This function is subject to array fusion.+-- value argument.  Subject to fusion.  Performs replacement on+-- invalid scalar values. -- -- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] scanl1 :: (Char -> Char -> Char) -> Text -> Text@@ -710,21 +785,24 @@                 Just (t,ts) -> scanl f t ts {-# INLINE scanl1 #-} --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs+-- replacement on invalid scalar values. -- -- > scanr f v == reverse . scanl (flip f) v . reverse scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f v = reverse . scanl (flip f) v . reverse+scanr f v = reverse . scanl g v . reverse+    where g a b = safe (f b a)  -- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument.+-- value argument.  Performs replacement on invalid scalar values. scanr1 :: (Char -> Char -> Char) -> Text -> Text scanr1 f t | null t    = empty            | otherwise = scanr f (last t) (init t)  -- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a -- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'.+-- parameter from left to right, and returns a final 'Text'.  Performs+-- replacement on invalid scalar values. mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) mapAccumL f = go   where@@ -738,7 +816,7 @@ -- a strict 'foldr'; it applies a function to each element of a -- 'Text', passing an accumulating parameter from right to left, and -- returning a final value of this accumulator together with the new--- 'Text'.+-- 'Text'.  Performs replacement on invalid scalar values. mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) mapAccumR f = go   where@@ -762,7 +840,7 @@ -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the -- value of every element. Subject to fusion. replicateChar :: Int64 -> Char -> Text-replicateChar n c = unstream (S.replicateCharI n c)+replicateChar n c = unstream (S.replicateCharI n (safe c)) {-# INLINE replicateChar #-}  {-# RULES@@ -775,9 +853,10 @@ -- 'Text' from a seed value. The function takes the element and -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.-unfoldr     :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr f s)+-- string, and @b@ is the seed value for further production.  Performs+-- replacement on invalid scalar values.+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) {-# INLINE unfoldr #-}  -- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed@@ -785,8 +864,9 @@ -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and -- correct, otherwise its performance is similar to 'unfoldr'.-unfoldrN     :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n f s)+-- Performs replacement on invalid scalar values.+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) {-# INLINE unfoldrN #-}  -- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the@@ -847,9 +927,9 @@             where len'  = fromIntegral len                   n'    = fromIntegral n --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', returns--- the longest prefix (possibly empty) of elements that satisfy @p@.--- This function is subject to array fusion.+-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',+-- returns the longest prefix (possibly empty) of elements that+-- satisfy @p@.  Subject to fusion. takeWhile :: (Char -> Bool) -> Text -> Text takeWhile p t0 = takeWhile' t0   where takeWhile' Empty        = Empty@@ -868,7 +948,7 @@   #-}  -- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@. This function is subject to array fusion.+-- 'takeWhile' @p@ @t@.  Subject to fusion. dropWhile :: (Char -> Bool) -> Text -> Text dropWhile p t0 = dropWhile' t0   where dropWhile' Empty        = Empty@@ -964,13 +1044,13 @@ -- -- Examples: ----- > break "::" "a::b::c" ==> ("a", "::b::c")--- > break "/" "foobar"   ==> ("foobar", "")+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")+-- > breakOn "/" "foobar"   ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack--- >   where (prefix, match) = break needle haystack+-- >   where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you -- want to break on every instance of a substring), use 'find'@@ -981,25 +1061,25 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-break :: Text -> Text -> (Text, Text)-break pat src-    | null pat  = emptyError "break"+breakOn :: Text -> Text -> (Text, Text)+breakOn pat src+    | null pat  = emptyError "breakOn"     | otherwise = case indices pat src of                     []    -> (src, empty)                     (x:_) -> let h :*: t = splitAtWord x src                              in  (h, t) --- | /O(n+m)/ Similar to 'break', but searches from the end of the string.+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@.  The second is the -- remainder of @haystack@, following the match. ----- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")-breakEnd :: Text -> Text -> (Text, Text)-breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")+breakOnEnd :: Text -> Text -> (Text, Text)+breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)                    in  (reverse b, reverse a)-{-# INLINE breakEnd #-}+{-# INLINE breakOnEnd #-}  -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@.  Each element of the returned list consists of a pair:@@ -1010,9 +1090,9 @@ -- -- Examples: ----- > find "::" ""+-- > breakOnAll "::" "" -- > ==> []--- > find "/" "a/b/c/"+-- > breakOnAll "/" "a/b/c/" -- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] -- -- This function is strict in its first argument, and lazy in its@@ -1022,11 +1102,11 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty.-find :: Text                    -- ^ @needle@ to search for-     -> Text                    -- ^ @haystack@ in which to search-     -> [(Text, Text)]-find pat src-    | null pat  = emptyError "find"+breakOnAll :: Text              -- ^ @needle@ to search for+           -> Text              -- ^ @haystack@ in which to search+           -> [(Text, Text)]+breakOnAll pat src+    | null pat  = emptyError "breakOnAll"     | otherwise = go 0 empty src (indices pat src)   where     go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s@@ -1034,10 +1114,10 @@                        in (h',t) : go x h' t xs     go _  _ _ _      = [] --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over+-- | /O(n)/ 'break' is like 'span', but the prefix returned is over -- elements that fail the predicate @p@.-breakBy :: (Char -> Bool) -> Text -> (Text, Text)-breakBy p t0 = break' t0+break :: (Char -> Bool) -> Text -> (Text, Text)+break p t0 = break' t0   where break' Empty          = (empty, empty)         break' c@(Chunk t ts) =           case T.findIndex p t of@@ -1047,13 +1127,13 @@                    | otherwise -> let (a,b) = T.splitAt n t                                   in (Chunk a Empty, Chunk b ts) --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list.-spanBy :: (Char -> Bool) -> Text -> (Text, Text)-spanBy p = breakBy (not . p)-{-# INLINE spanBy #-}+span :: (Char -> Bool) -> Text -> (Text, Text)+span p = break (not . p)+{-# INLINE span #-}  -- | The 'group' function takes a 'Text' and returns a list of 'Text's -- such that the concatenation of the result is equal to the argument.@@ -1072,7 +1152,7 @@ groupBy :: (Char -> Char -> Bool) -> Text -> [Text] groupBy _  Empty        = [] groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs-                          where (ys,zs) = spanBy (eq x) xs+                          where (ys,zs) = span (eq x) xs                                 x  = T.unsafeHead t                                 xs = chunk (T.unsafeTail t) ts @@ -1104,37 +1184,37 @@ -- -- Examples: ----- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > split "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]--- > split "x"    "x"                == ["",""]+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]+-- > splitOn "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]+-- > splitOn "x"    "x"                == ["",""] --  -- and ----- > intercalate s . split s         == id--- > split (singleton c)             == splitBy (==c)+-- > intercalate s . splitOn s         == id+-- > splitOn (singleton c)             == split (==c) -- -- This function is strict in its first argument, and lazy in its -- second. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-split :: Text                   -- ^ Text to split on-      -> Text                   -- ^ Input text-      -> [Text]-split pat src-    | null pat        = emptyError "split"-    | isSingleton pat = splitBy (== head pat) src+splitOn :: Text                 -- ^ Text to split on+        -> Text                 -- ^ Input text+        -> [Text]+splitOn pat src+    | null pat        = emptyError "splitOn"+    | isSingleton pat = split (== head pat) src     | otherwise       = go 0 (indices pat src) src   where     go  _ []     cs = [cs]     go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs                       in  h : go (x+l) xs (dropWords l t)     l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat-{-# INLINE [1] split #-}+{-# INLINE [1] splitOn #-}  {-# RULES-"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t.-    split (singleton c) t = splitBy (==c) t+"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.+    splitOn (singleton c) t = split (==c) t   #-}  -- | /O(n)/ Splits a 'Text' into components delimited by separators,@@ -1142,16 +1222,16 @@ -- resulting components do not contain the separators.  Two adjacent -- separators result in an empty component in the output.  eg. ----- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]--- > splitBy (=='a') []        == [""]-splitBy :: (Char -> Bool) -> Text -> [Text]-splitBy _ Empty = [Empty]-splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0+-- > split (=='a') "aabbaca" == ["","","bb","c",""]+-- > split (=='a') []        == [""]+split :: (Char -> Bool) -> Text -> [Text]+split _ Empty = [Empty]+split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0   where comb acc (s:[]) Empty        = revChunks (s:acc) : []-        comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts+        comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts         comb acc (s:ss) ts           = revChunks (s:acc) : comb [] ss ts-        comb _   []     _            = impossibleError "splitBy"-{-# INLINE splitBy #-}+        comb _   []     _            = impossibleError "split"+{-# INLINE split #-}  -- | /O(n)/ Splits a 'Text' into components of length @k@.  The last -- element may be shorter than the other chunks, depending on the@@ -1171,14 +1251,14 @@ -- newline 'Char's. The resulting strings do not contain newlines. lines :: Text -> [Text] lines Empty = []-lines t = let (l,t') = breakBy ((==) '\n') t+lines t = let (l,t') = break ((==) '\n') t           in l : if null t' then []                  else lines (tail t')  -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text]-words = L.filter (not . null) . splitBy isSpace+words = L.filter (not . null) . split isSpace {-# INLINE words #-}  -- | /O(n)/ Joins lines, after appending a terminating newline to@@ -1193,8 +1273,7 @@ {-# INLINE unwords #-}  -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.  This function is--- subject to fusion.+-- 'True' iff the first is a prefix of the second.  Subject to fusion. isPrefixOf :: Text -> Text -> Bool isPrefixOf Empty _  = True isPrefixOf _ Empty  = False@@ -1299,21 +1378,21 @@ filter p t = unstream (S.filter p (stream t)) {-# INLINE filter #-} --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and -- returns the first element in matching the predicate, or 'Nothing' -- if there is no such element.-findBy :: (Char -> Bool) -> Text -> Maybe Char-findBy p t = S.findBy p (stream t)-{-# INLINE findBy #-}+find :: (Char -> Bool) -> Text -> Maybe Char+find p t = S.findBy p (stream t)+{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. ----- > partitionBy p t == (filter p t, filter (not . p) t)-partitionBy :: (Char -> Bool) -> Text -> (Text, Text)-partitionBy p t = (filter p t, filter (not . p) t)-{-# INLINE partitionBy #-}+-- > partition p t == (filter p t, filter (not . p) t)+partition :: (Char -> Bool) -> Text -> (Text, Text)+partition p t = (filter p t, filter (not . p) t)+{-# INLINE partition #-}  -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. index :: Text -> Int64 -> Char@@ -1340,8 +1419,7 @@   #-}  -- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'. This function is subject--- to fusion.+-- query element appears in the given 'Text'.  Subject to fusion. countChar :: Char -> Text -> Int64 countChar c t = S.countChar c (stream t) @@ -1355,8 +1433,10 @@  -- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function -- given as the first argument, instead of a tupling function.+-- Performs replacement on invalid scalar values. zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2))+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))+    where g a b = safe (f a b) {-# INLINE [0] zipWith #-}  revChunks :: [T.Text] -> Text

@@ -10,8 +10,21 @@ -- Stability   : experimental -- Portability : portable to Hugs and GHC ----- Efficient construction of lazy texts.+-- Efficient construction of lazy @Text@ values.  The principal+-- operations on a @Builder@ are @singleton@, @fromText@, and+-- @fromLazyText@, which construct new builders, and 'mappend', which+-- concatenates two builders. --+-- To get maximum performance when building lazy @Text@ values using a builder, associate @mappend@ calls to the right.  For example, prefer+--+-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')+--+-- to+--+-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'+--+-- as the latter associates @mappend@ to the left.+-- -----------------------------------------------------------------------------  module Data.Text.Lazy.Builder@@ -46,15 +59,15 @@  ------------------------------------------------------------------------ --- | A 'Builder' is an efficient way to build lazy 'L.Text's.  There--- are several functions for constructing 'Builder's, but only one to--- inspect them: to extract any data, you have to turn them into lazy--- 'L.Text's using 'toLazyText'.+-- | A @Builder@ is an efficient way to build lazy @Text@ values.+-- There are several functions for constructing builders, but only one+-- to inspect them: to extract any data, you have to turn them into+-- lazy @Text@ values using @toLazyText@. ----- Internally, a 'Builder' constructs a lazy 'L.Text' by filling byte--- arrays piece by piece.  As each buffer is filled, it is \'popped\'--- off, to become a new chunk of the resulting lazy 'L.Text'.  All--- this is hidden from the user of the 'Builder'.+-- Internally, a builder constructs a lazy @Text@ by filling arrays+-- piece by piece.  As each buffer is filled, it is \'popped\' off, to+-- become a new chunk of the resulting lazy @Text@.  All this is+-- hidden from the user of the @Builder@. newtype Builder = Builder {      -- Invariant (from Data.Text.Lazy):      --      The lists include no null Texts.@@ -78,7 +91,7 @@  ------------------------------------------------------------------------ --- | /O(1)./ The empty Builder, satisfying+-- | /O(1)./ The empty @Builder@, satisfying -- --  * @'toLazyText' 'empty' = 'L.empty'@ --@@ -86,7 +99,7 @@ empty = Builder id {-# INLINE empty #-} --- | /O(1)./ A Builder taking a single character, satisfying+-- | /O(1)./ A @Builder@ taking a single character, satisfying -- --  * @'toLazyText' ('singleton' c) = 'L.singleton' c@ --@@ -96,7 +109,7 @@  ------------------------------------------------------------------------ --- | /O(1)./ The concatenation of two Builders, an associative+-- | /O(1)./ The concatenation of two builders, an associative -- operation with identity 'empty', satisfying -- --  * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@@@ -107,12 +120,12 @@  -- TODO: Experiment to find the right threshold. copyLimit :: Int-copyLimit =  128                                 +copyLimit = 128                                  --- This function attempts to merge small Texts instead of treating the--- text as its own chunk.  We may not always want this.+-- This function attempts to merge small @Text@ values instead of+-- treating each value as its own chunk.  We may not always want this. --- | /O(1)./ A Builder taking a 'S.Text', satisfying+-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying -- --  * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ --@@ -128,7 +141,7 @@         fromText (S.pack s) = fromString s  #-} --- | /O(1)./ A Builder taking a 'String', satisfying+-- | /O(1)./ A Builder taking a @String@, satisfying -- --  * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ --@@ -150,7 +163,7 @@     chunkSize = smallChunkSize {-# INLINE fromString #-} --- | /O(1)./ A Builder taking a lazy 'L.Text', satisfying+-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying -- --  * @'toLazyText' ('fromLazyText' t) = t@ --@@ -168,15 +181,15 @@  ------------------------------------------------------------------------ --- | /O(n)./ Extract a lazy 'L.Text' from a 'Builder' with a default+-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default -- buffer size.  The construction work takes place if and when the--- relevant part of the lazy 'L.Text' is demanded.+-- relevant part of the lazy @Text@ is demanded. toLazyText :: Builder -> L.Text toLazyText = toLazyTextWith smallChunkSize --- | /O(n)./ Extract a lazy 'L.Text' from a 'Builder', using the given+-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given -- size for the initial buffer.  The construction work takes place if--- and when the relevant part of the lazy 'L.Text' is demanded.+-- and when the relevant part of the lazy @Text@ is demanded. -- -- If the initial buffer is too small to hold all data, subsequent -- buffers will be the default buffer size.@@ -184,8 +197,8 @@ toLazyTextWith chunkSize m = L.fromChunks (runST $   newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) --- | /O(1)./ Pop the 'S.Text' we have constructed so far, if any,--- yielding a new chunk in the result lazy 'L.Text'.+-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any,+-- yielding a new chunk in the result lazy @Text@. flush :: Builder flush = Builder $ \ k buf@(Buffer p o u l) ->     if u == 0

@@ -12,12 +12,13 @@ -- Functions for converting lazy 'Text' values to and from lazy -- 'ByteString', using several standard encodings. ----- To make use of a much larger variety of encodings, use the @text-icu@--- package.+-- To gain access to a much larger variety of encodings, use the+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>  module Data.Text.Lazy.Encoding     (     -- * Decoding ByteStrings to Text+    -- $strict       decodeASCII     , decodeUtf8     , decodeUtf16LE@@ -51,6 +52,19 @@ import qualified Data.Text.Lazy.Encoding.Fusion as E import qualified Data.Text.Lazy.Fusion as F +-- $strict+--+-- All of the single-parameter functions for decoding bytestrings+-- encoded in one of the Unicode Transformation Formats (UTF) operate+-- in a /strict/ mode: each will throw an exception if given invalid+-- input.+--+-- Each function has a variant, whose name is suffixed with -'With',+-- that gives greater control over the handling of decoding errors.+-- For instance, 'decodeUtf8' will throw an exception, but+-- 'decodeUtf8With' allows the programmer to determine what to do on a+-- decoding error.+ -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: B.ByteString -> Text decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)@@ -95,6 +109,10 @@ {-# INLINE[0] decodeUtf8With #-}  -- | Decode a 'ByteString' containing UTF-8 encoded text.+--+-- If the input contains any invalid UTF-8 data, an exception will be+-- thrown.  For more control over the handling of invalid data, use+-- 'decodeUtf8With'. decodeUtf8 :: B.ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-}@@ -113,6 +131,10 @@ {-# INLINE decodeUtf16LEWith #-}  -- | Decode text from little endian UTF-16 encoding.+--+-- If the input contains any invalid little endian UTF-16 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: B.ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-}@@ -123,6 +145,10 @@ {-# INLINE decodeUtf16BEWith #-}  -- | Decode text from big endian UTF-16 encoding.+--+-- If the input contains any invalid big endian UTF-16 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: B.ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-}@@ -143,6 +169,10 @@ {-# INLINE decodeUtf32LEWith #-}  -- | Decode text from little endian UTF-32 encoding.+--+-- If the input contains any invalid little endian UTF-32 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: B.ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-}@@ -153,6 +183,10 @@ {-# INLINE decodeUtf32BEWith #-}  -- | Decode text from big endian UTF-32 encoding.+--+-- If the input contains any invalid big endian UTF-32 data, an+-- exception will be thrown.  For more control over the handling of+-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: B.ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-}

@@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for lazy text I\/O.+--+-- Skip past the synopsis for some important notes on performance and+-- portability across different versions of GHC.  module Data.Text.Lazy.IO     (+    -- * Performance+    -- $performance +     -- * Locale support     -- $locale     -- * File-at-a-time operations@@ -57,6 +63,21 @@ import System.IO.Error (isEOFError) import System.IO.Unsafe (unsafeInterleaveIO) #endif++-- $performance+--+-- The functions in this module obey the runtime system's locale,+-- character set encoding, and line ending conversion settings.+--+-- If you know in advance that you will be working with data that has+-- a specific encoding (e.g. UTF-8), and your application is highly+-- performance sensitive, you may find that it is faster to perform+-- I\/O with bytestrings and to encode and decode yourself than to use+-- the functions in this module.+--+-- Whether this will hold depends on the version of GHC you are using,+-- the platform you are working on, the data you are working with, and+-- the encodings you are using, so be sure to test for yourself.  -- | Read a file and return its contents as a string.  The file is -- read lazily, as with 'getContents'.

@@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt     | T.null h  = Left "input does not start with a digit"     | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.spanBy isDigit txt+  where (h,t)  = T.span isDigit txt         go n d = (n * 10 + fromIntegral (digitToInt d))  -- | Read a hexadecimal integer, consisting of an optional leading@@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-}@@ -74,7 +76,7 @@ hex txt     | T.null h  = Left "input does not start with a hexadecimal digit"     | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.spanBy isHexDigit txt+  where (h,t)  = T.span isHexDigit txt         go n d = (n * 16 + fromIntegral (hexDigitToInt d))  hexDigitToInt :: Char -> Int

@@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt     | T.null h  = Left "input does not start with a digit"     | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.spanBy isDigit txt+  where (h,t)  = T.span isDigit txt         go n d = (n * 10 + fromIntegral (digitToInt d))  -- | Read a hexadecimal integer, consisting of an optional leading@@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-}@@ -74,7 +76,7 @@ hex txt     | T.null h  = Left "input does not start with a hexadecimal digit"     | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.spanBy isHexDigit txt+  where (h,t)  = T.span isHexDigit txt         go n d = (n * 16 + fromIntegral (hexDigitToInt d))  hexDigitToInt :: Char -> Int@@ -99,7 +101,7 @@ -- by the 'read' function, with the exception that a trailing @\'.\'@ -- or @\'e\'@ /not/ followed by a number is not consumed. ----- Examples:+-- Examples (with behaviour identical to 'read'): -- -- >rational "3"     == Right (3.0, "") -- >rational "3.1"   == Right (3.1, "")

@@ -93,9 +93,6 @@ instance Show DecodeErr where     show (DE d _) = "DE " ++ d -instance CoArbitrary Word8 where-    coarbitrary = coarbitraryIntegral- instance Arbitrary DecodeErr where     arbitrary = oneof [ return $ DE "lenient" lenientDecode                       , return $ DE "ignore" ignore@@ -268,13 +265,13 @@ tl_reverse        = L.reverse `eqP` (unpackS . TL.reverse) t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) -t_replace s d     = (L.intercalate d . split s) `eqP`+t_replace s d     = (L.intercalate d . splitOn s) `eqP`                     (unpackS . T.replace (T.pack s) (T.pack d))-tl_replace s d     = (L.intercalate d . split s) `eqP`+tl_replace s d     = (L.intercalate d . splitOn s) `eqP`                      (unpackS . TL.replace (TL.pack s) (TL.pack d)) -split :: (Eq a) => [a] -> [a] -> [[a]]-split pat src0+splitOn :: (Eq a) => [a] -> [a] -> [[a]]+splitOn pat src0     | l == 0    = error "empty"     | otherwise = go src0   where@@ -485,25 +482,25 @@ tl_strip          = TL.dropAround isSpace `eq` TL.strip t_splitAt n       = L.splitAt n   `eqP` (unpack2 . T.splitAt n) tl_splitAt n      = L.splitAt n   `eqP` (unpack2 . TL.splitAt (fromIntegral n))-t_spanBy p        = L.span p      `eqP` (unpack2 . T.spanBy p)-tl_spanBy p       = L.span p      `eqP` (unpack2 . TL.spanBy p)+t_span p        = L.span p      `eqP` (unpack2 . T.span p)+tl_span p       = L.span p      `eqP` (unpack2 . TL.span p) -t_break_id s      = squid `eq` (uncurry T.append . T.break s)+t_breakOn_id s      = squid `eq` (uncurry T.append . T.breakOn s)   where squid t | T.null s  = error "empty"                 | otherwise = t-tl_break_id s     = squid `eq` (uncurry TL.append . TL.break s)+tl_breakOn_id s     = squid `eq` (uncurry TL.append . TL.breakOn s)   where squid t | TL.null s  = error "empty"                 | otherwise = t-t_break_start (NotEmpty s) t = let (_,m) = T.break s t+t_breakOn_start (NotEmpty s) t = let (_,m) = T.breakOn s t                                in T.null m || s `T.isPrefixOf` m-tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t+tl_breakOn_start (NotEmpty s) t = let (_,m) = TL.breakOn s t                                 in TL.null m || s `TL.isPrefixOf` m-t_breakEnd_end (NotEmpty s) t = let (m,_) = T.breakEnd s t+t_breakOnEnd_end (NotEmpty s) t = let (m,_) = T.breakOnEnd s t                                 in T.null m || s `T.isSuffixOf` m-tl_breakEnd_end (NotEmpty s) t = let (m,_) = TL.breakEnd s t+tl_breakOnEnd_end (NotEmpty s) t = let (m,_) = TL.breakOnEnd s t                                 in TL.null m || s `TL.isSuffixOf` m-t_breakBy p       = L.break p     `eqP` (unpack2 . T.breakBy p)-tl_breakBy p      = L.break p     `eqP` (unpack2 . TL.breakBy p)+t_break p       = L.break p     `eqP` (unpack2 . T.break p)+tl_break p      = L.break p     `eqP` (unpack2 . TL.break p) t_group           = L.group       `eqP` (map unpackS . T.group) tl_group          = L.group       `eqP` (map unpackS . TL.group) t_groupBy p       = L.groupBy p   `eqP` (map unpackS . T.groupBy p)@@ -514,33 +511,33 @@ tl_tails          = L.tails       `eqP` (map unpackS . TL.tails) t_findAppendId (NotEmpty s) = unsquare $ \ts ->     let t = T.intercalate s ts-    in all (==t) $ map (uncurry T.append) (T.find s t)+    in all (==t) $ map (uncurry T.append) (T.breakOnAll s t) tl_findAppendId (NotEmpty s) = unsquare $ \ts ->     let t = TL.intercalate s ts-    in all (==t) $ map (uncurry TL.append) (TL.find s t)-t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.find s .+    in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t)+t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.breakOnAll s .                               T.intercalate s tl_findContains (NotEmpty s) = all (TL.isPrefixOf s . snd) .-                               TL.find s . TL.intercalate s+                               TL.breakOnAll s . TL.intercalate s sl_filterCount c  = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c-t_findCount s     = (L.length . T.find s) `eq` T.count s-tl_findCount s    = (L.genericLength . TL.find s) `eq` TL.count s+t_findCount s     = (L.length . T.breakOnAll s) `eq` T.count s+tl_findCount s    = (L.genericLength . TL.breakOnAll s) `eq` TL.count s -t_split_split s         = (T.split s `eq` Slow.split s) . T.intercalate s-tl_split_split s        = ((TL.split (TL.fromStrict s) . TL.fromStrict) `eq`-                           (map TL.fromStrict . T.split s)) . T.intercalate s-t_split_i (NotEmpty t)  = id `eq` (T.intercalate t . T.split t)-tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)+t_splitOn_split s         = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s+tl_splitOn_split s        = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`+                           (map TL.fromStrict . T.splitOn s)) . T.intercalate s+t_splitOn_i (NotEmpty t)  = id `eq` (T.intercalate t . T.splitOn t)+tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) -t_splitBy p       = splitBy p `eqP` (map unpackS . T.splitBy p)-t_splitBy_count c = (L.length . T.splitBy (==c)) `eq`-                    ((1+) . T.count (T.singleton c))-t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c)-tl_splitBy p      = splitBy p `eqP` (map unpackS . TL.splitBy p)+t_split p       = split p `eqP` (map unpackS . T.split p)+t_split_count c = (L.length . T.split (==c)) `eq`+                  ((1+) . T.count (T.singleton c))+t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)+tl_split p      = split p `eqP` (map unpackS . TL.split p) -splitBy :: (a -> Bool) -> [a] -> [[a]]-splitBy _ [] =  [[]]-splitBy p xs = loop xs+split :: (a -> Bool) -> [a] -> [[a]]+split _ [] =  [[]]+split p xs = loop xs     where loop s | null s'   = [l]                  | otherwise = l : loop (tail s')               where (l, s') = break p s@@ -600,10 +597,10 @@ t_filter p        = L.filter p    `eqP` (unpackS . T.filter p) tl_filter p       = L.filter p    `eqP` (unpackS . TL.filter p) sf_findBy q p     = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)-t_findBy p        = L.find p      `eqP` T.findBy p-tl_findBy p       = L.find p      `eqP` TL.findBy p-t_partition p     = L.partition p `eqP` (unpack2 . T.partitionBy p)-tl_partition p    = L.partition p `eqP` (unpack2 . TL.partitionBy p)+t_find p          = L.find p      `eqP` T.find p+tl_find p         = L.find p      `eqP` TL.find p+t_partition p     = L.partition p `eqP` (unpack2 . T.partition p)+tl_partition p    = L.partition p `eqP` (unpack2 . TL.partition p)  sf_index p s      = forAll (choose (-l,l*2))                     ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s))@@ -616,8 +613,8 @@     where l = L.length s  t_findIndex p     = L.findIndex p `eqP` T.findIndex p-t_count (NotEmpty t)  = (subtract 1 . L.length . T.split t) `eq` T.count t-tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.split t) `eq`+t_count (NotEmpty t)  = (subtract 1 . L.length . T.splitOn t) `eq` T.count t+tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`                         TL.count t t_zip s           = L.zip s `eqP` T.zip (packS s) tl_zip s          = L.zip s `eqP` TL.zip (packS s)@@ -682,13 +679,13 @@  t_read_rational p tol (n::Double) s =     case p (T.pack (show n) `T.append` t) of-      Left err      -> False+      Left _err     -> False       Right (n',t') -> t == t' && abs (n-n') <= tol     where t = T.dropWhile isFloaty s  tl_read_rational p tol (n::Double) s =     case p (TL.pack (show n) `TL.append` t) of-      Left err      -> False+      Left _err     -> False       Right (n',t') -> t == t' && abs (n-n') <= tol     where t = TL.dropWhile isFloaty s @@ -1055,16 +1052,16 @@       testProperty "tl_strip" tl_strip,       testProperty "t_splitAt" t_splitAt,       testProperty "tl_splitAt" tl_splitAt,-      testProperty "t_spanBy" t_spanBy,-      testProperty "tl_spanBy" tl_spanBy,-      testProperty "t_break_id" t_break_id,-      testProperty "tl_break_id" tl_break_id,-      testProperty "t_break_start" t_break_start,-      testProperty "tl_break_start" tl_break_start,-      testProperty "t_breakEnd_end" t_breakEnd_end,-      testProperty "tl_breakEnd_end" tl_breakEnd_end,-      testProperty "t_breakBy" t_breakBy,-      testProperty "tl_breakBy" tl_breakBy,+      testProperty "t_span" t_span,+      testProperty "tl_span" tl_span,+      testProperty "t_breakOn_id" t_breakOn_id,+      testProperty "tl_breakOn_id" tl_breakOn_id,+      testProperty "t_breakOn_start" t_breakOn_start,+      testProperty "tl_breakOn_start" tl_breakOn_start,+      testProperty "t_breakOnEnd_end" t_breakOnEnd_end,+      testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,+      testProperty "t_break" t_break,+      testProperty "tl_break" tl_break,       testProperty "t_group" t_group,       testProperty "tl_group" tl_group,       testProperty "t_groupBy" t_groupBy,@@ -1083,14 +1080,14 @@       testProperty "sl_filterCount" sl_filterCount,       testProperty "t_findCount" t_findCount,       testProperty "tl_findCount" tl_findCount,-      testProperty "t_split_split" t_split_split,-      testProperty "tl_split_split" tl_split_split,-      testProperty "t_split_i" t_split_i,-      testProperty "tl_split_i" tl_split_i,-      testProperty "t_splitBy" t_splitBy,-      testProperty "t_splitBy_count" t_splitBy_count,-      testProperty "t_splitBy_split" t_splitBy_split,-      testProperty "tl_splitBy" tl_splitBy,+      testProperty "t_splitOn_split" t_splitOn_split,+      testProperty "tl_splitOn_split" tl_splitOn_split,+      testProperty "t_splitOn_i" t_splitOn_i,+      testProperty "tl_splitOn_i" tl_splitOn_i,+      testProperty "t_split" t_split,+      testProperty "t_split_count" t_split_count,+      testProperty "t_split_splitOn" t_split_splitOn,+      testProperty "tl_split" tl_split,       testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths,       testProperty "t_chunksOf_length" t_chunksOf_length,       testProperty "tl_chunksOf" tl_chunksOf@@ -1133,8 +1130,8 @@     testProperty "t_filter" t_filter,     testProperty "tl_filter" tl_filter,     testProperty "sf_findBy" sf_findBy,-    testProperty "t_findBy" t_findBy,-    testProperty "tl_findBy" tl_findBy,+    testProperty "t_find" t_find,+    testProperty "tl_find" tl_find,     testProperty "t_partition" t_partition,     testProperty "tl_partition" tl_partition   ],

@@ -5,8 +5,6 @@ import Control.Arrow (first) import Data.Char (chr) import Data.Bits ((.&.))-import Data.Int (Int64)-import Data.Word (Word8, Word16, Word32) import Data.String (IsString, fromString) import qualified Data.Text as T import Data.Text.Foreign (I16)@@ -15,13 +13,6 @@ import Test.QuickCheck hiding ((.&.)) import qualified Data.ByteString as B -instance Random Int64 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary Int64 where-    arbitrary     = choose (minBound,maxBound)- instance Random I16 where     randomR = integralRandomR     random  = randomR (minBound,maxBound)@@ -29,29 +20,8 @@ instance Arbitrary I16 where     arbitrary     = choose (minBound,maxBound) -instance Random Word8 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary Word8 where-    arbitrary     = choose (minBound,maxBound)- instance Arbitrary B.ByteString where     arbitrary     = B.pack `fmap` arbitrary--instance Random Word16 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary Word16 where-    arbitrary     = choose (minBound,maxBound)--instance Random Word32 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary Word32 where-    arbitrary     = choose (minBound,maxBound)  genUnicode :: IsString a => Gen a genUnicode = fmap fromString string where

@@ -3,7 +3,7 @@ module SlowFunctions     (       indices-    , split+    , splitOn     ) where  import qualified Data.Text as T@@ -23,12 +23,12 @@            where t = Text harr (hoff+i) (hlen-i)                  d = iter_ haystack i -split :: T.Text                 -- ^ Text to split on-      -> T.Text                 -- ^ Input text-      -> [T.Text]-split pat src0-    | T.null pat  = error "split: empty"-    | l == 1      = T.splitBy (== (unsafeHead pat)) src0+splitOn :: T.Text               -- ^ Text to split on+        -> T.Text               -- ^ Input text+        -> [T.Text]+splitOn pat src0+    | T.null pat  = error "splitOn: empty"+    | l == 1      = T.split (== (unsafeHead pat)) src0     | otherwise   = go src0   where     l      = T.length pat

@@ -1,5 +1,5 @@ name:           text-version:        0.10.0.0+version:        0.10.0.1 homepage:       http://bitbucket.org/bos/text bug-reports:    http://bitbucket.org/bos/text/issues synopsis:       An efficient packed Unicode text type.@@ -15,16 +15,21 @@     .     The 'Text' type provides character-encoding, type-safe case     conversion via whole-string case conversion functions. It also-    provides a range of functions for converting Text values to and from-    'ByteStrings', using several standard encodings (see the 'text-icu'-    package for a much larger variety of encoding functions).+    provides a range of functions for converting 'Text' values to and from+    'ByteStrings', using several standard encodings.     .     Efficient locale-sensitive support for text IO is also supported.     .-    This module is intended to be imported qualified, to avoid name+    These modules are intended to be imported qualified, to avoid name     clashes with Prelude functions, e.g.     .     > import qualified Data.Text as T+    .+    To use an extended and very rich family of functions for working+    with Unicode text (including normalization, regular expressions,+    non-standard encodings, text breaking, and locales), see+    the @text-icu@ package:+    <http://hackage.haskell.org/package/text-icu>     . license:        BSD3 license-file:   LICENSE