text 0.10.0.2 → 0.11.0.0
raw patch · 11 files changed
+332/−213 lines, 11 files
Files
- Data/Text.hs +69/−67
- Data/Text/Encoding.hs +38/−2
- Data/Text/IO.hs +22/−0
- Data/Text/Lazy.hs +66/−66
- Data/Text/Lazy/Encoding.hs +36/−2
- Data/Text/Lazy/IO.hs +21/−0
- Data/Text/Lazy/Read.hs +6/−4
- Data/Text/Read.hs +7/−5
- tests/Properties.hs +59/−59
- tests/SlowFunctions.hs +7/−7
- text.cabal +1/−1
Data/Text.hs view
@@ -128,10 +128,10 @@ , stripStart , stripEnd , splitAt- , spanBy+ , breakOn+ , breakOnEnd , break- , breakEnd- , breakBy+ , span , group , groupBy , inits@@ -139,8 +139,8 @@ -- ** Breaking into many substrings -- $split+ , splitOn , split- , splitBy , chunksOf -- ** Breaking into lines and words@@ -161,9 +161,9 @@ -- * Searching , filter+ , breakOnAll , find- , findBy- , partitionBy+ , partition -- , findSubstring @@ -598,7 +598,7 @@ -> Text -- ^ Replacement text -> Text -- ^ Input text -> Text-replace s d = intercalate d . split s+replace s d = intercalate d . splitOn s {-# INLINE replace #-} -- ----------------------------------------------------------------------------@@ -619,7 +619,8 @@ -- context-dependent operation. The case conversion functions in this -- module are /not/ locale sensitive. Programs that require locale -- sensitivity should use appropriate versions of the case mapping--- functions from the @text-icu@ package.+-- functions from the @text-icu@ package:+-- <http://hackage.haskell.org/package/text-icu> -- | /O(n)/ Convert a string to folded case. This function is mainly -- useful for performing caseless (also known as case insensitive)@@ -1072,23 +1073,23 @@ where d = iter_ t i {-# INLINE splitAt #-} --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list.-spanBy :: (Char -> Bool) -> Text -> (Text, Text)-spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))+span :: (Char -> Bool) -> Text -> (Text, Text)+span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k)) where k = loop 0 loop !i | i >= len || not (p c) = i | otherwise = loop (i+d) where Iter c d = iter t i-{-# INLINE spanBy #-}+{-# INLINE span #-} --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is+-- | /O(n)/ 'break' is like 'span', but the prefix returned is -- over elements that fail the predicate @p@.-breakBy :: (Char -> Bool) -> Text -> (Text, Text)-breakBy p = spanBy (not . p)-{-# INLINE breakBy #-}+break :: (Char -> Bool) -> Text -> (Text, Text)+break p = span (not . p)+{-# INLINE break #-} -- | /O(n)/ Group characters in a string according to a predicate. groupBy :: (Char -> Char -> Bool) -> Text -> [Text]@@ -1138,30 +1139,30 @@ -- -- Examples: ----- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]--- > split "x" "x" == ["",""]+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]+-- > splitOn "x" "x" == ["",""] -- -- and ----- > intercalate s . split s == id--- > split (singleton c) == splitBy (==c)+-- > intercalate s . splitOn s == id+-- > splitOn (singleton c) == split (==c) -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-split :: Text -> Text -> [Text]-split pat@(Text _ _ l) src@(Text arr off len)- | l <= 0 = emptyError "split"- | isSingleton pat = splitBy (== unsafeHead pat) src+splitOn :: Text -> Text -> [Text]+splitOn pat@(Text _ _ l) src@(Text arr off len)+ | l <= 0 = emptyError "splitOn"+ | isSingleton pat = split (== unsafeHead pat) src | otherwise = go 0 (indices pat src) where go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs go s _ = [textP arr (s+off) (len-s)]-{-# INLINE [1] split #-}+{-# INLINE [1] splitOn #-} {-# RULES-"TEXT split/singleton -> splitBy/==" [~1] forall c t.- split (singleton c) t = splitBy (==c) t+"TEXT splitOn/singleton -> split/==" [~1] forall c t.+ splitOn (singleton c) t = split (==c) t #-} -- | /O(n)/ Splits a 'Text' into components delimited by separators,@@ -1169,15 +1170,15 @@ -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. ----- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]--- > splitBy (=='a') "" == [""]-splitBy :: (Char -> Bool) -> Text -> [Text]-splitBy _ t@(Text _off _arr 0) = [t]-splitBy p t = loop t+-- > split (=='a') "aabbaca" == ["","","bb","c",""]+-- > split (=='a') "" == [""]+split :: (Char -> Bool) -> Text -> [Text]+split _ t@(Text _off _arr 0) = [t]+split p t = loop t where loop s | null s' = [l] | otherwise = l : loop (unsafeTail s')- where (l, s') = breakBy p s-{-# INLINE splitBy #-}+ where (l, s') = break p s+{-# INLINE split #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the@@ -1199,21 +1200,21 @@ ------------------------------------------------------------------------------- -- ** Searching with a predicate --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and--- returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.-findBy :: (Char -> Bool) -> Text -> Maybe Char-findBy p t = S.findBy p (stream t)-{-# INLINE findBy #-}+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and+-- returns the first element matching the predicate, or 'Nothing' if+-- there is no such element.+find :: (Char -> Bool) -> Text -> Maybe Char+find p t = S.findBy p (stream t)+{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. ----- > partitionBy p t == (filter p t, filter (not . p) t)-partitionBy :: (Char -> Bool) -> Text -> (Text, Text)-partitionBy p t = (filter p t, filter (not . p) t)-{-# INLINE partitionBy #-}+-- > partition p t == (filter p t, filter (not . p) t)+partition :: (Char -> Bool) -> Text -> (Text, Text)+partition p t = (filter p t, filter (not . p) t)+{-# INLINE partition #-} -- | /O(n)/ 'filter', applied to a predicate and a 'Text', -- returns a 'Text' containing those characters that satisfy the@@ -1229,39 +1230,40 @@ -- -- Examples: ----- > break "::" "a::b::c" ==> ("a", "::b::c")--- > break "/" "foobar" ==> ("foobar", "")+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")+-- > breakOn "/" "foobar" ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack--- > where (prefix, match) = break needle haystack+-- > where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'find'+-- want to break on every instance of a substring), use 'breakOnAll' -- instead, as it has lower startup overhead. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-break :: Text -> Text -> (Text, Text)-break pat src@(Text arr off len)- | null pat = emptyError "break"+breakOn :: Text -> Text -> (Text, Text)+breakOn pat src@(Text arr off len)+ | null pat = emptyError "breakOn" | otherwise = case indices pat src of [] -> (src, empty) (x:_) -> (textP arr off x, textP arr (off+x) (len-x))-{-# INLINE break #-}+{-# INLINE breakOn #-} --- | /O(n+m)/ Similar to 'break', but searches from the end of the string.+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the+-- string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@. The second is the -- remainder of @haystack@, following the match. ----- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")-breakEnd :: Text -> Text -> (Text, Text)-breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)- in (reverse b, reverse a)-{-# INLINE breakEnd #-}+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")+breakOnEnd :: Text -> Text -> (Text, Text)+breakOnEnd pat src = (reverse b, reverse a)+ where (a,b) = breakOn (reverse pat) (reverse src)+{-# INLINE breakOnEnd #-} -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@. Each element of the returned list consists of a pair:@@ -1281,16 +1283,16 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty.-find :: Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-find pat src@(Text arr off slen)- | null pat = emptyError "find"+breakOnAll :: Text -- ^ @needle@ to search for+ -> Text -- ^ @haystack@ in which to search+ -> [(Text, Text)]+breakOnAll pat src@(Text arr off slen)+ | null pat = emptyError "breakOnAll" | otherwise = L.map step (indices pat src) where step x = (chunk 0 x, chunk x (slen-x)) chunk !n !l = textP arr (n+off) l-{-# INLINE find #-}+{-# INLINE breakOnAll #-} ------------------------------------------------------------------------------- -- ** Indexing 'Text's@@ -1390,7 +1392,7 @@ | otherwise = h : if null t then [] else lines (unsafeTail t)- where (h,t) = spanBy (/= '\n') ps+ where (h,t) = span (/= '\n') ps {-# INLINE lines #-} {-
Data/Text/Encoding.hs view
@@ -14,18 +14,20 @@ -- Functions for converting 'Text' values to and from 'ByteString', -- using several standard encodings. ----- To make use of a much larger variety of encodings, use the @text-icu@--- package.+-- To gain access to a much larger family of encodings, use the+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu> module Data.Text.Encoding ( -- * Decoding ByteStrings to Text+ -- $strict decodeASCII , decodeUtf8 , decodeUtf16LE , decodeUtf16BE , decodeUtf32LE , decodeUtf32BE+ -- ** Controllable error handling , decodeUtf8With , decodeUtf16LEWith@@ -60,11 +62,25 @@ import qualified Data.Text.Encoding.Utf8 as U8 import qualified Data.Text.Fusion as F +-- $strict+--+-- All of the single-parameter functions for decoding bytestrings+-- encoded in one of the Unicode Transformation Formats (UTF) operate+-- in a /strict/ mode: each will throw an exception if given invalid+-- input.+--+-- Each function has a variant, whose name is suffixed with -'With',+-- that gives greater control over the handling of decoding errors.+-- For instance, 'decodeUtf8' will throw an exception, but+-- 'decodeUtf8With' allows the programmer to determine what to do on a+-- decoding error.+ -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: ByteString -> Text decodeASCII bs = F.unstream (E.streamASCII bs) {-# INLINE decodeASCII #-} +-- | Decode a 'ByteString' containing UTF-8 encoded text. decodeUtf8With :: OnDecodeError -> ByteString -> Text decodeUtf8With onErr bs = textP (fst a) 0 (snd a) where@@ -101,6 +117,10 @@ {-# INLINE[0] decodeUtf8With #-} -- | Decode a 'ByteString' containing UTF-8 encoded text.+--+-- If the input contains any invalid UTF-8 data, an exception will be+-- thrown. For more control over the handling of invalid data, use+-- 'decodeUtf8With'. decodeUtf8 :: ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-}@@ -156,6 +176,10 @@ {-# INLINE decodeUtf16LEWith #-} -- | Decode text from little endian UTF-16 encoding.+--+-- If the input contains any invalid little endian UTF-16 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-}@@ -166,6 +190,10 @@ {-# INLINE decodeUtf16BEWith #-} -- | Decode text from big endian UTF-16 encoding.+--+-- If the input contains any invalid big endian UTF-16 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-}@@ -186,6 +214,10 @@ {-# INLINE decodeUtf32LEWith #-} -- | Decode text from little endian UTF-32 encoding.+--+-- If the input contains any invalid little endian UTF-32 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-}@@ -196,6 +228,10 @@ {-# INLINE decodeUtf32BEWith #-} -- | Decode text from big endian UTF-32 encoding.+--+-- If the input contains any invalid big endian UTF-32 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-}
Data/Text/IO.hs view
@@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for text I\/O.+--+-- Skip past the synopsis for some important notes on performance and+-- portability across different versions of GHC. module Data.Text.IO (+ -- * Performance+ -- $performance + -- * Locale support -- $locale -- * File-at-a-time operations@@ -59,6 +65,22 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) import System.IO.Error (isEOFError) #endif++-- $performance+-- #performance#+--+-- The functions in this module obey the runtime system's locale,+-- character set encoding, and line ending conversion settings.+--+-- If you know in advance that you will be working with data that has+-- a specific encoding (e.g. UTF-8), and your application is highly+-- performance sensitive, you may find that it is faster to perform+-- I\/O with bytestrings and to encode and decode yourself than to use+-- the functions in this module.+--+-- Whether this will hold depends on the version of GHC you are using,+-- the platform you are working on, the data you are working with, and+-- the encodings you are using, so be sure to test for yourself. -- | The 'readFile' function reads a file and returns the contents of -- the file as a string. The entire file is read strictly, as with
Data/Text/Lazy.hs view
@@ -135,10 +135,10 @@ , stripStart , stripEnd , splitAt- , spanBy+ , span+ , breakOn+ , breakOnEnd , break- , breakEnd- , breakBy , group , groupBy , inits@@ -146,8 +146,8 @@ -- ** Breaking into many substrings -- $split+ , splitOn , split- , splitBy , chunksOf -- , breakSubstring @@ -169,8 +169,8 @@ -- * Searching , filter , find- , findBy- , partitionBy+ , breakOnAll+ , partition -- , findSubstring @@ -630,7 +630,7 @@ -> Text -- ^ Replacement text -> Text -- ^ Input text -> Text-replace s d = intercalate d . split s+replace s d = intercalate d . splitOn s {-# INLINE replace #-} -- ----------------------------------------------------------------------------@@ -1044,13 +1044,13 @@ -- -- Examples: ----- > break "::" "a::b::c" ==> ("a", "::b::c")--- > break "/" "foobar" ==> ("foobar", "")+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")+-- > breakOn "/" "foobar" ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack--- > where (prefix, match) = break needle haystack+-- > where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you -- want to break on every instance of a substring), use 'find'@@ -1061,25 +1061,25 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-break :: Text -> Text -> (Text, Text)-break pat src- | null pat = emptyError "break"+breakOn :: Text -> Text -> (Text, Text)+breakOn pat src+ | null pat = emptyError "breakOn" | otherwise = case indices pat src of [] -> (src, empty) (x:_) -> let h :*: t = splitAtWord x src in (h, t) --- | /O(n+m)/ Similar to 'break', but searches from the end of the string.+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@. The second is the -- remainder of @haystack@, following the match. ----- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")-breakEnd :: Text -> Text -> (Text, Text)-breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")+breakOnEnd :: Text -> Text -> (Text, Text)+breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) in (reverse b, reverse a)-{-# INLINE breakEnd #-}+{-# INLINE breakOnEnd #-} -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@. Each element of the returned list consists of a pair:@@ -1090,9 +1090,9 @@ -- -- Examples: ----- > find "::" ""+-- > breakOnAll "::" "" -- > ==> []--- > find "/" "a/b/c/"+-- > breakOnAll "/" "a/b/c/" -- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] -- -- This function is strict in its first argument, and lazy in its@@ -1102,11 +1102,11 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty.-find :: Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-find pat src- | null pat = emptyError "find"+breakOnAll :: Text -- ^ @needle@ to search for+ -> Text -- ^ @haystack@ in which to search+ -> [(Text, Text)]+breakOnAll pat src+ | null pat = emptyError "breakOnAll" | otherwise = go 0 empty src (indices pat src) where go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s@@ -1114,10 +1114,10 @@ in (h',t) : go x h' t xs go _ _ _ _ = [] --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over+-- | /O(n)/ 'break' is like 'span', but the prefix returned is over -- elements that fail the predicate @p@.-breakBy :: (Char -> Bool) -> Text -> (Text, Text)-breakBy p t0 = break' t0+break :: (Char -> Bool) -> Text -> (Text, Text)+break p t0 = break' t0 where break' Empty = (empty, empty) break' c@(Chunk t ts) = case T.findIndex p t of@@ -1127,13 +1127,13 @@ | otherwise -> let (a,b) = T.splitAt n t in (Chunk a Empty, Chunk b ts) --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list.-spanBy :: (Char -> Bool) -> Text -> (Text, Text)-spanBy p = breakBy (not . p)-{-# INLINE spanBy #-}+span :: (Char -> Bool) -> Text -> (Text, Text)+span p = break (not . p)+{-# INLINE span #-} -- | The 'group' function takes a 'Text' and returns a list of 'Text's -- such that the concatenation of the result is equal to the argument.@@ -1152,7 +1152,7 @@ groupBy :: (Char -> Char -> Bool) -> Text -> [Text] groupBy _ Empty = [] groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs- where (ys,zs) = spanBy (eq x) xs+ where (ys,zs) = span (eq x) xs x = T.unsafeHead t xs = chunk (T.unsafeTail t) ts @@ -1184,37 +1184,37 @@ -- -- Examples: ----- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]--- > split "x" "x" == ["",""]+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]+-- > splitOn "x" "x" == ["",""] -- -- and ----- > intercalate s . split s == id--- > split (singleton c) == splitBy (==c)+-- > intercalate s . splitOn s == id+-- > splitOn (singleton c) == split (==c) -- -- This function is strict in its first argument, and lazy in its -- second. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/.-split :: Text -- ^ Text to split on- -> Text -- ^ Input text- -> [Text]-split pat src- | null pat = emptyError "split"- | isSingleton pat = splitBy (== head pat) src+splitOn :: Text -- ^ Text to split on+ -> Text -- ^ Input text+ -> [Text]+splitOn pat src+ | null pat = emptyError "splitOn"+ | isSingleton pat = split (== head pat) src | otherwise = go 0 (indices pat src) src where go _ [] cs = [cs] go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs in h : go (x+l) xs (dropWords l t) l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat-{-# INLINE [1] split #-}+{-# INLINE [1] splitOn #-} {-# RULES-"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t.- split (singleton c) t = splitBy (==c) t+"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.+ splitOn (singleton c) t = split (==c) t #-} -- | /O(n)/ Splits a 'Text' into components delimited by separators,@@ -1222,16 +1222,16 @@ -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. ----- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]--- > splitBy (=='a') [] == [""]-splitBy :: (Char -> Bool) -> Text -> [Text]-splitBy _ Empty = [Empty]-splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0+-- > split (=='a') "aabbaca" == ["","","bb","c",""]+-- > split (=='a') [] == [""]+split :: (Char -> Bool) -> Text -> [Text]+split _ Empty = [Empty]+split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 where comb acc (s:[]) Empty = revChunks (s:acc) : []- comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts+ comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts- comb _ [] _ = impossibleError "splitBy"-{-# INLINE splitBy #-}+ comb _ [] _ = impossibleError "split"+{-# INLINE split #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the@@ -1251,14 +1251,14 @@ -- newline 'Char's. The resulting strings do not contain newlines. lines :: Text -> [Text] lines Empty = []-lines t = let (l,t') = breakBy ((==) '\n') t+lines t = let (l,t') = break ((==) '\n') t in l : if null t' then [] else lines (tail t') -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text]-words = L.filter (not . null) . splitBy isSpace+words = L.filter (not . null) . split isSpace {-# INLINE words #-} -- | /O(n)/ Joins lines, after appending a terminating newline to@@ -1378,21 +1378,21 @@ filter p t = unstream (S.filter p (stream t)) {-# INLINE filter #-} --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and -- returns the first element in matching the predicate, or 'Nothing' -- if there is no such element.-findBy :: (Char -> Bool) -> Text -> Maybe Char-findBy p t = S.findBy p (stream t)-{-# INLINE findBy #-}+find :: (Char -> Bool) -> Text -> Maybe Char+find p t = S.findBy p (stream t)+{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. ----- > partitionBy p t == (filter p t, filter (not . p) t)-partitionBy :: (Char -> Bool) -> Text -> (Text, Text)-partitionBy p t = (filter p t, filter (not . p) t)-{-# INLINE partitionBy #-}+-- > partition p t == (filter p t, filter (not . p) t)+partition :: (Char -> Bool) -> Text -> (Text, Text)+partition p t = (filter p t, filter (not . p) t)+{-# INLINE partition #-} -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. index :: Text -> Int64 -> Char
Data/Text/Lazy/Encoding.hs view
@@ -12,12 +12,13 @@ -- Functions for converting lazy 'Text' values to and from lazy -- 'ByteString', using several standard encodings. ----- To make use of a much larger variety of encodings, use the @text-icu@--- package.+-- To gain access to a much larger variety of encodings, use the+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu> module Data.Text.Lazy.Encoding ( -- * Decoding ByteStrings to Text+ -- $strict decodeASCII , decodeUtf8 , decodeUtf16LE@@ -51,6 +52,19 @@ import qualified Data.Text.Lazy.Encoding.Fusion as E import qualified Data.Text.Lazy.Fusion as F +-- $strict+--+-- All of the single-parameter functions for decoding bytestrings+-- encoded in one of the Unicode Transformation Formats (UTF) operate+-- in a /strict/ mode: each will throw an exception if given invalid+-- input.+--+-- Each function has a variant, whose name is suffixed with -'With',+-- that gives greater control over the handling of decoding errors.+-- For instance, 'decodeUtf8' will throw an exception, but+-- 'decodeUtf8With' allows the programmer to determine what to do on a+-- decoding error.+ -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: B.ByteString -> Text decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)@@ -95,6 +109,10 @@ {-# INLINE[0] decodeUtf8With #-} -- | Decode a 'ByteString' containing UTF-8 encoded text.+--+-- If the input contains any invalid UTF-8 data, an exception will be+-- thrown. For more control over the handling of invalid data, use+-- 'decodeUtf8With'. decodeUtf8 :: B.ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-}@@ -113,6 +131,10 @@ {-# INLINE decodeUtf16LEWith #-} -- | Decode text from little endian UTF-16 encoding.+--+-- If the input contains any invalid little endian UTF-16 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: B.ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-}@@ -123,6 +145,10 @@ {-# INLINE decodeUtf16BEWith #-} -- | Decode text from big endian UTF-16 encoding.+--+-- If the input contains any invalid big endian UTF-16 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: B.ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-}@@ -143,6 +169,10 @@ {-# INLINE decodeUtf32LEWith #-} -- | Decode text from little endian UTF-32 encoding.+--+-- If the input contains any invalid little endian UTF-32 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: B.ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-}@@ -153,6 +183,10 @@ {-# INLINE decodeUtf32BEWith #-} -- | Decode text from big endian UTF-32 encoding.+--+-- If the input contains any invalid big endian UTF-32 data, an+-- exception will be thrown. For more control over the handling of+-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: B.ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-}
Data/Text/Lazy/IO.hs view
@@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for lazy text I\/O.+--+-- Skip past the synopsis for some important notes on performance and+-- portability across different versions of GHC. module Data.Text.Lazy.IO (+ -- * Performance+ -- $performance + -- * Locale support -- $locale -- * File-at-a-time operations@@ -57,6 +63,21 @@ import System.IO.Error (isEOFError) import System.IO.Unsafe (unsafeInterleaveIO) #endif++-- $performance+--+-- The functions in this module obey the runtime system's locale,+-- character set encoding, and line ending conversion settings.+--+-- If you know in advance that you will be working with data that has+-- a specific encoding (e.g. UTF-8), and your application is highly+-- performance sensitive, you may find that it is faster to perform+-- I\/O with bytestrings and to encode and decode yourself than to use+-- the functions in this module.+--+-- Whether this will hold depends on the version of GHC you are using,+-- the platform you are working on, the data you are working with, and+-- the encodings you are using, so be sure to test for yourself. -- | Read a file and return its contents as a string. The file is -- read lazily, as with 'getContents'.
Data/Text/Lazy/Read.hs view
@@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results. If you are worried about overflow, use+-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t)- where (h,t) = T.spanBy isDigit txt+ where (h,t) = T.span isDigit txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading@@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results. If you are worried about overflow, use+-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-}@@ -74,7 +76,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t)- where (h,t) = T.spanBy isHexDigit txt+ where (h,t) = T.span isHexDigit txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) hexDigitToInt :: Char -> Int
Data/Text/Read.hs view
@@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results. If you are worried about overflow, use+-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t)- where (h,t) = T.spanBy isDigit txt+ where (h,t) = T.span isDigit txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading@@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.+-- incorrect results. If you are worried about overflow, use+-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-}@@ -74,7 +76,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t)- where (h,t) = T.spanBy isHexDigit txt+ where (h,t) = T.span isHexDigit txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) hexDigitToInt :: Char -> Int@@ -99,7 +101,7 @@ -- by the 'read' function, with the exception that a trailing @\'.\'@ -- or @\'e\'@ /not/ followed by a number is not consumed. ----- Examples:+-- Examples (with behaviour identical to 'read'): -- -- >rational "3" == Right (3.0, "") -- >rational "3.1" == Right (3.1, "")
tests/Properties.hs view
@@ -265,13 +265,13 @@ tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) -t_replace s d = (L.intercalate d . split s) `eqP`+t_replace s d = (L.intercalate d . splitOn s) `eqP` (unpackS . T.replace (T.pack s) (T.pack d))-tl_replace s d = (L.intercalate d . split s) `eqP`+tl_replace s d = (L.intercalate d . splitOn s) `eqP` (unpackS . TL.replace (TL.pack s) (TL.pack d)) -split :: (Eq a) => [a] -> [a] -> [[a]]-split pat src0+splitOn :: (Eq a) => [a] -> [a] -> [[a]]+splitOn pat src0 | l == 0 = error "empty" | otherwise = go src0 where@@ -482,25 +482,25 @@ tl_strip = TL.dropAround isSpace `eq` TL.strip t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n))-t_spanBy p = L.span p `eqP` (unpack2 . T.spanBy p)-tl_spanBy p = L.span p `eqP` (unpack2 . TL.spanBy p)+t_span p = L.span p `eqP` (unpack2 . T.span p)+tl_span p = L.span p `eqP` (unpack2 . TL.span p) -t_break_id s = squid `eq` (uncurry T.append . T.break s)+t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s) where squid t | T.null s = error "empty" | otherwise = t-tl_break_id s = squid `eq` (uncurry TL.append . TL.break s)+tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s) where squid t | TL.null s = error "empty" | otherwise = t-t_break_start (NotEmpty s) t = let (_,m) = T.break s t+t_breakOn_start (NotEmpty s) t = let (_,m) = T.breakOn s t in T.null m || s `T.isPrefixOf` m-tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t+tl_breakOn_start (NotEmpty s) t = let (_,m) = TL.breakOn s t in TL.null m || s `TL.isPrefixOf` m-t_breakEnd_end (NotEmpty s) t = let (m,_) = T.breakEnd s t+t_breakOnEnd_end (NotEmpty s) t = let (m,_) = T.breakOnEnd s t in T.null m || s `T.isSuffixOf` m-tl_breakEnd_end (NotEmpty s) t = let (m,_) = TL.breakEnd s t+tl_breakOnEnd_end (NotEmpty s) t = let (m,_) = TL.breakOnEnd s t in TL.null m || s `TL.isSuffixOf` m-t_breakBy p = L.break p `eqP` (unpack2 . T.breakBy p)-tl_breakBy p = L.break p `eqP` (unpack2 . TL.breakBy p)+t_break p = L.break p `eqP` (unpack2 . T.break p)+tl_break p = L.break p `eqP` (unpack2 . TL.break p) t_group = L.group `eqP` (map unpackS . T.group) tl_group = L.group `eqP` (map unpackS . TL.group) t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p)@@ -511,33 +511,33 @@ tl_tails = L.tails `eqP` (map unpackS . TL.tails) t_findAppendId (NotEmpty s) = unsquare $ \ts -> let t = T.intercalate s ts- in all (==t) $ map (uncurry T.append) (T.find s t)+ in all (==t) $ map (uncurry T.append) (T.breakOnAll s t) tl_findAppendId (NotEmpty s) = unsquare $ \ts -> let t = TL.intercalate s ts- in all (==t) $ map (uncurry TL.append) (TL.find s t)-t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.find s .+ in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t)+t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s tl_findContains (NotEmpty s) = all (TL.isPrefixOf s . snd) .- TL.find s . TL.intercalate s+ TL.breakOnAll s . TL.intercalate s sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c-t_findCount s = (L.length . T.find s) `eq` T.count s-tl_findCount s = (L.genericLength . TL.find s) `eq` TL.count s+t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s+tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s -t_split_split s = (T.split s `eq` Slow.split s) . T.intercalate s-tl_split_split s = ((TL.split (TL.fromStrict s) . TL.fromStrict) `eq`- (map TL.fromStrict . T.split s)) . T.intercalate s-t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)-tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)+t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s+tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`+ (map TL.fromStrict . T.splitOn s)) . T.intercalate s+t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t)+tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) -t_splitBy p = splitBy p `eqP` (map unpackS . T.splitBy p)-t_splitBy_count c = (L.length . T.splitBy (==c)) `eq`- ((1+) . T.count (T.singleton c))-t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c)-tl_splitBy p = splitBy p `eqP` (map unpackS . TL.splitBy p)+t_split p = split p `eqP` (map unpackS . T.split p)+t_split_count c = (L.length . T.split (==c)) `eq`+ ((1+) . T.count (T.singleton c))+t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)+tl_split p = split p `eqP` (map unpackS . TL.split p) -splitBy :: (a -> Bool) -> [a] -> [[a]]-splitBy _ [] = [[]]-splitBy p xs = loop xs+split :: (a -> Bool) -> [a] -> [[a]]+split _ [] = [[]]+split p xs = loop xs where loop s | null s' = [l] | otherwise = l : loop (tail s') where (l, s') = break p s@@ -597,10 +597,10 @@ t_filter p = L.filter p `eqP` (unpackS . T.filter p) tl_filter p = L.filter p `eqP` (unpackS . TL.filter p) sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)-t_findBy p = L.find p `eqP` T.findBy p-tl_findBy p = L.find p `eqP` TL.findBy p-t_partition p = L.partition p `eqP` (unpack2 . T.partitionBy p)-tl_partition p = L.partition p `eqP` (unpack2 . TL.partitionBy p)+t_find p = L.find p `eqP` T.find p+tl_find p = L.find p `eqP` TL.find p+t_partition p = L.partition p `eqP` (unpack2 . T.partition p)+tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p) sf_index p s = forAll (choose (-l,l*2)) ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s))@@ -613,8 +613,8 @@ where l = L.length s t_findIndex p = L.findIndex p `eqP` T.findIndex p-t_count (NotEmpty t) = (subtract 1 . L.length . T.split t) `eq` T.count t-tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.split t) `eq`+t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t+tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` TL.count t t_zip s = L.zip s `eqP` T.zip (packS s) tl_zip s = L.zip s `eqP` TL.zip (packS s)@@ -1052,16 +1052,16 @@ testProperty "tl_strip" tl_strip, testProperty "t_splitAt" t_splitAt, testProperty "tl_splitAt" tl_splitAt,- testProperty "t_spanBy" t_spanBy,- testProperty "tl_spanBy" tl_spanBy,- testProperty "t_break_id" t_break_id,- testProperty "tl_break_id" tl_break_id,- testProperty "t_break_start" t_break_start,- testProperty "tl_break_start" tl_break_start,- testProperty "t_breakEnd_end" t_breakEnd_end,- testProperty "tl_breakEnd_end" tl_breakEnd_end,- testProperty "t_breakBy" t_breakBy,- testProperty "tl_breakBy" tl_breakBy,+ testProperty "t_span" t_span,+ testProperty "tl_span" tl_span,+ testProperty "t_breakOn_id" t_breakOn_id,+ testProperty "tl_breakOn_id" tl_breakOn_id,+ testProperty "t_breakOn_start" t_breakOn_start,+ testProperty "tl_breakOn_start" tl_breakOn_start,+ testProperty "t_breakOnEnd_end" t_breakOnEnd_end,+ testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,+ testProperty "t_break" t_break,+ testProperty "tl_break" tl_break, testProperty "t_group" t_group, testProperty "tl_group" tl_group, testProperty "t_groupBy" t_groupBy,@@ -1080,14 +1080,14 @@ testProperty "sl_filterCount" sl_filterCount, testProperty "t_findCount" t_findCount, testProperty "tl_findCount" tl_findCount,- testProperty "t_split_split" t_split_split,- testProperty "tl_split_split" tl_split_split,- testProperty "t_split_i" t_split_i,- testProperty "tl_split_i" tl_split_i,- testProperty "t_splitBy" t_splitBy,- testProperty "t_splitBy_count" t_splitBy_count,- testProperty "t_splitBy_split" t_splitBy_split,- testProperty "tl_splitBy" tl_splitBy,+ testProperty "t_splitOn_split" t_splitOn_split,+ testProperty "tl_splitOn_split" tl_splitOn_split,+ testProperty "t_splitOn_i" t_splitOn_i,+ testProperty "tl_splitOn_i" tl_splitOn_i,+ testProperty "t_split" t_split,+ testProperty "t_split_count" t_split_count,+ testProperty "t_split_splitOn" t_split_splitOn,+ testProperty "tl_split" tl_split, testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths, testProperty "t_chunksOf_length" t_chunksOf_length, testProperty "tl_chunksOf" tl_chunksOf@@ -1130,8 +1130,8 @@ testProperty "t_filter" t_filter, testProperty "tl_filter" tl_filter, testProperty "sf_findBy" sf_findBy,- testProperty "t_findBy" t_findBy,- testProperty "tl_findBy" tl_findBy,+ testProperty "t_find" t_find,+ testProperty "tl_find" tl_find, testProperty "t_partition" t_partition, testProperty "tl_partition" tl_partition ],
tests/SlowFunctions.hs view
@@ -3,7 +3,7 @@ module SlowFunctions ( indices- , split+ , splitOn ) where import qualified Data.Text as T@@ -23,12 +23,12 @@ where t = Text harr (hoff+i) (hlen-i) d = iter_ haystack i -split :: T.Text -- ^ Text to split on- -> T.Text -- ^ Input text- -> [T.Text]-split pat src0- | T.null pat = error "split: empty"- | l == 1 = T.splitBy (== (unsafeHead pat)) src0+splitOn :: T.Text -- ^ Text to split on+ -> T.Text -- ^ Input text+ -> [T.Text]+splitOn pat src0+ | T.null pat = error "splitOn: empty"+ | l == 1 = T.split (== (unsafeHead pat)) src0 | otherwise = go src0 where l = T.length pat
text.cabal view
@@ -1,5 +1,5 @@ name: text-version: 0.10.0.2+version: 0.11.0.0 homepage: http://bitbucket.org/bos/text bug-reports: http://bitbucket.org/bos/text/issues synopsis: An efficient packed Unicode text type.