diff --git a/Data/Text.hs b/Data/Text.hs --- a/Data/Text.hs +++ b/Data/Text.hs @@ -13,20 +13,31 @@ -- Stability : experimental -- Portability : GHC -- --- A time and space-efficient implementation of Unicode text using --- packed Word16 arrays. Suitable for performance critical use, both --- in terms of large data quantities and high speed. +-- A time and space-efficient implementation of Unicode text. +-- Suitable for performance critical use, both in terms of large data +-- quantities and high speed. -- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- -- This module is intended to be imported @qualified@, to avoid name -- clashes with "Prelude" functions, e.g. -- -- > import qualified Data.Text as T +-- +-- To use an extended and very rich family of functions for working +-- with Unicode text (including normalization, regular expressions, +-- non-standard encodings, text breaking, and locales), see the +-- @text-icu@ package: module Data.Text ( -- * Strict vs lazy types -- $strict + -- * Acceptable data + -- $replacement + -- * Fusion -- $fusion @@ -117,10 +128,10 @@ , stripStart , stripEnd , splitAt - , spanBy + , breakOn + , breakOnEnd , break - , breakEnd - , breakBy + , span , group , groupBy , inits @@ -128,8 +139,8 @@ -- ** Breaking into many substrings -- $split + , splitOn , split - , splitBy , chunksOf -- ** Breaking into lines and words @@ -150,9 +161,9 @@ -- * Searching , filter + , breakOnAll , find - , findBy - , partitionBy + , partition -- , findSubstring @@ -196,7 +207,7 @@ import qualified Data.Text.Fusion as S import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion (stream, reverseStream, unstream) -import Data.Text.Internal (Text(..), empty, text, textP) +import Data.Text.Internal (Text(..), empty, firstf, safe, text, textP) import qualified Prelude as P import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter, unsafeHead, unsafeTail) @@ -230,6 +241,33 @@ -- difference being that the strict module uses 'Int' values for -- lengths and counts, while the lazy module uses 'Int64' lengths. +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in §3.9, definition D76 of the Unicode 5.2 standard: +-- . As +-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF +-- inclusive. Haskell implementations admit all Unicode code points +-- (§3.4, definition D10) as 'Char' values, including code points +-- from this invalid range. This means that there are some 'Char' +-- values that are not valid Unicode scalar values, and the functions +-- in this module must handle those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". +-- +-- (One reason for this policy of replacement is that internally, a +-- 'Text' value is represented as packed UTF-16 data. Values in the +-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate +-- code points, and so cannot be represented. The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see Unicode Technical Report 36, §3.5: +-- ) + -- $fusion -- -- Most of the functions in this module are subject to /fusion/, @@ -240,6 +278,7 @@ -- -- > import Data.Text as T -- > import Data.Text.Encoding as E +-- > import Data.ByteString (ByteString) -- > -- > countChars :: ByteString -> Int -- > countChars = T.length . T.toUpper . E.decodeUtf8 @@ -251,7 +290,7 @@ -- function will be compiled down to a single loop over the source -- 'ByteString'. -- --- Functions that can be fused by the compiler are marked with the +-- Functions that can be fused by the compiler are documented with the -- phrase \"Subject to fusion\". instance Eq Text where @@ -320,9 +359,10 @@ -- ----------------------------------------------------------------------------- -- * Conversion to/from 'Text' --- | /O(n)/ Convert a 'String' into a 'Text'. Subject to fusion. +-- | /O(n)/ Convert a 'String' into a 'Text'. Subject to +-- fusion. Performs replacement on invalid scalar values. pack :: String -> Text -pack = unstream . S.streamList +pack = unstream . S.streamList . L.map safe {-# INLINE [1] pack #-} -- | /O(n)/ Convert a Text into a String. Subject to fusion. @@ -330,10 +370,10 @@ unpack = S.unstreamList . stream {-# INLINE [1] unpack #-} --- | /O(1)/ Convert a character into a Text. --- Subject to fusion. +-- | /O(1)/ Convert a character into a Text. Subject to fusion. +-- Performs replacement on invalid scalar values. singleton :: Char -> Text -singleton = unstream . S.singleton +singleton = unstream . S.singleton . safe {-# INLINE [1] singleton #-} -- ----------------------------------------------------------------------------- @@ -341,15 +381,17 @@ -- | /O(n)/ Adds a character to the front of a 'Text'. This function -- is more costly than its 'List' counterpart because it requires --- copying a new array. Subject to fusion. +-- copying a new array. Subject to fusion. Performs replacement on +-- invalid scalar values. cons :: Char -> Text -> Text -cons c t = unstream (S.cons c (stream t)) +cons c t = unstream (S.cons (safe c) (stream t)) {-# INLINE cons #-} -- | /O(n)/ Adds a character to the end of a 'Text'. This copies the -- entire array in the process, unless fused. Subject to fusion. +-- Performs replacement on invalid scalar values. snoc :: Text -> Char -> Text -snoc t c = unstream (S.snoc (stream t) c) +snoc t c = unstream (S.snoc (stream t) (safe c)) {-# INLINE snoc #-} -- | /O(n)/ Appends one 'Text' to the other by copying both of them @@ -523,9 +565,10 @@ -- ----------------------------------------------------------------------------- -- * Transformations -- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Subject to fusion. +-- each element of @t@. Subject to fusion. Performs replacement on +-- invalid scalar values. map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map f (stream t)) +map f t = unstream (S.map (safe . f) (stream t)) {-# INLINE [1] map #-} -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of @@ -536,9 +579,10 @@ {-# INLINE intercalate #-} -- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Subject to fusion. +-- between the characters of a 'Text'. Subject to fusion. Performs +-- replacement on invalid scalar values. intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse c (stream t)) +intersperse c t = unstream (S.intersperse (safe c) (stream t)) {-# INLINE intersperse #-} -- | /O(n)/ Reverse the characters of a string. Subject to fusion. @@ -554,7 +598,7 @@ -> Text -- ^ Replacement text -> Text -- ^ Input text -> Text -replace s d = intercalate d . split s +replace s d = intercalate d . splitOn s {-# INLINE replace #-} -- ---------------------------------------------------------------------------- @@ -575,7 +619,8 @@ -- context-dependent operation. The case conversion functions in this -- module are /not/ locale sensitive. Programs that require locale -- sensitivity should use appropriate versions of the case mapping --- functions from the @text-icu@ package. +-- functions from the @text-icu@ package: +-- -- | /O(n)/ Convert a string to folded case. This function is mainly -- useful for performing caseless (also known as case insensitive) @@ -614,8 +659,11 @@ {-# INLINE toUpper #-} -- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. Examples: +-- specified fill character on the right. Subject to fusion. +-- Performs replacement on invalid scalar values. -- +-- Examples: +-- -- > justifyLeft 7 'x' "foo" == "fooxxxx" -- > justifyLeft 3 'x' "foobar" == "foobar" justifyLeft :: Int -> Char -> Text -> Text @@ -633,8 +681,11 @@ #-} -- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Examples: +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. -- +-- Examples: +-- -- > justifyRight 7 'x' "bar" == "xxxxbar" -- > justifyRight 3 'x' "foobar" == "foobar" justifyRight :: Int -> Char -> Text -> Text @@ -644,9 +695,12 @@ where len = length t {-# INLINE justifyRight #-} --- | /O(n)/ Center a string to the given length, using the --- specified fill character on either side. Examples: +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. -- +-- Examples: +-- -- > center 8 'x' "HS" = "xxxHSxxx" center :: Int -> Char -> Text -> Text center k c t @@ -760,6 +814,7 @@ -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of -- successive reduced values from the left. Subject to fusion. +-- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] -- @@ -767,11 +822,13 @@ -- -- > last (scanl f z xs) == foldl f z xs. scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl f z (stream t)) +scanl f z t = unstream (S.scanl g z (stream t)) + where g a b = safe (f a b) {-# INLINE scanl #-} -- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. Subject to fusion. +-- value argument. Subject to fusion. Performs replacement on +-- invalid scalar values. -- -- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] scanl1 :: (Char -> Char -> Char) -> Text -> Text @@ -779,15 +836,18 @@ | otherwise = scanl f (unsafeHead t) (unsafeTail t) {-# INLINE scanl1 #-} --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. -- -- > scanr f v == reverse . scanl (flip f) v . reverse scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f z = S.reverse . S.reverseScanr f z . reverseStream +scanr f z = S.reverse . S.reverseScanr g z . reverseStream + where g a b = safe (f a b) {-# INLINE scanr #-} -- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. Subject to fusion. +-- value argument. Subject to fusion. Performs replacement on +-- invalid scalar values. scanr1 :: (Char -> Char -> Char) -> Text -> Text scanr1 f t | null t = empty | otherwise = scanr f (last t) (init t) @@ -795,9 +855,11 @@ -- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a -- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumL f z0 = S.mapAccumL f z0 . stream +mapAccumL f z0 = S.mapAccumL g z0 . stream + where g a b = second safe (f a b) {-# INLINE mapAccumL #-} -- | The 'mapAccumR' function behaves like a combination of 'map' and @@ -805,8 +867,10 @@ -- 'Text', passing an accumulating parameter from right to left, and -- returning a final value of this accumulator together with the new -- 'Text'. +-- Performs replacement on invalid scalar values. mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumR f z0 = second reverse . S.mapAccumL f z0 . reverseStream +mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream + where g a b = second safe (f a b) {-# INLINE mapAccumR #-} -- ----------------------------------------------------------------------------- @@ -839,7 +903,7 @@ -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the -- value of every element. Subject to fusion. replicateChar :: Int -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n c) +replicateChar n c = unstream (S.replicateCharI n (safe c)) {-# INLINE replicateChar #-} -- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' @@ -848,9 +912,9 @@ -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the -- string, and @b@ is the seed value for further production. Subject --- to fusion. +-- to fusion. Performs replacement on invalid scalar values. unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr f s) +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) {-# INLINE unfoldr #-} -- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed @@ -858,9 +922,9 @@ -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and -- correct, otherwise its performance is similar to 'unfoldr'. Subject --- to fusion. +-- to fusion. Performs replacement on invalid scalar values. unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n f s) +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) {-# INLINE unfoldrN #-} -- ----------------------------------------------------------------------------- @@ -1009,23 +1073,23 @@ where d = iter_ t i {-# INLINE splitAt #-} --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list. -spanBy :: (Char -> Bool) -> Text -> (Text, Text) -spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k)) +span :: (Char -> Bool) -> Text -> (Text, Text) +span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k)) where k = loop 0 loop !i | i >= len || not (p c) = i | otherwise = loop (i+d) where Iter c d = iter t i -{-# INLINE spanBy #-} +{-# INLINE span #-} --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is +-- | /O(n)/ 'break' is like 'span', but the prefix returned is -- over elements that fail the predicate @p@. -breakBy :: (Char -> Bool) -> Text -> (Text, Text) -breakBy p = spanBy (not . p) -{-# INLINE breakBy #-} +break :: (Char -> Bool) -> Text -> (Text, Text) +break p = span (not . p) +{-# INLINE break #-} -- | /O(n)/ Group characters in a string according to a predicate. groupBy :: (Char -> Char -> Bool) -> Text -> [Text] @@ -1075,30 +1139,30 @@ -- -- Examples: -- --- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] --- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] --- > split "x" "x" == ["",""] +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] -- -- and -- --- > intercalate s . split s == id --- > split (singleton c) == splitBy (==c) +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -split :: Text -> Text -> [Text] -split pat@(Text _ _ l) src@(Text arr off len) - | l <= 0 = emptyError "split" - | isSingleton pat = splitBy (== unsafeHead pat) src +splitOn :: Text -> Text -> [Text] +splitOn pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "splitOn" + | isSingleton pat = split (== unsafeHead pat) src | otherwise = go 0 (indices pat src) where go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs go s _ = [textP arr (s+off) (len-s)] -{-# INLINE [1] split #-} +{-# INLINE [1] splitOn #-} {-# RULES -"TEXT split/singleton -> splitBy/==" [~1] forall c t. - split (singleton c) t = splitBy (==c) t +"TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t #-} -- | /O(n)/ Splits a 'Text' into components delimited by separators, @@ -1106,15 +1170,15 @@ -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. -- --- > splitBy (=='a') "aabbaca" == ["","","bb","c",""] --- > splitBy (=='a') "" == [""] -splitBy :: (Char -> Bool) -> Text -> [Text] -splitBy _ t@(Text _off _arr 0) = [t] -splitBy p t = loop t +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') "" == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ t@(Text _off _arr 0) = [t] +split p t = loop t where loop s | null s' = [l] | otherwise = l : loop (unsafeTail s') - where (l, s') = breakBy p s -{-# INLINE splitBy #-} + where (l, s') = break p s +{-# INLINE split #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the @@ -1136,21 +1200,21 @@ ------------------------------------------------------------------------------- -- ** Searching with a predicate --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and --- returns the first element in matching the predicate, or 'Nothing' --- if there is no such element. -findBy :: (Char -> Bool) -> Text -> Maybe Char -findBy p t = S.findBy p (stream t) -{-# INLINE findBy #-} +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element matching the predicate, or 'Nothing' if +-- there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text', +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. -- --- > partitionBy p t == (filter p t, filter (not . p) t) -partitionBy :: (Char -> Bool) -> Text -> (Text, Text) -partitionBy p t = (filter p t, filter (not . p) t) -{-# INLINE partitionBy #-} +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} -- | /O(n)/ 'filter', applied to a predicate and a 'Text', -- returns a 'Text' containing those characters that satisfy the @@ -1166,39 +1230,40 @@ -- -- Examples: -- --- > break "::" "a::b::c" ==> ("a", "::b::c") --- > break "/" "foobar" ==> ("foobar", "") +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack --- > where (prefix, match) = break needle haystack +-- > where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you --- want to break on every instance of a substring), use 'find' +-- want to break on every instance of a substring), use 'breakOnAll' -- instead, as it has lower startup overhead. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -break :: Text -> Text -> (Text, Text) -break pat src@(Text arr off len) - | null pat = emptyError "break" +breakOn :: Text -> Text -> (Text, Text) +breakOn pat src@(Text arr off len) + | null pat = emptyError "breakOn" | otherwise = case indices pat src of [] -> (src, empty) (x:_) -> (textP arr off x, textP arr (off+x) (len-x)) -{-# INLINE break #-} +{-# INLINE breakOn #-} --- | /O(n+m)/ Similar to 'break', but searches from the end of the string. +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the +-- string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@. The second is the -- remainder of @haystack@, following the match. -- --- > breakEnd "::" "a::b::c" ==> ("a::b::", "c") -breakEnd :: Text -> Text -> (Text, Text) -breakEnd pat src = let (a,b) = break (reverse pat) (reverse src) - in (reverse b, reverse a) -{-# INLINE breakEnd #-} +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: Text -> Text -> (Text, Text) +breakOnEnd pat src = (reverse b, reverse a) + where (a,b) = breakOn (reverse pat) (reverse src) +{-# INLINE breakOnEnd #-} -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@. Each element of the returned list consists of a pair: @@ -1218,16 +1283,16 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty. -find :: Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -find pat src@(Text arr off slen) - | null pat = emptyError "find" +breakOnAll :: Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src@(Text arr off slen) + | null pat = emptyError "breakOnAll" | otherwise = L.map step (indices pat src) where step x = (chunk 0 x, chunk x (slen-x)) chunk !n !l = textP arr (n+off) l -{-# INLINE find #-} +{-# INLINE breakOnAll #-} ------------------------------------------------------------------------------- -- ** Indexing 'Text's @@ -1297,8 +1362,10 @@ -- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function -- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2)) +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) {-# INLINE [0] zipWith #-} -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's @@ -1325,7 +1392,7 @@ | otherwise = h : if null t then [] else lines (unsafeTail t) - where (h,t) = spanBy (/= '\n') ps + where (h,t) = span (/= '\n') ps {-# INLINE lines #-} {- @@ -1362,8 +1429,7 @@ {-# INLINE unwords #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. This function is --- subject to fusion. +-- 'True' iff the first is a prefix of the second. Subject to fusion. isPrefixOf :: Text -> Text -> Bool isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) = alen <= blen && S.isPrefixOf (stream a) (stream b) diff --git a/Data/Text/Encoding.hs b/Data/Text/Encoding.hs --- a/Data/Text/Encoding.hs +++ b/Data/Text/Encoding.hs @@ -14,18 +14,20 @@ -- Functions for converting 'Text' values to and from 'ByteString', -- using several standard encodings. -- --- To make use of a much larger variety of encodings, use the @text-icu@ --- package. +-- To gain access to a much larger family of encodings, use the +-- @text-icu@ package: module Data.Text.Encoding ( -- * Decoding ByteStrings to Text + -- $strict decodeASCII , decodeUtf8 , decodeUtf16LE , decodeUtf16BE , decodeUtf32LE , decodeUtf32BE + -- ** Controllable error handling , decodeUtf8With , decodeUtf16LEWith @@ -60,11 +62,25 @@ import qualified Data.Text.Encoding.Utf8 as U8 import qualified Data.Text.Fusion as F +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: ByteString -> Text decodeASCII bs = F.unstream (E.streamASCII bs) {-# INLINE decodeASCII #-} +-- | Decode a 'ByteString' containing UTF-8 encoded text. decodeUtf8With :: OnDecodeError -> ByteString -> Text decodeUtf8With onErr bs = textP (fst a) 0 (snd a) where @@ -101,6 +117,10 @@ {-# INLINE[0] decodeUtf8With #-} -- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown. For more control over the handling of invalid data, use +-- 'decodeUtf8With'. decodeUtf8 :: ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-} @@ -156,6 +176,10 @@ {-# INLINE decodeUtf16LEWith #-} -- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-} @@ -166,6 +190,10 @@ {-# INLINE decodeUtf16BEWith #-} -- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-} @@ -186,6 +214,10 @@ {-# INLINE decodeUtf32LEWith #-} -- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-} @@ -196,6 +228,10 @@ {-# INLINE decodeUtf32BEWith #-} -- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-} diff --git a/Data/Text/IO.hs b/Data/Text/IO.hs --- a/Data/Text/IO.hs +++ b/Data/Text/IO.hs @@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for text I\/O. +-- +-- Skip past the synopsis for some important notes on performance and +-- portability across different versions of GHC. module Data.Text.IO ( + -- * Performance + -- $performance + -- * Locale support -- $locale -- * File-at-a-time operations @@ -59,6 +65,22 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) import System.IO.Error (isEOFError) #endif + +-- $performance +-- #performance# +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding (e.g. UTF-8), and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. +-- +-- Whether this will hold depends on the version of GHC you are using, +-- the platform you are working on, the data you are working with, and +-- the encodings you are using, so be sure to test for yourself. -- | The 'readFile' function reads a file and returns the contents of -- the file as a string. The entire file is read strictly, as with diff --git a/Data/Text/Internal.hs b/Data/Text/Internal.hs --- a/Data/Text/Internal.hs +++ b/Data/Text/Internal.hs @@ -22,8 +22,12 @@ -- * Construction , text , textP + -- * Safety + , safe -- * Code that must be here for accessibility , empty + -- * Utilities + , firstf -- * Debugging , showText ) where @@ -31,7 +35,9 @@ #if defined(ASSERTS) import Control.Exception (assert) #endif +import Data.Bits ((.&.)) import qualified Data.Text.Array as A +import Data.Text.UnsafeChar (ord) import Data.Typeable (Typeable) -- | A space efficient, packed, unboxed Unicode text type. @@ -72,3 +78,21 @@ showText (Text arr off len) = "Text " ++ show (A.toList arr off len) ++ ' ' : show off ++ ' ' : show len + +-- | Map a 'Char' to a 'Text'-safe value. +-- +-- UTF-16 surrogate code points are not included in the set of Unicode +-- scalar values, but are unfortunately admitted as valid 'Char' +-- values by Haskell. They cannot be represented in a 'Text'. This +-- function remaps those code points to the Unicode replacement +-- character \"�\", and leaves other code points unchanged. +safe :: Char -> Char +safe c + | ord c .&. 0x1ff800 /= 0xd800 = c + | otherwise = '\xfffd' +{-# INLINE safe #-} + +-- | Apply a function to the first element of an optional pair. +firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) +firstf f (Just (a, b)) = Just (f a, b) +firstf _ Nothing = Nothing diff --git a/Data/Text/Lazy.hs b/Data/Text/Lazy.hs --- a/Data/Text/Lazy.hs +++ b/Data/Text/Lazy.hs @@ -11,18 +11,23 @@ -- Portability : GHC -- -- A time and space-efficient implementation of Unicode text using --- lists of packed arrays. This representation is suitable for high +-- lists of packed arrays. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- The representation used by this module is suitable for high -- performance use and for streaming large quantities of data. It -- provides a means to manipulate a large body of text without -- requiring that the entire content be resident in memory. -- -- Some operations, such as 'concat', 'append', 'reverse' and 'cons', --- have better complexity than their "Data.Text" equivalents, due to --- optimisations resulting from the list spine structure. And for --- other operations lazy 'Text's are usually within a few percent of --- strict ones, but with better heap usage. For data larger than --- available memory, or if you have tight memory constraints, this --- module will be the only option. +-- have better time complexity than their "Data.Text" equivalents, due +-- to the underlying representation being a list of chunks. For other +-- operations, lazy 'Text's are usually within a few percent of strict +-- ones, but often with better heap usage if used in a streaming +-- fashion. For data larger than available memory, or if you have +-- tight memory constraints, this module will be the only option. -- -- This module is intended to be imported @qualified@, to avoid name -- clashes with "Prelude" functions. eg. @@ -31,7 +36,15 @@ module Data.Text.Lazy ( + -- * Fusion + -- $fusion + + -- * Acceptable data + -- $replacement + + -- * Types Text + -- * Creation and elimination , pack , unpack @@ -122,10 +135,10 @@ , stripStart , stripEnd , splitAt - , spanBy + , span + , breakOn + , breakOnEnd , break - , breakEnd - , breakBy , group , groupBy , inits @@ -133,8 +146,8 @@ -- ** Breaking into many substrings -- $split + , splitOn , split - , splitBy , chunksOf -- , breakSubstring @@ -156,8 +169,8 @@ -- * Searching , filter , find - , findBy - , partitionBy + , breakOnAll + , partition -- , findSubstring @@ -200,10 +213,62 @@ import Data.Text.Fusion.Internal (PairS(..)) import Data.Text.Lazy.Fusion (stream, unstream) import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks) -import Data.Text.Internal (textP) +import Data.Text.Internal (firstf, safe, textP) import qualified Data.Text.Util as U import Data.Text.Lazy.Search (indices) +-- $fusion +-- +-- Most of the functions in this module are subject to /fusion/, +-- meaning that a pipeline of such functions will usually allocate at +-- most one 'Text' value. +-- +-- As an example, consider the following pipeline: +-- +-- > import Data.Text.Lazy as T +-- > import Data.Text.Lazy.Encoding as E +-- > import Data.ByteString.Lazy (ByteString) +-- > +-- > countChars :: ByteString -> Int +-- > countChars = T.length . T.toUpper . E.decodeUtf8 +-- +-- From the type signatures involved, this looks like it should +-- allocate one 'ByteString' value, and two 'Text' values. However, +-- when a module is compiled with optimisation enabled under GHC, the +-- two intermediate 'Text' values will be optimised away, and the +-- function will be compiled down to a single loop over the source +-- 'ByteString'. +-- +-- Functions that can be fused by the compiler are documented with the +-- phrase \"Subject to fusion\". + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in §3.9, definition D76 of the Unicode 5.2 standard: +-- . As +-- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF +-- inclusive. Haskell implementations admit all Unicode code points +-- (§3.4, definition D10) as 'Char' values, including code points +-- from this invalid range. This means that there are some 'Char' +-- values that are not valid Unicode scalar values, and the functions +-- in this module must handle those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". +-- +-- (One reason for this policy of replacement is that internally, a +-- 'Text' value is represented as packed UTF-16 data. Values in the +-- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate +-- code points, and so cannot be represented. The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see Unicode Technical Report 36, §3.5: +-- ) + equal :: Text -> Text -> Bool equal Empty Empty = True equal Empty _ = False @@ -274,19 +339,19 @@ -- | /O(n)/ Convert a 'String' into a 'Text'. -- --- This function is subject to array fusion. +-- Subject to fusion. Performs replacement on invalid scalar values. pack :: String -> Text -pack s = unstream (S.streamList s) +pack = unstream . S.streamList . L.map safe {-# INLINE [1] pack #-} -- | /O(n)/ Convert a 'Text' into a 'String'. --- Subject to array fusion. +-- Subject to fusion. unpack :: Text -> String unpack t = S.unstreamList (stream t) {-# INLINE [1] unpack #-} --- | /O(1)/ Convert a character into a Text. --- Subject to fusion. +-- | /O(1)/ Convert a character into a Text. Subject to fusion. +-- Performs replacement on invalid scalar values. singleton :: Char -> Text singleton c = Chunk (T.singleton c) Empty {-# INLINE [1] singleton #-} @@ -346,8 +411,7 @@ unstream (S.snoc (stream t) c) = snoc t c #-} --- | /O(n\/c)/ Appends one 'Text' to another. Subject to array --- fusion. +-- | /O(n\/c)/ Appends one 'Text' to another. Subject to fusion. append :: Text -> Text -> Text append xs ys = foldrChunks Chunk ys xs {-# INLINE [1] append #-} @@ -360,7 +424,7 @@ #-} -- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. Subject to array fusion. +-- 'Nothing' if empty. Subject to fusion. uncons :: Text -> Maybe (Char, Text) uncons Empty = Nothing uncons (Chunk t ts) = Just (T.unsafeHead t, ts') @@ -369,13 +433,13 @@ {-# INLINE uncons #-} -- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. Subject to array fusion. +-- non-empty. Subject to fusion. head :: Text -> Char head t = S.head (stream t) {-# INLINE head #-} -- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. Subject to array fusion. +-- must be non-empty. Subject to fusion. tail :: Text -> Text tail (Chunk t ts) = chunk (T.tail t) ts tail Empty = emptyError "tail" @@ -389,7 +453,7 @@ #-} -- | /O(1)/ Returns all but the last character of a 'Text', which must --- be non-empty. Subject to array fusion. +-- be non-empty. Subject to fusion. init :: Text -> Text init (Chunk t0 ts0) = go t0 ts0 where go t (Chunk t' ts) = Chunk t (go t' ts) @@ -404,7 +468,7 @@ unstream (S.init (stream t)) = init t #-} --- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to array +-- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to -- fusion. null :: Text -> Bool null Empty = True @@ -425,7 +489,7 @@ {-# INLINE isSingleton #-} -- | /O(1)/ Returns the last character of a 'Text', which must be --- non-empty. Subject to array fusion. +-- non-empty. Subject to fusion. last :: Text -> Char last Empty = emptyError "last" last (Chunk t ts) = go t ts @@ -469,9 +533,10 @@ -- properties of code. -- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Subject to array fusion. +-- each element of @t@. Subject to fusion. Performs replacement on +-- invalid scalar values. map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map f (stream t)) +map f t = unstream (S.map (safe . f) (stream t)) {-# INLINE [1] map #-} -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of @@ -482,14 +547,18 @@ {-# INLINE intercalate #-} -- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Subject to array fusion. -intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse c (stream t)) +-- between the characters of a 'Text'. Subject to fusion. Performs +-- replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t = unstream (S.intersperse (safe c) (stream t)) {-# INLINE intersperse #-} -- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. Examples: +-- specified fill character on the right. Subject to fusion. Performs +-- replacement on invalid scalar values. -- +-- Examples: +-- -- > justifyLeft 7 'x' "foo" == "fooxxxx" -- > justifyLeft 3 'x' "foobar" == "foobar" justifyLeft :: Int64 -> Char -> Text -> Text @@ -507,8 +576,11 @@ #-} -- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Examples: +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. -- +-- Examples: +-- -- > justifyRight 7 'x' "bar" == "xxxxbar" -- > justifyRight 3 'x' "foobar" == "foobar" justifyRight :: Int64 -> Char -> Text -> Text @@ -518,9 +590,12 @@ where len = length t {-# INLINE justifyRight #-} --- | /O(n)/ Center a string to the given length, using the --- specified fill character on either side. Examples: +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. -- +-- Examples: +-- -- > center 8 'x' "HS" = "xxxHSxxx" center :: Int64 -> Char -> Text -> Text center k c t @@ -555,7 +630,7 @@ -> Text -- ^ Replacement text -> Text -- ^ Input text -> Text -replace s d = intercalate d . split s +replace s d = intercalate d . splitOn s {-# INLINE replace #-} -- ---------------------------------------------------------------------------- @@ -608,26 +683,24 @@ -- | /O(n)/ 'foldl', applied to a binary operator, a starting value -- (typically the left-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from left to right. --- Subject to array fusion. +-- Subject to fusion. foldl :: (a -> Char -> a) -> a -> Text -> a foldl f z t = S.foldl f z (stream t) {-# INLINE foldl #-} -- | /O(n)/ A strict version of 'foldl'. --- Subject to array fusion. +-- Subject to fusion. foldl' :: (a -> Char -> a) -> a -> Text -> a foldl' f z t = S.foldl' f z (stream t) {-# INLINE foldl' #-} -- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to array --- fusion. +-- and thus must be applied to a non-empty 'Text'. Subject to fusion. foldl1 :: (Char -> Char -> Char) -> Text -> Char foldl1 f t = S.foldl1 f (stream t) {-# INLINE foldl1 #-} --- | /O(n)/ A strict version of 'foldl1'. --- Subject to array fusion. +-- | /O(n)/ A strict version of 'foldl1'. Subject to fusion. foldl1' :: (Char -> Char -> Char) -> Text -> Char foldl1' f t = S.foldl1' f (stream t) {-# INLINE foldl1' #-} @@ -635,13 +708,13 @@ -- | /O(n)/ 'foldr', applied to a binary operator, a starting value -- (typically the right-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from right to left. --- Subject to array fusion. +-- Subject to fusion. foldr :: (Char -> a -> a) -> a -> Text -> a foldr f z t = S.foldr f z (stream t) {-# INLINE foldr #-} --- | /O(n)/ A variant of 'foldr' that has no starting value argument, and --- thust must be applied to a non-empty 'Text'. Subject to array +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thust must be applied to a non-empty 'Text'. Subject to -- fusion. foldr1 :: (Char -> Char -> Char) -> Text -> Char foldr1 f t = S.foldr1 f (stream t) @@ -664,32 +737,32 @@ {-# INLINE concatMap #-} -- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisifes the predicate @p@. Subject to array fusion. +-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion. any :: (Char -> Bool) -> Text -> Bool any p t = S.any p (stream t) {-# INLINE any #-} -- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisify the predicate @p@. Subject to array fusion. +-- 'Text' @t@ satisify the predicate @p@. Subject to fusion. all :: (Char -> Bool) -> Text -> Bool all p t = S.all p (stream t) {-# INLINE all #-} -- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. Subject to array fusion. +-- must be non-empty. Subject to fusion. maximum :: Text -> Char maximum t = S.maximum (stream t) {-# INLINE maximum #-} -- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. Subject to array fusion. +-- must be non-empty. Subject to fusion. minimum :: Text -> Char minimum t = S.minimum (stream t) {-# INLINE minimum #-} -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. This function is subject --- to array fusion. +-- successive reduced values from the left. Subject to fusion. +-- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] -- @@ -697,11 +770,13 @@ -- -- > last (scanl f z xs) == foldl f z xs. scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl f z (stream t)) +scanl f z t = unstream (S.scanl g z (stream t)) + where g a b = safe (f a b) {-# INLINE scanl #-} -- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. This function is subject to array fusion. +-- value argument. Subject to fusion. Performs replacement on +-- invalid scalar values. -- -- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] scanl1 :: (Char -> Char -> Char) -> Text -> Text @@ -710,21 +785,24 @@ Just (t,ts) -> scanl f t ts {-# INLINE scanl1 #-} --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. -- -- > scanr f v == reverse . scanl (flip f) v . reverse scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f v = reverse . scanl (flip f) v . reverse +scanr f v = reverse . scanl g v . reverse + where g a b = safe (f b a) -- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. +-- value argument. Performs replacement on invalid scalar values. scanr1 :: (Char -> Char -> Char) -> Text -> Text scanr1 f t | null t = empty | otherwise = scanr f (last t) (init t) -- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a -- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) mapAccumL f = go where @@ -738,7 +816,7 @@ -- a strict 'foldr'; it applies a function to each element of a -- 'Text', passing an accumulating parameter from right to left, and -- returning a final value of this accumulator together with the new --- 'Text'. +-- 'Text'. Performs replacement on invalid scalar values. mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) mapAccumR f = go where @@ -762,7 +840,7 @@ -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the -- value of every element. Subject to fusion. replicateChar :: Int64 -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n c) +replicateChar n c = unstream (S.replicateCharI n (safe c)) {-# INLINE replicateChar #-} {-# RULES @@ -775,9 +853,10 @@ -- 'Text' from a seed value. The function takes the element and -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr f s) +-- string, and @b@ is the seed value for further production. Performs +-- replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) {-# INLINE unfoldr #-} -- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed @@ -785,8 +864,9 @@ -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and -- correct, otherwise its performance is similar to 'unfoldr'. -unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n f s) +-- Performs replacement on invalid scalar values. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) {-# INLINE unfoldrN #-} -- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the @@ -847,9 +927,9 @@ where len' = fromIntegral len n' = fromIntegral n --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', returns --- the longest prefix (possibly empty) of elements that satisfy @p@. --- This function is subject to array fusion. +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. Subject to fusion. takeWhile :: (Char -> Bool) -> Text -> Text takeWhile p t0 = takeWhile' t0 where takeWhile' Empty = Empty @@ -868,7 +948,7 @@ #-} -- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. This function is subject to array fusion. +-- 'takeWhile' @p@ @t@. Subject to fusion. dropWhile :: (Char -> Bool) -> Text -> Text dropWhile p t0 = dropWhile' t0 where dropWhile' Empty = Empty @@ -964,13 +1044,13 @@ -- -- Examples: -- --- > break "::" "a::b::c" ==> ("a", "::b::c") --- > break "/" "foobar" ==> ("foobar", "") +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") -- -- Laws: -- -- > append prefix match == haystack --- > where (prefix, match) = break needle haystack +-- > where (prefix, match) = breakOn needle haystack -- -- If you need to break a string by a substring repeatedly (e.g. you -- want to break on every instance of a substring), use 'find' @@ -981,25 +1061,25 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -break :: Text -> Text -> (Text, Text) -break pat src - | null pat = emptyError "break" +breakOn :: Text -> Text -> (Text, Text) +breakOn pat src + | null pat = emptyError "breakOn" | otherwise = case indices pat src of [] -> (src, empty) (x:_) -> let h :*: t = splitAtWord x src in (h, t) --- | /O(n+m)/ Similar to 'break', but searches from the end of the string. +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. -- -- The first element of the returned tuple is the prefix of @haystack@ -- up to and including the last match of @needle@. The second is the -- remainder of @haystack@, following the match. -- --- > breakEnd "::" "a::b::c" ==> ("a::b::", "c") -breakEnd :: Text -> Text -> (Text, Text) -breakEnd pat src = let (a,b) = break (reverse pat) (reverse src) +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: Text -> Text -> (Text, Text) +breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) in (reverse b, reverse a) -{-# INLINE breakEnd #-} +{-# INLINE breakOnEnd #-} -- | /O(n+m)/ Find all non-overlapping instances of @needle@ in -- @haystack@. Each element of the returned list consists of a pair: @@ -1010,9 +1090,9 @@ -- -- Examples: -- --- > find "::" "" +-- > breakOnAll "::" "" -- > ==> [] --- > find "/" "a/b/c/" +-- > breakOnAll "/" "a/b/c/" -- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] -- -- This function is strict in its first argument, and lazy in its @@ -1022,11 +1102,11 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty. -find :: Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -find pat src - | null pat = emptyError "find" +breakOnAll :: Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src + | null pat = emptyError "breakOnAll" | otherwise = go 0 empty src (indices pat src) where go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s @@ -1034,10 +1114,10 @@ in (h',t) : go x h' t xs go _ _ _ _ = [] --- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over +-- | /O(n)/ 'break' is like 'span', but the prefix returned is over -- elements that fail the predicate @p@. -breakBy :: (Char -> Bool) -> Text -> (Text, Text) -breakBy p t0 = break' t0 +break :: (Char -> Bool) -> Text -> (Text, Text) +break p t0 = break' t0 where break' Empty = (empty, empty) break' c@(Chunk t ts) = case T.findIndex p t of @@ -1047,13 +1127,13 @@ | otherwise -> let (a,b) = T.splitAt n t in (Chunk a Empty, Chunk b ts) --- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) -- of @t@ of elements that satisfy @p@, and whose second is the -- remainder of the list. -spanBy :: (Char -> Bool) -> Text -> (Text, Text) -spanBy p = breakBy (not . p) -{-# INLINE spanBy #-} +span :: (Char -> Bool) -> Text -> (Text, Text) +span p = break (not . p) +{-# INLINE span #-} -- | The 'group' function takes a 'Text' and returns a list of 'Text's -- such that the concatenation of the result is equal to the argument. @@ -1072,7 +1152,7 @@ groupBy :: (Char -> Char -> Bool) -> Text -> [Text] groupBy _ Empty = [] groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs - where (ys,zs) = spanBy (eq x) xs + where (ys,zs) = span (eq x) xs x = T.unsafeHead t xs = chunk (T.unsafeTail t) ts @@ -1104,37 +1184,37 @@ -- -- Examples: -- --- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] --- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] --- > split "x" "x" == ["",""] +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] -- -- and -- --- > intercalate s . split s == id --- > split (singleton c) == splitBy (==c) +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) -- -- This function is strict in its first argument, and lazy in its -- second. -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -split :: Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -split pat src - | null pat = emptyError "split" - | isSingleton pat = splitBy (== head pat) src +splitOn :: Text -- ^ Text to split on + -> Text -- ^ Input text + -> [Text] +splitOn pat src + | null pat = emptyError "splitOn" + | isSingleton pat = split (== head pat) src | otherwise = go 0 (indices pat src) src where go _ [] cs = [cs] go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs in h : go (x+l) xs (dropWords l t) l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat -{-# INLINE [1] split #-} +{-# INLINE [1] splitOn #-} {-# RULES -"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t. - split (singleton c) t = splitBy (==c) t +"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t #-} -- | /O(n)/ Splits a 'Text' into components delimited by separators, @@ -1142,16 +1222,16 @@ -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. -- --- > splitBy (=='a') "aabbaca" == ["","","bb","c",""] --- > splitBy (=='a') [] == [""] -splitBy :: (Char -> Bool) -> Text -> [Text] -splitBy _ Empty = [Empty] -splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0 +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') [] == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ Empty = [Empty] +split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 where comb acc (s:[]) Empty = revChunks (s:acc) : [] - comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts - comb _ [] _ = impossibleError "splitBy" -{-# INLINE splitBy #-} + comb _ [] _ = impossibleError "split" +{-# INLINE split #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the @@ -1171,14 +1251,14 @@ -- newline 'Char's. The resulting strings do not contain newlines. lines :: Text -> [Text] lines Empty = [] -lines t = let (l,t') = breakBy ((==) '\n') t +lines t = let (l,t') = break ((==) '\n') t in l : if null t' then [] else lines (tail t') -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text] -words = L.filter (not . null) . splitBy isSpace +words = L.filter (not . null) . split isSpace {-# INLINE words #-} -- | /O(n)/ Joins lines, after appending a terminating newline to @@ -1193,8 +1273,7 @@ {-# INLINE unwords #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. This function is --- subject to fusion. +-- 'True' iff the first is a prefix of the second. Subject to fusion. isPrefixOf :: Text -> Text -> Bool isPrefixOf Empty _ = True isPrefixOf _ Empty = False @@ -1299,21 +1378,21 @@ filter p t = unstream (S.filter p (stream t)) {-# INLINE filter #-} --- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and -- returns the first element in matching the predicate, or 'Nothing' -- if there is no such element. -findBy :: (Char -> Bool) -> Text -> Maybe Char -findBy p t = S.findBy p (stream t) -{-# INLINE findBy #-} +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} --- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text', +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. -- --- > partitionBy p t == (filter p t, filter (not . p) t) -partitionBy :: (Char -> Bool) -> Text -> (Text, Text) -partitionBy p t = (filter p t, filter (not . p) t) -{-# INLINE partitionBy #-} +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. index :: Text -> Int64 -> Char @@ -1340,8 +1419,7 @@ #-} -- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. This function is subject --- to fusion. +-- query element appears in the given 'Text'. Subject to fusion. countChar :: Char -> Text -> Int64 countChar c t = S.countChar c (stream t) @@ -1355,8 +1433,10 @@ -- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function -- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith f (stream t1) (stream t2)) +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) {-# INLINE [0] zipWith #-} revChunks :: [T.Text] -> Text diff --git a/Data/Text/Lazy/Builder.hs b/Data/Text/Lazy/Builder.hs --- a/Data/Text/Lazy/Builder.hs +++ b/Data/Text/Lazy/Builder.hs @@ -10,8 +10,21 @@ -- Stability : experimental -- Portability : portable to Hugs and GHC -- --- Efficient construction of lazy texts. +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. -- +-- To get maximum performance when building lazy @Text@ values using a builder, associate @mappend@ calls to the right. For example, prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. +-- ----------------------------------------------------------------------------- module Data.Text.Lazy.Builder @@ -46,15 +59,15 @@ ------------------------------------------------------------------------ --- | A 'Builder' is an efficient way to build lazy 'L.Text's. There --- are several functions for constructing 'Builder's, but only one to --- inspect them: to extract any data, you have to turn them into lazy --- 'L.Text's using 'toLazyText'. +-- | A @Builder@ is an efficient way to build lazy @Text@ values. +-- There are several functions for constructing builders, but only one +-- to inspect them: to extract any data, you have to turn them into +-- lazy @Text@ values using @toLazyText@. -- --- Internally, a 'Builder' constructs a lazy 'L.Text' by filling byte --- arrays piece by piece. As each buffer is filled, it is \'popped\' --- off, to become a new chunk of the resulting lazy 'L.Text'. All --- this is hidden from the user of the 'Builder'. +-- Internally, a builder constructs a lazy @Text@ by filling arrays +-- piece by piece. As each buffer is filled, it is \'popped\' off, to +-- become a new chunk of the resulting lazy @Text@. All this is +-- hidden from the user of the @Builder@. newtype Builder = Builder { -- Invariant (from Data.Text.Lazy): -- The lists include no null Texts. @@ -78,7 +91,7 @@ ------------------------------------------------------------------------ --- | /O(1)./ The empty Builder, satisfying +-- | /O(1)./ The empty @Builder@, satisfying -- -- * @'toLazyText' 'empty' = 'L.empty'@ -- @@ -86,7 +99,7 @@ empty = Builder id {-# INLINE empty #-} --- | /O(1)./ A Builder taking a single character, satisfying +-- | /O(1)./ A @Builder@ taking a single character, satisfying -- -- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ -- @@ -96,7 +109,7 @@ ------------------------------------------------------------------------ --- | /O(1)./ The concatenation of two Builders, an associative +-- | /O(1)./ The concatenation of two builders, an associative -- operation with identity 'empty', satisfying -- -- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ @@ -107,12 +120,12 @@ -- TODO: Experiment to find the right threshold. copyLimit :: Int -copyLimit = 128 +copyLimit = 128 --- This function attempts to merge small Texts instead of treating the --- text as its own chunk. We may not always want this. +-- This function attempts to merge small @Text@ values instead of +-- treating each value as its own chunk. We may not always want this. --- | /O(1)./ A Builder taking a 'S.Text', satisfying +-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying -- -- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ -- @@ -128,7 +141,7 @@ fromText (S.pack s) = fromString s #-} --- | /O(1)./ A Builder taking a 'String', satisfying +-- | /O(1)./ A Builder taking a @String@, satisfying -- -- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ -- @@ -150,7 +163,7 @@ chunkSize = smallChunkSize {-# INLINE fromString #-} --- | /O(1)./ A Builder taking a lazy 'L.Text', satisfying +-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying -- -- * @'toLazyText' ('fromLazyText' t) = t@ -- @@ -168,15 +181,15 @@ ------------------------------------------------------------------------ --- | /O(n)./ Extract a lazy 'L.Text' from a 'Builder' with a default +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default -- buffer size. The construction work takes place if and when the --- relevant part of the lazy 'L.Text' is demanded. +-- relevant part of the lazy @Text@ is demanded. toLazyText :: Builder -> L.Text toLazyText = toLazyTextWith smallChunkSize --- | /O(n)./ Extract a lazy 'L.Text' from a 'Builder', using the given +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given -- size for the initial buffer. The construction work takes place if --- and when the relevant part of the lazy 'L.Text' is demanded. +-- and when the relevant part of the lazy @Text@ is demanded. -- -- If the initial buffer is too small to hold all data, subsequent -- buffers will be the default buffer size. @@ -184,8 +197,8 @@ toLazyTextWith chunkSize m = L.fromChunks (runST $ newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) --- | /O(1)./ Pop the 'S.Text' we have constructed so far, if any, --- yielding a new chunk in the result lazy 'L.Text'. +-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, +-- yielding a new chunk in the result lazy @Text@. flush :: Builder flush = Builder $ \ k buf@(Buffer p o u l) -> if u == 0 diff --git a/Data/Text/Lazy/Encoding.hs b/Data/Text/Lazy/Encoding.hs --- a/Data/Text/Lazy/Encoding.hs +++ b/Data/Text/Lazy/Encoding.hs @@ -12,12 +12,13 @@ -- Functions for converting lazy 'Text' values to and from lazy -- 'ByteString', using several standard encodings. -- --- To make use of a much larger variety of encodings, use the @text-icu@ --- package. +-- To gain access to a much larger variety of encodings, use the +-- @text-icu@ package: module Data.Text.Lazy.Encoding ( -- * Decoding ByteStrings to Text + -- $strict decodeASCII , decodeUtf8 , decodeUtf16LE @@ -51,6 +52,19 @@ import qualified Data.Text.Lazy.Encoding.Fusion as E import qualified Data.Text.Lazy.Fusion as F +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + -- | Decode a 'ByteString' containing 7-bit ASCII encoded text. decodeASCII :: B.ByteString -> Text decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs) @@ -95,6 +109,10 @@ {-# INLINE[0] decodeUtf8With #-} -- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown. For more control over the handling of invalid data, use +-- 'decodeUtf8With'. decodeUtf8 :: B.ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-} @@ -113,6 +131,10 @@ {-# INLINE decodeUtf16LEWith #-} -- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. decodeUtf16LE :: B.ByteString -> Text decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-} @@ -123,6 +145,10 @@ {-# INLINE decodeUtf16BEWith #-} -- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. decodeUtf16BE :: B.ByteString -> Text decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-} @@ -143,6 +169,10 @@ {-# INLINE decodeUtf32LEWith #-} -- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. decodeUtf32LE :: B.ByteString -> Text decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-} @@ -153,6 +183,10 @@ {-# INLINE decodeUtf32BEWith #-} -- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. decodeUtf32BE :: B.ByteString -> Text decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-} diff --git a/Data/Text/Lazy/IO.hs b/Data/Text/Lazy/IO.hs --- a/Data/Text/Lazy/IO.hs +++ b/Data/Text/Lazy/IO.hs @@ -9,9 +9,15 @@ -- Portability : GHC -- -- Efficient locale-sensitive support for lazy text I\/O. +-- +-- Skip past the synopsis for some important notes on performance and +-- portability across different versions of GHC. module Data.Text.Lazy.IO ( + -- * Performance + -- $performance + -- * Locale support -- $locale -- * File-at-a-time operations @@ -57,6 +63,21 @@ import System.IO.Error (isEOFError) import System.IO.Unsafe (unsafeInterleaveIO) #endif + +-- $performance +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding (e.g. UTF-8), and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. +-- +-- Whether this will hold depends on the version of GHC you are using, +-- the platform you are working on, the data you are working with, and +-- the encodings you are using, so be sure to test for yourself. -- | Read a file and return its contents as a string. The file is -- read lazily, as with 'getContents'. diff --git a/Data/Text/Lazy/Read.hs b/Data/Text/Lazy/Read.hs --- a/Data/Text/Lazy/Read.hs +++ b/Data/Text/Lazy/Read.hs @@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.spanBy isDigit txt + where (h,t) = T.span isDigit txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading @@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-} @@ -74,7 +76,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.spanBy isHexDigit txt + where (h,t) = T.span isHexDigit txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) hexDigitToInt :: Char -> Int diff --git a/Data/Text/Read.hs b/Data/Text/Read.hs --- a/Data/Text/Read.hs +++ b/Data/Text/Read.hs @@ -39,14 +39,15 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. decimal :: Integral a => Reader a {-# SPECIALIZE decimal :: Reader Int #-} {-# SPECIALIZE decimal :: Reader Integer #-} decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.spanBy isDigit txt + where (h,t) = T.span isDigit txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading @@ -59,7 +60,8 @@ -- -- /Note/: For fixed-width integer types, this function does not -- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. hexadecimal :: Integral a => Reader a {-# SPECIALIZE hexadecimal :: Reader Int #-} {-# SPECIALIZE hexadecimal :: Reader Integer #-} @@ -74,7 +76,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.spanBy isHexDigit txt + where (h,t) = T.span isHexDigit txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) hexDigitToInt :: Char -> Int @@ -99,7 +101,7 @@ -- by the 'read' function, with the exception that a trailing @\'.\'@ -- or @\'e\'@ /not/ followed by a number is not consumed. -- --- Examples: +-- Examples (with behaviour identical to 'read'): -- -- >rational "3" == Right (3.0, "") -- >rational "3.1" == Right (3.1, "") diff --git a/tests/Properties.hs b/tests/Properties.hs --- a/tests/Properties.hs +++ b/tests/Properties.hs @@ -93,9 +93,6 @@ instance Show DecodeErr where show (DE d _) = "DE " ++ d -instance CoArbitrary Word8 where - coarbitrary = coarbitraryIntegral - instance Arbitrary DecodeErr where arbitrary = oneof [ return $ DE "lenient" lenientDecode , return $ DE "ignore" ignore @@ -268,13 +265,13 @@ tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) -t_replace s d = (L.intercalate d . split s) `eqP` +t_replace s d = (L.intercalate d . splitOn s) `eqP` (unpackS . T.replace (T.pack s) (T.pack d)) -tl_replace s d = (L.intercalate d . split s) `eqP` +tl_replace s d = (L.intercalate d . splitOn s) `eqP` (unpackS . TL.replace (TL.pack s) (TL.pack d)) -split :: (Eq a) => [a] -> [a] -> [[a]] -split pat src0 +splitOn :: (Eq a) => [a] -> [a] -> [[a]] +splitOn pat src0 | l == 0 = error "empty" | otherwise = go src0 where @@ -485,25 +482,25 @@ tl_strip = TL.dropAround isSpace `eq` TL.strip t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n)) -t_spanBy p = L.span p `eqP` (unpack2 . T.spanBy p) -tl_spanBy p = L.span p `eqP` (unpack2 . TL.spanBy p) +t_span p = L.span p `eqP` (unpack2 . T.span p) +tl_span p = L.span p `eqP` (unpack2 . TL.span p) -t_break_id s = squid `eq` (uncurry T.append . T.break s) +t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s) where squid t | T.null s = error "empty" | otherwise = t -tl_break_id s = squid `eq` (uncurry TL.append . TL.break s) +tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s) where squid t | TL.null s = error "empty" | otherwise = t -t_break_start (NotEmpty s) t = let (_,m) = T.break s t +t_breakOn_start (NotEmpty s) t = let (_,m) = T.breakOn s t in T.null m || s `T.isPrefixOf` m -tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t +tl_breakOn_start (NotEmpty s) t = let (_,m) = TL.breakOn s t in TL.null m || s `TL.isPrefixOf` m -t_breakEnd_end (NotEmpty s) t = let (m,_) = T.breakEnd s t +t_breakOnEnd_end (NotEmpty s) t = let (m,_) = T.breakOnEnd s t in T.null m || s `T.isSuffixOf` m -tl_breakEnd_end (NotEmpty s) t = let (m,_) = TL.breakEnd s t +tl_breakOnEnd_end (NotEmpty s) t = let (m,_) = TL.breakOnEnd s t in TL.null m || s `TL.isSuffixOf` m -t_breakBy p = L.break p `eqP` (unpack2 . T.breakBy p) -tl_breakBy p = L.break p `eqP` (unpack2 . TL.breakBy p) +t_break p = L.break p `eqP` (unpack2 . T.break p) +tl_break p = L.break p `eqP` (unpack2 . TL.break p) t_group = L.group `eqP` (map unpackS . T.group) tl_group = L.group `eqP` (map unpackS . TL.group) t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p) @@ -514,33 +511,33 @@ tl_tails = L.tails `eqP` (map unpackS . TL.tails) t_findAppendId (NotEmpty s) = unsquare $ \ts -> let t = T.intercalate s ts - in all (==t) $ map (uncurry T.append) (T.find s t) + in all (==t) $ map (uncurry T.append) (T.breakOnAll s t) tl_findAppendId (NotEmpty s) = unsquare $ \ts -> let t = TL.intercalate s ts - in all (==t) $ map (uncurry TL.append) (TL.find s t) -t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.find s . + in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t) +t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s tl_findContains (NotEmpty s) = all (TL.isPrefixOf s . snd) . - TL.find s . TL.intercalate s + TL.breakOnAll s . TL.intercalate s sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c -t_findCount s = (L.length . T.find s) `eq` T.count s -tl_findCount s = (L.genericLength . TL.find s) `eq` TL.count s +t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s +tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s -t_split_split s = (T.split s `eq` Slow.split s) . T.intercalate s -tl_split_split s = ((TL.split (TL.fromStrict s) . TL.fromStrict) `eq` - (map TL.fromStrict . T.split s)) . T.intercalate s -t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t) -tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t) +t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s +tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq` + (map TL.fromStrict . T.splitOn s)) . T.intercalate s +t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t) +tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) -t_splitBy p = splitBy p `eqP` (map unpackS . T.splitBy p) -t_splitBy_count c = (L.length . T.splitBy (==c)) `eq` - ((1+) . T.count (T.singleton c)) -t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c) -tl_splitBy p = splitBy p `eqP` (map unpackS . TL.splitBy p) +t_split p = split p `eqP` (map unpackS . T.split p) +t_split_count c = (L.length . T.split (==c)) `eq` + ((1+) . T.count (T.singleton c)) +t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c) +tl_split p = split p `eqP` (map unpackS . TL.split p) -splitBy :: (a -> Bool) -> [a] -> [[a]] -splitBy _ [] = [[]] -splitBy p xs = loop xs +split :: (a -> Bool) -> [a] -> [[a]] +split _ [] = [[]] +split p xs = loop xs where loop s | null s' = [l] | otherwise = l : loop (tail s') where (l, s') = break p s @@ -600,10 +597,10 @@ t_filter p = L.filter p `eqP` (unpackS . T.filter p) tl_filter p = L.filter p `eqP` (unpackS . TL.filter p) sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) -t_findBy p = L.find p `eqP` T.findBy p -tl_findBy p = L.find p `eqP` TL.findBy p -t_partition p = L.partition p `eqP` (unpack2 . T.partitionBy p) -tl_partition p = L.partition p `eqP` (unpack2 . TL.partitionBy p) +t_find p = L.find p `eqP` T.find p +tl_find p = L.find p `eqP` TL.find p +t_partition p = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p) sf_index p s = forAll (choose (-l,l*2)) ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) @@ -616,8 +613,8 @@ where l = L.length s t_findIndex p = L.findIndex p `eqP` T.findIndex p -t_count (NotEmpty t) = (subtract 1 . L.length . T.split t) `eq` T.count t -tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.split t) `eq` +t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t +tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` TL.count t t_zip s = L.zip s `eqP` T.zip (packS s) tl_zip s = L.zip s `eqP` TL.zip (packS s) @@ -682,13 +679,13 @@ t_read_rational p tol (n::Double) s = case p (T.pack (show n) `T.append` t) of - Left err -> False + Left _err -> False Right (n',t') -> t == t' && abs (n-n') <= tol where t = T.dropWhile isFloaty s tl_read_rational p tol (n::Double) s = case p (TL.pack (show n) `TL.append` t) of - Left err -> False + Left _err -> False Right (n',t') -> t == t' && abs (n-n') <= tol where t = TL.dropWhile isFloaty s @@ -1055,16 +1052,16 @@ testProperty "tl_strip" tl_strip, testProperty "t_splitAt" t_splitAt, testProperty "tl_splitAt" tl_splitAt, - testProperty "t_spanBy" t_spanBy, - testProperty "tl_spanBy" tl_spanBy, - testProperty "t_break_id" t_break_id, - testProperty "tl_break_id" tl_break_id, - testProperty "t_break_start" t_break_start, - testProperty "tl_break_start" tl_break_start, - testProperty "t_breakEnd_end" t_breakEnd_end, - testProperty "tl_breakEnd_end" tl_breakEnd_end, - testProperty "t_breakBy" t_breakBy, - testProperty "tl_breakBy" tl_breakBy, + testProperty "t_span" t_span, + testProperty "tl_span" tl_span, + testProperty "t_breakOn_id" t_breakOn_id, + testProperty "tl_breakOn_id" tl_breakOn_id, + testProperty "t_breakOn_start" t_breakOn_start, + testProperty "tl_breakOn_start" tl_breakOn_start, + testProperty "t_breakOnEnd_end" t_breakOnEnd_end, + testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end, + testProperty "t_break" t_break, + testProperty "tl_break" tl_break, testProperty "t_group" t_group, testProperty "tl_group" tl_group, testProperty "t_groupBy" t_groupBy, @@ -1083,14 +1080,14 @@ testProperty "sl_filterCount" sl_filterCount, testProperty "t_findCount" t_findCount, testProperty "tl_findCount" tl_findCount, - testProperty "t_split_split" t_split_split, - testProperty "tl_split_split" tl_split_split, - testProperty "t_split_i" t_split_i, - testProperty "tl_split_i" tl_split_i, - testProperty "t_splitBy" t_splitBy, - testProperty "t_splitBy_count" t_splitBy_count, - testProperty "t_splitBy_split" t_splitBy_split, - testProperty "tl_splitBy" tl_splitBy, + testProperty "t_splitOn_split" t_splitOn_split, + testProperty "tl_splitOn_split" tl_splitOn_split, + testProperty "t_splitOn_i" t_splitOn_i, + testProperty "tl_splitOn_i" tl_splitOn_i, + testProperty "t_split" t_split, + testProperty "t_split_count" t_split_count, + testProperty "t_split_splitOn" t_split_splitOn, + testProperty "tl_split" tl_split, testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths, testProperty "t_chunksOf_length" t_chunksOf_length, testProperty "tl_chunksOf" tl_chunksOf @@ -1133,8 +1130,8 @@ testProperty "t_filter" t_filter, testProperty "tl_filter" tl_filter, testProperty "sf_findBy" sf_findBy, - testProperty "t_findBy" t_findBy, - testProperty "tl_findBy" tl_findBy, + testProperty "t_find" t_find, + testProperty "tl_find" tl_find, testProperty "t_partition" t_partition, testProperty "tl_partition" tl_partition ], diff --git a/tests/QuickCheckUtils.hs b/tests/QuickCheckUtils.hs --- a/tests/QuickCheckUtils.hs +++ b/tests/QuickCheckUtils.hs @@ -5,8 +5,6 @@ import Control.Arrow (first) import Data.Char (chr) import Data.Bits ((.&.)) -import Data.Int (Int64) -import Data.Word (Word8, Word16, Word32) import Data.String (IsString, fromString) import qualified Data.Text as T import Data.Text.Foreign (I16) @@ -15,13 +13,6 @@ import Test.QuickCheck hiding ((.&.)) import qualified Data.ByteString as B -instance Random Int64 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Int64 where - arbitrary = choose (minBound,maxBound) - instance Random I16 where randomR = integralRandomR random = randomR (minBound,maxBound) @@ -29,29 +20,8 @@ instance Arbitrary I16 where arbitrary = choose (minBound,maxBound) -instance Random Word8 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Word8 where - arbitrary = choose (minBound,maxBound) - instance Arbitrary B.ByteString where arbitrary = B.pack `fmap` arbitrary - -instance Random Word16 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Word16 where - arbitrary = choose (minBound,maxBound) - -instance Random Word32 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Word32 where - arbitrary = choose (minBound,maxBound) genUnicode :: IsString a => Gen a genUnicode = fmap fromString string where diff --git a/tests/SlowFunctions.hs b/tests/SlowFunctions.hs --- a/tests/SlowFunctions.hs +++ b/tests/SlowFunctions.hs @@ -3,7 +3,7 @@ module SlowFunctions ( indices - , split + , splitOn ) where import qualified Data.Text as T @@ -23,12 +23,12 @@ where t = Text harr (hoff+i) (hlen-i) d = iter_ haystack i -split :: T.Text -- ^ Text to split on - -> T.Text -- ^ Input text - -> [T.Text] -split pat src0 - | T.null pat = error "split: empty" - | l == 1 = T.splitBy (== (unsafeHead pat)) src0 +splitOn :: T.Text -- ^ Text to split on + -> T.Text -- ^ Input text + -> [T.Text] +splitOn pat src0 + | T.null pat = error "splitOn: empty" + | l == 1 = T.split (== (unsafeHead pat)) src0 | otherwise = go src0 where l = T.length pat diff --git a/text.cabal b/text.cabal --- a/text.cabal +++ b/text.cabal @@ -1,5 +1,5 @@ name: text -version: 0.10.0.0 +version: 0.10.0.1 homepage: http://bitbucket.org/bos/text bug-reports: http://bitbucket.org/bos/text/issues synopsis: An efficient packed Unicode text type. @@ -15,16 +15,21 @@ . The 'Text' type provides character-encoding, type-safe case conversion via whole-string case conversion functions. It also - provides a range of functions for converting Text values to and from - 'ByteStrings', using several standard encodings (see the 'text-icu' - package for a much larger variety of encoding functions). + provides a range of functions for converting 'Text' values to and from + 'ByteStrings', using several standard encodings. . Efficient locale-sensitive support for text IO is also supported. . - This module is intended to be imported qualified, to avoid name + These modules are intended to be imported qualified, to avoid name clashes with Prelude functions, e.g. . > import qualified Data.Text as T + . + To use an extended and very rich family of functions for working + with Unicode text (including normalization, regular expressions, + non-standard encodings, text breaking, and locales), see + the @text-icu@ package: + . license: BSD3 license-file: LICENSE