diff --git a/Data/Text.hs b/Data/Text.hs deleted file mode 100644 --- a/Data/Text.hs +++ /dev/null @@ -1,1755 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types, UnboxedTuples #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif -#if __GLASGOW_HASKELL__ >= 708 -{-# LANGUAGE TypeFamilies #-} -#endif - --- | --- Module : Data.Text --- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts, --- (c) 2008, 2009 Tom Harper --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- A time and space-efficient implementation of Unicode text. --- Suitable for performance critical use, both in terms of large data --- quantities and high speed. --- --- /Note/: Read below the synopsis for important notes on the use of --- this module. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions, e.g. --- --- > import qualified Data.Text as T --- --- To use an extended and very rich family of functions for working --- with Unicode text (including normalization, regular expressions, --- non-standard encodings, text breaking, and locales), see --- . - -module Data.Text - ( - -- * Strict vs lazy types - -- $strict - - -- * Acceptable data - -- $replacement - - -- * Fusion - -- $fusion - - -- * Types - Text - - -- * Creation and elimination - , pack - , unpack - , singleton - , empty - - -- * Basic interface - , cons - , snoc - , append - , uncons - , head - , last - , tail - , init - , null - , length - , compareLength - - -- * Transformations - , map - , intercalate - , intersperse - , transpose - , reverse - , replace - - -- ** Case conversion - -- $case - , toCaseFold - , toLower - , toUpper - , toTitle - - -- ** Justification - , justifyLeft - , justifyRight - , center - - -- * Folds - , foldl - , foldl' - , foldl1 - , foldl1' - , foldr - , foldr1 - - -- ** Special folds - , concat - , concatMap - , any - , all - , maximum - , minimum - - -- * Construction - - -- ** Scans - , scanl - , scanl1 - , scanr - , scanr1 - - -- ** Accumulating maps - , mapAccumL - , mapAccumR - - -- ** Generation and unfolding - , replicate - , unfoldr - , unfoldrN - - -- * Substrings - - -- ** Breaking strings - , take - , takeEnd - , drop - , dropEnd - , takeWhile - , takeWhileEnd - , dropWhile - , dropWhileEnd - , dropAround - , strip - , stripStart - , stripEnd - , splitAt - , breakOn - , breakOnEnd - , break - , span - , group - , groupBy - , inits - , tails - - -- ** Breaking into many substrings - -- $split - , splitOn - , split - , chunksOf - - -- ** Breaking into lines and words - , lines - --, lines' - , words - , unlines - , unwords - - -- * Predicates - , isPrefixOf - , isSuffixOf - , isInfixOf - - -- ** View patterns - , stripPrefix - , stripSuffix - , commonPrefixes - - -- * Searching - , filter - , breakOnAll - , find - , partition - - -- , findSubstring - - -- * Indexing - -- $index - , index - , findIndex - , count - - -- * Zipping - , zip - , zipWith - - -- -* Ordered text - -- , sort - - -- * Low level operations - , copy - , unpackCString# - ) where - -import Prelude (Char, Bool(..), Int, Maybe(..), String, - Eq(..), Ord(..), Ordering(..), (++), - Read(..), - (&&), (||), (+), (-), (.), ($), ($!), (>>), - not, return, otherwise, quot) -#if defined(HAVE_DEEPSEQ) -import Control.DeepSeq (NFData(rnf)) -#endif -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.Char (isSpace) -import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, - Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) -import Control.Monad (foldM) -import Control.Monad.ST (ST) -import qualified Data.Text.Array as A -import qualified Data.List as L -import Data.Binary (Binary(get, put)) -import Data.Monoid (Monoid(..)) -#if MIN_VERSION_base(4,9,0) -import Data.Semigroup (Semigroup(..)) -#endif -import Data.String (IsString(..)) -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import Data.Text.Encoding (decodeUtf8', encodeUtf8) -import Data.Text.Internal.Fusion (stream, reverseStream, unstream) -import Data.Text.Internal.Private (span_) -import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text) -import Data.Text.Show (singleton, unpack, unpackCString#) -import qualified Prelude as P -import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter, - reverseIter_, unsafeHead, unsafeTail) -import Data.Text.Internal.Unsafe.Char (unsafeChr) -import qualified Data.Text.Internal.Functions as F -import qualified Data.Text.Internal.Encoding.Utf16 as U16 -import Data.Text.Internal.Search (indices) -#if defined(__HADDOCK__) -import Data.ByteString (ByteString) -import qualified Data.Text.Lazy as L -import Data.Int (Int64) -#endif -import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) -#if __GLASGOW_HASKELL__ >= 708 -import qualified GHC.Exts as Exts -#endif -#if MIN_VERSION_base(4,7,0) -import Text.Printf (PrintfArg, formatArg, formatString) -#endif - --- $strict --- --- This package provides both strict and lazy 'Text' types. The --- strict type is provided by the "Data.Text" module, while the lazy --- type is provided by the "Data.Text.Lazy" module. Internally, the --- lazy @Text@ type consists of a list of strict chunks. --- --- The strict 'Text' type requires that an entire string fit into --- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of --- streaming strings that are larger than memory using a small memory --- footprint. In many cases, the overhead of chunked streaming makes --- the lazy 'Data.Text.Lazy.Text' type slower than its strict --- counterpart, but this is not always the case. Sometimes, the time --- complexity of a function in one module may be different from the --- other, due to their differing internal structures. --- --- Each module provides an almost identical API, with the main --- difference being that the strict module uses 'Int' values for --- lengths and counts, while the lazy module uses 'Data.Int.Int64' --- lengths. - --- $replacement --- --- A 'Text' value is a sequence of Unicode scalar values, as defined --- in --- . --- As such, a 'Text' cannot contain values in the range U+D800 to --- U+DFFF inclusive. Haskell implementations admit all Unicode code --- points --- () --- as 'Char' values, including code points from this invalid range. --- This means that there are some 'Char' values that are not valid --- Unicode scalar values, and the functions in this module must handle --- those cases. --- --- Within this module, many functions construct a 'Text' from one or --- more 'Char' values. Those functions will substitute 'Char' values --- that are not valid Unicode scalar values with the replacement --- character \"�\" (U+FFFD). Functions that perform this --- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". --- --- (One reason for this policy of replacement is that internally, a --- 'Text' value is represented as packed UTF-16 data. Values in the --- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate --- code points, and so cannot be represented. The functions replace --- invalid scalar values, instead of dropping them, as a security --- measure. For details, see --- .) - --- $fusion --- --- Most of the functions in this module are subject to /fusion/, --- meaning that a pipeline of such functions will usually allocate at --- most one 'Text' value. --- --- As an example, consider the following pipeline: --- --- > import Data.Text as T --- > import Data.Text.Encoding as E --- > import Data.ByteString (ByteString) --- > --- > countChars :: ByteString -> Int --- > countChars = T.length . T.toUpper . E.decodeUtf8 --- --- From the type signatures involved, this looks like it should --- allocate one 'Data.ByteString.ByteString' value, and two 'Text' --- values. However, when a module is compiled with optimisation --- enabled under GHC, the two intermediate 'Text' values will be --- optimised away, and the function will be compiled down to a single --- loop over the source 'Data.ByteString.ByteString'. --- --- Functions that can be fused by the compiler are documented with the --- phrase \"Subject to fusion\". - -instance Eq Text where - Text arrA offA lenA == Text arrB offB lenB - | lenA == lenB = A.equal arrA offA arrB offB lenA - | otherwise = False - {-# INLINE (==) #-} - -instance Ord Text where - compare = compareText - -instance Read Text where - readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] - -#if MIN_VERSION_base(4,9,0) --- Semigroup orphan instances for older GHCs are provided by --- 'semigroups` package - -instance Semigroup Text where - (<>) = append -#endif - -instance Monoid Text where - mempty = empty -#if MIN_VERSION_base(4,9,0) - mappend = (<>) -- future-proof definition -#else - mappend = append -#endif - mconcat = concat - -instance IsString Text where - fromString = pack - -#if __GLASGOW_HASKELL__ >= 708 -instance Exts.IsList Text where - type Item Text = Char - fromList = pack - toList = unpack -#endif - -#if defined(HAVE_DEEPSEQ) -instance NFData Text where rnf !_ = () -#endif - -instance Binary Text where - put t = put (encodeUtf8 t) - get = do - bs <- get - case decodeUtf8' bs of - P.Left exn -> P.fail (P.show exn) - P.Right a -> P.return a - --- | This instance preserves data abstraction at the cost of inefficiency. --- We omit reflection services for the sake of data abstraction. --- --- This instance was created by copying the updated behavior of --- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you --- feel a mistake has been made, please feel free to submit --- improvements. --- --- The original discussion is archived here: --- --- --- The followup discussion that changed the behavior of 'Data.Set.Set' --- and 'Data.Map.Map' is archived here: --- - -instance Data Text where - gfoldl f z txt = z pack `f` (unpack txt) - toConstr _ = packConstr - gunfold k z c = case constrIndex c of - 1 -> k (z pack) - _ -> P.error "gunfold" - dataTypeOf _ = textDataType - -#if MIN_VERSION_base(4,7,0) --- | Only defined for @base-4.7.0.0@ and later -instance PrintfArg Text where - formatArg txt = formatString $ unpack txt -#endif - -packConstr :: Constr -packConstr = mkConstr textDataType "pack" [] Prefix - -textDataType :: DataType -textDataType = mkDataType "Data.Text.Text" [packConstr] - --- | /O(n)/ Compare two 'Text' values lexicographically. -compareText :: Text -> Text -> Ordering -compareText ta@(Text _arrA _offA lenA) tb@(Text _arrB _offB lenB) - | lenA == 0 && lenB == 0 = EQ - | otherwise = go 0 0 - where - go !i !j - | i >= lenA || j >= lenB = compare lenA lenB - | a < b = LT - | a > b = GT - | otherwise = go (i+di) (j+dj) - where Iter a di = iter ta i - Iter b dj = iter tb j - --- ----------------------------------------------------------------------------- --- * Conversion to/from 'Text' - --- | /O(n)/ Convert a 'String' into a 'Text'. Subject to --- fusion. Performs replacement on invalid scalar values. -pack :: String -> Text -pack = unstream . S.map safe . S.streamList -{-# INLINE [1] pack #-} - --- ----------------------------------------------------------------------------- --- * Basic functions - --- | /O(n)/ Adds a character to the front of a 'Text'. This function --- is more costly than its 'List' counterpart because it requires --- copying a new array. Subject to fusion. Performs replacement on --- invalid scalar values. -cons :: Char -> Text -> Text -cons c t = unstream (S.cons (safe c) (stream t)) -{-# INLINE cons #-} - -infixr 5 `cons` - --- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process, unless fused. Subject to fusion. --- Performs replacement on invalid scalar values. -snoc :: Text -> Char -> Text -snoc t c = unstream (S.snoc (stream t) (safe c)) -{-# INLINE snoc #-} - --- | /O(n)/ Appends one 'Text' to the other by copying both of them --- into a new 'Text'. Subject to fusion. -append :: Text -> Text -> Text -append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) - | len1 == 0 = b - | len2 == 0 = a - | len > 0 = Text (A.run x) 0 len - | otherwise = overflowError "append" - where - len = len1+len2 - x :: ST s (A.MArray s) - x = do - arr <- A.new len - A.copyI arr 0 arr1 off1 len1 - A.copyI arr len1 arr2 off2 len - return arr -{-# NOINLINE append #-} - -{-# RULES -"TEXT append -> fused" [~1] forall t1 t2. - append t1 t2 = unstream (S.append (stream t1) (stream t2)) -"TEXT append -> unfused" [1] forall t1 t2. - unstream (S.append (stream t1) (stream t2)) = append t1 t2 - #-} - --- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. Subject to fusion. -head :: Text -> Char -head t = S.head (stream t) -{-# INLINE head #-} - --- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. Subject to fusion. -uncons :: Text -> Maybe (Char, Text) -uncons t@(Text arr off len) - | len <= 0 = Nothing - | otherwise = Just $ let !(Iter c d) = iter t 0 - in (c, text arr (off+d) (len-d)) -{-# INLINE [1] uncons #-} - --- | Lifted from Control.Arrow and specialized. -second :: (b -> c) -> (a,b) -> (a,c) -second f (a, b) = (a, f b) - --- | /O(1)/ Returns the last character of a 'Text', which must be --- non-empty. Subject to fusion. -last :: Text -> Char -last (Text arr off len) - | len <= 0 = emptyError "last" - | n < 0xDC00 || n > 0xDFFF = unsafeChr n - | otherwise = U16.chr2 n0 n - where n = A.unsafeIndex arr (off+len-1) - n0 = A.unsafeIndex arr (off+len-2) -{-# INLINE [1] last #-} - -{-# RULES -"TEXT last -> fused" [~1] forall t. - last t = S.last (stream t) -"TEXT last -> unfused" [1] forall t. - S.last (stream t) = last t - #-} - --- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. Subject to fusion. -tail :: Text -> Text -tail t@(Text arr off len) - | len <= 0 = emptyError "tail" - | otherwise = text arr (off+d) (len-d) - where d = iter_ t 0 -{-# INLINE [1] tail #-} - -{-# RULES -"TEXT tail -> fused" [~1] forall t. - tail t = unstream (S.tail (stream t)) -"TEXT tail -> unfused" [1] forall t. - unstream (S.tail (stream t)) = tail t - #-} - --- | /O(1)/ Returns all but the last character of a 'Text', which must --- be non-empty. Subject to fusion. -init :: Text -> Text -init (Text arr off len) | len <= 0 = emptyError "init" - | n >= 0xDC00 && n <= 0xDFFF = text arr off (len-2) - | otherwise = text arr off (len-1) - where - n = A.unsafeIndex arr (off+len-1) -{-# INLINE [1] init #-} - -{-# RULES -"TEXT init -> fused" [~1] forall t. - init t = unstream (S.init (stream t)) -"TEXT init -> unfused" [1] forall t. - unstream (S.init (stream t)) = init t - #-} - --- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to --- fusion. -null :: Text -> Bool -null (Text _arr _off len) = -#if defined(ASSERTS) - assert (len >= 0) $ -#endif - len <= 0 -{-# INLINE [1] null #-} - -{-# RULES -"TEXT null -> fused" [~1] forall t. - null t = S.null (stream t) -"TEXT null -> unfused" [1] forall t. - S.null (stream t) = null t - #-} - --- | /O(1)/ Tests whether a 'Text' contains exactly one character. --- Subject to fusion. -isSingleton :: Text -> Bool -isSingleton = S.isSingleton . stream -{-# INLINE isSingleton #-} - --- | /O(n)/ Returns the number of characters in a 'Text'. --- Subject to fusion. -length :: Text -> Int -length t = S.length (stream t) -{-# INLINE [0] length #-} --- length needs to be phased after the compareN/length rules otherwise --- it may inline before the rules have an opportunity to fire. - --- | /O(n)/ Compare the count of characters in a 'Text' to a number. --- Subject to fusion. --- --- This function gives the same answer as comparing against the result --- of 'length', but can short circuit if the count of characters is --- greater than the number, and hence be more efficient. -compareLength :: Text -> Int -> Ordering -compareLength t n = S.compareLengthI (stream t) n -{-# INLINE [1] compareLength #-} - -{-# RULES -"TEXT compareN/length -> compareLength" [~1] forall t n. - compare (length t) n = compareLength t n - #-} - -{-# RULES -"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. - eqInt (length t) n = compareLength t n == EQ - #-} - -{-# RULES -"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. - neInt (length t) n = compareLength t n /= EQ - #-} - -{-# RULES -"TEXT compareLength/==LT" [~1] forall t n. - ltInt (length t) n = compareLength t n == LT - #-} - -{-# RULES -"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. - leInt (length t) n = compareLength t n /= GT - #-} - -{-# RULES -"TEXT >N/length -> compareLength/==GT" [~1] forall t n. - gtInt (length t) n = compareLength t n == GT - #-} - -{-# RULES -"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. - geInt (length t) n = compareLength t n /= LT - #-} - --- ----------------------------------------------------------------------------- --- * Transformations --- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Subject to fusion. Performs replacement on --- invalid scalar values. -map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map (safe . f) (stream t)) -{-# INLINE [1] map #-} - --- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of --- 'Text's and concatenates the list after interspersing the first --- argument between each element of the list. -intercalate :: Text -> [Text] -> Text -intercalate t = concat . (F.intersperse t) -{-# INLINE intercalate #-} - --- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Subject to fusion. Performs --- replacement on invalid scalar values. -intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse (safe c) (stream t)) -{-# INLINE intersperse #-} - --- | /O(n)/ Reverse the characters of a string. Subject to fusion. -reverse :: Text -> Text -reverse t = S.reverse (stream t) -{-# INLINE reverse #-} - --- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in --- @haystack@ with @replacement@. --- --- This function behaves as though it was defined as follows: --- --- @ --- replace needle replacement haystack = --- 'intercalate' replacement ('splitOn' needle haystack) --- @ --- --- As this suggests, each occurrence is replaced exactly once. So if --- @needle@ occurs in @replacement@, that occurrence will /not/ itself --- be replaced recursively: --- --- > replace "oo" "foo" "oo" == "foo" --- --- In cases where several instances of @needle@ overlap, only the --- first one will be replaced: --- --- > replace "ofo" "bar" "ofofo" == "barfo" --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -replace :: Text - -- ^ @needle@ to search for. If this string is empty, an - -- error will occur. - -> Text - -- ^ @replacement@ to replace @needle@ with. - -> Text - -- ^ @haystack@ in which to search. - -> Text -replace needle@(Text _ _ neeLen) - (Text repArr repOff repLen) - haystack@(Text hayArr hayOff hayLen) - | neeLen == 0 = emptyError "replace" - | L.null ixs = haystack - | len > 0 = Text (A.run x) 0 len - | otherwise = empty - where - ixs = indices needle haystack - len = hayLen - (neeLen - repLen) `mul` L.length ixs - x :: ST s (A.MArray s) - x = do - marr <- A.new len - let loop (i:is) o d = do - let d0 = d + i - o - d1 = d0 + repLen - A.copyI marr d hayArr (hayOff+o) d0 - A.copyI marr d0 repArr repOff d1 - loop is (i + neeLen) d1 - loop [] o d = A.copyI marr d hayArr (hayOff+o) len - loop ixs 0 0 - return marr - --- ---------------------------------------------------------------------------- --- ** Case conversions (folds) - --- $case --- --- When case converting 'Text' values, do not use combinators like --- @map toUpper@ to case convert each character of a string --- individually, as this gives incorrect results according to the --- rules of some writing systems. The whole-string case conversion --- functions from this module, such as @toUpper@, obey the correct --- case conversion rules. As a result, these functions may map one --- input character to two or three output characters. For examples, --- see the documentation of each function. --- --- /Note/: In some languages, case conversion is a locale- and --- context-dependent operation. The case conversion functions in this --- module are /not/ locale sensitive. Programs that require locale --- sensitivity should use appropriate versions of the --- . - --- | /O(n)/ Convert a string to folded case. Subject to fusion. --- --- This function is mainly useful for performing caseless (also known --- as case insensitive) string comparisons. --- --- A string @x@ is a caseless match for a string @y@ if and only if: --- --- @toCaseFold x == toCaseFold y@ --- --- The result string may be longer than the input string, and may --- differ from applying 'toLower' to the input string. For instance, --- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case --- folded to the sequence \"մ\" (men, U+0574) followed by --- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, --- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) --- instead of itself. -toCaseFold :: Text -> Text -toCaseFold t = unstream (S.toCaseFold (stream t)) -{-# INLINE toCaseFold #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. Subject to fusion. --- --- The result string may be longer than the input string. For --- instance, \"İ\" (Latin capital letter I with dot above, --- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) --- followed by \" ̇\" (combining dot above, U+0307). -toLower :: Text -> Text -toLower t = unstream (S.toLower (stream t)) -{-# INLINE toLower #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. Subject to fusion. --- --- The result string may be longer than the input string. For --- instance, the German \"ß\" (eszett, U+00DF) maps to the --- two-letter sequence \"SS\". -toUpper :: Text -> Text -toUpper t = unstream (S.toUpper (stream t)) -{-# INLINE toUpper #-} - --- | /O(n)/ Convert a string to title case, using simple case --- conversion. Subject to fusion. --- --- The first letter of the input is converted to title case, as is --- every subsequent letter that immediately follows a non-letter. --- Every letter that immediately follows another letter is converted --- to lower case. --- --- The result string may be longer than the input string. For example, --- the Latin small ligature fl (U+FB02) is converted to the --- sequence Latin capital letter F (U+0046) followed by Latin small --- letter l (U+006C). --- --- /Note/: this function does not take language or culture specific --- rules into account. For instance, in English, different style --- guides disagree on whether the book name \"The Hill of the Red --- Fox\" is correctly title cased—but this function will --- capitalize /every/ word. -toTitle :: Text -> Text -toTitle t = unstream (S.toTitle (stream t)) -{-# INLINE toTitle #-} - --- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. --- Performs replacement on invalid scalar values. --- --- Examples: --- --- > justifyLeft 7 'x' "foo" == "fooxxxx" --- > justifyLeft 3 'x' "foobar" == "foobar" -justifyLeft :: Int -> Char -> Text -> Text -justifyLeft k c t - | len >= k = t - | otherwise = t `append` replicateChar (k-len) c - where len = length t -{-# INLINE [1] justifyLeft #-} - -{-# RULES -"TEXT justifyLeft -> fused" [~1] forall k c t. - justifyLeft k c t = unstream (S.justifyLeftI k c (stream t)) -"TEXT justifyLeft -> unfused" [1] forall k c t. - unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t - #-} - --- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Performs replacement on --- invalid scalar values. --- --- Examples: --- --- > justifyRight 7 'x' "bar" == "xxxxbar" --- > justifyRight 3 'x' "foobar" == "foobar" -justifyRight :: Int -> Char -> Text -> Text -justifyRight k c t - | len >= k = t - | otherwise = replicateChar (k-len) c `append` t - where len = length t -{-# INLINE justifyRight #-} - --- | /O(n)/ Center a string to the given length, using the specified --- fill character on either side. Performs replacement on invalid --- scalar values. --- --- Examples: --- --- > center 8 'x' "HS" = "xxxHSxxx" -center :: Int -> Char -> Text -> Text -center k c t - | len >= k = t - | otherwise = replicateChar l c `append` t `append` replicateChar r c - where len = length t - d = k - len - r = d `quot` 2 - l = d - r -{-# INLINE center #-} - --- | /O(n)/ The 'transpose' function transposes the rows and columns --- of its 'Text' argument. Note that this function uses 'pack', --- 'unpack', and the list version of transpose, and is thus not very --- efficient. -transpose :: [Text] -> [Text] -transpose ts = P.map pack (L.transpose (P.map unpack ts)) - --- ----------------------------------------------------------------------------- --- * Reducing 'Text's (folds) - --- | /O(n)/ 'foldl', applied to a binary operator, a starting value --- (typically the left-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from left to right. --- Subject to fusion. -foldl :: (a -> Char -> a) -> a -> Text -> a -foldl f z t = S.foldl f z (stream t) -{-# INLINE foldl #-} - --- | /O(n)/ A strict version of 'foldl'. Subject to fusion. -foldl' :: (a -> Char -> a) -> a -> Text -> a -foldl' f z t = S.foldl' f z (stream t) -{-# INLINE foldl' #-} - --- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to fusion. -foldl1 :: (Char -> Char -> Char) -> Text -> Char -foldl1 f t = S.foldl1 f (stream t) -{-# INLINE foldl1 #-} - --- | /O(n)/ A strict version of 'foldl1'. Subject to fusion. -foldl1' :: (Char -> Char -> Char) -> Text -> Char -foldl1' f t = S.foldl1' f (stream t) -{-# INLINE foldl1' #-} - --- | /O(n)/ 'foldr', applied to a binary operator, a starting value --- (typically the right-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from right to left. --- Subject to fusion. -foldr :: (Char -> a -> a) -> a -> Text -> a -foldr f z t = S.foldr f z (stream t) -{-# INLINE foldr #-} - --- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to --- fusion. -foldr1 :: (Char -> Char -> Char) -> Text -> Char -foldr1 f t = S.foldr1 f (stream t) -{-# INLINE foldr1 #-} - --- ----------------------------------------------------------------------------- --- ** Special folds - --- | /O(n)/ Concatenate a list of 'Text's. -concat :: [Text] -> Text -concat ts = case ts' of - [] -> empty - [t] -> t - _ -> Text (A.run go) 0 len - where - ts' = L.filter (not . null) ts - len = sumP "concat" $ L.map lengthWord16 ts' - go :: ST s (A.MArray s) - go = do - arr <- A.new len - let step i (Text a o l) = - let !j = i + l in A.copyI arr i a o j >> return j - foldM step 0 ts' >> return arr - --- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and --- concatenate the results. -concatMap :: (Char -> Text) -> Text -> Text -concatMap f = concat . foldr ((:) . f) [] -{-# INLINE concatMap #-} - --- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. Subject to fusion. -any :: (Char -> Bool) -> Text -> Bool -any p t = S.any p (stream t) -{-# INLINE any #-} - --- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. Subject to fusion. -all :: (Char -> Bool) -> Text -> Bool -all p t = S.all p (stream t) -{-# INLINE all #-} - --- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. Subject to fusion. -maximum :: Text -> Char -maximum t = S.maximum (stream t) -{-# INLINE maximum #-} - --- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. Subject to fusion. -minimum :: Text -> Char -minimum t = S.minimum (stream t) -{-# INLINE minimum #-} - --- ----------------------------------------------------------------------------- --- * Building 'Text's - --- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. Subject to fusion. --- Performs replacement on invalid scalar values. --- --- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --- --- Note that --- --- > last (scanl f z xs) == foldl f z xs. -scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl g z (stream t)) - where g a b = safe (f a b) -{-# INLINE scanl #-} - --- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. Subject to fusion. Performs replacement on --- invalid scalar values. --- --- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] -scanl1 :: (Char -> Char -> Char) -> Text -> Text -scanl1 f t | null t = empty - | otherwise = scanl f (unsafeHead t) (unsafeTail t) -{-# INLINE scanl1 #-} - --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs --- replacement on invalid scalar values. --- --- > scanr f v == reverse . scanl (flip f) v . reverse -scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f z = S.reverse . S.reverseScanr g z . reverseStream - where g a b = safe (f a b) -{-# INLINE scanr #-} - --- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. Subject to fusion. Performs replacement on --- invalid scalar values. -scanr1 :: (Char -> Char -> Char) -> Text -> Text -scanr1 f t | null t = empty - | otherwise = scanr f (last t) (init t) -{-# INLINE scanr1 #-} - --- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a --- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. Performs --- replacement on invalid scalar values. -mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumL f z0 = S.mapAccumL g z0 . stream - where g a b = second safe (f a b) -{-# INLINE mapAccumL #-} - --- | The 'mapAccumR' function behaves like a combination of 'map' and --- a strict 'foldr'; it applies a function to each element of a --- 'Text', passing an accumulating parameter from right to left, and --- returning a final value of this accumulator together with the new --- 'Text'. --- Performs replacement on invalid scalar values. -mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream - where g a b = second safe (f a b) -{-# INLINE mapAccumR #-} - --- ----------------------------------------------------------------------------- --- ** Generating and unfolding 'Text's - --- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input --- @t@ repeated @n@ times. -replicate :: Int -> Text -> Text -replicate n t@(Text a o l) - | n <= 0 || l <= 0 = empty - | n == 1 = t - | isSingleton t = replicateChar n (unsafeHead t) - | otherwise = Text (A.run x) 0 len - where - len = l `mul` n - x :: ST s (A.MArray s) - x = do - arr <- A.new len - let loop !d !i | i >= n = return arr - | otherwise = let m = d + l - in A.copyI arr d a o m >> loop m (i+1) - loop 0 0 -{-# INLINE [1] replicate #-} - -{-# RULES -"TEXT replicate/singleton -> replicateChar" [~1] forall n c. - replicate n (singleton c) = replicateChar n c - #-} - --- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the --- value of every element. Subject to fusion. -replicateChar :: Int -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n (safe c)) -{-# INLINE replicateChar #-} - --- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' --- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a --- 'Text' from a seed value. The function takes the element and --- returns 'Nothing' if it is done producing the 'Text', otherwise --- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. Subject --- to fusion. Performs replacement on invalid scalar values. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) -{-# INLINE unfoldr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed --- value. However, the length of the result should be limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the maximum length of the result is known and --- correct, otherwise its performance is similar to 'unfoldr'. Subject --- to fusion. Performs replacement on invalid scalar values. -unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) -{-# INLINE unfoldrN #-} - --- ----------------------------------------------------------------------------- --- * Substrings - --- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the --- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. Subject to fusion. -take :: Int -> Text -> Text -take n t@(Text arr off len) - | n <= 0 = empty - | n >= len = t - | otherwise = text arr off (iterN n t) -{-# INLINE [1] take #-} - -iterN :: Int -> Text -> Int -iterN n t@(Text _arr _off len) = loop 0 0 - where loop !i !cnt - | i >= len || cnt >= n = i - | otherwise = loop (i+d) (cnt+1) - where d = iter_ t i - -{-# RULES -"TEXT take -> fused" [~1] forall n t. - take n t = unstream (S.take n (stream t)) -"TEXT take -> unfused" [1] forall n t. - unstream (S.take n (stream t)) = take n t - #-} - --- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after --- taking @n@ characters from the end of @t@. --- --- Examples: --- --- > takeEnd 3 "foobar" == "bar" -takeEnd :: Int -> Text -> Text -takeEnd n t@(Text arr off len) - | n <= 0 = empty - | n >= len = t - | otherwise = text arr (off+i) (len-i) - where i = iterNEnd n t - -iterNEnd :: Int -> Text -> Int -iterNEnd n t@(Text _arr _off len) = loop (len-1) n - where loop i !m - | m <= 0 = i+1 - | i <= 0 = 0 - | otherwise = loop (i+d) (m-1) - where d = reverseIter_ t i - --- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the --- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. Subject to fusion. -drop :: Int -> Text -> Text -drop n t@(Text arr off len) - | n <= 0 = t - | n >= len = empty - | otherwise = text arr (off+i) (len-i) - where i = iterN n t -{-# INLINE [1] drop #-} - -{-# RULES -"TEXT drop -> fused" [~1] forall n t. - drop n t = unstream (S.drop n (stream t)) -"TEXT drop -> unfused" [1] forall n t. - unstream (S.drop n (stream t)) = drop n t - #-} - --- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after --- dropping @n@ characters from the end of @t@. --- --- Examples: --- --- > dropEnd 3 "foobar" == "foo" -dropEnd :: Int -> Text -> Text -dropEnd n t@(Text arr off len) - | n <= 0 = t - | n >= len = empty - | otherwise = text arr off (iterNEnd n t) - --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', --- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. Subject to fusion. -takeWhile :: (Char -> Bool) -> Text -> Text -takeWhile p t@(Text arr off len) = loop 0 - where loop !i | i >= len = t - | p c = loop (i+d) - | otherwise = text arr off i - where Iter c d = iter t i -{-# INLINE [1] takeWhile #-} - -{-# RULES -"TEXT takeWhile -> fused" [~1] forall p t. - takeWhile p t = unstream (S.takeWhile p (stream t)) -"TEXT takeWhile -> unfused" [1] forall p t. - unstream (S.takeWhile p (stream t)) = takeWhile p t - #-} - --- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', --- returns the longest suffix (possibly empty) of elements that --- satisfy @p@. Subject to fusion. --- Examples: --- --- > takeWhileEnd (=='o') "foo" == "oo" -takeWhileEnd :: (Char -> Bool) -> Text -> Text -takeWhileEnd p t@(Text arr off len) = loop (len-1) len - where loop !i !l | l <= 0 = t - | p c = loop (i+d) (l+d) - | otherwise = text arr (off+l) (len-l) - where (c,d) = reverseIter t i -{-# INLINE [1] takeWhileEnd #-} - -{-# RULES -"TEXT takeWhileEnd -> fused" [~1] forall p t. - takeWhileEnd p t = S.reverse (S.takeWhile p (S.reverseStream t)) -"TEXT takeWhileEnd -> unfused" [1] forall p t. - S.reverse (S.takeWhile p (S.reverseStream t)) = takeWhileEnd p t - #-} - --- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. Subject to fusion. -dropWhile :: (Char -> Bool) -> Text -> Text -dropWhile p t@(Text arr off len) = loop 0 0 - where loop !i !l | l >= len = empty - | p c = loop (i+d) (l+d) - | otherwise = Text arr (off+i) (len-l) - where Iter c d = iter t i -{-# INLINE [1] dropWhile #-} - -{-# RULES -"TEXT dropWhile -> fused" [~1] forall p t. - dropWhile p t = unstream (S.dropWhile p (stream t)) -"TEXT dropWhile -> unfused" [1] forall p t. - unstream (S.dropWhile p (stream t)) = dropWhile p t - #-} - --- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after --- dropping characters that satisfy the predicate @p@ from the end of --- @t@. Subject to fusion. --- --- Examples: --- --- > dropWhileEnd (=='.') "foo..." == "foo" -dropWhileEnd :: (Char -> Bool) -> Text -> Text -dropWhileEnd p t@(Text arr off len) = loop (len-1) len - where loop !i !l | l <= 0 = empty - | p c = loop (i+d) (l+d) - | otherwise = Text arr off l - where (c,d) = reverseIter t i -{-# INLINE [1] dropWhileEnd #-} - -{-# RULES -"TEXT dropWhileEnd -> fused" [~1] forall p t. - dropWhileEnd p t = S.reverse (S.dropWhile p (S.reverseStream t)) -"TEXT dropWhileEnd -> unfused" [1] forall p t. - S.reverse (S.dropWhile p (S.reverseStream t)) = dropWhileEnd p t - #-} - --- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after --- dropping characters that satisfy the predicate @p@ from both the --- beginning and end of @t@. Subject to fusion. -dropAround :: (Char -> Bool) -> Text -> Text -dropAround p = dropWhile p . dropWhileEnd p -{-# INLINE [1] dropAround #-} - --- | /O(n)/ Remove leading white space from a string. Equivalent to: --- --- > dropWhile isSpace -stripStart :: Text -> Text -stripStart = dropWhile isSpace -{-# INLINE [1] stripStart #-} - --- | /O(n)/ Remove trailing white space from a string. Equivalent to: --- --- > dropWhileEnd isSpace -stripEnd :: Text -> Text -stripEnd = dropWhileEnd isSpace -{-# INLINE [1] stripEnd #-} - --- | /O(n)/ Remove leading and trailing white space from a string. --- Equivalent to: --- --- > dropAround isSpace -strip :: Text -> Text -strip = dropAround isSpace -{-# INLINE [1] strip #-} - --- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a --- prefix of @t@ of length @n@, and whose second is the remainder of --- the string. It is equivalent to @('take' n t, 'drop' n t)@. -splitAt :: Int -> Text -> (Text, Text) -splitAt n t@(Text arr off len) - | n <= 0 = (empty, t) - | n >= len = (t, empty) - | otherwise = let k = iterN n t - in (text arr off k, text arr (off+k) (len-k)) - --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns --- a pair whose first element is the longest prefix (possibly empty) --- of @t@ of elements that satisfy @p@, and whose second is the --- remainder of the list. -span :: (Char -> Bool) -> Text -> (Text, Text) -span p t = case span_ p t of - (# hd,tl #) -> (hd,tl) -{-# INLINE span #-} - --- | /O(n)/ 'break' is like 'span', but the prefix returned is --- over elements that fail the predicate @p@. -break :: (Char -> Bool) -> Text -> (Text, Text) -break p = span (not . p) -{-# INLINE break #-} - --- | /O(n)/ Group characters in a string according to a predicate. -groupBy :: (Char -> Char -> Bool) -> Text -> [Text] -groupBy p = loop - where - loop t@(Text arr off len) - | null t = [] - | otherwise = text arr off n : loop (text arr (off+n) (len-n)) - where Iter c d = iter t 0 - n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) - --- | Returns the /array/ index (in units of 'Word16') at which a --- character may be found. This is /not/ the same as the logical --- index returned by e.g. 'findIndex'. -findAIndexOrEnd :: (Char -> Bool) -> Text -> Int -findAIndexOrEnd q t@(Text _arr _off len) = go 0 - where go !i | i >= len || q c = i - | otherwise = go (i+d) - where Iter c d = iter t i - --- | /O(n)/ Group characters in a string by equality. -group :: Text -> [Text] -group = groupBy (==) - --- | /O(n)/ Return all initial segments of the given 'Text', shortest --- first. -inits :: Text -> [Text] -inits t@(Text arr off len) = loop 0 - where loop i | i >= len = [t] - | otherwise = Text arr off i : loop (i + iter_ t i) - --- | /O(n)/ Return all final segments of the given 'Text', longest --- first. -tails :: Text -> [Text] -tails t | null t = [empty] - | otherwise = t : tails (unsafeTail t) - --- $split --- --- Splitting functions in this library do not perform character-wise --- copies to create substrings; they just construct new 'Text's that --- are slices of the original. - --- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' --- argument (which cannot be empty), consuming the delimiter. An empty --- delimiter is invalid, and will cause an error to be raised. --- --- Examples: --- --- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] --- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] --- > splitOn "x" "x" == ["",""] --- --- and --- --- > intercalate s . splitOn s == id --- > splitOn (singleton c) == split (==c) --- --- (Note: the string @s@ to split on above cannot be empty.) --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -splitOn :: Text - -- ^ String to split on. If this string is empty, an error - -- will occur. - -> Text - -- ^ Input text. - -> [Text] -splitOn pat@(Text _ _ l) src@(Text arr off len) - | l <= 0 = emptyError "splitOn" - | isSingleton pat = split (== unsafeHead pat) src - | otherwise = go 0 (indices pat src) - where - go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs - go s _ = [text arr (s+off) (len-s)] -{-# INLINE [1] splitOn #-} - -{-# RULES -"TEXT splitOn/singleton -> split/==" [~1] forall c t. - splitOn (singleton c) t = split (==c) t - #-} - --- | /O(n)/ Splits a 'Text' into components delimited by separators, --- where the predicate returns True for a separator element. The --- resulting components do not contain the separators. Two adjacent --- separators result in an empty component in the output. eg. --- --- > split (=='a') "aabbaca" == ["","","bb","c",""] --- > split (=='a') "" == [""] -split :: (Char -> Bool) -> Text -> [Text] -split _ t@(Text _off _arr 0) = [t] -split p t = loop t - where loop s | null s' = [l] - | otherwise = l : loop (unsafeTail s') - where (# l, s' #) = span_ (not . p) s -{-# INLINE split #-} - --- | /O(n)/ Splits a 'Text' into components of length @k@. The last --- element may be shorter than the other chunks, depending on the --- length of the input. Examples: --- --- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] --- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] -chunksOf :: Int -> Text -> [Text] -chunksOf k = go - where - go t = case splitAt k t of - (a,b) | null a -> [] - | otherwise -> a : go b -{-# INLINE chunksOf #-} - --- ---------------------------------------------------------------------------- --- * Searching - -------------------------------------------------------------------------------- --- ** Searching with a predicate - --- | /O(n)/ The 'find' function takes a predicate and a 'Text', and --- returns the first element matching the predicate, or 'Nothing' if --- there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.findBy p (stream t) -{-# INLINE find #-} - --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', --- and returns the pair of 'Text's with elements which do and do not --- satisfy the predicate, respectively; i.e. --- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} - --- | /O(n)/ 'filter', applied to a predicate and a 'Text', --- returns a 'Text' containing those characters that satisfy the --- predicate. -filter :: (Char -> Bool) -> Text -> Text -filter p t = unstream (S.filter p (stream t)) -{-# INLINE filter #-} - --- | /O(n+m)/ Find the first instance of @needle@ (which must be --- non-'null') in @haystack@. The first element of the returned tuple --- is the prefix of @haystack@ before @needle@ is matched. The second --- is the remainder of @haystack@, starting with the match. --- --- Examples: --- --- > breakOn "::" "a::b::c" ==> ("a", "::b::c") --- > breakOn "/" "foobar" ==> ("foobar", "") --- --- Laws: --- --- > append prefix match == haystack --- > where (prefix, match) = breakOn needle haystack --- --- If you need to break a string by a substring repeatedly (e.g. you --- want to break on every instance of a substring), use 'breakOnAll' --- instead, as it has lower startup overhead. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -breakOn :: Text -> Text -> (Text, Text) -breakOn pat src@(Text arr off len) - | null pat = emptyError "breakOn" - | otherwise = case indices pat src of - [] -> (src, empty) - (x:_) -> (text arr off x, text arr (off+x) (len-x)) -{-# INLINE breakOn #-} - --- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the --- string. --- --- The first element of the returned tuple is the prefix of @haystack@ --- up to and including the last match of @needle@. The second is the --- remainder of @haystack@, following the match. --- --- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") -breakOnEnd :: Text -> Text -> (Text, Text) -breakOnEnd pat src = (reverse b, reverse a) - where (a,b) = breakOn (reverse pat) (reverse src) -{-# INLINE breakOnEnd #-} - --- | /O(n+m)/ Find all non-overlapping instances of @needle@ in --- @haystack@. Each element of the returned list consists of a pair: --- --- * The entire string prior to the /k/th match (i.e. the prefix) --- --- * The /k/th match, followed by the remainder of the string --- --- Examples: --- --- > breakOnAll "::" "" --- > ==> [] --- > breakOnAll "/" "a/b/c/" --- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. --- --- The @needle@ parameter may not be empty. -breakOnAll :: Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -breakOnAll pat src@(Text arr off slen) - | null pat = emptyError "breakOnAll" - | otherwise = L.map step (indices pat src) - where - step x = (chunk 0 x, chunk x (slen-x)) - chunk !n !l = text arr (n+off) l -{-# INLINE breakOnAll #-} - -------------------------------------------------------------------------------- --- ** Indexing 'Text's - --- $index --- --- If you think of a 'Text' value as an array of 'Char' values (which --- it is not), you run the risk of writing inefficient code. --- --- An idiom that is common in some languages is to find the numeric --- offset of a character or substring, then use that number to split --- or trim the searched string. With a 'Text' value, this approach --- would require two /O(n)/ operations: one to perform the search, and --- one to operate from wherever the search ended. --- --- For example, suppose you have a string that you want to split on --- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of --- searching for the index of @\"::\"@ and taking the substrings --- before and after that index, you would instead use @breakOnAll \"::\"@. - --- | /O(n)/ 'Text' index (subscript) operator, starting from 0. -index :: Text -> Int -> Char -index t n = S.index (stream t) n -{-# INLINE index #-} - --- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' --- and returns the index of the first element in the 'Text' satisfying --- the predicate. Subject to fusion. -findIndex :: (Char -> Bool) -> Text -> Maybe Int -findIndex p t = S.findIndex p (stream t) -{-# INLINE findIndex #-} - --- | /O(n+m)/ The 'count' function returns the number of times the --- query string appears in the given 'Text'. An empty query string is --- invalid, and will cause an error to be raised. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -count :: Text -> Text -> Int -count pat src - | null pat = emptyError "count" - | isSingleton pat = countChar (unsafeHead pat) src - | otherwise = L.length (indices pat src) -{-# INLINE [1] count #-} - -{-# RULES -"TEXT count/singleton -> countChar" [~1] forall c t. - count (singleton c) t = countChar c t - #-} - --- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. Subject to fusion. -countChar :: Char -> Text -> Int -countChar c t = S.countChar c (stream t) -{-# INLINE countChar #-} - -------------------------------------------------------------------------------- --- * Zipping - --- | /O(n)/ 'zip' takes two 'Text's and returns a list of --- corresponding pairs of bytes. If one input 'Text' is short, --- excess elements of the longer 'Text' are discarded. This is --- equivalent to a pair of 'unpack' operations. -zip :: Text -> Text -> [(Char,Char)] -zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) -{-# INLINE zip #-} - --- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function --- given as the first argument, instead of a tupling function. --- Performs replacement on invalid scalar values. -zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) - where g a b = safe (f a b) -{-# INLINE zipWith #-} - --- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's --- representing white space. -words :: Text -> [Text] -words t@(Text arr off len) = loop 0 0 - where - loop !start !n - | n >= len = if start == n - then [] - else [Text arr (start+off) (n-start)] - | isSpace c = - if start == n - then loop (start+1) (start+1) - else Text arr (start+off) (n-start) : loop (n+d) (n+d) - | otherwise = loop start (n+d) - where Iter c d = iter t n -{-# INLINE words #-} - --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at --- newline 'Char's. The resulting strings do not contain newlines. -lines :: Text -> [Text] -lines ps | null ps = [] - | otherwise = h : if null t - then [] - else lines (unsafeTail t) - where (# h,t #) = span_ (/= '\n') ps -{-# INLINE lines #-} - -{- --- | /O(n)/ Portably breaks a 'Text' up into a list of 'Text's at line --- boundaries. --- --- A line boundary is considered to be either a line feed, a carriage --- return immediately followed by a line feed, or a carriage return. --- This accounts for both Unix and Windows line ending conventions, --- and for the old convention used on Mac OS 9 and earlier. -lines' :: Text -> [Text] -lines' ps | null ps = [] - | otherwise = h : case uncons t of - Nothing -> [] - Just (c,t') - | c == '\n' -> lines t' - | c == '\r' -> case uncons t' of - Just ('\n',t'') -> lines t'' - _ -> lines t' - where (h,t) = span notEOL ps - notEOL c = c /= '\n' && c /= '\r' -{-# INLINE lines' #-} --} - --- | /O(n)/ Joins lines, after appending a terminating newline to --- each. -unlines :: [Text] -> Text -unlines = concat . L.map (`snoc` '\n') -{-# INLINE unlines #-} - --- | /O(n)/ Joins words using single space characters. -unwords :: [Text] -> Text -unwords = intercalate (singleton ' ') -{-# INLINE unwords #-} - --- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. Subject to fusion. -isPrefixOf :: Text -> Text -> Bool -isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) = - alen <= blen && S.isPrefixOf (stream a) (stream b) -{-# INLINE [1] isPrefixOf #-} - -{-# RULES -"TEXT isPrefixOf -> fused" [~1] forall s t. - isPrefixOf s t = S.isPrefixOf (stream s) (stream t) - #-} - --- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns --- 'True' iff the first is a suffix of the second. -isSuffixOf :: Text -> Text -> Bool -isSuffixOf a@(Text _aarr _aoff alen) b@(Text barr boff blen) = - d >= 0 && a == b' - where d = blen - alen - b' | d == 0 = b - | otherwise = Text barr (boff+d) alen -{-# INLINE isSuffixOf #-} - --- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns --- 'True' iff the first is contained, wholly and intact, anywhere --- within the second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -isInfixOf :: Text -> Text -> Bool -isInfixOf needle haystack - | null needle = True - | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack - | otherwise = not . L.null . indices needle $ haystack -{-# INLINE [1] isInfixOf #-} - -{-# RULES -"TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. - isInfixOf (singleton n) h = S.elem n (S.stream h) - #-} - -------------------------------------------------------------------------------- --- * View patterns - --- | /O(n)/ Return the suffix of the second string if its prefix --- matches the entire first string. --- --- Examples: --- --- > stripPrefix "foo" "foobar" == Just "bar" --- > stripPrefix "" "baz" == Just "baz" --- > stripPrefix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text as T --- > --- > fnordLength :: Text -> Int --- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf --- > fnordLength _ = -1 -stripPrefix :: Text -> Text -> Maybe Text -stripPrefix p@(Text _arr _off plen) t@(Text arr off len) - | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) - | otherwise = Nothing - --- | /O(n)/ Find the longest non-empty common prefix of two strings --- and return it, along with the suffixes of each string at which they --- no longer match. --- --- If the strings do not have a common prefix or either one is empty, --- this function returns 'Nothing'. --- --- Examples: --- --- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") --- > commonPrefixes "veeble" "fetzer" == Nothing --- > commonPrefixes "" "baz" == Nothing -commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) -commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0 - where - go !i !j | i < len0 && j < len1 && a == b = go (i+d0) (j+d1) - | i > 0 = Just (Text arr0 off0 i, - text arr0 (off0+i) (len0-i), - text arr1 (off1+j) (len1-j)) - | otherwise = Nothing - where Iter a d0 = iter t0 i - Iter b d1 = iter t1 j - --- | /O(n)/ Return the prefix of the second string if its suffix --- matches the entire first string. --- --- Examples: --- --- > stripSuffix "bar" "foobar" == Just "foo" --- > stripSuffix "" "baz" == Just "baz" --- > stripSuffix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text as T --- > --- > quuxLength :: Text -> Int --- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre --- > quuxLength _ = -1 -stripSuffix :: Text -> Text -> Maybe Text -stripSuffix p@(Text _arr _off plen) t@(Text arr off len) - | p `isSuffixOf` t = Just $! text arr off (len-plen) - | otherwise = Nothing - --- | Add a list of non-negative numbers. Errors out on overflow. -sumP :: String -> [Int] -> Int -sumP fun = go 0 - where go !a (x:xs) - | ax >= 0 = go ax xs - | otherwise = overflowError fun - where ax = a + x - go a _ = a - -emptyError :: String -> a -emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" - -overflowError :: String -> a -overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" - --- | /O(n)/ Make a distinct copy of the given string, sharing no --- storage with the original string. --- --- As an example, suppose you read a large string, of which you need --- only a small portion. If you do not use 'copy', the entire original --- array will be kept alive in memory by the smaller string. Making a --- copy \"breaks the link\" to the original array, allowing it to be --- garbage collected if there are no other live references to it. -copy :: Text -> Text -copy (Text arr off len) = Text (A.run go) 0 len - where - go :: ST s (A.MArray s) - go = do - marr <- A.new len - A.copyI marr 0 arr off len - return marr diff --git a/Data/Text/Array.hs b/Data/Text/Array.hs deleted file mode 100644 --- a/Data/Text/Array.hs +++ /dev/null @@ -1,246 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, MagicHash, Rank2Types, - RecordWildCards, UnboxedTuples, UnliftedFFITypes #-} -{-# OPTIONS_GHC -fno-warn-unused-matches #-} --- | --- Module : Data.Text.Array --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- Packed, unboxed, heap-resident arrays. Suitable for performance --- critical use, both in terms of large data quantities and high --- speed. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions, e.g. --- --- > import qualified Data.Text.Array as A --- --- The names in this module resemble those in the 'Data.Array' family --- of modules, but are shorter due to the assumption of qualified --- naming. -module Data.Text.Array - ( - -- * Types - Array(Array, aBA) - , MArray(MArray, maBA) - - -- * Functions - , copyM - , copyI - , empty - , equal -#if defined(ASSERTS) - , length -#endif - , run - , run2 - , toList - , unsafeFreeze - , unsafeIndex - , new - , unsafeWrite - ) where - -#if defined(ASSERTS) --- This fugly hack is brought by GHC's apparent reluctance to deal --- with MagicHash and UnboxedTuples when inferring types. Eek! -# define CHECK_BOUNDS(_func_,_len_,_k_) \ -if (_k_) < 0 || (_k_) >= (_len_) then error ("Data.Text.Array." ++ (_func_) ++ ": bounds error, offset " ++ show (_k_) ++ ", length " ++ show (_len_)) else -#else -# define CHECK_BOUNDS(_func_,_len_,_k_) -#endif - -#include "MachDeps.h" - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -#if __GLASGOW_HASKELL__ >= 702 -import Control.Monad.ST.Unsafe (unsafeIOToST) -#else -import Control.Monad.ST (unsafeIOToST) -#endif -import Data.Bits ((.&.), xor) -import Data.Text.Internal.Unsafe (inlinePerformIO) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) -#if __GLASGOW_HASKELL__ >= 703 -import Foreign.C.Types (CInt(CInt), CSize(CSize)) -#else -import Foreign.C.Types (CInt, CSize) -#endif -import GHC.Base (ByteArray#, MutableByteArray#, Int(..), - indexWord16Array#, newByteArray#, - unsafeFreezeByteArray#, writeWord16Array#) -import GHC.ST (ST(..), runST) -import GHC.Word (Word16(..)) -import Prelude hiding (length, read) - --- | Immutable array type. -data Array = Array { - aBA :: ByteArray# -#if defined(ASSERTS) - , aLen :: {-# UNPACK #-} !Int -- length (in units of Word16, not bytes) -#endif - } - --- | Mutable array type, for use in the ST monad. -data MArray s = MArray { - maBA :: MutableByteArray# s -#if defined(ASSERTS) - , maLen :: {-# UNPACK #-} !Int -- length (in units of Word16, not bytes) -#endif - } - -#if defined(ASSERTS) --- | Operations supported by all arrays. -class IArray a where - -- | Return the length of an array. - length :: a -> Int - -instance IArray Array where - length = aLen - {-# INLINE length #-} - -instance IArray (MArray s) where - length = maLen - {-# INLINE length #-} -#endif - --- | Create an uninitialized mutable array. -new :: forall s. Int -> ST s (MArray s) -new n - | n < 0 || n .&. highBit /= 0 = array_size_error - | otherwise = ST $ \s1# -> - case newByteArray# len# s1# of - (# s2#, marr# #) -> (# s2#, MArray marr# -#if defined(ASSERTS) - n -#endif - #) - where !(I# len#) = bytesInArray n - highBit = maxBound `xor` (maxBound `shiftR` 1) -{-# INLINE new #-} - -array_size_error :: a -array_size_error = error "Data.Text.Array.new: size overflow" - --- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! -unsafeFreeze :: MArray s -> ST s Array -unsafeFreeze MArray{..} = ST $ \s1# -> - case unsafeFreezeByteArray# maBA s1# of - (# s2#, ba# #) -> (# s2#, Array ba# -#if defined(ASSERTS) - maLen -#endif - #) -{-# INLINE unsafeFreeze #-} - --- | Indicate how many bytes would be used for an array of the given --- size. -bytesInArray :: Int -> Int -bytesInArray n = n `shiftL` 1 -{-# INLINE bytesInArray #-} - --- | Unchecked read of an immutable array. May return garbage or --- crash on an out-of-bounds access. -unsafeIndex :: Array -> Int -> Word16 -unsafeIndex Array{..} i@(I# i#) = - CHECK_BOUNDS("unsafeIndex",aLen,i) - case indexWord16Array# aBA i# of r# -> (W16# r#) -{-# INLINE unsafeIndex #-} - --- | Unchecked write of a mutable array. May return garbage or crash --- on an out-of-bounds access. -unsafeWrite :: MArray s -> Int -> Word16 -> ST s () -unsafeWrite MArray{..} i@(I# i#) (W16# e#) = ST $ \s1# -> - CHECK_BOUNDS("unsafeWrite",maLen,i) - case writeWord16Array# maBA i# e# s1# of - s2# -> (# s2#, () #) -{-# INLINE unsafeWrite #-} - --- | Convert an immutable array to a list. -toList :: Array -> Int -> Int -> [Word16] -toList ary off len = loop 0 - where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) - | otherwise = [] - --- | An empty immutable array. -empty :: Array -empty = runST (new 0 >>= unsafeFreeze) - --- | Run an action in the ST monad and return an immutable array of --- its result. -run :: (forall s. ST s (MArray s)) -> Array -run k = runST (k >>= unsafeFreeze) - --- | Run an action in the ST monad and return an immutable array of --- its result paired with whatever else the action returns. -run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) -run2 k = runST (do - (marr,b) <- k - arr <- unsafeFreeze marr - return (arr,b)) -{-# INLINE run2 #-} - --- | Copy some elements of a mutable array. -copyM :: MArray s -- ^ Destination - -> Int -- ^ Destination offset - -> MArray s -- ^ Source - -> Int -- ^ Source offset - -> Int -- ^ Count - -> ST s () -copyM dest didx src sidx count - | count <= 0 = return () - | otherwise = -#if defined(ASSERTS) - assert (sidx + count <= length src) . - assert (didx + count <= length dest) . -#endif - unsafeIOToST $ memcpyM (maBA dest) (fromIntegral didx) - (maBA src) (fromIntegral sidx) - (fromIntegral count) -{-# INLINE copyM #-} - --- | Copy some elements of an immutable array. -copyI :: MArray s -- ^ Destination - -> Int -- ^ Destination offset - -> Array -- ^ Source - -> Int -- ^ Source offset - -> Int -- ^ First offset in destination /not/ to - -- copy (i.e. /not/ length) - -> ST s () -copyI dest i0 src j0 top - | i0 >= top = return () - | otherwise = unsafeIOToST $ - memcpyI (maBA dest) (fromIntegral i0) - (aBA src) (fromIntegral j0) - (fromIntegral (top-i0)) -{-# INLINE copyI #-} - --- | Compare portions of two arrays for equality. No bounds checking --- is performed. -equal :: Array -- ^ First - -> Int -- ^ Offset into first - -> Array -- ^ Second - -> Int -- ^ Offset into second - -> Int -- ^ Count - -> Bool -equal arrA offA arrB offB count = inlinePerformIO $ do - i <- memcmp (aBA arrA) (fromIntegral offA) - (aBA arrB) (fromIntegral offB) (fromIntegral count) - return $! i == 0 -{-# INLINE equal #-} - -foreign import ccall unsafe "_hs_text_memcpy" memcpyI - :: MutableByteArray# s -> CSize -> ByteArray# -> CSize -> CSize -> IO () - -foreign import ccall unsafe "_hs_text_memcmp" memcmp - :: ByteArray# -> CSize -> ByteArray# -> CSize -> CSize -> IO CInt - -foreign import ccall unsafe "_hs_text_memcpy" memcpyM - :: MutableByteArray# s -> CSize -> MutableByteArray# s -> CSize -> CSize - -> IO () diff --git a/Data/Text/Encoding.hs b/Data/Text/Encoding.hs deleted file mode 100644 --- a/Data/Text/Encoding.hs +++ /dev/null @@ -1,479 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash, - UnliftedFFITypes #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif --- | --- Module : Data.Text.Encoding --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts, --- (c) 2008, 2009 Tom Harper --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- Functions for converting 'Text' values to and from 'ByteString', --- using several standard encodings. --- --- To gain access to a much larger family of encodings, use the --- @text-icu@ package: - -module Data.Text.Encoding - ( - -- * Decoding ByteStrings to Text - -- $strict - decodeASCII - , decodeLatin1 - , decodeUtf8 - , decodeUtf16LE - , decodeUtf16BE - , decodeUtf32LE - , decodeUtf32BE - - -- ** Catchable failure - , decodeUtf8' - - -- ** Controllable error handling - , decodeUtf8With - , decodeUtf16LEWith - , decodeUtf16BEWith - , decodeUtf32LEWith - , decodeUtf32BEWith - - -- ** Stream oriented decoding - -- $stream - , streamDecodeUtf8 - , streamDecodeUtf8With - , Decoding(..) - - -- * Encoding Text to ByteStrings - , encodeUtf8 - , encodeUtf16LE - , encodeUtf16BE - , encodeUtf32LE - , encodeUtf32BE - - -- * Encoding Text using ByteString Builders - , encodeUtf8Builder - , encodeUtf8BuilderEscaped - ) where - -#if __GLASGOW_HASKELL__ >= 702 -import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) -#else -import Control.Monad.ST (unsafeIOToST, unsafeSTToIO) -#endif - -import Control.Exception (evaluate, try) -import Control.Monad.ST (runST) -import Data.Bits ((.&.)) -import Data.ByteString as B -import Data.ByteString.Internal as B hiding (c2w) -import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) -import Data.Text.Internal (Text(..), safe, text) -import Data.Text.Internal.Private (runText) -import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftR) -import Data.Text.Show () -import Data.Text.Unsafe (unsafeDupablePerformIO) -import Data.Word (Word8, Word32) -import Foreign.C.Types (CSize(..)) -import Foreign.ForeignPtr (withForeignPtr) -import Foreign.Marshal.Utils (with) -import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr) -import Foreign.Storable (Storable, peek, poke) -import GHC.Base (ByteArray#, MutableByteArray#) -import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) -import qualified Data.ByteString.Builder.Prim as BP -import qualified Data.ByteString.Builder.Prim.Internal as BP -import qualified Data.Text.Array as A -import qualified Data.Text.Internal.Encoding.Fusion as E -import qualified Data.Text.Internal.Encoding.Utf16 as U16 -import qualified Data.Text.Internal.Fusion as F - -#include "text_cbits.h" - --- $strict --- --- All of the single-parameter functions for decoding bytestrings --- encoded in one of the Unicode Transformation Formats (UTF) operate --- in a /strict/ mode: each will throw an exception if given invalid --- input. --- --- Each function has a variant, whose name is suffixed with -'With', --- that gives greater control over the handling of decoding errors. --- For instance, 'decodeUtf8' will throw an exception, but --- 'decodeUtf8With' allows the programmer to determine what to do on a --- decoding error. - --- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII --- encoded text. -decodeASCII :: ByteString -> Text -decodeASCII = decodeUtf8 -{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-} - --- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. --- --- 'decodeLatin1' is semantically equivalent to --- @Data.Text.pack . Data.ByteString.Char8.unpack@ -decodeLatin1 :: ByteString -> Text -decodeLatin1 (PS fp off len) = text a 0 len - where - a = A.run (A.new len >>= unsafeIOToST . go) - go dest = withForeignPtr fp $ \ptr -> do - c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len)) - return dest - --- | Decode a 'ByteString' containing UTF-8 encoded text. -decodeUtf8With :: OnDecodeError -> ByteString -> Text -decodeUtf8With onErr (PS fp off len) = runText $ \done -> do - let go dest = withForeignPtr fp $ \ptr -> - with (0::CSize) $ \destOffPtr -> do - let end = ptr `plusPtr` (off + len) - loop curPtr = do - curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end - if curPtr' == end - then do - n <- peek destOffPtr - unsafeSTToIO (done dest (fromIntegral n)) - else do - x <- peek curPtr' - case onErr desc (Just x) of - Nothing -> loop $ curPtr' `plusPtr` 1 - Just c -> do - destOff <- peek destOffPtr - w <- unsafeSTToIO $ - unsafeWrite dest (fromIntegral destOff) (safe c) - poke destOffPtr (destOff + fromIntegral w) - loop $ curPtr' `plusPtr` 1 - loop (ptr `plusPtr` off) - (unsafeIOToST . go) =<< A.new len - where - desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream" -{- INLINE[0] decodeUtf8With #-} - --- $stream --- --- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept --- a 'ByteString' that represents a possibly incomplete input (e.g. a --- packet from a network stream) that may not end on a UTF-8 boundary. --- --- 1. The maximal prefix of 'Text' that could be decoded from the --- given input. --- --- 2. The suffix of the 'ByteString' that could not be decoded due to --- insufficient input. --- --- 3. A function that accepts another 'ByteString'. That string will --- be assumed to directly follow the string that was passed as --- input to the original function, and it will in turn be decoded. --- --- To help understand the use of these functions, consider the Unicode --- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi --- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. --- --- Now suppose that we receive this encoded string as 3 packets that --- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", --- \"\\x83\"]@. We cannot decode the entire Unicode string until we --- have received all three packets, but we would like to make progress --- as we receive each one. --- --- @ --- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" --- ghci> s0 --- 'Some' \"hi \" \"\\xe2\" _ --- @ --- --- We use the continuation @f0@ to decode our second packet. --- --- @ --- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" --- ghci> s1 --- 'Some' \"\" \"\\xe2\\x98\" --- @ --- --- We could not give @f0@ enough input to decode anything, so it --- returned an empty string. Once we feed our second continuation @f1@ --- the last byte of input, it will make progress. --- --- @ --- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" --- ghci> s2 --- 'Some' \"\\x2603\" \"\" _ --- @ --- --- If given invalid input, an exception will be thrown by the function --- or continuation where it is encountered. - --- | A stream oriented decoding result. -data Decoding = Some Text ByteString (ByteString -> Decoding) - -instance Show Decoding where - showsPrec d (Some t bs _) = showParen (d > prec) $ - showString "Some " . showsPrec prec' t . - showChar ' ' . showsPrec prec' bs . - showString " _" - where prec = 10; prec' = prec + 1 - -newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable) -newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable) - --- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8 --- encoded text that is known to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown (either by this function or a continuation) that cannot be --- caught in pure code. For more control over the handling of invalid --- data, use 'streamDecodeUtf8With'. -streamDecodeUtf8 :: ByteString -> Decoding -streamDecodeUtf8 = streamDecodeUtf8With strictDecode - --- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8 --- encoded text. -streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding -streamDecodeUtf8With onErr = decodeChunk B.empty 0 0 - where - -- We create a slightly larger than necessary buffer to accommodate a - -- potential surrogate pair started in the last buffer - decodeChunk :: ByteString -> CodePoint -> DecoderState -> ByteString - -> Decoding - decodeChunk undecoded0 codepoint0 state0 bs@(PS fp off len) = - runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1) - where - decodeChunkToBuffer :: A.MArray s -> IO Decoding - decodeChunkToBuffer dest = withForeignPtr fp $ \ptr -> - with (0::CSize) $ \destOffPtr -> - with codepoint0 $ \codepointPtr -> - with state0 $ \statePtr -> - with nullPtr $ \curPtrPtr -> - let end = ptr `plusPtr` (off + len) - loop curPtr = do - poke curPtrPtr curPtr - curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr - curPtrPtr end codepointPtr statePtr - state <- peek statePtr - case state of - UTF8_REJECT -> do - -- We encountered an encoding error - x <- peek curPtr' - poke statePtr 0 - case onErr desc (Just x) of - Nothing -> loop $ curPtr' `plusPtr` 1 - Just c -> do - destOff <- peek destOffPtr - w <- unsafeSTToIO $ - unsafeWrite dest (fromIntegral destOff) (safe c) - poke destOffPtr (destOff + fromIntegral w) - loop $ curPtr' `plusPtr` 1 - - _ -> do - -- We encountered the end of the buffer while decoding - n <- peek destOffPtr - codepoint <- peek codepointPtr - chunkText <- unsafeSTToIO $ do - arr <- A.unsafeFreeze dest - return $! text arr 0 (fromIntegral n) - lastPtr <- peek curPtrPtr - let left = lastPtr `minusPtr` curPtr - !undecoded = case state of - UTF8_ACCEPT -> B.empty - _ -> B.append undecoded0 (B.drop left bs) - return $ Some chunkText undecoded - (decodeChunk undecoded codepoint state) - in loop (ptr `plusPtr` off) - desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream" - --- | Decode a 'ByteString' containing UTF-8 encoded text that is known --- to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown that cannot be caught in pure code. For more control over --- the handling of invalid data, use 'decodeUtf8'' or --- 'decodeUtf8With'. -decodeUtf8 :: ByteString -> Text -decodeUtf8 = decodeUtf8With strictDecode -{-# INLINE[0] decodeUtf8 #-} -{-# RULES "STREAM stream/decodeUtf8 fusion" [1] - forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-} - --- | Decode a 'ByteString' containing UTF-8 encoded text. --- --- If the input contains any invalid UTF-8 data, the relevant --- exception will be returned, otherwise the decoded text. -decodeUtf8' :: ByteString -> Either UnicodeException Text -decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode -{-# INLINE decodeUtf8' #-} - --- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. -encodeUtf8Builder :: Text -> B.Builder -encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8) - --- | Encode text using UTF-8 encoding and escape the ASCII characters using --- a 'BP.BoundedPrim'. --- --- Use this function is to implement efficient encoders for text-based formats --- like JSON or HTML. -{-# INLINE encodeUtf8BuilderEscaped #-} --- TODO: Extend documentation with references to source code in @blaze-html@ --- or @aeson@ that uses this function. -encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder -encodeUtf8BuilderEscaped be = - -- manual eta-expansion to ensure inlining works as expected - \txt -> B.builder (mkBuildstep txt) - where - bound = max 4 $ BP.sizeBound be - - mkBuildstep (Text arr off len) !k = - outerLoop off - where - iend = off + len - - outerLoop !i0 !br@(B.BufferRange op0 ope) - | i0 >= iend = k br - | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) - -- TODO: Use a loop with an integrated bound's check if outRemaining - -- is smaller than 8, as this will save on divisions. - | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) - where - outRemaining = (ope `minusPtr` op0) `div` bound - inpRemaining = iend - i0 - - goPartial !iendTmp = go i0 op0 - where - go !i !op - | i < iendTmp = case A.unsafeIndex arr i of - w | w <= 0x7F -> do - BP.runB be (fromIntegral w) op >>= go (i + 1) - | w <= 0x7FF -> do - poke8 0 $ (w `shiftR` 6) + 0xC0 - poke8 1 $ (w .&. 0x3f) + 0x80 - go (i + 1) (op `plusPtr` 2) - | 0xD800 <= w && w <= 0xDBFF -> do - let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1)) - poke8 0 $ (c `shiftR` 18) + 0xF0 - poke8 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80 - poke8 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80 - poke8 3 $ (c .&. 0x3F) + 0x80 - go (i + 2) (op `plusPtr` 4) - | otherwise -> do - poke8 0 $ (w `shiftR` 12) + 0xE0 - poke8 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80 - poke8 2 $ (w .&. 0x3F) + 0x80 - go (i + 1) (op `plusPtr` 3) - | otherwise = - outerLoop i (B.BufferRange op ope) - where - poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8) - --- | Encode text using UTF-8 encoding. -encodeUtf8 :: Text -> ByteString -encodeUtf8 (Text arr off len) - | len == 0 = B.empty - | otherwise = unsafeDupablePerformIO $ do - fp <- mallocByteString (len*4) - withForeignPtr fp $ \ptr -> - with ptr $ \destPtr -> do - c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len) - newDest <- peek destPtr - let utf8len = newDest `minusPtr` ptr - if utf8len >= len `shiftR` 1 - then return (PS fp 0 utf8len) - else do - fp' <- mallocByteString utf8len - withForeignPtr fp' $ \ptr' -> do - memcpy ptr' ptr (fromIntegral utf8len) - return (PS fp' 0 utf8len) - --- | Decode text from little endian UTF-16 encoding. -decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text -decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) -{-# INLINE decodeUtf16LEWith #-} - --- | Decode text from little endian UTF-16 encoding. --- --- If the input contains any invalid little endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16LEWith'. -decodeUtf16LE :: ByteString -> Text -decodeUtf16LE = decodeUtf16LEWith strictDecode -{-# INLINE decodeUtf16LE #-} - --- | Decode text from big endian UTF-16 encoding. -decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text -decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) -{-# INLINE decodeUtf16BEWith #-} - --- | Decode text from big endian UTF-16 encoding. --- --- If the input contains any invalid big endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16BEWith'. -decodeUtf16BE :: ByteString -> Text -decodeUtf16BE = decodeUtf16BEWith strictDecode -{-# INLINE decodeUtf16BE #-} - --- | Encode text using little endian UTF-16 encoding. -encodeUtf16LE :: Text -> ByteString -encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) -{-# INLINE encodeUtf16LE #-} - --- | Encode text using big endian UTF-16 encoding. -encodeUtf16BE :: Text -> ByteString -encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) -{-# INLINE encodeUtf16BE #-} - --- | Decode text from little endian UTF-32 encoding. -decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text -decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) -{-# INLINE decodeUtf32LEWith #-} - --- | Decode text from little endian UTF-32 encoding. --- --- If the input contains any invalid little endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32LEWith'. -decodeUtf32LE :: ByteString -> Text -decodeUtf32LE = decodeUtf32LEWith strictDecode -{-# INLINE decodeUtf32LE #-} - --- | Decode text from big endian UTF-32 encoding. -decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text -decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) -{-# INLINE decodeUtf32BEWith #-} - --- | Decode text from big endian UTF-32 encoding. --- --- If the input contains any invalid big endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32BEWith'. -decodeUtf32BE :: ByteString -> Text -decodeUtf32BE = decodeUtf32BEWith strictDecode -{-# INLINE decodeUtf32BE #-} - --- | Encode text using little endian UTF-32 encoding. -encodeUtf32LE :: Text -> ByteString -encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) -{-# INLINE encodeUtf32LE #-} - --- | Encode text using big endian UTF-32 encoding. -encodeUtf32BE :: Text -> ByteString -encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) -{-# INLINE encodeUtf32BE #-} - -foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8 - :: MutableByteArray# s -> Ptr CSize - -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8) - -foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state - :: MutableByteArray# s -> Ptr CSize - -> Ptr (Ptr Word8) -> Ptr Word8 - -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8) - -foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1 - :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO () - -foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8 - :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO () diff --git a/Data/Text/Encoding/Error.hs b/Data/Text/Encoding/Error.hs deleted file mode 100644 --- a/Data/Text/Encoding/Error.hs +++ /dev/null @@ -1,125 +0,0 @@ -{-# LANGUAGE CPP, DeriveDataTypeable #-} -#if __GLASGOW_HASKELL__ >= 704 -{-# LANGUAGE Safe #-} -#elif __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif --- | --- Module : Data.Text.Encoding.Error --- Copyright : (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Types and functions for dealing with encoding and decoding errors --- in Unicode text. --- --- The standard functions for encoding and decoding text are strict, --- which is to say that they throw exceptions on invalid input. This --- is often unhelpful on real world input, so alternative functions --- exist that accept custom handlers for dealing with invalid inputs. --- These 'OnError' handlers are normal Haskell functions. You can use --- one of the presupplied functions in this module, or you can write a --- custom handler of your own. - -module Data.Text.Encoding.Error - ( - -- * Error handling types - UnicodeException(..) - , OnError - , OnDecodeError - , OnEncodeError - -- * Useful error handling functions - , lenientDecode - , strictDecode - , strictEncode - , ignore - , replace - ) where - -import Control.DeepSeq (NFData (..)) -import Control.Exception (Exception, throw) -import Data.Typeable (Typeable) -import Data.Word (Word8) -import Numeric (showHex) - --- | Function type for handling a coding error. It is supplied with --- two inputs: --- --- * A 'String' that describes the error. --- --- * The input value that caused the error. If the error arose --- because the end of input was reached or could not be identified --- precisely, this value will be 'Nothing'. --- --- If the handler returns a value wrapped with 'Just', that value will --- be used in the output as the replacement for the invalid input. If --- it returns 'Nothing', no value will be used in the output. --- --- Should the handler need to abort processing, it should use 'error' --- or 'throw' an exception (preferably a 'UnicodeException'). It may --- use the description provided to construct a more helpful error --- report. -type OnError a b = String -> Maybe a -> Maybe b - --- | A handler for a decoding error. -type OnDecodeError = OnError Word8 Char - --- | A handler for an encoding error. -{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} -type OnEncodeError = OnError Char Word8 - --- | An exception type for representing Unicode encoding errors. -data UnicodeException = - DecodeError String (Maybe Word8) - -- ^ Could not decode a byte sequence because it was invalid under - -- the given encoding, or ran out of input in mid-decode. - | EncodeError String (Maybe Char) - -- ^ Tried to encode a character that could not be represented - -- under the given encoding, or ran out of input in mid-encode. - deriving (Eq, Typeable) - -{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} - -showUnicodeException :: UnicodeException -> String -showUnicodeException (DecodeError desc (Just w)) - = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) -showUnicodeException (DecodeError desc Nothing) - = "Cannot decode input: " ++ desc -showUnicodeException (EncodeError desc (Just c)) - = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) -showUnicodeException (EncodeError desc Nothing) - = "Cannot encode input: " ++ desc - -instance Show UnicodeException where - show = showUnicodeException - -instance Exception UnicodeException - -instance NFData UnicodeException where - rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () - rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () - --- | Throw a 'UnicodeException' if decoding fails. -strictDecode :: OnDecodeError -strictDecode desc c = throw (DecodeError desc c) - --- | Replace an invalid input byte with the Unicode replacement --- character U+FFFD. -lenientDecode :: OnDecodeError -lenientDecode _ _ = Just '\xfffd' - --- | Throw a 'UnicodeException' if encoding fails. -{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} -strictEncode :: OnEncodeError -strictEncode desc c = throw (EncodeError desc c) - --- | Ignore an invalid input, substituting nothing in the output. -ignore :: OnError a b -ignore _ _ = Nothing - --- | Replace an invalid input with a valid output. -replace :: b -> OnError a b -replace c _ _ = Just c diff --git a/Data/Text/Foreign.hs b/Data/Text/Foreign.hs deleted file mode 100644 --- a/Data/Text/Foreign.hs +++ /dev/null @@ -1,173 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving #-} --- | --- Module : Data.Text.Foreign --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Support for using 'Text' data with native code via the Haskell --- foreign function interface. - -module Data.Text.Foreign - ( - -- * Interoperability with native code - -- $interop - I16 - -- * Safe conversion functions - , fromPtr - , useAsPtr - , asForeignPtr - -- ** Encoding as UTF-8 - , peekCStringLen - , withCStringLen - -- * Unsafe conversion code - , lengthWord16 - , unsafeCopyToPtr - -- * Low-level manipulation - -- $lowlevel - , dropWord16 - , takeWord16 - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -#if __GLASGOW_HASKELL__ >= 702 -import Control.Monad.ST.Unsafe (unsafeIOToST) -#else -import Control.Monad.ST (unsafeIOToST) -#endif -import Data.ByteString.Unsafe (unsafePackCStringLen, unsafeUseAsCStringLen) -import Data.Text.Encoding (decodeUtf8, encodeUtf8) -import Data.Text.Internal (Text(..), empty) -import Data.Text.Unsafe (lengthWord16) -import Data.Word (Word16) -import Foreign.C.String (CStringLen) -import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray, withForeignPtr) -import Foreign.Marshal.Alloc (allocaBytes) -import Foreign.Ptr (Ptr, castPtr, plusPtr) -import Foreign.Storable (peek, poke) -import qualified Data.Text.Array as A - --- $interop --- --- The 'Text' type is implemented using arrays that are not guaranteed --- to have a fixed address in the Haskell heap. All communication with --- native code must thus occur by copying data back and forth. --- --- The 'Text' type's internal representation is UTF-16, using the --- platform's native endianness. This makes copied data suitable for --- use with native libraries that use a similar representation, such --- as ICU. To interoperate with native libraries that use different --- internal representations, such as UTF-8 or UTF-32, consider using --- the functions in the 'Data.Text.Encoding' module. - --- | A type representing a number of UTF-16 code units. -newtype I16 = I16 Int - deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) - --- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word16' by copying the --- contents of the array. -fromPtr :: Ptr Word16 -- ^ source array - -> I16 -- ^ length of source array (in 'Word16' units) - -> IO Text -fromPtr _ (I16 0) = return empty -fromPtr ptr (I16 len) = -#if defined(ASSERTS) - assert (len > 0) $ -#endif - return $! Text arr 0 len - where - arr = A.run (A.new len >>= copy) - copy marr = loop ptr 0 - where - loop !p !i | i == len = return marr - | otherwise = do - A.unsafeWrite marr i =<< unsafeIOToST (peek p) - loop (p `plusPtr` 2) (i + 1) - --- $lowlevel --- --- Foreign functions that use UTF-16 internally may return indices in --- units of 'Word16' instead of characters. These functions may --- safely be used with such indices, as they will adjust offsets if --- necessary to preserve the validity of a Unicode string. - --- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word16' units in --- length. --- --- If @n@ would cause the 'Text' to end inside a surrogate pair, the --- end of the prefix will be advanced by one additional 'Word16' unit --- to maintain its validity. -takeWord16 :: I16 -> Text -> Text -takeWord16 (I16 n) t@(Text arr off len) - | n <= 0 = empty - | n >= len || m >= len = t - | otherwise = Text arr off m - where - m | w < 0xD800 || w > 0xDBFF = n - | otherwise = n+1 - w = A.unsafeIndex arr (off+n-1) - --- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word16' units --- dropped from its beginning. --- --- If @n@ would cause the 'Text' to begin inside a surrogate pair, the --- beginning of the suffix will be advanced by one additional 'Word16' --- unit to maintain its validity. -dropWord16 :: I16 -> Text -> Text -dropWord16 (I16 n) t@(Text arr off len) - | n <= 0 = t - | n >= len || m >= len = empty - | otherwise = Text arr (off+m) (len-m) - where - m | w < 0xD800 || w > 0xDBFF = n - | otherwise = n+1 - w = A.unsafeIndex arr (off+n-1) - --- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big --- enough to hold the contents of the entire 'Text'. -unsafeCopyToPtr :: Text -> Ptr Word16 -> IO () -unsafeCopyToPtr (Text arr off len) ptr = loop ptr off - where - end = off + len - loop !p !i | i == end = return () - | otherwise = do - poke p (A.unsafeIndex arr i) - loop (p `plusPtr` 2) (i + 1) - --- | /O(n)/ Perform an action on a temporary, mutable copy of a --- 'Text'. The copy is freed as soon as the action returns. -useAsPtr :: Text -> (Ptr Word16 -> I16 -> IO a) -> IO a -useAsPtr t@(Text _arr _off len) action = - allocaBytes (len * 2) $ \buf -> do - unsafeCopyToPtr t buf - action (castPtr buf) (fromIntegral len) - --- | /O(n)/ Make a mutable copy of a 'Text'. -asForeignPtr :: Text -> IO (ForeignPtr Word16, I16) -asForeignPtr t@(Text _arr _off len) = do - fp <- mallocForeignPtrArray len - withForeignPtr fp $ unsafeCopyToPtr t - return (fp, I16 len) - --- | /O(n)/ Decode a C string with explicit length, which is assumed --- to have been encoded as UTF-8. If decoding fails, a --- 'UnicodeException' is thrown. -peekCStringLen :: CStringLen -> IO Text -peekCStringLen cs = do - bs <- unsafePackCStringLen cs - return $! decodeUtf8 bs - --- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary --- storage, with explicit length information. The encoded string may --- contain NUL bytes, and is not followed by a trailing NUL byte. --- --- The temporary storage is freed when the subcomputation terminates --- (either normally or via an exception), so the pointer to the --- temporary storage must /not/ be used after this function returns. -withCStringLen :: Text -> (CStringLen -> IO a) -> IO a -withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act diff --git a/Data/Text/IO.hs b/Data/Text/IO.hs deleted file mode 100644 --- a/Data/Text/IO.hs +++ /dev/null @@ -1,338 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif --- | --- Module : Data.Text.IO --- Copyright : (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Simon Marlow --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Efficient locale-sensitive support for text I\/O. --- --- Skip past the synopsis for some important notes on performance and --- portability across different versions of GHC. - -module Data.Text.IO - ( - -- * Performance - -- $performance - - -- * Locale support - -- $locale - -- * File-at-a-time operations - readFile - , writeFile - , appendFile - -- * Operations on handles - , hGetContents - , hGetChunk - , hGetLine - , hPutStr - , hPutStrLn - -- * Special cases for standard input and output - , interact - , getContents - , getLine - , putStr - , putStrLn - ) where - -import Data.Text (Text) -import Prelude hiding (appendFile, getContents, getLine, interact, - putStr, putStrLn, readFile, writeFile) -import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout, - withFile) -import qualified Control.Exception as E -import Control.Monad (liftM2, when) -import Data.IORef (readIORef, writeIORef) -import qualified Data.Text as T -import Data.Text.Internal.Fusion (stream) -import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) -import Data.Text.Internal.IO (hGetLineWith, readChunk) -import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBufElem, CharBuffer, - RawCharBuffer, emptyBuffer, isEmptyBuffer, newCharBuffer, - writeCharBuf) -import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) -import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle, - wantWritableHandle) -import GHC.IO.Handle.Text (commitBuffer') -import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), - HandleType(..), Newline(..)) -import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) -import System.IO.Error (isEOFError) - --- $performance --- #performance# --- --- The functions in this module obey the runtime system's locale, --- character set encoding, and line ending conversion settings. --- --- If you know in advance that you will be working with data that has --- a specific encoding (e.g. UTF-8), and your application is highly --- performance sensitive, you may find that it is faster to perform --- I\/O with bytestrings and to encode and decode yourself than to use --- the functions in this module. --- --- Whether this will hold depends on the version of GHC you are using, --- the platform you are working on, the data you are working with, and --- the encodings you are using, so be sure to test for yourself. - --- | The 'readFile' function reads a file and returns the contents of --- the file as a string. The entire file is read strictly, as with --- 'getContents'. -readFile :: FilePath -> IO Text -readFile name = openFile name ReadMode >>= hGetContents - --- | Write a string to a file. The file is truncated to zero length --- before writing begins. -writeFile :: FilePath -> Text -> IO () -writeFile p = withFile p WriteMode . flip hPutStr - --- | Write a string the end of a file. -appendFile :: FilePath -> Text -> IO () -appendFile p = withFile p AppendMode . flip hPutStr - -catchError :: String -> Handle -> Handle__ -> IOError -> IO Text -catchError caller h Handle__{..} err - | isEOFError err = do - buf <- readIORef haCharBuffer - return $ if isEmptyBuffer buf - then T.empty - else T.singleton '\r' - | otherwise = E.throwIO (augmentIOError err caller h) - --- | /Experimental./ Read a single chunk of strict text from a --- 'Handle'. The size of the chunk depends on the amount of input --- currently buffered. --- --- This function blocks only if there is no data available, and EOF --- has not yet been reached. Once EOF is reached, this function --- returns an empty string instead of throwing an exception. -hGetChunk :: Handle -> IO Text -hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk - where - readSingleChunk hh@Handle__{..} = do - buf <- readIORef haCharBuffer - t <- readChunk hh buf `E.catch` catchError "hGetChunk" h hh - return (hh, t) - --- | Read the remaining contents of a 'Handle' as a string. The --- 'Handle' is closed once the contents have been read, or if an --- exception is thrown. --- --- Internally, this function reads a chunk at a time from the --- lower-level buffering abstraction, and concatenates the chunks into --- a single string once the entire file has been read. --- --- As a result, it requires approximately twice as much memory as its --- result to construct its result. For files more than a half of --- available RAM in size, this may result in memory exhaustion. -hGetContents :: Handle -> IO Text -hGetContents h = do - chooseGoodBuffering h - wantReadableHandle "hGetContents" h readAll - where - readAll hh@Handle__{..} = do - let readChunks = do - buf <- readIORef haCharBuffer - t <- readChunk hh buf `E.catch` catchError "hGetContents" h hh - if T.null t - then return [t] - else (t:) `fmap` readChunks - ts <- readChunks - (hh', _) <- hClose_help hh - return (hh'{haType=ClosedHandle}, T.concat ts) - --- | Use a more efficient buffer size if we're reading in --- block-buffered mode with the default buffer size. When we can --- determine the size of the handle we're reading, set the buffer size --- to that, so that we can read the entire file in one chunk. --- Otherwise, use a buffer size of at least 16KB. -chooseGoodBuffering :: Handle -> IO () -chooseGoodBuffering h = do - bufMode <- hGetBuffering h - case bufMode of - BlockBuffering Nothing -> do - d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> - if ioe_type e == InappropriateType - then return 16384 -- faster than the 2KB default - else E.throwIO e - when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromIntegral $ d - _ -> return () - --- | Read a single line from a handle. -hGetLine :: Handle -> IO Text -hGetLine = hGetLineWith T.concat - --- | Write a string to a handle. -hPutStr :: Handle -> Text -> IO () --- This function is lifted almost verbatim from GHC.IO.Handle.Text. -hPutStr h t = do - (buffer_mode, nl) <- - wantWritableHandle "hPutStr" h $ \h_ -> do - bmode <- getSpareBuffer h_ - return (bmode, haOutputNL h_) - let str = stream t - case buffer_mode of - (NoBuffering, _) -> hPutChars h str - (LineBuffering, buf) -> writeLines h nl buf str - (BlockBuffering _, buf) - | nl == CRLF -> writeBlocksCRLF h buf str - | otherwise -> writeBlocksRaw h buf str - -hPutChars :: Handle -> Stream Char -> IO () -hPutChars h (Stream next0 s0 _len) = loop s0 - where - loop !s = case next0 s of - Done -> return () - Skip s' -> loop s' - Yield x s' -> hPutChar h x >> loop s' - --- The following functions are largely lifted from GHC.IO.Handle.Text, --- but adapted to a coinductive stream of data instead of an inductive --- list. --- --- We have several variations of more or less the same code for --- performance reasons. Splitting the original buffered write --- function into line- and block-oriented versions gave us a 2.1x --- performance improvement. Lifting out the raw/cooked newline --- handling gave a few more percent on top. - -writeLines :: Handle -> Newline -> Buffer CharBufElem -> Stream Char -> IO () -writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | x == '\n' -> do - n' <- if nl == CRLF - then do n1 <- writeCharBuf raw n '\r' - writeCharBuf raw n1 '\n' - else writeCharBuf raw n x - commit n' True{-needs flush-} False >>= outer s' - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - -writeBlocksCRLF :: Handle -> Buffer CharBufElem -> Stream Char -> IO () -writeBlocksCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | x == '\n' -> do n1 <- writeCharBuf raw n '\r' - writeCharBuf raw n1 '\n' >>= inner s' - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - -writeBlocksRaw :: Handle -> Buffer CharBufElem -> Stream Char -> IO () -writeBlocksRaw h buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - --- This function is completely lifted from GHC.IO.Handle.Text. -getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) -getSpareBuffer Handle__{haCharBuffer=ref, - haBuffers=spare_ref, - haBufferMode=mode} - = do - case mode of - NoBuffering -> return (mode, error "no buffer!") - _ -> do - bufs <- readIORef spare_ref - buf <- readIORef ref - case bufs of - BufferListCons b rest -> do - writeIORef spare_ref rest - return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) - BufferListNil -> do - new_buf <- newCharBuffer (bufSize buf) WriteBuffer - return (mode, new_buf) - - --- This function is completely lifted from GHC.IO.Handle.Text. -commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool - -> IO CharBuffer -commitBuffer hdl !raw !sz !count flush release = - wantWritableHandle "commitAndReleaseBuffer" hdl $ - commitBuffer' raw sz count flush release -{-# INLINE commitBuffer #-} - --- | Write a string to a handle, followed by a newline. -hPutStrLn :: Handle -> Text -> IO () -hPutStrLn h t = hPutStr h t >> hPutChar h '\n' - --- | The 'interact' function takes a function of type @Text -> Text@ --- as its argument. The entire input from the standard input device is --- passed to this function as its argument, and the resulting string --- is output on the standard output device. -interact :: (Text -> Text) -> IO () -interact f = putStr . f =<< getContents - --- | Read all user input on 'stdin' as a single string. -getContents :: IO Text -getContents = hGetContents stdin - --- | Read a single line of user input from 'stdin'. -getLine :: IO Text -getLine = hGetLine stdin - --- | Write a string to 'stdout'. -putStr :: Text -> IO () -putStr = hPutStr stdout - --- | Write a string to 'stdout', followed by a newline. -putStrLn :: Text -> IO () -putStrLn = hPutStrLn stdout - --- $locale --- --- /Note/: The behaviour of functions in this module depends on the --- version of GHC you are using. --- --- Beginning with GHC 6.12, text I\/O is performed using the system or --- handle's current locale and line ending conventions. --- --- Under GHC 6.10 and earlier, the system I\/O libraries do not --- support locale-sensitive I\/O or line ending conversion. On these --- versions of GHC, functions in this library all use UTF-8. What --- does this mean in practice? --- --- * All data that is read will be decoded as UTF-8. --- --- * Before data is written, it is first encoded as UTF-8. --- --- * On both reading and writing, the platform's native newline --- conversion is performed. --- --- If you must use a non-UTF-8 locale on an older version of GHC, you --- will have to perform the transcoding yourself, e.g. as follows: --- --- > import qualified Data.ByteString as B --- > import Data.Text (Text) --- > import Data.Text.Encoding (encodeUtf16) --- > --- > putStr_Utf16LE :: Text -> IO () --- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t) diff --git a/Data/Text/Internal.hs b/Data/Text/Internal.hs deleted file mode 100644 --- a/Data/Text/Internal.hs +++ /dev/null @@ -1,188 +0,0 @@ -{-# LANGUAGE CPP, DeriveDataTypeable, UnboxedTuples #-} -{-# OPTIONS_HADDOCK not-home #-} - --- | --- Module : Data.Text.Internal --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- A module containing private 'Text' internals. This exposes the --- 'Text' representation and low level construction functions. --- Modules which extend the 'Text' system may need to use this module. --- --- You should not use this module unless you are determined to monkey --- with the internals, as the functions here do just about nothing to --- preserve data invariants. You have been warned! - -#if defined(__GLASGOW_HASKELL__) && !defined(__HADDOCK__) -#include "MachDeps.h" -#endif - -module Data.Text.Internal - ( - -- * Types - -- $internals - Text(..) - -- * Construction - , text - , textP - -- * Safety - , safe - -- * Code that must be here for accessibility - , empty - , empty_ - -- * Utilities - , firstf - -- * Checked multiplication - , mul - , mul32 - , mul64 - -- * Debugging - , showText - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.Bits -import Data.Int (Int32, Int64) -import Data.Text.Internal.Unsafe.Char (ord) -import Data.Typeable (Typeable) -import qualified Data.Text.Array as A - --- | A space efficient, packed, unboxed Unicode text type. -data Text = Text - {-# UNPACK #-} !A.Array -- payload (Word16 elements) - {-# UNPACK #-} !Int -- offset (units of Word16, not Char) - {-# UNPACK #-} !Int -- length (units of Word16, not Char) - deriving (Typeable) - --- | Smart constructor. -text_ :: A.Array -> Int -> Int -> Text -text_ arr off len = -#if defined(ASSERTS) - let c = A.unsafeIndex arr off - alen = A.length arr - in assert (len >= 0) . - assert (off >= 0) . - assert (alen == 0 || len == 0 || off < alen) . - assert (len == 0 || c < 0xDC00 || c > 0xDFFF) $ -#endif - Text arr off len -{-# INLINE text_ #-} - --- | /O(1)/ The empty 'Text'. -empty :: Text -empty = Text A.empty 0 0 -{-# INLINE [1] empty #-} - --- | A non-inlined version of 'empty'. -empty_ :: Text -empty_ = Text A.empty 0 0 -{-# NOINLINE empty_ #-} - --- | Construct a 'Text' without invisibly pinning its byte array in --- memory if its length has dwindled to zero. -text :: A.Array -> Int -> Int -> Text -text arr off len | len == 0 = empty - | otherwise = text_ arr off len -{-# INLINE text #-} - -textP :: A.Array -> Int -> Int -> Text -{-# DEPRECATED textP "Use text instead" #-} -textP = text - --- | A useful 'show'-like function for debugging purposes. -showText :: Text -> String -showText (Text arr off len) = - "Text " ++ show (A.toList arr off len) ++ ' ' : - show off ++ ' ' : show len - --- | Map a 'Char' to a 'Text'-safe value. --- --- UTF-16 surrogate code points are not included in the set of Unicode --- scalar values, but are unfortunately admitted as valid 'Char' --- values by Haskell. They cannot be represented in a 'Text'. This --- function remaps those code points to the Unicode replacement --- character (U+FFFD, \'�\'), and leaves other code points --- unchanged. -safe :: Char -> Char -safe c - | ord c .&. 0x1ff800 /= 0xd800 = c - | otherwise = '\xfffd' -{-# INLINE [0] safe #-} - --- | Apply a function to the first element of an optional pair. -firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) -firstf f (Just (a, b)) = Just (f a, b) -firstf _ Nothing = Nothing - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul :: Int -> Int -> Int -#if WORD_SIZE_IN_BITS == 64 -mul a b = fromIntegral $ fromIntegral a `mul64` fromIntegral b -#else -mul a b = fromIntegral $ fromIntegral a `mul32` fromIntegral b -#endif -{-# INLINE mul #-} -infixl 7 `mul` - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul64 :: Int64 -> Int64 -> Int64 -mul64 a b - | a >= 0 && b >= 0 = mul64_ a b - | a >= 0 = -mul64_ a (-b) - | b >= 0 = -mul64_ (-a) b - | otherwise = mul64_ (-a) (-b) -{-# INLINE mul64 #-} -infixl 7 `mul64` - -mul64_ :: Int64 -> Int64 -> Int64 -mul64_ a b - | ahi > 0 && bhi > 0 = error "overflow" - | top > 0x7fffffff = error "overflow" - | total < 0 = error "overflow" - | otherwise = total - where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) - (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) - top = ahi * blo + alo * bhi - total = (top `shiftL` 32) + alo * blo -{-# INLINE mul64_ #-} - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul32 :: Int32 -> Int32 -> Int32 -mul32 a b = case fromIntegral a * fromIntegral b of - ab | ab < min32 || ab > max32 -> error "overflow" - | otherwise -> fromIntegral ab - where min32 = -0x80000000 :: Int64 - max32 = 0x7fffffff -{-# INLINE mul32 #-} -infixl 7 `mul32` - --- $internals --- --- Internally, the 'Text' type is represented as an array of 'Word16' --- UTF-16 code units. The offset and length fields in the constructor --- are in these units, /not/ units of 'Char'. --- --- Invariants that all functions must maintain: --- --- * Since the 'Text' type uses UTF-16 internally, it cannot represent --- characters in the reserved surrogate code point range U+D800 to --- U+DFFF. To maintain this invariant, the 'safe' function maps --- 'Char' values in this range to the replacement character (U+FFFD, --- \'�\'). --- --- * A leading (or \"high\") surrogate code unit (0xD800–0xDBFF) must --- always be followed by a trailing (or \"low\") surrogate code unit --- (0xDC00-0xDFFF). A trailing surrogate code unit must always be --- preceded by a leading surrogate code unit. diff --git a/Data/Text/Internal/Builder.hs b/Data/Text/Internal/Builder.hs deleted file mode 100644 --- a/Data/Text/Internal/Builder.hs +++ /dev/null @@ -1,329 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, Rank2Types #-} -{-# OPTIONS_HADDOCK not-home #-} - ------------------------------------------------------------------------------ --- | --- Module : Data.Text.Internal.Builder --- Copyright : (c) 2013 Bryan O'Sullivan --- (c) 2010 Johan Tibell --- License : BSD-style (see LICENSE) --- --- Maintainer : Johan Tibell --- Stability : experimental --- Portability : portable to Hugs and GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Efficient construction of lazy @Text@ values. The principal --- operations on a @Builder@ are @singleton@, @fromText@, and --- @fromLazyText@, which construct new builders, and 'mappend', which --- concatenates two builders. --- --- To get maximum performance when building lazy @Text@ values using a --- builder, associate @mappend@ calls to the right. For example, --- prefer --- --- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') --- --- to --- --- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' --- --- as the latter associates @mappend@ to the left. --- ------------------------------------------------------------------------------ - -module Data.Text.Internal.Builder - ( -- * Public API - -- ** The Builder type - Builder - , toLazyText - , toLazyTextWith - - -- ** Constructing Builders - , singleton - , fromText - , fromLazyText - , fromString - - -- ** Flushing the buffer state - , flush - - -- * Internal functions - , append' - , ensureFree - , writeN - ) where - -import Control.Monad.ST (ST, runST) -import Data.Monoid (Monoid(..)) -#if MIN_VERSION_base(4,9,0) -import Data.Semigroup (Semigroup(..)) -#endif -import Data.Text.Internal (Text(..)) -import Data.Text.Internal.Lazy (smallChunkSize) -import Data.Text.Unsafe (inlineInterleaveST) -import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import Prelude hiding (map, putChar) - -import qualified Data.String as String -import qualified Data.Text as S -import qualified Data.Text.Array as A -import qualified Data.Text.Lazy as L - ------------------------------------------------------------------------- - --- | A @Builder@ is an efficient way to build lazy @Text@ values. --- There are several functions for constructing builders, but only one --- to inspect them: to extract any data, you have to turn them into --- lazy @Text@ values using @toLazyText@. --- --- Internally, a builder constructs a lazy @Text@ by filling arrays --- piece by piece. As each buffer is filled, it is \'popped\' off, to --- become a new chunk of the resulting lazy @Text@. All this is --- hidden from the user of the @Builder@. -newtype Builder = Builder { - -- Invariant (from Data.Text.Lazy): - -- The lists include no null Texts. - runBuilder :: forall s. (Buffer s -> ST s [S.Text]) - -> Buffer s - -> ST s [S.Text] - } - -#if MIN_VERSION_base(4,9,0) -instance Semigroup Builder where - (<>) = append - {-# INLINE (<>) #-} -#endif - -instance Monoid Builder where - mempty = empty - {-# INLINE mempty #-} -#if MIN_VERSION_base(4,9,0) - mappend = (<>) -- future-proof definition -#else - mappend = append -#endif - {-# INLINE mappend #-} - mconcat = foldr mappend Data.Monoid.mempty - {-# INLINE mconcat #-} - -instance String.IsString Builder where - fromString = fromString - {-# INLINE fromString #-} - -instance Show Builder where - show = show . toLazyText - -instance Eq Builder where - a == b = toLazyText a == toLazyText b - -instance Ord Builder where - a <= b = toLazyText a <= toLazyText b - ------------------------------------------------------------------------- - --- | /O(1)./ The empty @Builder@, satisfying --- --- * @'toLazyText' 'empty' = 'L.empty'@ --- -empty :: Builder -empty = Builder (\ k buf -> k buf) -{-# INLINE empty #-} - --- | /O(1)./ A @Builder@ taking a single character, satisfying --- --- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ --- -singleton :: Char -> Builder -singleton c = writeAtMost 2 $ \ marr o -> unsafeWrite marr o c -{-# INLINE singleton #-} - ------------------------------------------------------------------------- - --- | /O(1)./ The concatenation of two builders, an associative --- operation with identity 'empty', satisfying --- --- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ --- -append :: Builder -> Builder -> Builder -append (Builder f) (Builder g) = Builder (f . g) -{-# INLINE [0] append #-} - --- TODO: Experiment to find the right threshold. -copyLimit :: Int -copyLimit = 128 - --- This function attempts to merge small @Text@ values instead of --- treating each value as its own chunk. We may not always want this. - --- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying --- --- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ --- -fromText :: S.Text -> Builder -fromText t@(Text arr off l) - | S.null t = empty - | l <= copyLimit = writeN l $ \marr o -> A.copyI marr o arr off (l+o) - | otherwise = flush `append` mapBuilder (t :) -{-# INLINE [1] fromText #-} - -{-# RULES -"fromText/pack" forall s . - fromText (S.pack s) = fromString s - #-} - --- | /O(1)./ A Builder taking a @String@, satisfying --- --- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ --- -fromString :: String -> Builder -fromString str = Builder $ \k (Buffer p0 o0 u0 l0) -> - let loop !marr !o !u !l [] = k (Buffer marr o u l) - loop marr o u l s@(c:cs) - | l <= 1 = do - arr <- A.unsafeFreeze marr - let !t = Text arr o u - marr' <- A.new chunkSize - ts <- inlineInterleaveST (loop marr' 0 0 chunkSize s) - return $ t : ts - | otherwise = do - n <- unsafeWrite marr (o+u) c - loop marr o (u+n) (l-n) cs - in loop p0 o0 u0 l0 str - where - chunkSize = smallChunkSize -{-# INLINE fromString #-} - --- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying --- --- * @'toLazyText' ('fromLazyText' t) = t@ --- -fromLazyText :: L.Text -> Builder -fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) -{-# INLINE fromLazyText #-} - ------------------------------------------------------------------------- - --- Our internal buffer type -data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) - {-# UNPACK #-} !Int -- offset - {-# UNPACK #-} !Int -- used units - {-# UNPACK #-} !Int -- length left - ------------------------------------------------------------------------- - --- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default --- buffer size. The construction work takes place if and when the --- relevant part of the lazy @Text@ is demanded. -toLazyText :: Builder -> L.Text -toLazyText = toLazyTextWith smallChunkSize - --- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given --- size for the initial buffer. The construction work takes place if --- and when the relevant part of the lazy @Text@ is demanded. --- --- If the initial buffer is too small to hold all data, subsequent --- buffers will be the default buffer size. -toLazyTextWith :: Int -> Builder -> L.Text -toLazyTextWith chunkSize m = L.fromChunks (runST $ - newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) - --- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, --- yielding a new chunk in the result lazy @Text@. -flush :: Builder -flush = Builder $ \ k buf@(Buffer p o u l) -> - if u == 0 - then k buf - else do arr <- A.unsafeFreeze p - let !b = Buffer p (o+u) 0 l - !t = Text arr o u - ts <- inlineInterleaveST (k b) - return $! t : ts -{-# INLINE [1] flush #-} --- defer inlining so that flush/flush rule may fire. - ------------------------------------------------------------------------- - --- | Sequence an ST operation on the buffer -withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder -withBuffer f = Builder $ \k buf -> f buf >>= k -{-# INLINE withBuffer #-} - --- | Get the size of the buffer -withSize :: (Int -> Builder) -> Builder -withSize f = Builder $ \ k buf@(Buffer _ _ _ l) -> - runBuilder (f l) k buf -{-# INLINE withSize #-} - --- | Map the resulting list of texts. -mapBuilder :: ([S.Text] -> [S.Text]) -> Builder -mapBuilder f = Builder (fmap f .) - ------------------------------------------------------------------------- - --- | Ensure that there are at least @n@ many elements available. -ensureFree :: Int -> Builder -ensureFree !n = withSize $ \ l -> - if n <= l - then empty - else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) -{-# INLINE [0] ensureFree #-} - -writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder -writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) -{-# INLINE [0] writeAtMost #-} - --- | Ensure that @n@ many elements are available, and then use @f@ to --- write some elements into the memory. -writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder -writeN n f = writeAtMost n (\ p o -> f p o >> return n) -{-# INLINE writeN #-} - -writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) -writeBuffer f (Buffer p o u l) = do - n <- f p (o+u) - return $! Buffer p o (u+n) (l-n) -{-# INLINE writeBuffer #-} - -newBuffer :: Int -> ST s (Buffer s) -newBuffer size = do - arr <- A.new size - return $! Buffer arr 0 0 size -{-# INLINE newBuffer #-} - ------------------------------------------------------------------------- --- Some nice rules for Builder - --- This function makes GHC understand that 'writeN' and 'ensureFree' --- are *not* recursive in the precense of the rewrite rules below. --- This is not needed with GHC 7+. -append' :: Builder -> Builder -> Builder -append' (Builder f) (Builder g) = Builder (f . g) -{-# INLINE append' #-} - -{-# RULES - -"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) - (g::forall s. A.MArray s -> Int -> ST s Int) ws. - append (writeAtMost a f) (append (writeAtMost b g) ws) = - append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> - g marr (o+n) >>= \ m -> - let s = n+m in s `seq` return s)) ws - -"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) - (g::forall s. A.MArray s -> Int -> ST s Int). - append (writeAtMost a f) (writeAtMost b g) = - writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> - g marr (o+n) >>= \ m -> - let s = n+m in s `seq` return s) - -"ensureFree/ensureFree" forall a b . - append (ensureFree a) (ensureFree b) = ensureFree (max a b) - -"flush/flush" - append flush flush = flush - - #-} diff --git a/Data/Text/Internal/Builder/Functions.hs b/Data/Text/Internal/Builder/Functions.hs deleted file mode 100644 --- a/Data/Text/Internal/Builder/Functions.hs +++ /dev/null @@ -1,40 +0,0 @@ -{-# LANGUAGE MagicHash #-} - --- | --- Module : Data.Text.Internal.Builder.Functions --- Copyright : (c) 2011 MailRank, Inc. --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Useful functions and combinators. - -module Data.Text.Internal.Builder.Functions - ( - (<>) - , i2d - ) where - -import Data.Monoid (mappend) -import Data.Text.Lazy.Builder (Builder) -import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#)) -import Prelude () - --- | Unsafe conversion for decimal digits. -{-# INLINE i2d #-} -i2d :: Int -> Char -i2d (I# i#) = C# (chr# (ord# '0'# +# i#)) - --- | The normal 'mappend' function with right associativity instead of --- left. -(<>) :: Builder -> Builder -> Builder -(<>) = mappend -{-# INLINE (<>) #-} - -infixr 4 <> diff --git a/Data/Text/Internal/Builder/Int/Digits.hs b/Data/Text/Internal/Builder/Int/Digits.hs deleted file mode 100644 --- a/Data/Text/Internal/Builder/Int/Digits.hs +++ /dev/null @@ -1,26 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - --- Module: Data.Text.Internal.Builder.Int.Digits --- Copyright: (c) 2013 Bryan O'Sullivan --- License: BSD-style --- Maintainer: Bryan O'Sullivan --- Stability: experimental --- Portability: portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- This module exists because the C preprocessor does things that we --- shall not speak of when confronted with Haskell multiline strings. - -module Data.Text.Internal.Builder.Int.Digits (digits) where - -import Data.ByteString.Char8 (ByteString) - -digits :: ByteString -digits = "0001020304050607080910111213141516171819\ - \2021222324252627282930313233343536373839\ - \4041424344454647484950515253545556575859\ - \6061626364656667686970717273747576777879\ - \8081828384858687888990919293949596979899" diff --git a/Data/Text/Internal/Builder/RealFloat/Functions.hs b/Data/Text/Internal/Builder/RealFloat/Functions.hs deleted file mode 100644 --- a/Data/Text/Internal/Builder/RealFloat/Functions.hs +++ /dev/null @@ -1,57 +0,0 @@ -{-# LANGUAGE CPP #-} - --- | --- Module: Data.Text.Internal.Builder.RealFloat.Functions --- Copyright: (c) The University of Glasgow 1994-2002 --- License: see libraries/base/LICENSE --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! - -module Data.Text.Internal.Builder.RealFloat.Functions - ( - roundTo - ) where - -roundTo :: Int -> [Int] -> (Int,[Int]) - -#if MIN_VERSION_base(4,6,0) - -roundTo d is = - case f d True is of - x@(0,_) -> x - (1,xs) -> (1, 1:xs) - _ -> error "roundTo: bad Value" - where - b2 = base `quot` 2 - - f n _ [] = (0, replicate n 0) - f 0 e (x:xs) | x == b2 && e && all (== 0) xs = (0, []) -- Round to even when at exactly half the base - | otherwise = (if x >= b2 then 1 else 0, []) - f n _ (i:xs) - | i' == base = (1,0:ds) - | otherwise = (0,i':ds) - where - (c,ds) = f (n-1) (even i) xs - i' = c + i - base = 10 - -#else - -roundTo d is = - case f d is of - x@(0,_) -> x - (1,xs) -> (1, 1:xs) - _ -> error "roundTo: bad Value" - where - f n [] = (0, replicate n 0) - f 0 (x:_) = (if x >= 5 then 1 else 0, []) - f n (i:xs) - | i' == 10 = (1,0:ds) - | otherwise = (0,i':ds) - where - (c,ds) = f (n-1) xs - i' = c + i - -#endif diff --git a/Data/Text/Internal/Encoding/Fusion.hs b/Data/Text/Internal/Encoding/Fusion.hs deleted file mode 100644 --- a/Data/Text/Internal/Encoding/Fusion.hs +++ /dev/null @@ -1,208 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, Rank2Types #-} - --- | --- Module : Data.Text.Internal.Encoding.Fusion --- Copyright : (c) Tom Harper 2008-2009, --- (c) Bryan O'Sullivan 2009, --- (c) Duncan Coutts 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fusible 'Stream'-oriented functions for converting between 'Text' --- and several common encodings. - -module Data.Text.Internal.Encoding.Fusion - ( - -- * Streaming - streamASCII - , streamUtf8 - , streamUtf16LE - , streamUtf16BE - , streamUtf32LE - , streamUtf32BE - - -- * Unstreaming - , unstream - - , module Data.Text.Internal.Encoding.Fusion.Common - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy) -import Data.Text.Internal.Fusion (Step(..), Stream(..)) -import Data.Text.Internal.Fusion.Size -import Data.Text.Encoding.Error -import Data.Text.Internal.Encoding.Fusion.Common -import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) -import Data.Word (Word8, Word16, Word32) -import Foreign.ForeignPtr (withForeignPtr, ForeignPtr) -import Foreign.Storable (pokeByteOff) -import qualified Data.ByteString as B -import qualified Data.ByteString.Unsafe as B -import qualified Data.Text.Internal.Encoding.Utf8 as U8 -import qualified Data.Text.Internal.Encoding.Utf16 as U16 -import qualified Data.Text.Internal.Encoding.Utf32 as U32 -import Data.Text.Unsafe (unsafeDupablePerformIO) - -streamASCII :: ByteString -> Stream Char -streamASCII bs = Stream next 0 (maxSize l) - where - l = B.length bs - {-# INLINE next #-} - next i - | i >= l = Done - | otherwise = Yield (unsafeChr8 x1) (i+1) - where - x1 = B.unsafeIndex bs i -{-# DEPRECATED streamASCII "Do not use this function" #-} -{-# INLINE [0] streamASCII #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8 --- encoding. -streamUtf8 :: OnDecodeError -> ByteString -> Stream Char -streamUtf8 onErr bs = Stream next 0 (maxSize l) - where - l = B.length bs - next i - | i >= l = Done - | U8.validate1 x1 = Yield (unsafeChr8 x1) (i+1) - | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (i+2) - | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (i+3) - | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (i+4) - | otherwise = decodeError "streamUtf8" "UTF-8" onErr (Just x1) (i+1) - where - x1 = idx i - x2 = idx (i + 1) - x3 = idx (i + 2) - x4 = idx (i + 3) - idx = B.unsafeIndex bs -{-# INLINE [0] streamUtf8 #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little --- endian UTF-16 encoding. -streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16LE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) - where - l = B.length bs - {-# INLINE next #-} - next i - | i >= l = Done - | i+1 < l && U16.validate1 x1 = Yield (unsafeChr x1) (i+2) - | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) - | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1) - where - x1 = idx i + (idx (i + 1) `shiftL` 8) - x2 = idx (i + 2) + (idx (i + 3) `shiftL` 8) - idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16 -{-# INLINE [0] streamUtf16LE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big --- endian UTF-16 encoding. -streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16BE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) - where - l = B.length bs - {-# INLINE next #-} - next i - | i >= l = Done - | i+1 < l && U16.validate1 x1 = Yield (unsafeChr x1) (i+2) - | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) - | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1) - where - x1 = (idx i `shiftL` 8) + idx (i + 1) - x2 = (idx (i + 2) `shiftL` 8) + idx (i + 3) - idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16 -{-# INLINE [0] streamUtf16BE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big --- endian UTF-32 encoding. -streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32BE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) - where - l = B.length bs - {-# INLINE next #-} - next i - | i >= l = Done - | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4) - | otherwise = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (i+1) - where - x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 - x1 = idx i - x2 = idx (i+1) - x3 = idx (i+2) - x4 = idx (i+3) - idx = fromIntegral . B.unsafeIndex bs :: Int -> Word32 -{-# INLINE [0] streamUtf32BE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little --- endian UTF-32 encoding. -streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32LE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) - where - l = B.length bs - {-# INLINE next #-} - next i - | i >= l = Done - | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4) - | otherwise = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (i+1) - where - x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 - x1 = idx i - x2 = idx $ i+1 - x3 = idx $ i+2 - x4 = idx $ i+3 - idx = fromIntegral . B.unsafeIndex bs :: Int -> Word32 -{-# INLINE [0] streamUtf32LE #-} - --- | /O(n)/ Convert a 'Stream' 'Word8' to a 'ByteString'. -unstream :: Stream Word8 -> ByteString -unstream (Stream next s0 len) = unsafeDupablePerformIO $ do - let mlen = upperBound 4 len - mallocByteString mlen >>= loop mlen 0 s0 - where - loop !n !off !s fp = case next s of - Done -> trimUp fp n off - Skip s' -> loop n off s' fp - Yield x s' - | off == n -> realloc fp n off s' x - | otherwise -> do - withForeignPtr fp $ \p -> pokeByteOff p off x - loop n (off+1) s' fp - {-# NOINLINE realloc #-} - realloc fp n off s x = do - let n' = n+n - fp' <- copy0 fp n n' - withForeignPtr fp' $ \p -> pokeByteOff p off x - loop n' (off+1) s fp' - {-# NOINLINE trimUp #-} - trimUp fp _ off = return $! PS fp 0 off - copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8) - copy0 !src !srcLen !destLen = -#if defined(ASSERTS) - assert (srcLen <= destLen) $ -#endif - do - dest <- mallocByteString destLen - withForeignPtr src $ \src' -> - withForeignPtr dest $ \dest' -> - memcpy dest' src' (fromIntegral srcLen) - return dest - -decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8 - -> s -> Step s Char -decodeError func kind onErr mb i = - case onErr desc mb of - Nothing -> Skip i - Just c -> Yield c i - where desc = "Data.Text.Internal.Encoding.Fusion." ++ func ++ ": Invalid " ++ - kind ++ " stream" diff --git a/Data/Text/Internal/Encoding/Fusion/Common.hs b/Data/Text/Internal/Encoding/Fusion/Common.hs deleted file mode 100644 --- a/Data/Text/Internal/Encoding/Fusion/Common.hs +++ /dev/null @@ -1,123 +0,0 @@ -{-# LANGUAGE BangPatterns #-} - --- | --- Module : Data.Text.Internal.Encoding.Fusion.Common --- Copyright : (c) Tom Harper 2008-2009, --- (c) Bryan O'Sullivan 2009, --- (c) Duncan Coutts 2009, --- (c) Jasper Van der Jeugt 2011 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Use at your own risk! --- --- Fusible 'Stream'-oriented functions for converting between 'Text' --- and several common encodings. - -module Data.Text.Internal.Encoding.Fusion.Common - ( - -- * Restreaming - -- Restreaming is the act of converting from one 'Stream' - -- representation to another. - restreamUtf16LE - , restreamUtf16BE - , restreamUtf32LE - , restreamUtf32BE - ) where - -import Data.Bits ((.&.)) -import Data.Text.Internal.Fusion (Step(..), Stream(..)) -import Data.Text.Internal.Fusion.Types (RS(..)) -import Data.Text.Internal.Unsafe.Char (ord) -import Data.Text.Internal.Unsafe.Shift (shiftR) -import Data.Word (Word8) - -restreamUtf16BE :: Stream Char -> Stream Word8 -restreamUtf16BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) - where - next (RS0 s) = case next0 s of - Done -> Done - Skip s' -> Skip (RS0 s') - Yield x s' - | n < 0x10000 -> Yield (fromIntegral $ n `shiftR` 8) $ - RS1 s' (fromIntegral n) - | otherwise -> Yield c1 $ RS3 s' c2 c3 c4 - where - n = ord x - n1 = n - 0x10000 - c1 = fromIntegral (n1 `shiftR` 18 + 0xD8) - c2 = fromIntegral (n1 `shiftR` 10) - n2 = n1 .&. 0x3FF - c3 = fromIntegral (n2 `shiftR` 8 + 0xDC) - c4 = fromIntegral n2 - next (RS1 s x2) = Yield x2 (RS0 s) - next (RS2 s x2 x3) = Yield x2 (RS1 s x3) - next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) - {-# INLINE next #-} -{-# INLINE restreamUtf16BE #-} - -restreamUtf16LE :: Stream Char -> Stream Word8 -restreamUtf16LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) - where - next (RS0 s) = case next0 s of - Done -> Done - Skip s' -> Skip (RS0 s') - Yield x s' - | n < 0x10000 -> Yield (fromIntegral n) $ - RS1 s' (fromIntegral $ shiftR n 8) - | otherwise -> Yield c1 $ RS3 s' c2 c3 c4 - where - n = ord x - n1 = n - 0x10000 - c2 = fromIntegral (shiftR n1 18 + 0xD8) - c1 = fromIntegral (shiftR n1 10) - n2 = n1 .&. 0x3FF - c4 = fromIntegral (shiftR n2 8 + 0xDC) - c3 = fromIntegral n2 - next (RS1 s x2) = Yield x2 (RS0 s) - next (RS2 s x2 x3) = Yield x2 (RS1 s x3) - next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) - {-# INLINE next #-} -{-# INLINE restreamUtf16LE #-} - -restreamUtf32BE :: Stream Char -> Stream Word8 -restreamUtf32BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) - where - next (RS0 s) = case next0 s of - Done -> Done - Skip s' -> Skip (RS0 s') - Yield x s' -> Yield c1 (RS3 s' c2 c3 c4) - where - n = ord x - c1 = fromIntegral $ shiftR n 24 - c2 = fromIntegral $ shiftR n 16 - c3 = fromIntegral $ shiftR n 8 - c4 = fromIntegral n - next (RS1 s x2) = Yield x2 (RS0 s) - next (RS2 s x2 x3) = Yield x2 (RS1 s x3) - next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) - {-# INLINE next #-} -{-# INLINE restreamUtf32BE #-} - -restreamUtf32LE :: Stream Char -> Stream Word8 -restreamUtf32LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) - where - next (RS0 s) = case next0 s of - Done -> Done - Skip s' -> Skip (RS0 s') - Yield x s' -> Yield c1 (RS3 s' c2 c3 c4) - where - n = ord x - c4 = fromIntegral $ shiftR n 24 - c3 = fromIntegral $ shiftR n 16 - c2 = fromIntegral $ shiftR n 8 - c1 = fromIntegral n - next (RS1 s x2) = Yield x2 (RS0 s) - next (RS2 s x2 x3) = Yield x2 (RS1 s x3) - next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) - {-# INLINE next #-} -{-# INLINE restreamUtf32LE #-} diff --git a/Data/Text/Internal/Encoding/Utf16.hs b/Data/Text/Internal/Encoding/Utf16.hs deleted file mode 100644 --- a/Data/Text/Internal/Encoding/Utf16.hs +++ /dev/null @@ -1,45 +0,0 @@ -{-# LANGUAGE MagicHash, BangPatterns #-} - --- | --- Module : Data.Text.Internal.Encoding.Utf16 --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Basic UTF-16 validation and character manipulation. -module Data.Text.Internal.Encoding.Utf16 - ( - chr2 - , validate1 - , validate2 - ) where - -import GHC.Exts -import GHC.Word (Word16(..)) - -chr2 :: Word16 -> Word16 -> Char -chr2 (W16# a#) (W16# b#) = C# (chr# (upper# +# lower# +# 0x10000#)) - where - !x# = word2Int# a# - !y# = word2Int# b# - !upper# = uncheckedIShiftL# (x# -# 0xD800#) 10# - !lower# = y# -# 0xDC00# -{-# INLINE chr2 #-} - -validate1 :: Word16 -> Bool -validate1 x1 = x1 < 0xD800 || x1 > 0xDFFF -{-# INLINE validate1 #-} - -validate2 :: Word16 -> Word16 -> Bool -validate2 x1 x2 = x1 >= 0xD800 && x1 <= 0xDBFF && - x2 >= 0xDC00 && x2 <= 0xDFFF -{-# INLINE validate2 #-} diff --git a/Data/Text/Internal/Encoding/Utf32.hs b/Data/Text/Internal/Encoding/Utf32.hs deleted file mode 100644 --- a/Data/Text/Internal/Encoding/Utf32.hs +++ /dev/null @@ -1,26 +0,0 @@ --- | --- Module : Data.Text.Internal.Encoding.Utf32 --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Basic UTF-32 validation. -module Data.Text.Internal.Encoding.Utf32 - ( - validate - ) where - -import Data.Word (Word32) - -validate :: Word32 -> Bool -validate x1 = x1 < 0xD800 || (x1 > 0xDFFF && x1 <= 0x10FFFF) -{-# INLINE validate #-} diff --git a/Data/Text/Internal/Encoding/Utf8.hs b/Data/Text/Internal/Encoding/Utf8.hs deleted file mode 100644 --- a/Data/Text/Internal/Encoding/Utf8.hs +++ /dev/null @@ -1,168 +0,0 @@ -{-# LANGUAGE CPP, MagicHash, BangPatterns #-} - --- | --- Module : Data.Text.Internal.Encoding.Utf8 --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Basic UTF-8 validation and character manipulation. -module Data.Text.Internal.Encoding.Utf8 - ( - -- Decomposition - ord2 - , ord3 - , ord4 - -- Construction - , chr2 - , chr3 - , chr4 - -- * Validation - , validate1 - , validate2 - , validate3 - , validate4 - ) where - -#if defined(TEST_SUITE) -# undef ASSERTS -#endif - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.Bits ((.&.)) -import Data.Text.Internal.Unsafe.Char (ord) -import Data.Text.Internal.Unsafe.Shift (shiftR) -import GHC.Exts -import GHC.Word (Word8(..)) - -default(Int) - -between :: Word8 -- ^ byte to check - -> Word8 -- ^ lower bound - -> Word8 -- ^ upper bound - -> Bool -between x y z = x >= y && x <= z -{-# INLINE between #-} - -ord2 :: Char -> (Word8,Word8) -ord2 c = -#if defined(ASSERTS) - assert (n >= 0x80 && n <= 0x07ff) -#endif - (x1,x2) - where - n = ord c - x1 = fromIntegral $ (n `shiftR` 6) + 0xC0 - x2 = fromIntegral $ (n .&. 0x3F) + 0x80 - -ord3 :: Char -> (Word8,Word8,Word8) -ord3 c = -#if defined(ASSERTS) - assert (n >= 0x0800 && n <= 0xffff) -#endif - (x1,x2,x3) - where - n = ord c - x1 = fromIntegral $ (n `shiftR` 12) + 0xE0 - x2 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80 - x3 = fromIntegral $ (n .&. 0x3F) + 0x80 - -ord4 :: Char -> (Word8,Word8,Word8,Word8) -ord4 c = -#if defined(ASSERTS) - assert (n >= 0x10000) -#endif - (x1,x2,x3,x4) - where - n = ord c - x1 = fromIntegral $ (n `shiftR` 18) + 0xF0 - x2 = fromIntegral $ ((n `shiftR` 12) .&. 0x3F) + 0x80 - x3 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80 - x4 = fromIntegral $ (n .&. 0x3F) + 0x80 - -chr2 :: Word8 -> Word8 -> Char -chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) - where - !y1# = word2Int# x1# - !y2# = word2Int# x2# - !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# - !z2# = y2# -# 0x80# -{-# INLINE chr2 #-} - -chr3 :: Word8 -> Word8 -> Word8 -> Char -chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) - where - !y1# = word2Int# x1# - !y2# = word2Int# x2# - !y3# = word2Int# x3# - !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# - !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# - !z3# = y3# -# 0x80# -{-# INLINE chr3 #-} - -chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char -chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = - C# (chr# (z1# +# z2# +# z3# +# z4#)) - where - !y1# = word2Int# x1# - !y2# = word2Int# x2# - !y3# = word2Int# x3# - !y4# = word2Int# x4# - !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# - !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# - !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# - !z4# = y4# -# 0x80# -{-# INLINE chr4 #-} - -validate1 :: Word8 -> Bool -validate1 x1 = x1 <= 0x7F -{-# INLINE validate1 #-} - -validate2 :: Word8 -> Word8 -> Bool -validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF -{-# INLINE validate2 #-} - -validate3 :: Word8 -> Word8 -> Word8 -> Bool -{-# INLINE validate3 #-} -validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 - where - validate3_1 = (x1 == 0xE0) && - between x2 0xA0 0xBF && - between x3 0x80 0xBF - validate3_2 = between x1 0xE1 0xEC && - between x2 0x80 0xBF && - between x3 0x80 0xBF - validate3_3 = x1 == 0xED && - between x2 0x80 0x9F && - between x3 0x80 0xBF - validate3_4 = between x1 0xEE 0xEF && - between x2 0x80 0xBF && - between x3 0x80 0xBF - -validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool -{-# INLINE validate4 #-} -validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 - where - validate4_1 = x1 == 0xF0 && - between x2 0x90 0xBF && - between x3 0x80 0xBF && - between x4 0x80 0xBF - validate4_2 = between x1 0xF1 0xF3 && - between x2 0x80 0xBF && - between x3 0x80 0xBF && - between x4 0x80 0xBF - validate4_3 = x1 == 0xF4 && - between x2 0x80 0x8F && - between x3 0x80 0xBF && - between x4 0x80 0xBF diff --git a/Data/Text/Internal/Functions.hs b/Data/Text/Internal/Functions.hs deleted file mode 100644 --- a/Data/Text/Internal/Functions.hs +++ /dev/null @@ -1,31 +0,0 @@ -{-# LANGUAGE CPP, DeriveDataTypeable #-} - --- | --- Module : Data.Text.Internal.Functions --- Copyright : 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Useful functions. - -module Data.Text.Internal.Functions - ( - intersperse - ) where - --- | A lazier version of Data.List.intersperse. The other version --- causes space leaks! -intersperse :: a -> [a] -> [a] -intersperse _ [] = [] -intersperse sep (x:xs) = x : go xs - where - go [] = [] - go (y:ys) = sep : y: go ys -{-# INLINE intersperse #-} diff --git a/Data/Text/Internal/Fusion.hs b/Data/Text/Internal/Fusion.hs deleted file mode 100644 --- a/Data/Text/Internal/Fusion.hs +++ /dev/null @@ -1,244 +0,0 @@ -{-# LANGUAGE BangPatterns, MagicHash #-} - --- | --- Module : Data.Text.Internal.Fusion --- Copyright : (c) Tom Harper 2008-2009, --- (c) Bryan O'Sullivan 2009-2010, --- (c) Duncan Coutts 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Text manipulation functions represented as fusible operations over --- streams. -module Data.Text.Internal.Fusion - ( - -- * Types - Stream(..) - , Step(..) - - -- * Creation and elimination - , stream - , unstream - , reverseStream - - , length - - -- * Transformations - , reverse - - -- * Construction - -- ** Scans - , reverseScanr - - -- ** Accumulating maps - , mapAccumL - - -- ** Generation and unfolding - , unfoldrN - - -- * Indexing - , index - , findIndex - , countChar - ) where - -import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int, - Num(..), Ord(..), ($), (&&), - fromIntegral, otherwise) -import Data.Bits ((.&.)) -import Data.Text.Internal (Text(..)) -import Data.Text.Internal.Private (runText) -import Data.Text.Internal.Unsafe.Char (ord, unsafeChr, unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) -import qualified Data.Text.Array as A -import qualified Data.Text.Internal.Fusion.Common as S -import Data.Text.Internal.Fusion.Types -import Data.Text.Internal.Fusion.Size -import qualified Data.Text.Internal as I -import qualified Data.Text.Internal.Encoding.Utf16 as U16 - -default(Int) - --- | /O(n)/ Convert a 'Text' into a 'Stream Char'. -stream :: Text -> Stream Char -stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 1) len) - where - !end = off+len - next !i - | i >= end = Done - | n >= 0xD800 && n <= 0xDBFF = Yield (U16.chr2 n n2) (i + 2) - | otherwise = Yield (unsafeChr n) (i + 1) - where - n = A.unsafeIndex arr i - n2 = A.unsafeIndex arr (i + 1) -{-# INLINE [0] stream #-} - --- | /O(n)/ Convert a 'Text' into a 'Stream Char', but iterate --- backwards. -reverseStream :: Text -> Stream Char -reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 1) len) - where - {-# INLINE next #-} - next !i - | i < off = Done - | n >= 0xDC00 && n <= 0xDFFF = Yield (U16.chr2 n2 n) (i - 2) - | otherwise = Yield (unsafeChr n) (i - 1) - where - n = A.unsafeIndex arr i - n2 = A.unsafeIndex arr (i - 1) -{-# INLINE [0] reverseStream #-} - --- | /O(n)/ Convert a 'Stream Char' into a 'Text'. -unstream :: Stream Char -> Text -unstream (Stream next0 s0 len) = runText $ \done -> do - -- Before encoding each char we perform a buffer realloc check assuming - -- worst case encoding size of two 16-bit units for the char. Just add an - -- extra space to the buffer so that we do not end up reallocating even when - -- all the chars are encoded as single unit. - let mlen = upperBound 4 len + 1 - arr0 <- A.new mlen - let outer !arr !maxi = encode - where - -- keep the common case loop as small as possible - encode !si !di = - case next0 si of - Done -> done arr di - Skip si' -> encode si' di - Yield c si' - -- simply check for the worst case - | maxi < di + 1 -> realloc si di - | otherwise -> do - n <- unsafeWrite arr di c - encode si' (di + n) - - -- keep uncommon case separate from the common case code - {-# NOINLINE realloc #-} - realloc !si !di = do - let newlen = (maxi + 1) * 2 - arr' <- A.new newlen - A.copyM arr' 0 arr 0 di - outer arr' (newlen - 1) si di - - outer arr0 (mlen - 1) s0 0 -{-# INLINE [0] unstream #-} -{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-} - - --- ---------------------------------------------------------------------------- --- * Basic stream functions - -length :: Stream Char -> Int -length = S.lengthI -{-# INLINE[0] length #-} - --- | /O(n)/ Reverse the characters of a string. -reverse :: Stream Char -> Text -reverse (Stream next s len0) - | isEmpty len0 = I.empty - | otherwise = I.text arr off' len' - where - len0' = upperBound 4 (larger len0 4) - (arr, (off', len')) = A.run2 (A.new len0' >>= loop s (len0'-1) len0') - loop !s0 !i !len marr = - case next s0 of - Done -> return (marr, (j, len-j)) - where j = i + 1 - Skip s1 -> loop s1 i len marr - Yield x s1 | i < least -> {-# SCC "reverse/resize" #-} do - let newLen = len `shiftL` 1 - marr' <- A.new newLen - A.copyM marr' (newLen-len) marr 0 len - write s1 (len+i) newLen marr' - | otherwise -> write s1 i len marr - where n = ord x - least | n < 0x10000 = 0 - | otherwise = 1 - m = n - 0x10000 - lo = fromIntegral $ (m `shiftR` 10) + 0xD800 - hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00 - write t j l mar - | n < 0x10000 = do - A.unsafeWrite mar j (fromIntegral n) - loop t (j-1) l mar - | otherwise = do - A.unsafeWrite mar (j-1) lo - A.unsafeWrite mar j hi - loop t (j-2) l mar -{-# INLINE [0] reverse #-} - --- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with --- the input and result reversed. -reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char -reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low - where - {-# INLINE next #-} - next (Scan1 z s) = Yield z (Scan2 z s) - next (Scan2 z s) = case next0 s of - Yield x s' -> let !x' = f x z - in Yield x' (Scan2 x' s') - Skip s' -> Skip (Scan2 z s') - Done -> Done -{-# INLINE reverseScanr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed --- value. However, the length of the result is limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the length of the result is known. -unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Stream Char -unfoldrN n = S.unfoldrNI n -{-# INLINE [0] unfoldrN #-} - -------------------------------------------------------------------------------- --- ** Indexing streams - --- | /O(n)/ stream index (subscript) operator, starting from 0. -index :: Stream Char -> Int -> Char -index = S.indexI -{-# INLINE [0] index #-} - --- | The 'findIndex' function takes a predicate and a stream and --- returns the index of the first element in the stream --- satisfying the predicate. -findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int -findIndex = S.findIndexI -{-# INLINE [0] findIndex #-} - --- | /O(n)/ The 'count' function returns the number of times the query --- element appears in the given stream. -countChar :: Char -> Stream Char -> Int -countChar = S.countCharI -{-# INLINE [0] countChar #-} - --- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a --- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. -mapAccumL :: (a -> Char -> (a,Char)) -> a -> Stream Char -> (a, Text) -mapAccumL f z0 (Stream next0 s0 len) = (nz, I.text na 0 nl) - where - (na,(nz,nl)) = A.run2 (A.new mlen >>= \arr -> outer arr mlen z0 s0 0) - where mlen = upperBound 4 len - outer arr top = loop - where - loop !z !s !i = - case next0 s of - Done -> return (arr, (z,i)) - Skip s' -> loop z s' i - Yield x s' - | j >= top -> {-# SCC "mapAccumL/resize" #-} do - let top' = (top + 1) `shiftL` 1 - arr' <- A.new top' - A.copyM arr' 0 arr 0 top - outer arr' top' z s i - | otherwise -> do d <- unsafeWrite arr i c - loop z' s' (i+d) - where (z',c) = f z x - j | ord c < 0x10000 = i - | otherwise = i + 1 -{-# INLINE [0] mapAccumL #-} diff --git a/Data/Text/Internal/Fusion/CaseMapping.hs b/Data/Text/Internal/Fusion/CaseMapping.hs deleted file mode 100644 --- a/Data/Text/Internal/Fusion/CaseMapping.hs +++ /dev/null @@ -1,842 +0,0 @@ -{-# LANGUAGE Rank2Types #-} --- AUTOMATICALLY GENERATED - DO NOT EDIT --- Generated by scripts/CaseMapping.hs --- CaseFolding-8.0.0.txt --- Date: 2015-01-13, 18:16:36 GMT [MD] --- SpecialCasing-8.0.0.txt --- Date: 2014-12-16, 23:08:04 GMT [MD] - -module Data.Text.Internal.Fusion.CaseMapping where -import Data.Char -import Data.Text.Internal.Fusion.Types - -upperMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE upperMapping #-} --- LATIN SMALL LETTER SHARP S -upperMapping '\x00df' s = Yield '\x0053' (CC s '\x0053' '\x0000') --- LATIN SMALL LIGATURE FF -upperMapping '\xfb00' s = Yield '\x0046' (CC s '\x0046' '\x0000') --- LATIN SMALL LIGATURE FI -upperMapping '\xfb01' s = Yield '\x0046' (CC s '\x0049' '\x0000') --- LATIN SMALL LIGATURE FL -upperMapping '\xfb02' s = Yield '\x0046' (CC s '\x004c' '\x0000') --- LATIN SMALL LIGATURE FFI -upperMapping '\xfb03' s = Yield '\x0046' (CC s '\x0046' '\x0049') --- LATIN SMALL LIGATURE FFL -upperMapping '\xfb04' s = Yield '\x0046' (CC s '\x0046' '\x004c') --- LATIN SMALL LIGATURE LONG S T -upperMapping '\xfb05' s = Yield '\x0053' (CC s '\x0054' '\x0000') --- LATIN SMALL LIGATURE ST -upperMapping '\xfb06' s = Yield '\x0053' (CC s '\x0054' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -upperMapping '\x0587' s = Yield '\x0535' (CC s '\x0552' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -upperMapping '\xfb13' s = Yield '\x0544' (CC s '\x0546' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -upperMapping '\xfb14' s = Yield '\x0544' (CC s '\x0535' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -upperMapping '\xfb15' s = Yield '\x0544' (CC s '\x053b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -upperMapping '\xfb16' s = Yield '\x054e' (CC s '\x0546' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -upperMapping '\xfb17' s = Yield '\x0544' (CC s '\x053d' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -upperMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -upperMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -upperMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- LATIN SMALL LETTER J WITH CARON -upperMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -upperMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -upperMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -upperMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -upperMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -upperMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -upperMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -upperMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -upperMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -upperMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -upperMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -upperMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -upperMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -upperMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -upperMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -upperMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -upperMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -upperMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -upperMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -upperMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f80' s = Yield '\x1f08' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f81' s = Yield '\x1f09' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f82' s = Yield '\x1f0a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f83' s = Yield '\x1f0b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f84' s = Yield '\x1f0c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f85' s = Yield '\x1f0d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f86' s = Yield '\x1f0e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f87' s = Yield '\x1f0f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f88' s = Yield '\x1f08' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f89' s = Yield '\x1f09' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8a' s = Yield '\x1f0a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8b' s = Yield '\x1f0b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8c' s = Yield '\x1f0c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8d' s = Yield '\x1f0d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8e' s = Yield '\x1f0e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8f' s = Yield '\x1f0f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f90' s = Yield '\x1f28' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f91' s = Yield '\x1f29' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f92' s = Yield '\x1f2a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f93' s = Yield '\x1f2b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f94' s = Yield '\x1f2c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f95' s = Yield '\x1f2d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f96' s = Yield '\x1f2e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f97' s = Yield '\x1f2f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f98' s = Yield '\x1f28' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f99' s = Yield '\x1f29' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9a' s = Yield '\x1f2a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9b' s = Yield '\x1f2b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9c' s = Yield '\x1f2c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9d' s = Yield '\x1f2d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9e' s = Yield '\x1f2e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9f' s = Yield '\x1f2f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1fa0' s = Yield '\x1f68' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1fa1' s = Yield '\x1f69' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa2' s = Yield '\x1f6a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa3' s = Yield '\x1f6b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa4' s = Yield '\x1f6c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa5' s = Yield '\x1f6d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa6' s = Yield '\x1f6e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa7' s = Yield '\x1f6f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1fa8' s = Yield '\x1f68' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1fa9' s = Yield '\x1f69' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1faa' s = Yield '\x1f6a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1fab' s = Yield '\x1f6b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fac' s = Yield '\x1f6c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fad' s = Yield '\x1f6d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1fae' s = Yield '\x1f6e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1faf' s = Yield '\x1f6f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -upperMapping '\x1fb3' s = Yield '\x0391' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -upperMapping '\x1fbc' s = Yield '\x0391' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -upperMapping '\x1fc3' s = Yield '\x0397' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -upperMapping '\x1fcc' s = Yield '\x0397' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -upperMapping '\x1ff3' s = Yield '\x03a9' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -upperMapping '\x1ffc' s = Yield '\x03a9' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0399') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0399') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0399') -upperMapping c s = Yield (toUpper c) (CC s '\0' '\0') -lowerMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE lowerMapping #-} --- LATIN CAPITAL LETTER I WITH DOT ABOVE -lowerMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000') -lowerMapping c s = Yield (toLower c) (CC s '\0' '\0') -titleMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE titleMapping #-} --- LATIN SMALL LETTER SHARP S -titleMapping '\x00df' s = Yield '\x0053' (CC s '\x0073' '\x0000') --- LATIN SMALL LIGATURE FF -titleMapping '\xfb00' s = Yield '\x0046' (CC s '\x0066' '\x0000') --- LATIN SMALL LIGATURE FI -titleMapping '\xfb01' s = Yield '\x0046' (CC s '\x0069' '\x0000') --- LATIN SMALL LIGATURE FL -titleMapping '\xfb02' s = Yield '\x0046' (CC s '\x006c' '\x0000') --- LATIN SMALL LIGATURE FFI -titleMapping '\xfb03' s = Yield '\x0046' (CC s '\x0066' '\x0069') --- LATIN SMALL LIGATURE FFL -titleMapping '\xfb04' s = Yield '\x0046' (CC s '\x0066' '\x006c') --- LATIN SMALL LIGATURE LONG S T -titleMapping '\xfb05' s = Yield '\x0053' (CC s '\x0074' '\x0000') --- LATIN SMALL LIGATURE ST -titleMapping '\xfb06' s = Yield '\x0053' (CC s '\x0074' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -titleMapping '\x0587' s = Yield '\x0535' (CC s '\x0582' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -titleMapping '\xfb13' s = Yield '\x0544' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -titleMapping '\xfb14' s = Yield '\x0544' (CC s '\x0565' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -titleMapping '\xfb15' s = Yield '\x0544' (CC s '\x056b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -titleMapping '\xfb16' s = Yield '\x054e' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -titleMapping '\xfb17' s = Yield '\x0544' (CC s '\x056d' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -titleMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -titleMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -titleMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- LATIN SMALL LETTER J WITH CARON -titleMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -titleMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -titleMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -titleMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -titleMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -titleMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -titleMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -titleMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -titleMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -titleMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -titleMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -titleMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -titleMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -titleMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -titleMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -titleMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -titleMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -titleMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -titleMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -titleMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -titleMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -titleMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0345') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0345') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0345') -titleMapping c s = Yield (toTitle c) (CC s '\0' '\0') -foldMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE foldMapping #-} --- MICRO SIGN -foldMapping '\x00b5' s = Yield '\x03bc' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER SHARP S -foldMapping '\x00df' s = Yield '\x0073' (CC s '\x0073' '\x0000') --- LATIN CAPITAL LETTER I WITH DOT ABOVE -foldMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -foldMapping '\x0149' s = Yield '\x02bc' (CC s '\x006e' '\x0000') --- LATIN SMALL LETTER LONG S -foldMapping '\x017f' s = Yield '\x0073' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER J WITH CARON -foldMapping '\x01f0' s = Yield '\x006a' (CC s '\x030c' '\x0000') --- COMBINING GREEK YPOGEGRAMMENI -foldMapping '\x0345' s = Yield '\x03b9' (CC s '\x0000' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -foldMapping '\x0390' s = Yield '\x03b9' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -foldMapping '\x03b0' s = Yield '\x03c5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER FINAL SIGMA -foldMapping '\x03c2' s = Yield '\x03c3' (CC s '\x0000' '\x0000') --- GREEK BETA SYMBOL -foldMapping '\x03d0' s = Yield '\x03b2' (CC s '\x0000' '\x0000') --- GREEK THETA SYMBOL -foldMapping '\x03d1' s = Yield '\x03b8' (CC s '\x0000' '\x0000') --- GREEK PHI SYMBOL -foldMapping '\x03d5' s = Yield '\x03c6' (CC s '\x0000' '\x0000') --- GREEK PI SYMBOL -foldMapping '\x03d6' s = Yield '\x03c0' (CC s '\x0000' '\x0000') --- GREEK KAPPA SYMBOL -foldMapping '\x03f0' s = Yield '\x03ba' (CC s '\x0000' '\x0000') --- GREEK RHO SYMBOL -foldMapping '\x03f1' s = Yield '\x03c1' (CC s '\x0000' '\x0000') --- GREEK LUNATE EPSILON SYMBOL -foldMapping '\x03f5' s = Yield '\x03b5' (CC s '\x0000' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -foldMapping '\x0587' s = Yield '\x0565' (CC s '\x0582' '\x0000') --- CHEROKEE SMALL LETTER YE -foldMapping '\x13f8' s = Yield '\x13f0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YI -foldMapping '\x13f9' s = Yield '\x13f1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YO -foldMapping '\x13fa' s = Yield '\x13f2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YU -foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YV -foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MV -foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -foldMapping '\x1e97' s = Yield '\x0074' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -foldMapping '\x1e98' s = Yield '\x0077' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -foldMapping '\x1e99' s = Yield '\x0079' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -foldMapping '\x1e9a' s = Yield '\x0061' (CC s '\x02be' '\x0000') --- LATIN SMALL LETTER LONG S WITH DOT ABOVE -foldMapping '\x1e9b' s = Yield '\x1e61' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER SHARP S -foldMapping '\x1e9e' s = Yield '\x0073' (CC s '\x0073' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -foldMapping '\x1f50' s = Yield '\x03c5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -foldMapping '\x1f52' s = Yield '\x03c5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -foldMapping '\x1f54' s = Yield '\x03c5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -foldMapping '\x1f56' s = Yield '\x03c5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f80' s = Yield '\x1f00' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f81' s = Yield '\x1f01' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f82' s = Yield '\x1f02' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f83' s = Yield '\x1f03' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f84' s = Yield '\x1f04' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f85' s = Yield '\x1f05' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f86' s = Yield '\x1f06' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f87' s = Yield '\x1f07' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f88' s = Yield '\x1f00' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f89' s = Yield '\x1f01' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8a' s = Yield '\x1f02' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8b' s = Yield '\x1f03' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8c' s = Yield '\x1f04' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8d' s = Yield '\x1f05' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8e' s = Yield '\x1f06' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8f' s = Yield '\x1f07' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f90' s = Yield '\x1f20' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f91' s = Yield '\x1f21' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f92' s = Yield '\x1f22' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f93' s = Yield '\x1f23' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f94' s = Yield '\x1f24' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f95' s = Yield '\x1f25' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f96' s = Yield '\x1f26' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f97' s = Yield '\x1f27' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f98' s = Yield '\x1f20' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f99' s = Yield '\x1f21' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9a' s = Yield '\x1f22' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9b' s = Yield '\x1f23' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9c' s = Yield '\x1f24' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9d' s = Yield '\x1f25' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9e' s = Yield '\x1f26' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9f' s = Yield '\x1f27' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1fa0' s = Yield '\x1f60' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1fa1' s = Yield '\x1f61' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa2' s = Yield '\x1f62' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa3' s = Yield '\x1f63' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa4' s = Yield '\x1f64' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa5' s = Yield '\x1f65' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa6' s = Yield '\x1f66' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa7' s = Yield '\x1f67' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1fa8' s = Yield '\x1f60' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1fa9' s = Yield '\x1f61' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1faa' s = Yield '\x1f62' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1fab' s = Yield '\x1f63' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fac' s = Yield '\x1f64' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fad' s = Yield '\x1f65' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1fae' s = Yield '\x1f66' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1faf' s = Yield '\x1f67' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fb2' s = Yield '\x1f70' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -foldMapping '\x1fb3' s = Yield '\x03b1' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fb4' s = Yield '\x03ac' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -foldMapping '\x1fb6' s = Yield '\x03b1' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fb7' s = Yield '\x03b1' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -foldMapping '\x1fbc' s = Yield '\x03b1' (CC s '\x03b9' '\x0000') --- GREEK PROSGEGRAMMENI -foldMapping '\x1fbe' s = Yield '\x03b9' (CC s '\x0000' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fc2' s = Yield '\x1f74' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -foldMapping '\x1fc3' s = Yield '\x03b7' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fc4' s = Yield '\x03ae' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -foldMapping '\x1fc6' s = Yield '\x03b7' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fc7' s = Yield '\x03b7' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -foldMapping '\x1fcc' s = Yield '\x03b7' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -foldMapping '\x1fd2' s = Yield '\x03b9' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -foldMapping '\x1fd3' s = Yield '\x03b9' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -foldMapping '\x1fd6' s = Yield '\x03b9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fd7' s = Yield '\x03b9' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -foldMapping '\x1fe2' s = Yield '\x03c5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -foldMapping '\x1fe3' s = Yield '\x03c5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -foldMapping '\x1fe4' s = Yield '\x03c1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -foldMapping '\x1fe6' s = Yield '\x03c5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fe7' s = Yield '\x03c5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1ff2' s = Yield '\x1f7c' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -foldMapping '\x1ff3' s = Yield '\x03c9' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1ff4' s = Yield '\x03ce' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000') --- LATIN CAPITAL LETTER J WITH CROSSED-TAIL -foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER CHI -foldMapping '\xa7b3' s = Yield '\xab53' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER BETA -foldMapping '\xa7b4' s = Yield '\xa7b5' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER OMEGA -foldMapping '\xa7b6' s = Yield '\xa7b7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER A -foldMapping '\xab70' s = Yield '\x13a0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER E -foldMapping '\xab71' s = Yield '\x13a1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER I -foldMapping '\xab72' s = Yield '\x13a2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER O -foldMapping '\xab73' s = Yield '\x13a3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER U -foldMapping '\xab74' s = Yield '\x13a4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER V -foldMapping '\xab75' s = Yield '\x13a5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GA -foldMapping '\xab76' s = Yield '\x13a6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER KA -foldMapping '\xab77' s = Yield '\x13a7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GE -foldMapping '\xab78' s = Yield '\x13a8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GI -foldMapping '\xab79' s = Yield '\x13a9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GO -foldMapping '\xab7a' s = Yield '\x13aa' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GU -foldMapping '\xab7b' s = Yield '\x13ab' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GV -foldMapping '\xab7c' s = Yield '\x13ac' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HA -foldMapping '\xab7d' s = Yield '\x13ad' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HE -foldMapping '\xab7e' s = Yield '\x13ae' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HI -foldMapping '\xab7f' s = Yield '\x13af' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HO -foldMapping '\xab80' s = Yield '\x13b0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HU -foldMapping '\xab81' s = Yield '\x13b1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HV -foldMapping '\xab82' s = Yield '\x13b2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LA -foldMapping '\xab83' s = Yield '\x13b3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LE -foldMapping '\xab84' s = Yield '\x13b4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LI -foldMapping '\xab85' s = Yield '\x13b5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LO -foldMapping '\xab86' s = Yield '\x13b6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LU -foldMapping '\xab87' s = Yield '\x13b7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LV -foldMapping '\xab88' s = Yield '\x13b8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MA -foldMapping '\xab89' s = Yield '\x13b9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER ME -foldMapping '\xab8a' s = Yield '\x13ba' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MI -foldMapping '\xab8b' s = Yield '\x13bb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MO -foldMapping '\xab8c' s = Yield '\x13bc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MU -foldMapping '\xab8d' s = Yield '\x13bd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NA -foldMapping '\xab8e' s = Yield '\x13be' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HNA -foldMapping '\xab8f' s = Yield '\x13bf' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NAH -foldMapping '\xab90' s = Yield '\x13c0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NE -foldMapping '\xab91' s = Yield '\x13c1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NI -foldMapping '\xab92' s = Yield '\x13c2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NO -foldMapping '\xab93' s = Yield '\x13c3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NU -foldMapping '\xab94' s = Yield '\x13c4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NV -foldMapping '\xab95' s = Yield '\x13c5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUA -foldMapping '\xab96' s = Yield '\x13c6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUE -foldMapping '\xab97' s = Yield '\x13c7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUI -foldMapping '\xab98' s = Yield '\x13c8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUO -foldMapping '\xab99' s = Yield '\x13c9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUU -foldMapping '\xab9a' s = Yield '\x13ca' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUV -foldMapping '\xab9b' s = Yield '\x13cb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SA -foldMapping '\xab9c' s = Yield '\x13cc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER S -foldMapping '\xab9d' s = Yield '\x13cd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SE -foldMapping '\xab9e' s = Yield '\x13ce' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SI -foldMapping '\xab9f' s = Yield '\x13cf' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SO -foldMapping '\xaba0' s = Yield '\x13d0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SU -foldMapping '\xaba1' s = Yield '\x13d1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SV -foldMapping '\xaba2' s = Yield '\x13d2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DA -foldMapping '\xaba3' s = Yield '\x13d3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TA -foldMapping '\xaba4' s = Yield '\x13d4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DE -foldMapping '\xaba5' s = Yield '\x13d5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TE -foldMapping '\xaba6' s = Yield '\x13d6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DI -foldMapping '\xaba7' s = Yield '\x13d7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TI -foldMapping '\xaba8' s = Yield '\x13d8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DO -foldMapping '\xaba9' s = Yield '\x13d9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DU -foldMapping '\xabaa' s = Yield '\x13da' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DV -foldMapping '\xabab' s = Yield '\x13db' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DLA -foldMapping '\xabac' s = Yield '\x13dc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLA -foldMapping '\xabad' s = Yield '\x13dd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLE -foldMapping '\xabae' s = Yield '\x13de' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLI -foldMapping '\xabaf' s = Yield '\x13df' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLO -foldMapping '\xabb0' s = Yield '\x13e0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLU -foldMapping '\xabb1' s = Yield '\x13e1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLV -foldMapping '\xabb2' s = Yield '\x13e2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSA -foldMapping '\xabb3' s = Yield '\x13e3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSE -foldMapping '\xabb4' s = Yield '\x13e4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSI -foldMapping '\xabb5' s = Yield '\x13e5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSO -foldMapping '\xabb6' s = Yield '\x13e6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSU -foldMapping '\xabb7' s = Yield '\x13e7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSV -foldMapping '\xabb8' s = Yield '\x13e8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WA -foldMapping '\xabb9' s = Yield '\x13e9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WE -foldMapping '\xabba' s = Yield '\x13ea' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WI -foldMapping '\xabbb' s = Yield '\x13eb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WO -foldMapping '\xabbc' s = Yield '\x13ec' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WU -foldMapping '\xabbd' s = Yield '\x13ed' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WV -foldMapping '\xabbe' s = Yield '\x13ee' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YA -foldMapping '\xabbf' s = Yield '\x13ef' (CC s '\x0000' '\x0000') --- LATIN SMALL LIGATURE FF -foldMapping '\xfb00' s = Yield '\x0066' (CC s '\x0066' '\x0000') --- LATIN SMALL LIGATURE FI -foldMapping '\xfb01' s = Yield '\x0066' (CC s '\x0069' '\x0000') --- LATIN SMALL LIGATURE FL -foldMapping '\xfb02' s = Yield '\x0066' (CC s '\x006c' '\x0000') --- LATIN SMALL LIGATURE FFI -foldMapping '\xfb03' s = Yield '\x0066' (CC s '\x0066' '\x0069') --- LATIN SMALL LIGATURE FFL -foldMapping '\xfb04' s = Yield '\x0066' (CC s '\x0066' '\x006c') --- LATIN SMALL LIGATURE LONG S T -foldMapping '\xfb05' s = Yield '\x0073' (CC s '\x0074' '\x0000') --- LATIN SMALL LIGATURE ST -foldMapping '\xfb06' s = Yield '\x0073' (CC s '\x0074' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -foldMapping '\xfb13' s = Yield '\x0574' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -foldMapping '\xfb14' s = Yield '\x0574' (CC s '\x0565' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER A -foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER AA -foldMapping '\x10c81' s = Yield '\x10cc1' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EB -foldMapping '\x10c82' s = Yield '\x10cc2' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER AMB -foldMapping '\x10c83' s = Yield '\x10cc3' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EC -foldMapping '\x10c84' s = Yield '\x10cc4' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ENC -foldMapping '\x10c85' s = Yield '\x10cc5' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ECS -foldMapping '\x10c86' s = Yield '\x10cc6' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ED -foldMapping '\x10c87' s = Yield '\x10cc7' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER AND -foldMapping '\x10c88' s = Yield '\x10cc8' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER E -foldMapping '\x10c89' s = Yield '\x10cc9' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER CLOSE E -foldMapping '\x10c8a' s = Yield '\x10cca' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EE -foldMapping '\x10c8b' s = Yield '\x10ccb' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EF -foldMapping '\x10c8c' s = Yield '\x10ccc' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EG -foldMapping '\x10c8d' s = Yield '\x10ccd' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EGY -foldMapping '\x10c8e' s = Yield '\x10cce' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EH -foldMapping '\x10c8f' s = Yield '\x10ccf' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER I -foldMapping '\x10c90' s = Yield '\x10cd0' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER II -foldMapping '\x10c91' s = Yield '\x10cd1' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EJ -foldMapping '\x10c92' s = Yield '\x10cd2' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EK -foldMapping '\x10c93' s = Yield '\x10cd3' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER AK -foldMapping '\x10c94' s = Yield '\x10cd4' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER UNK -foldMapping '\x10c95' s = Yield '\x10cd5' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EL -foldMapping '\x10c96' s = Yield '\x10cd6' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ELY -foldMapping '\x10c97' s = Yield '\x10cd7' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EM -foldMapping '\x10c98' s = Yield '\x10cd8' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EN -foldMapping '\x10c99' s = Yield '\x10cd9' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ENY -foldMapping '\x10c9a' s = Yield '\x10cda' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER O -foldMapping '\x10c9b' s = Yield '\x10cdb' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER OO -foldMapping '\x10c9c' s = Yield '\x10cdc' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE -foldMapping '\x10c9d' s = Yield '\x10cdd' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE -foldMapping '\x10c9e' s = Yield '\x10cde' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER OEE -foldMapping '\x10c9f' s = Yield '\x10cdf' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EP -foldMapping '\x10ca0' s = Yield '\x10ce0' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EMP -foldMapping '\x10ca1' s = Yield '\x10ce1' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ER -foldMapping '\x10ca2' s = Yield '\x10ce2' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER SHORT ER -foldMapping '\x10ca3' s = Yield '\x10ce3' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ES -foldMapping '\x10ca4' s = Yield '\x10ce4' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ESZ -foldMapping '\x10ca5' s = Yield '\x10ce5' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ET -foldMapping '\x10ca6' s = Yield '\x10ce6' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ENT -foldMapping '\x10ca7' s = Yield '\x10ce7' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ETY -foldMapping '\x10ca8' s = Yield '\x10ce8' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ECH -foldMapping '\x10ca9' s = Yield '\x10ce9' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER U -foldMapping '\x10caa' s = Yield '\x10cea' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER UU -foldMapping '\x10cab' s = Yield '\x10ceb' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE -foldMapping '\x10cac' s = Yield '\x10cec' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE -foldMapping '\x10cad' s = Yield '\x10ced' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EV -foldMapping '\x10cae' s = Yield '\x10cee' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EZ -foldMapping '\x10caf' s = Yield '\x10cef' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER EZS -foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN -foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000') --- OLD HUNGARIAN CAPITAL LETTER US -foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000') -foldMapping c s = Yield (toLower c) (CC s '\0' '\0') diff --git a/Data/Text/Internal/Fusion/Common.hs b/Data/Text/Internal/Fusion/Common.hs deleted file mode 100644 --- a/Data/Text/Internal/Fusion/Common.hs +++ /dev/null @@ -1,939 +0,0 @@ -{-# LANGUAGE BangPatterns, MagicHash, Rank2Types #-} --- | --- Module : Data.Text.Internal.Fusion.Common --- Copyright : (c) Bryan O'Sullivan 2009, 2012 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Common stream fusion functionality for text. - -module Data.Text.Internal.Fusion.Common - ( - -- * Creation and elimination - singleton - , streamList - , unstreamList - , streamCString# - - -- * Basic interface - , cons - , snoc - , append - , head - , uncons - , last - , tail - , init - , null - , lengthI - , compareLengthI - , isSingleton - - -- * Transformations - , map - , intercalate - , intersperse - - -- ** Case conversion - -- $case - , toCaseFold - , toLower - , toTitle - , toUpper - - -- ** Justification - , justifyLeftI - - -- * Folds - , foldl - , foldl' - , foldl1 - , foldl1' - , foldr - , foldr1 - - -- ** Special folds - , concat - , concatMap - , any - , all - , maximum - , minimum - - -- * Construction - -- ** Scans - , scanl - - -- ** Generation and unfolding - , replicateCharI - , replicateI - , unfoldr - , unfoldrNI - - -- * Substrings - -- ** Breaking strings - , take - , drop - , takeWhile - , dropWhile - - -- * Predicates - , isPrefixOf - - -- * Searching - , elem - , filter - - -- * Indexing - , findBy - , indexI - , findIndexI - , countCharI - - -- * Zipping and unzipping - , zipWith - ) where - -import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..), - Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++), - (&&), fromIntegral, otherwise) -import qualified Data.List as L -import qualified Prelude as P -import Data.Bits (shiftL) -import Data.Char (isLetter, isSpace) -import Data.Int (Int64) -import Data.Text.Internal.Fusion.Types -import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping, - upperMapping) -import Data.Text.Internal.Fusion.Size -import GHC.Prim (Addr#, chr#, indexCharOffAddr#, ord#) -import GHC.Types (Char(..), Int(..)) - -singleton :: Char -> Stream Char -singleton c = Stream next False 1 - where next False = Yield c True - next True = Done -{-# INLINE [0] singleton #-} - -streamList :: [a] -> Stream a -{-# INLINE [0] streamList #-} -streamList s = Stream next s unknownSize - where next [] = Done - next (x:xs) = Yield x xs - -unstreamList :: Stream a -> [a] -unstreamList (Stream next s0 _len) = unfold s0 - where unfold !s = case next s of - Done -> [] - Skip s' -> unfold s' - Yield x s' -> x : unfold s' -{-# INLINE [0] unstreamList #-} - -{-# RULES "STREAM streamList/unstreamList fusion" forall s. streamList (unstreamList s) = s #-} - --- | Stream the UTF-8-like packed encoding used by GHC to represent --- constant strings in generated code. --- --- This encoding uses the byte sequence "\xc0\x80" to represent NUL, --- and the string is NUL-terminated. -streamCString# :: Addr# -> Stream Char -streamCString# addr = Stream step 0 unknownSize - where - step !i - | b == 0 = Done - | b <= 0x7f = Yield (C# b#) (i+1) - | b <= 0xdf = let !c = chr $ ((b-0xc0) `shiftL` 6) + next 1 - in Yield c (i+2) - | b <= 0xef = let !c = chr $ ((b-0xe0) `shiftL` 12) + - (next 1 `shiftL` 6) + - next 2 - in Yield c (i+3) - | otherwise = let !c = chr $ ((b-0xf0) `shiftL` 18) + - (next 1 `shiftL` 12) + - (next 2 `shiftL` 6) + - next 3 - in Yield c (i+4) - where b = I# (ord# b#) - next n = I# (ord# (at# (i+n))) - 0x80 - !b# = at# i - at# (I# i#) = indexCharOffAddr# addr i# - chr (I# i#) = C# (chr# i#) -{-# INLINE [0] streamCString# #-} - --- ---------------------------------------------------------------------------- --- * Basic stream functions - -data C s = C0 !s - | C1 !s - --- | /O(n)/ Adds a character to the front of a Stream Char. -cons :: Char -> Stream Char -> Stream Char -cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len+1) - where - next (C1 s) = Yield w (C0 s) - next (C0 s) = case next0 s of - Done -> Done - Skip s' -> Skip (C0 s') - Yield x s' -> Yield x (C0 s') -{-# INLINE [0] cons #-} - -data Snoc a = N - | J !a - --- | /O(n)/ Adds a character to the end of a stream. -snoc :: Stream Char -> Char -> Stream Char -snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+1) - where - next (J xs) = case next0 xs of - Done -> Yield w N - Skip xs' -> Skip (J xs') - Yield x xs' -> Yield x (J xs') - next N = Done -{-# INLINE [0] snoc #-} - -data E l r = L !l - | R !r - --- | /O(n)/ Appends one Stream to the other. -append :: Stream Char -> Stream Char -> Stream Char -append (Stream next0 s01 len1) (Stream next1 s02 len2) = - Stream next (L s01) (len1 + len2) - where - next (L s1) = case next0 s1 of - Done -> Skip (R s02) - Skip s1' -> Skip (L s1') - Yield x s1' -> Yield x (L s1') - next (R s2) = case next1 s2 of - Done -> Done - Skip s2' -> Skip (R s2') - Yield x s2' -> Yield x (R s2') -{-# INLINE [0] append #-} - --- | /O(1)/ Returns the first character of a Text, which must be non-empty. --- Subject to array fusion. -head :: Stream Char -> Char -head (Stream next s0 _len) = loop_head s0 - where - loop_head !s = case next s of - Yield x _ -> x - Skip s' -> loop_head s' - Done -> head_empty -{-# INLINE [0] head #-} - -head_empty :: a -head_empty = streamError "head" "Empty stream" -{-# NOINLINE head_empty #-} - --- | /O(1)/ Returns the first character and remainder of a 'Stream --- Char', or 'Nothing' if empty. Subject to array fusion. -uncons :: Stream Char -> Maybe (Char, Stream Char) -uncons (Stream next s0 len) = loop_uncons s0 - where - loop_uncons !s = case next s of - Yield x s1 -> Just (x, Stream next s1 (len-1)) - Skip s' -> loop_uncons s' - Done -> Nothing -{-# INLINE [0] uncons #-} - --- | /O(n)/ Returns the last character of a 'Stream Char', which must --- be non-empty. -last :: Stream Char -> Char -last (Stream next s0 _len) = loop0_last s0 - where - loop0_last !s = case next s of - Done -> emptyError "last" - Skip s' -> loop0_last s' - Yield x s' -> loop_last x s' - loop_last !x !s = case next s of - Done -> x - Skip s' -> loop_last x s' - Yield x' s' -> loop_last x' s' -{-# INLINE[0] last #-} - --- | /O(1)/ Returns all characters after the head of a Stream Char, which must --- be non-empty. -tail :: Stream Char -> Stream Char -tail (Stream next0 s0 len) = Stream next (C0 s0) (len-1) - where - next (C0 s) = case next0 s of - Done -> emptyError "tail" - Skip s' -> Skip (C0 s') - Yield _ s' -> Skip (C1 s') - next (C1 s) = case next0 s of - Done -> Done - Skip s' -> Skip (C1 s') - Yield x s' -> Yield x (C1 s') -{-# INLINE [0] tail #-} - -data Init s = Init0 !s - | Init1 {-# UNPACK #-} !Char !s - --- | /O(1)/ Returns all but the last character of a Stream Char, which --- must be non-empty. -init :: Stream Char -> Stream Char -init (Stream next0 s0 len) = Stream next (Init0 s0) (len-1) - where - next (Init0 s) = case next0 s of - Done -> emptyError "init" - Skip s' -> Skip (Init0 s') - Yield x s' -> Skip (Init1 x s') - next (Init1 x s) = case next0 s of - Done -> Done - Skip s' -> Skip (Init1 x s') - Yield x' s' -> Yield x (Init1 x' s') -{-# INLINE [0] init #-} - --- | /O(1)/ Tests whether a Stream Char is empty or not. -null :: Stream Char -> Bool -null (Stream next s0 _len) = loop_null s0 - where - loop_null !s = case next s of - Done -> True - Yield _ _ -> False - Skip s' -> loop_null s' -{-# INLINE[0] null #-} - --- | /O(n)/ Returns the number of characters in a string. -lengthI :: Integral a => Stream Char -> a -lengthI (Stream next s0 _len) = loop_length 0 s0 - where - loop_length !z s = case next s of - Done -> z - Skip s' -> loop_length z s' - Yield _ s' -> loop_length (z + 1) s' -{-# INLINE[0] lengthI #-} - --- | /O(n)/ Compares the count of characters in a string to a number. --- Subject to fusion. --- --- This function gives the same answer as comparing against the result --- of 'lengthI', but can short circuit if the count of characters is --- greater than the number or if the stream can't possibly be as long --- as the number supplied, and hence be more efficient. -compareLengthI :: Integral a => Stream Char -> a -> Ordering -compareLengthI (Stream next s0 len) n = - case compareSize len (fromIntegral n) of - Just o -> o - Nothing -> loop_cmp 0 s0 - where - loop_cmp !z s = case next s of - Done -> compare z n - Skip s' -> loop_cmp z s' - Yield _ s' | z > n -> GT - | otherwise -> loop_cmp (z + 1) s' -{-# INLINE[0] compareLengthI #-} - --- | /O(n)/ Indicate whether a string contains exactly one element. -isSingleton :: Stream Char -> Bool -isSingleton (Stream next s0 _len) = loop 0 s0 - where - loop !z s = case next s of - Done -> z == (1::Int) - Skip s' -> loop z s' - Yield _ s' - | z >= 1 -> False - | otherwise -> loop (z+1) s' -{-# INLINE[0] isSingleton #-} - --- ---------------------------------------------------------------------------- --- * Stream transformations - --- | /O(n)/ 'map' @f @xs is the Stream Char obtained by applying @f@ --- to each element of @xs@. -map :: (Char -> Char) -> Stream Char -> Stream Char -map f (Stream next0 s0 len) = Stream next s0 len - where - next !s = case next0 s of - Done -> Done - Skip s' -> Skip s' - Yield x s' -> Yield (f x) s' -{-# INLINE [0] map #-} - -{-# - RULES "STREAM map/map fusion" forall f g s. - map f (map g s) = map (\x -> f (g x)) s - #-} - -data I s = I1 !s - | I2 !s {-# UNPACK #-} !Char - | I3 !s - --- | /O(n)/ Take a character and place it between each of the --- characters of a 'Stream Char'. -intersperse :: Char -> Stream Char -> Stream Char -intersperse c (Stream next0 s0 len) = Stream next (I1 s0) len - where - next (I1 s) = case next0 s of - Done -> Done - Skip s' -> Skip (I1 s') - Yield x s' -> Skip (I2 s' x) - next (I2 s x) = Yield x (I3 s) - next (I3 s) = case next0 s of - Done -> Done - Skip s' -> Skip (I3 s') - Yield x s' -> Yield c (I2 s' x) -{-# INLINE [0] intersperse #-} - --- ---------------------------------------------------------------------------- --- ** Case conversions (folds) - --- $case --- --- With Unicode text, it is incorrect to use combinators like @map --- toUpper@ to case convert each character of a string individually. --- Instead, use the whole-string case conversion functions from this --- module. For correctness in different writing systems, these --- functions may map one input character to two or three output --- characters. - -caseConvert :: (forall s. Char -> s -> Step (CC s) Char) - -> Stream Char -> Stream Char -caseConvert remap (Stream next0 s0 len) = Stream next (CC s0 '\0' '\0') len - where - next (CC s '\0' _) = - case next0 s of - Done -> Done - Skip s' -> Skip (CC s' '\0' '\0') - Yield c s' -> remap c s' - next (CC s a b) = Yield a (CC s b '\0') - --- | /O(n)/ Convert a string to folded case. This function is mainly --- useful for performing caseless (or case insensitive) string --- comparisons. --- --- A string @x@ is a caseless match for a string @y@ if and only if: --- --- @toCaseFold x == toCaseFold y@ --- --- The result string may be longer than the input string, and may --- differ from applying 'toLower' to the input string. For instance, --- the Armenian small ligature men now (U+FB13) is case folded to the --- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is --- case folded to the Greek small letter letter mu (U+03BC) instead of --- itself. -toCaseFold :: Stream Char -> Stream Char -toCaseFold = caseConvert foldMapping -{-# INLINE [0] toCaseFold #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. The result string may be longer than the input string. --- For instance, the German eszett (U+00DF) maps to the two-letter --- sequence SS. -toUpper :: Stream Char -> Stream Char -toUpper = caseConvert upperMapping -{-# INLINE [0] toUpper #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. The result string may be longer than the input string. --- For instance, the Latin capital letter I with dot above (U+0130) --- maps to the sequence Latin small letter i (U+0069) followed by --- combining dot above (U+0307). -toLower :: Stream Char -> Stream Char -toLower = caseConvert lowerMapping -{-# INLINE [0] toLower #-} - --- | /O(n)/ Convert a string to title case, using simple case --- conversion. --- --- The first letter of the input is converted to title case, as is --- every subsequent letter that immediately follows a non-letter. --- Every letter that immediately follows another letter is converted --- to lower case. --- --- The result string may be longer than the input string. For example, --- the Latin small ligature fl (U+FB02) is converted to the --- sequence Latin capital letter F (U+0046) followed by Latin small --- letter l (U+006C). --- --- /Note/: this function does not take language or culture specific --- rules into account. For instance, in English, different style --- guides disagree on whether the book name \"The Hill of the Red --- Fox\" is correctly title cased—but this function will --- capitalize /every/ word. -toTitle :: Stream Char -> Stream Char -toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len - where - next (CC (letter :*: s) '\0' _) = - case next0 s of - Done -> Done - Skip s' -> Skip (CC (letter :*: s') '\0' '\0') - Yield c s' - | nonSpace -> if letter - then lowerMapping c (nonSpace :*: s') - else titleMapping c (letter' :*: s') - | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0') - where nonSpace = P.not (isSpace c) - letter' = isLetter c - next (CC s a b) = Yield a (CC s b '\0') -{-# INLINE [0] toTitle #-} - -data Justify i s = Just1 !i !s - | Just2 !i !s - -justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char -justifyLeftI k c (Stream next0 s0 len) = - Stream next (Just1 0 s0) (larger (fromIntegral k) len) - where - next (Just1 n s) = - case next0 s of - Done -> next (Just2 n s) - Skip s' -> Skip (Just1 n s') - Yield x s' -> Yield x (Just1 (n+1) s') - next (Just2 n s) - | n < k = Yield c (Just2 (n+1) s) - | otherwise = Done - {-# INLINE next #-} -{-# INLINE [0] justifyLeftI #-} - --- ---------------------------------------------------------------------------- --- * Reducing Streams (folds) - --- | foldl, applied to a binary operator, a starting value (typically the --- left-identity of the operator), and a Stream, reduces the Stream using the --- binary operator, from left to right. -foldl :: (b -> Char -> b) -> b -> Stream Char -> b -foldl f z0 (Stream next s0 _len) = loop_foldl z0 s0 - where - loop_foldl z !s = case next s of - Done -> z - Skip s' -> loop_foldl z s' - Yield x s' -> loop_foldl (f z x) s' -{-# INLINE [0] foldl #-} - --- | A strict version of foldl. -foldl' :: (b -> Char -> b) -> b -> Stream Char -> b -foldl' f z0 (Stream next s0 _len) = loop_foldl' z0 s0 - where - loop_foldl' !z !s = case next s of - Done -> z - Skip s' -> loop_foldl' z s' - Yield x s' -> loop_foldl' (f z x) s' -{-# INLINE [0] foldl' #-} - --- | foldl1 is a variant of foldl that has no starting value argument, --- and thus must be applied to non-empty Streams. -foldl1 :: (Char -> Char -> Char) -> Stream Char -> Char -foldl1 f (Stream next s0 _len) = loop0_foldl1 s0 - where - loop0_foldl1 !s = case next s of - Skip s' -> loop0_foldl1 s' - Yield x s' -> loop_foldl1 x s' - Done -> emptyError "foldl1" - loop_foldl1 z !s = case next s of - Done -> z - Skip s' -> loop_foldl1 z s' - Yield x s' -> loop_foldl1 (f z x) s' -{-# INLINE [0] foldl1 #-} - --- | A strict version of foldl1. -foldl1' :: (Char -> Char -> Char) -> Stream Char -> Char -foldl1' f (Stream next s0 _len) = loop0_foldl1' s0 - where - loop0_foldl1' !s = case next s of - Skip s' -> loop0_foldl1' s' - Yield x s' -> loop_foldl1' x s' - Done -> emptyError "foldl1" - loop_foldl1' !z !s = case next s of - Done -> z - Skip s' -> loop_foldl1' z s' - Yield x s' -> loop_foldl1' (f z x) s' -{-# INLINE [0] foldl1' #-} - --- | 'foldr', applied to a binary operator, a starting value (typically the --- right-identity of the operator), and a stream, reduces the stream using the --- binary operator, from right to left. -foldr :: (Char -> b -> b) -> b -> Stream Char -> b -foldr f z (Stream next s0 _len) = loop_foldr s0 - where - loop_foldr !s = case next s of - Done -> z - Skip s' -> loop_foldr s' - Yield x s' -> f x (loop_foldr s') -{-# INLINE [0] foldr #-} - --- | foldr1 is a variant of 'foldr' that has no starting value argument, --- and thus must be applied to non-empty streams. --- Subject to array fusion. -foldr1 :: (Char -> Char -> Char) -> Stream Char -> Char -foldr1 f (Stream next s0 _len) = loop0_foldr1 s0 - where - loop0_foldr1 !s = case next s of - Done -> emptyError "foldr1" - Skip s' -> loop0_foldr1 s' - Yield x s' -> loop_foldr1 x s' - - loop_foldr1 x !s = case next s of - Done -> x - Skip s' -> loop_foldr1 x s' - Yield x' s' -> f x (loop_foldr1 x' s') -{-# INLINE [0] foldr1 #-} - -intercalate :: Stream Char -> [Stream Char] -> Stream Char -intercalate s = concat . (L.intersperse s) -{-# INLINE [0] intercalate #-} - --- ---------------------------------------------------------------------------- --- ** Special folds - --- | /O(n)/ Concatenate a list of streams. Subject to array fusion. -concat :: [Stream Char] -> Stream Char -concat = L.foldr append empty -{-# INLINE [0] concat #-} - --- | Map a function over a stream that results in a stream and concatenate the --- results. -concatMap :: (Char -> Stream Char) -> Stream Char -> Stream Char -concatMap f = foldr (append . f) empty -{-# INLINE [0] concatMap #-} - --- | /O(n)/ any @p @xs determines if any character in the stream --- @xs@ satisfies the predicate @p@. -any :: (Char -> Bool) -> Stream Char -> Bool -any p (Stream next0 s0 _len) = loop_any s0 - where - loop_any !s = case next0 s of - Done -> False - Skip s' -> loop_any s' - Yield x s' | p x -> True - | otherwise -> loop_any s' -{-# INLINE [0] any #-} - --- | /O(n)/ all @p @xs determines if all characters in the 'Text' --- @xs@ satisfy the predicate @p@. -all :: (Char -> Bool) -> Stream Char -> Bool -all p (Stream next0 s0 _len) = loop_all s0 - where - loop_all !s = case next0 s of - Done -> True - Skip s' -> loop_all s' - Yield x s' | p x -> loop_all s' - | otherwise -> False -{-# INLINE [0] all #-} - --- | /O(n)/ maximum returns the maximum value from a stream, which must be --- non-empty. -maximum :: Stream Char -> Char -maximum (Stream next0 s0 _len) = loop0_maximum s0 - where - loop0_maximum !s = case next0 s of - Done -> emptyError "maximum" - Skip s' -> loop0_maximum s' - Yield x s' -> loop_maximum x s' - loop_maximum !z !s = case next0 s of - Done -> z - Skip s' -> loop_maximum z s' - Yield x s' - | x > z -> loop_maximum x s' - | otherwise -> loop_maximum z s' -{-# INLINE [0] maximum #-} - --- | /O(n)/ minimum returns the minimum value from a 'Text', which must be --- non-empty. -minimum :: Stream Char -> Char -minimum (Stream next0 s0 _len) = loop0_minimum s0 - where - loop0_minimum !s = case next0 s of - Done -> emptyError "minimum" - Skip s' -> loop0_minimum s' - Yield x s' -> loop_minimum x s' - loop_minimum !z !s = case next0 s of - Done -> z - Skip s' -> loop_minimum z s' - Yield x s' - | x < z -> loop_minimum x s' - | otherwise -> loop_minimum z s' -{-# INLINE [0] minimum #-} - --- ----------------------------------------------------------------------------- --- * Building streams - -scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char -scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low - where - {-# INLINE next #-} - next (Scan1 z s) = Yield z (Scan2 z s) - next (Scan2 z s) = case next0 s of - Yield x s' -> let !x' = f z x - in Yield x' (Scan2 x' s') - Skip s' -> Skip (Scan2 z s') - Done -> Done -{-# INLINE [0] scanl #-} - --- ----------------------------------------------------------------------------- --- ** Generating and unfolding streams - -replicateCharI :: Integral a => a -> Char -> Stream Char -replicateCharI !n !c - | n < 0 = empty - | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low - where - next !i | i >= n = Done - | otherwise = Yield c (i + 1) -{-# INLINE [0] replicateCharI #-} - -data RI s = RI !s {-# UNPACK #-} !Int64 - -replicateI :: Int64 -> Stream Char -> Stream Char -replicateI n (Stream next0 s0 len) = - Stream next (RI s0 0) (fromIntegral (max 0 n) * len) - where - next (RI s k) - | k >= n = Done - | otherwise = case next0 s of - Done -> Skip (RI s0 (k+1)) - Skip s' -> Skip (RI s' k) - Yield x s' -> Yield x (RI s' k) -{-# INLINE [0] replicateI #-} - --- | /O(n)/, where @n@ is the length of the result. The unfoldr function --- is analogous to the List 'unfoldr'. unfoldr builds a stream --- from a seed value. The function takes the element and returns --- Nothing if it is done producing the stream or returns Just --- (a,b), in which case, a is the next Char in the string, and b is --- the seed value for further production. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char -unfoldr f s0 = Stream next s0 1 -- HINT maybe too low - where - {-# INLINE next #-} - next !s = case f s of - Nothing -> Done - Just (w, s') -> Yield w s' -{-# INLINE [0] unfoldr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrNI' builds a stream from a seed --- value. However, the length of the result is limited by the --- first argument to 'unfoldrNI'. This function is more efficient than --- 'unfoldr' when the length of the result is known. -unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char -unfoldrNI n f s0 | n < 0 = empty - | otherwise = Stream next (0 :*: s0) (fromIntegral (n*2)) -- HINT maybe too high - where - {-# INLINE next #-} - next (z :*: s) = case f s of - Nothing -> Done - Just (w, s') | z >= n -> Done - | otherwise -> Yield w ((z + 1) :*: s') -{-# INLINE unfoldrNI #-} - -------------------------------------------------------------------------------- --- * Substreams - --- | /O(n)/ take n, applied to a stream, returns the prefix of the --- stream of length @n@, or the stream itself if @n@ is greater than the --- length of the stream. -take :: Integral a => a -> Stream Char -> Stream Char -take n0 (Stream next0 s0 len) = - Stream next (n0 :*: s0) (smaller len (fromIntegral (max 0 n0))) - where - {-# INLINE next #-} - next (n :*: s) | n <= 0 = Done - | otherwise = case next0 s of - Done -> Done - Skip s' -> Skip (n :*: s') - Yield x s' -> Yield x ((n-1) :*: s') -{-# INLINE [0] take #-} - -data Drop a s = NS !s - | JS !a !s - --- | /O(n)/ drop n, applied to a stream, returns the suffix of the --- stream after the first @n@ characters, or the empty stream if @n@ --- is greater than the length of the stream. -drop :: Integral a => a -> Stream Char -> Stream Char -drop n0 (Stream next0 s0 len) = - Stream next (JS n0 s0) (len - fromIntegral (max 0 n0)) - where - {-# INLINE next #-} - next (JS n s) - | n <= 0 = Skip (NS s) - | otherwise = case next0 s of - Done -> Done - Skip s' -> Skip (JS n s') - Yield _ s' -> Skip (JS (n-1) s') - next (NS s) = case next0 s of - Done -> Done - Skip s' -> Skip (NS s') - Yield x s' -> Yield x (NS s') -{-# INLINE [0] drop #-} - --- | takeWhile, applied to a predicate @p@ and a stream, returns the --- longest prefix (possibly empty) of elements that satisfy p. -takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char -takeWhile p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high - where - {-# INLINE next #-} - next !s = case next0 s of - Done -> Done - Skip s' -> Skip s' - Yield x s' | p x -> Yield x s' - | otherwise -> Done -{-# INLINE [0] takeWhile #-} - --- | dropWhile @p @xs returns the suffix remaining after takeWhile @p @xs. -dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char -dropWhile p (Stream next0 s0 len) = Stream next (L s0) len -- HINT maybe too high - where - {-# INLINE next #-} - next (L s) = case next0 s of - Done -> Done - Skip s' -> Skip (L s') - Yield x s' | p x -> Skip (L s') - | otherwise -> Yield x (R s') - next (R s) = case next0 s of - Done -> Done - Skip s' -> Skip (R s') - Yield x s' -> Yield x (R s') -{-# INLINE [0] dropWhile #-} - --- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns --- 'True' iff the first is a prefix of the second. -isPrefixOf :: (Eq a) => Stream a -> Stream a -> Bool -isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) - where - loop Done _ = True - loop _ Done = False - loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') - loop (Skip s1') x2 = loop (next1 s1') x2 - loop x1 (Skip s2') = loop x1 (next2 s2') - loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 && - loop (next1 s1') (next2 s2') -{-# INLINE [0] isPrefixOf #-} - --- ---------------------------------------------------------------------------- --- * Searching - -------------------------------------------------------------------------------- --- ** Searching by equality - --- | /O(n)/ elem is the stream membership predicate. -elem :: Char -> Stream Char -> Bool -elem w (Stream next s0 _len) = loop_elem s0 - where - loop_elem !s = case next s of - Done -> False - Skip s' -> loop_elem s' - Yield x s' | x == w -> True - | otherwise -> loop_elem s' -{-# INLINE [0] elem #-} - -------------------------------------------------------------------------------- --- ** Searching with a predicate - --- | /O(n)/ The 'findBy' function takes a predicate and a stream, --- and returns the first element in matching the predicate, or 'Nothing' --- if there is no such element. - -findBy :: (Char -> Bool) -> Stream Char -> Maybe Char -findBy p (Stream next s0 _len) = loop_find s0 - where - loop_find !s = case next s of - Done -> Nothing - Skip s' -> loop_find s' - Yield x s' | p x -> Just x - | otherwise -> loop_find s' -{-# INLINE [0] findBy #-} - --- | /O(n)/ Stream index (subscript) operator, starting from 0. -indexI :: Integral a => Stream Char -> a -> Char -indexI (Stream next s0 _len) n0 - | n0 < 0 = streamError "index" "Negative index" - | otherwise = loop_index n0 s0 - where - loop_index !n !s = case next s of - Done -> streamError "index" "Index too large" - Skip s' -> loop_index n s' - Yield x s' | n == 0 -> x - | otherwise -> loop_index (n-1) s' -{-# INLINE [0] indexI #-} - --- | /O(n)/ 'filter', applied to a predicate and a stream, --- returns a stream containing those characters that satisfy the --- predicate. -filter :: (Char -> Bool) -> Stream Char -> Stream Char -filter p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high - where - next !s = case next0 s of - Done -> Done - Skip s' -> Skip s' - Yield x s' | p x -> Yield x s' - | otherwise -> Skip s' -{-# INLINE [0] filter #-} - -{-# RULES - "STREAM filter/filter fusion" forall p q s. - filter p (filter q s) = filter (\x -> q x && p x) s - #-} - --- | The 'findIndexI' function takes a predicate and a stream and --- returns the index of the first element in the stream satisfying the --- predicate. -findIndexI :: Integral a => (Char -> Bool) -> Stream Char -> Maybe a -findIndexI p s = case findIndicesI p s of - (i:_) -> Just i - _ -> Nothing -{-# INLINE [0] findIndexI #-} - --- | The 'findIndicesI' function takes a predicate and a stream and --- returns all indices of the elements in the stream satisfying the --- predicate. -findIndicesI :: Integral a => (Char -> Bool) -> Stream Char -> [a] -findIndicesI p (Stream next s0 _len) = loop_findIndex 0 s0 - where - loop_findIndex !i !s = case next s of - Done -> [] - Skip s' -> loop_findIndex i s' -- hmm. not caught by QC - Yield x s' | p x -> i : loop_findIndex (i+1) s' - | otherwise -> loop_findIndex (i+1) s' -{-# INLINE [0] findIndicesI #-} - -------------------------------------------------------------------------------- --- * Zipping - --- | Strict triple. -data Zip a b m = Z1 !a !b - | Z2 !a !b !m - --- | zipWith generalises 'zip' by zipping with the function given as --- the first argument, instead of a tupling function. -zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b -zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) = - Stream next (Z1 sa0 sb0) (smaller len1 len2) - where - next (Z1 sa sb) = case next0 sa of - Done -> Done - Skip sa' -> Skip (Z1 sa' sb) - Yield a sa' -> Skip (Z2 sa' sb a) - - next (Z2 sa' sb a) = case next1 sb of - Done -> Done - Skip sb' -> Skip (Z2 sa' sb' a) - Yield b sb' -> Yield (f a b) (Z1 sa' sb') -{-# INLINE [0] zipWith #-} - --- | /O(n)/ The 'countCharI' function returns the number of times the --- query element appears in the given stream. -countCharI :: Integral a => Char -> Stream Char -> a -countCharI a (Stream next s0 _len) = loop 0 s0 - where - loop !i !s = case next s of - Done -> i - Skip s' -> loop i s' - Yield x s' | a == x -> loop (i+1) s' - | otherwise -> loop i s' -{-# INLINE [0] countCharI #-} - -streamError :: String -> String -> a -streamError func msg = P.error $ "Data.Text.Internal.Fusion.Common." ++ func ++ ": " ++ msg - -emptyError :: String -> a -emptyError func = internalError func "Empty input" - -internalError :: String -> a -internalError func = streamError func "Internal error" diff --git a/Data/Text/Internal/Fusion/Size.hs b/Data/Text/Internal/Fusion/Size.hs deleted file mode 100644 --- a/Data/Text/Internal/Fusion/Size.hs +++ /dev/null @@ -1,157 +0,0 @@ -{-# LANGUAGE CPP #-} -{-# OPTIONS_GHC -fno-warn-missing-methods #-} --- | --- Module : Data.Text.Internal.Fusion.Internal --- Copyright : (c) Roman Leshchinskiy 2008, --- (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Size hints. - -module Data.Text.Internal.Fusion.Size - ( - Size - , exactly - , exactSize - , maxSize - , betweenSize - , unknownSize - , smaller - , larger - , upperBound - , lowerBound - , compareSize - , isEmpty - ) where - -import Data.Text.Internal (mul) -#if defined(ASSERTS) -import Control.Exception (assert) -#endif - -data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size. - | Unknown -- ^ Unknown size. - deriving (Eq, Show) - -exactly :: Size -> Maybe Int -exactly (Between na nb) | na == nb = Just na -exactly _ = Nothing -{-# INLINE exactly #-} - -exactSize :: Int -> Size -exactSize n = -#if defined(ASSERTS) - assert (n >= 0) -#endif - Between n n -{-# INLINE exactSize #-} - -maxSize :: Int -> Size -maxSize n = -#if defined(ASSERTS) - assert (n >= 0) -#endif - Between 0 n -{-# INLINE maxSize #-} - -betweenSize :: Int -> Int -> Size -betweenSize m n = -#if defined(ASSERTS) - assert (m >= 0) - assert (n >= m) -#endif - Between m n -{-# INLINE betweenSize #-} - -unknownSize :: Size -unknownSize = Unknown -{-# INLINE unknownSize #-} - -instance Num Size where - (+) = addSize - (-) = subtractSize - (*) = mulSize - - fromInteger = f where f = exactSize . fromInteger - {-# INLINE f #-} - -add :: Int -> Int -> Int -add m n | mn >= 0 = mn - | otherwise = overflowError - where mn = m + n -{-# INLINE add #-} - -addSize :: Size -> Size -> Size -addSize (Between ma mb) (Between na nb) = Between (add ma na) (add mb nb) -addSize _ _ = Unknown -{-# INLINE addSize #-} - -subtractSize :: Size -> Size -> Size -subtractSize (Between ma mb) (Between na nb) = Between (max (ma-nb) 0) (max (mb-na) 0) -subtractSize a@(Between 0 _) Unknown = a -subtractSize (Between _ mb) Unknown = Between 0 mb -subtractSize _ _ = Unknown -{-# INLINE subtractSize #-} - -mulSize :: Size -> Size -> Size -mulSize (Between ma mb) (Between na nb) = Between (mul ma na) (mul mb nb) -mulSize _ _ = Unknown -{-# INLINE mulSize #-} - --- | Minimum of two size hints. -smaller :: Size -> Size -> Size -smaller a@(Between ma mb) b@(Between na nb) - | mb <= na = a - | nb <= ma = b - | otherwise = Between (ma `min` na) (mb `min` nb) -smaller a@(Between 0 _) Unknown = a -smaller (Between _ mb) Unknown = Between 0 mb -smaller Unknown b@(Between 0 _) = b -smaller Unknown (Between _ nb) = Between 0 nb -smaller Unknown Unknown = Unknown -{-# INLINE smaller #-} - --- | Maximum of two size hints. -larger :: Size -> Size -> Size -larger a@(Between ma mb) b@(Between na nb) - | ma >= nb = a - | na >= mb = b - | otherwise = Between (ma `max` na) (mb `max` nb) -larger _ _ = Unknown -{-# INLINE larger #-} - --- | Compute the maximum size from a size hint, if possible. -upperBound :: Int -> Size -> Int -upperBound _ (Between _ n) = n -upperBound k _ = k -{-# INLINE upperBound #-} - --- | Compute the maximum size from a size hint, if possible. -lowerBound :: Int -> Size -> Int -lowerBound _ (Between n _) = n -lowerBound k _ = k -{-# INLINE lowerBound #-} - -compareSize :: Size -> Int -> Maybe Ordering -compareSize (Between ma mb) n - | mb < n = Just LT - | ma > n = Just GT - | ma == n && mb == n = Just EQ -compareSize _ _ = Nothing - - -isEmpty :: Size -> Bool -isEmpty (Between _ n) = n <= 0 -isEmpty _ = False -{-# INLINE isEmpty #-} - -overflowError :: Int -overflowError = error "Data.Text.Internal.Fusion.Size: size overflow" diff --git a/Data/Text/Internal/Fusion/Types.hs b/Data/Text/Internal/Fusion/Types.hs deleted file mode 100644 --- a/Data/Text/Internal/Fusion/Types.hs +++ /dev/null @@ -1,122 +0,0 @@ -{-# LANGUAGE BangPatterns, ExistentialQuantification #-} --- | --- Module : Data.Text.Internal.Fusion.Types --- Copyright : (c) Tom Harper 2008-2009, --- (c) Bryan O'Sullivan 2009, --- (c) Duncan Coutts 2009, --- (c) Jasper Van der Jeugt 2011 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Core stream fusion functionality for text. - -module Data.Text.Internal.Fusion.Types - ( - CC(..) - , PairS(..) - , Scan(..) - , RS(..) - , Step(..) - , Stream(..) - , empty - ) where - -import Data.Text.Internal.Fusion.Size -import Data.Word (Word8) - --- | Specialised tuple for case conversion. -data CC s = CC !s {-# UNPACK #-} !Char {-# UNPACK #-} !Char - --- | Restreaming state. -data RS s - = RS0 !s - | RS1 !s {-# UNPACK #-} !Word8 - | RS2 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 - | RS3 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 - --- | Strict pair. -data PairS a b = !a :*: !b - -- deriving (Eq, Ord, Show) -infixl 2 :*: - --- | An intermediate result in a scan. -data Scan s = Scan1 {-# UNPACK #-} !Char !s - | Scan2 {-# UNPACK #-} !Char !s - --- | Intermediate result in a processing pipeline. -data Step s a = Done - | Skip !s - | Yield !a !s - -{- -instance (Show a) => Show (Step s a) - where show Done = "Done" - show (Skip _) = "Skip" - show (Yield x _) = "Yield " ++ show x --} - -instance (Eq a) => Eq (Stream a) where - (==) = eq - -instance (Ord a) => Ord (Stream a) where - compare = cmp - --- The length hint in a Stream has two roles. If its value is zero, --- we trust it, and treat the stream as empty. Otherwise, we treat it --- as a hint: it should usually be accurate, so we use it when --- unstreaming to decide what size array to allocate. However, the --- unstreaming functions must be able to cope with the hint being too --- small or too large. --- --- The size hint tries to track the UTF-16 code points in a stream, --- but often counts the number of characters instead. It can easily --- undercount if, for instance, a transformed stream contains astral --- plane characters (those above 0x10000). - -data Stream a = - forall s. Stream - (s -> Step s a) -- stepper function - !s -- current state - !Size -- size hint - --- | /O(n)/ Determines if two streams are equal. -eq :: (Eq a) => Stream a -> Stream a -> Bool -eq (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) - where - loop Done Done = True - loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') - loop (Skip s1') x2 = loop (next1 s1') x2 - loop x1 (Skip s2') = loop x1 (next2 s2') - loop Done _ = False - loop _ Done = False - loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 && - loop (next1 s1') (next2 s2') -{-# INLINE [0] eq #-} - -cmp :: (Ord a) => Stream a -> Stream a -> Ordering -cmp (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) - where - loop Done Done = EQ - loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') - loop (Skip s1') x2 = loop (next1 s1') x2 - loop x1 (Skip s2') = loop x1 (next2 s2') - loop Done _ = LT - loop _ Done = GT - loop (Yield x1 s1') (Yield x2 s2') = - case compare x1 x2 of - EQ -> loop (next1 s1') (next2 s2') - other -> other -{-# INLINE [0] cmp #-} - --- | The empty stream. -empty :: Stream a -empty = Stream next () 0 - where next _ = Done -{-# INLINE [0] empty #-} diff --git a/Data/Text/Internal/IO.hs b/Data/Text/Internal/IO.hs deleted file mode 100644 --- a/Data/Text/Internal/IO.hs +++ /dev/null @@ -1,166 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-} --- | --- Module : Data.Text.Internal.IO --- Copyright : (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Simon Marlow --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Low-level support for text I\/O. - -module Data.Text.Internal.IO - ( - hGetLineWith - , readChunk - ) where - -import qualified Control.Exception as E -import Data.IORef (readIORef, writeIORef) -import Data.Text (Text) -import Data.Text.Internal.Fusion (unstream) -import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) -import Data.Text.Internal.Fusion.Size (exactSize, maxSize) -import Data.Text.Unsafe (inlinePerformIO) -import Foreign.Storable (peekElemOff) -import GHC.IO.Buffer (Buffer(..), CharBuffer, RawCharBuffer, bufferAdjustL, - bufferElems, charSize, isEmptyBuffer, readCharBuf, - withRawBuffer, writeCharBuf) -import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_) -import GHC.IO.Handle.Types (Handle__(..), Newline(..)) -import System.IO (Handle) -import System.IO.Error (isEOFError) -import qualified Data.Text as T - --- | Read a single line of input from a handle, constructing a list of --- decoded chunks as we go. When we're done, transform them into the --- destination type. -hGetLineWith :: ([Text] -> t) -> Handle -> IO t -hGetLineWith f h = wantReadableHandle_ "hGetLine" h go - where - go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] - -hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] -hGetLineLoop hh@Handle__{..} = go where - go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do - let findEOL raw r | r == w = return (False, w) - | otherwise = do - (c,r') <- readCharBuf raw r - if c == '\n' - then return (True, r) - else findEOL raw r' - (eol, off) <- findEOL raw0 r0 - (t,r') <- if haInputNL == CRLF - then unpack_nl raw0 r0 off - else do t <- unpack raw0 r0 off - return (t,off) - if eol - then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) - return $ reverse (t:ts) - else do - let buf1 = bufferAdjustL r' buf - maybe_buf <- maybeFillReadBuffer hh buf1 - case maybe_buf of - -- Nothing indicates we caught an EOF, and we may have a - -- partial line to return. - Nothing -> do - -- we reached EOF. There might be a lone \r left - -- in the buffer, so check for that and - -- append it to the line if necessary. - let pre | isEmptyBuffer buf1 = T.empty - | otherwise = T.singleton '\r' - writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } - let str = reverse . filter (not . T.null) $ pre:t:ts - if null str - then ioe_EOF - else return str - Just new_buf -> go (t:ts) new_buf - --- This function is lifted almost verbatim from GHC.IO.Handle.Text. -maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) -maybeFillReadBuffer handle_ buf - = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> - if isEOFError e - then return Nothing - else ioError e - -unpack :: RawCharBuffer -> Int -> Int -> IO Text -unpack !buf !r !w - | charSize /= 4 = sizeError "unpack" - | r >= w = return T.empty - | otherwise = withRawBuffer buf go - where - go pbuf = return $! unstream (Stream next r (exactSize (w-r))) - where - next !i | i >= w = Done - | otherwise = Yield (ix i) (i+1) - ix i = inlinePerformIO $ peekElemOff pbuf i - -unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) -unpack_nl !buf !r !w - | charSize /= 4 = sizeError "unpack_nl" - | r >= w = return (T.empty, 0) - | otherwise = withRawBuffer buf $ go - where - go pbuf = do - let !t = unstream (Stream next r (maxSize (w-r))) - w' = w - 1 - return $ if ix w' == '\r' - then (t,w') - else (t,w) - where - next !i | i >= w = Done - | c == '\r' = let i' = i + 1 - in if i' < w - then if ix i' == '\n' - then Yield '\n' (i+2) - else Yield '\n' i' - else Done - | otherwise = Yield c (i+1) - where c = ix i - ix i = inlinePerformIO $ peekElemOff pbuf i - --- This function is completely lifted from GHC.IO.Handle.Text. -getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer -getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = - case bufferElems buf of - -- buffer empty: read some more - 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf - - -- if the buffer has a single '\r' in it and we're doing newline - -- translation: read some more - 1 | haInputNL == CRLF -> do - (c,_) <- readCharBuf bufRaw bufL - if c == '\r' - then do -- shuffle the '\r' to the beginning. This is only safe - -- if we're about to call readTextDevice, otherwise it - -- would mess up flushCharBuffer. - -- See [note Buffer Flushing], GHC.IO.Handle.Types - _ <- writeCharBuf bufRaw 0 '\r' - let buf' = buf{ bufL=0, bufR=1 } - readTextDevice handle_ buf' - else do - return buf - - -- buffer has some chars in it already: just return it - _otherwise -> {-# SCC "otherwise" #-} return buf - --- | Read a single chunk of strict text from a buffer. Used by both --- the strict and lazy implementations of hGetContents. -readChunk :: Handle__ -> CharBuffer -> IO Text -readChunk hh@Handle__{..} buf = do - buf'@Buffer{..} <- getSomeCharacters hh buf - (t,r) <- if haInputNL == CRLF - then unpack_nl bufRaw bufL bufR - else do t <- unpack bufRaw bufL bufR - return (t,bufR) - writeIORef haCharBuffer (bufferAdjustL r buf') - return t - -sizeError :: String -> a -sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size" diff --git a/Data/Text/Internal/Lazy.hs b/Data/Text/Internal/Lazy.hs deleted file mode 100644 --- a/Data/Text/Internal/Lazy.hs +++ /dev/null @@ -1,119 +0,0 @@ -{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} -{-# OPTIONS_HADDOCK not-home #-} - --- | --- Module : Data.Text.Internal.Lazy --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- A module containing private 'Text' internals. This exposes the --- 'Text' representation and low level construction functions. --- Modules which extend the 'Text' system may need to use this module. - -module Data.Text.Internal.Lazy - ( - Text(..) - , chunk - , empty - , foldrChunks - , foldlChunks - -- * Data type invariant and abstraction functions - - -- $invariant - , strictInvariant - , lazyInvariant - , showStructure - - -- * Chunk allocation sizes - , defaultChunkSize - , smallChunkSize - , chunkOverhead - ) where - -import Data.Text () -import Data.Text.Internal.Unsafe.Shift (shiftL) -import Data.Typeable (Typeable) -import Foreign.Storable (sizeOf) -import qualified Data.Text.Internal as T - -data Text = Empty - | Chunk {-# UNPACK #-} !T.Text Text - deriving (Typeable) - --- $invariant --- --- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or --- consists of non-null 'T.Text's. All functions must preserve this, --- and the QC properties must check this. - --- | Check the invariant strictly. -strictInvariant :: Text -> Bool -strictInvariant Empty = True -strictInvariant x@(Chunk (T.Text _ _ len) cs) - | len > 0 = strictInvariant cs - | otherwise = error $ "Data.Text.Lazy: invariant violation: " - ++ showStructure x - --- | Check the invariant lazily. -lazyInvariant :: Text -> Text -lazyInvariant Empty = Empty -lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) - | len > 0 = Chunk c (lazyInvariant cs) - | otherwise = error $ "Data.Text.Lazy: invariant violation: " - ++ showStructure x - --- | Display the internal structure of a lazy 'Text'. -showStructure :: Text -> String -showStructure Empty = "Empty" -showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" -showStructure (Chunk t ts) = - "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" - --- | Smart constructor for 'Chunk'. Guarantees the data type invariant. -chunk :: T.Text -> Text -> Text -{-# INLINE chunk #-} -chunk t@(T.Text _ _ len) ts | len == 0 = ts - | otherwise = Chunk t ts - --- | Smart constructor for 'Empty'. -empty :: Text -{-# INLINE [0] empty #-} -empty = Empty - --- | Consume the chunks of a lazy 'Text' with a natural right fold. -foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a -foldrChunks f z = go - where go Empty = z - go (Chunk c cs) = f c (go cs) -{-# INLINE foldrChunks #-} - --- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, --- accumulating left fold. -foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a -foldlChunks f z = go z - where go !a Empty = a - go !a (Chunk c cs) = go (f a c) cs -{-# INLINE foldlChunks #-} - --- | Currently set to 16 KiB, less the memory management overhead. -defaultChunkSize :: Int -defaultChunkSize = 16384 - chunkOverhead -{-# INLINE defaultChunkSize #-} - --- | Currently set to 128 bytes, less the memory management overhead. -smallChunkSize :: Int -smallChunkSize = 128 - chunkOverhead -{-# INLINE smallChunkSize #-} - --- | The memory management overhead. Currently this is tuned for GHC only. -chunkOverhead :: Int -chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 -{-# INLINE chunkOverhead #-} diff --git a/Data/Text/Internal/Lazy/Encoding/Fusion.hs b/Data/Text/Internal/Lazy/Encoding/Fusion.hs deleted file mode 100644 --- a/Data/Text/Internal/Lazy/Encoding/Fusion.hs +++ /dev/null @@ -1,324 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, Rank2Types #-} - --- | --- Module : Data.Text.Lazy.Encoding.Fusion --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fusible 'Stream'-oriented functions for converting between lazy --- 'Text' and several common encodings. - -module Data.Text.Internal.Lazy.Encoding.Fusion - ( - -- * Streaming - -- streamASCII - streamUtf8 - , streamUtf16LE - , streamUtf16BE - , streamUtf32LE - , streamUtf32BE - - -- * Unstreaming - , unstream - - , module Data.Text.Internal.Encoding.Fusion.Common - ) where - -import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize) -import qualified Data.ByteString as B -import qualified Data.ByteString.Unsafe as B -import Data.Text.Internal.Encoding.Fusion.Common -import Data.Text.Encoding.Error -import Data.Text.Internal.Fusion (Step(..), Stream(..)) -import Data.Text.Internal.Fusion.Size -import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32) -import Data.Text.Internal.Unsafe.Shift (shiftL) -import Data.Word (Word8, Word16, Word32) -import qualified Data.Text.Internal.Encoding.Utf8 as U8 -import qualified Data.Text.Internal.Encoding.Utf16 as U16 -import qualified Data.Text.Internal.Encoding.Utf32 as U32 -import Data.Text.Unsafe (unsafeDupablePerformIO) -import Foreign.ForeignPtr (withForeignPtr, ForeignPtr) -import Foreign.Storable (pokeByteOff) -import Data.ByteString.Internal (mallocByteString, memcpy) -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import qualified Data.ByteString.Internal as B - -data S = S0 - | S1 {-# UNPACK #-} !Word8 - | S2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 - | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 - | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 - -data T = T !ByteString !S {-# UNPACK #-} !Int - --- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using --- UTF-8 encoding. -streamUtf8 :: OnDecodeError -> ByteString -> Stream Char -streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize - where - next (T bs@(Chunk ps _) S0 i) - | i < len && U8.validate1 a = - Yield (unsafeChr8 a) (T bs S0 (i+1)) - | i + 1 < len && U8.validate2 a b = - Yield (U8.chr2 a b) (T bs S0 (i+2)) - | i + 2 < len && U8.validate3 a b c = - Yield (U8.chr3 a b c) (T bs S0 (i+3)) - | i + 3 < len && U8.validate4 a b c d = - Yield (U8.chr4 a b c d) (T bs S0 (i+4)) - where len = B.length ps - a = B.unsafeIndex ps i - b = B.unsafeIndex ps (i+1) - c = B.unsafeIndex ps (i+2) - d = B.unsafeIndex ps (i+3) - next st@(T bs s i) = - case s of - S1 a | U8.validate1 a -> Yield (unsafeChr8 a) es - S2 a b | U8.validate2 a b -> Yield (U8.chr2 a b) es - S3 a b c | U8.validate3 a b c -> Yield (U8.chr3 a b c) es - S4 a b c d | U8.validate4 a b c d -> Yield (U8.chr4 a b c d) es - _ -> consume st - where es = T bs S0 i - consume (T bs@(Chunk ps rest) s i) - | i >= B.length ps = consume (T rest s 0) - | otherwise = - case s of - S0 -> next (T bs (S1 x) (i+1)) - S1 a -> next (T bs (S2 a x) (i+1)) - S2 a b -> next (T bs (S3 a b x) (i+1)) - S3 a b c -> next (T bs (S4 a b c x) (i+1)) - S4 a b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a) - (T bs (S3 b c d) (i+1)) - where x = B.unsafeIndex ps i - consume (T Empty S0 _) = Done - consume st = decodeError "streamUtf8" "UTF-8" onErr Nothing st -{-# INLINE [0] streamUtf8 #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little --- endian UTF-16 encoding. -streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize - where - next (T bs@(Chunk ps _) S0 i) - | i + 1 < len && U16.validate1 x1 = - Yield (unsafeChr x1) (T bs S0 (i+2)) - | i + 3 < len && U16.validate2 x1 x2 = - Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) - where len = B.length ps - x1 = c (idx i) (idx (i + 1)) - x2 = c (idx (i + 2)) (idx (i + 3)) - c w1 w2 = w1 + (w2 `shiftL` 8) - idx = fromIntegral . B.unsafeIndex ps :: Int -> Word16 - next st@(T bs s i) = - case s of - S2 w1 w2 | U16.validate1 (c w1 w2) -> - Yield (unsafeChr (c w1 w2)) es - S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> - Yield (U16.chr2 (c w1 w2) (c w3 w4)) es - _ -> consume st - where es = T bs S0 i - c :: Word8 -> Word8 -> Word16 - c w1 w2 = fromIntegral w1 + (fromIntegral w2 `shiftL` 8) - consume (T bs@(Chunk ps rest) s i) - | i >= B.length ps = consume (T rest s 0) - | otherwise = - case s of - S0 -> next (T bs (S1 x) (i+1)) - S1 w1 -> next (T bs (S2 w1 x) (i+1)) - S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) - S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) - S4 w1 w2 w3 w4 -> decodeError "streamUtf16LE" "UTF-16LE" onErr (Just w1) - (T bs (S3 w2 w3 w4) (i+1)) - where x = B.unsafeIndex ps i - consume (T Empty S0 _) = Done - consume st = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing st -{-# INLINE [0] streamUtf16LE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big --- endian UTF-16 encoding. -streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize - where - next (T bs@(Chunk ps _) S0 i) - | i + 1 < len && U16.validate1 x1 = - Yield (unsafeChr x1) (T bs S0 (i+2)) - | i + 3 < len && U16.validate2 x1 x2 = - Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) - where len = B.length ps - x1 = c (idx i) (idx (i + 1)) - x2 = c (idx (i + 2)) (idx (i + 3)) - c w1 w2 = (w1 `shiftL` 8) + w2 - idx = fromIntegral . B.unsafeIndex ps :: Int -> Word16 - next st@(T bs s i) = - case s of - S2 w1 w2 | U16.validate1 (c w1 w2) -> - Yield (unsafeChr (c w1 w2)) es - S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> - Yield (U16.chr2 (c w1 w2) (c w3 w4)) es - _ -> consume st - where es = T bs S0 i - c :: Word8 -> Word8 -> Word16 - c w1 w2 = (fromIntegral w1 `shiftL` 8) + fromIntegral w2 - consume (T bs@(Chunk ps rest) s i) - | i >= B.length ps = consume (T rest s 0) - | otherwise = - case s of - S0 -> next (T bs (S1 x) (i+1)) - S1 w1 -> next (T bs (S2 w1 x) (i+1)) - S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) - S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) - S4 w1 w2 w3 w4 -> decodeError "streamUtf16BE" "UTF-16BE" onErr (Just w1) - (T bs (S3 w2 w3 w4) (i+1)) - where x = B.unsafeIndex ps i - consume (T Empty S0 _) = Done - consume st = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing st -{-# INLINE [0] streamUtf16BE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big --- endian UTF-32 encoding. -streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize - where - next (T bs@(Chunk ps _) S0 i) - | i + 3 < len && U32.validate x = - Yield (unsafeChr32 x) (T bs S0 (i+4)) - where len = B.length ps - x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 - x1 = idx i - x2 = idx (i+1) - x3 = idx (i+2) - x4 = idx (i+3) - idx = fromIntegral . B.unsafeIndex ps :: Int -> Word32 - next st@(T bs s i) = - case s of - S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) -> - Yield (unsafeChr32 (c w1 w2 w3 w4)) es - _ -> consume st - where es = T bs S0 i - c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32 - c w1 w2 w3 w4 = shifted - where - shifted = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 - x1 = fromIntegral w1 - x2 = fromIntegral w2 - x3 = fromIntegral w3 - x4 = fromIntegral w4 - consume (T bs@(Chunk ps rest) s i) - | i >= B.length ps = consume (T rest s 0) - | otherwise = - case s of - S0 -> next (T bs (S1 x) (i+1)) - S1 w1 -> next (T bs (S2 w1 x) (i+1)) - S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) - S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) - S4 w1 w2 w3 w4 -> decodeError "streamUtf32BE" "UTF-32BE" onErr (Just w1) - (T bs (S3 w2 w3 w4) (i+1)) - where x = B.unsafeIndex ps i - consume (T Empty S0 _) = Done - consume st = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing st -{-# INLINE [0] streamUtf32BE #-} - --- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little --- endian UTF-32 encoding. -streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize - where - next (T bs@(Chunk ps _) S0 i) - | i + 3 < len && U32.validate x = - Yield (unsafeChr32 x) (T bs S0 (i+4)) - where len = B.length ps - x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 - x1 = idx i - x2 = idx (i+1) - x3 = idx (i+2) - x4 = idx (i+3) - idx = fromIntegral . B.unsafeIndex ps :: Int -> Word32 - next st@(T bs s i) = - case s of - S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) -> - Yield (unsafeChr32 (c w1 w2 w3 w4)) es - _ -> consume st - where es = T bs S0 i - c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32 - c w1 w2 w3 w4 = shifted - where - shifted = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 - x1 = fromIntegral w1 - x2 = fromIntegral w2 - x3 = fromIntegral w3 - x4 = fromIntegral w4 - consume (T bs@(Chunk ps rest) s i) - | i >= B.length ps = consume (T rest s 0) - | otherwise = - case s of - S0 -> next (T bs (S1 x) (i+1)) - S1 w1 -> next (T bs (S2 w1 x) (i+1)) - S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) - S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) - S4 w1 w2 w3 w4 -> decodeError "streamUtf32LE" "UTF-32LE" onErr (Just w1) - (T bs (S3 w2 w3 w4) (i+1)) - where x = B.unsafeIndex ps i - consume (T Empty S0 _) = Done - consume st = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing st -{-# INLINE [0] streamUtf32LE #-} - --- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'. -unstreamChunks :: Int -> Stream Word8 -> ByteString -unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 (upperBound 4 len0) - where chunk s1 len1 = unsafeDupablePerformIO $ do - let len = max 4 (min len1 chunkSize) - mallocByteString len >>= loop len 0 s1 - where - loop !n !off !s fp = case next s of - Done | off == 0 -> return Empty - | otherwise -> return $! Chunk (trimUp fp off) Empty - Skip s' -> loop n off s' fp - Yield x s' - | off == chunkSize -> do - let !newLen = n - off - return $! Chunk (trimUp fp off) (chunk s newLen) - | off == n -> realloc fp n off s' x - | otherwise -> do - withForeignPtr fp $ \p -> pokeByteOff p off x - loop n (off+1) s' fp - {-# NOINLINE realloc #-} - realloc fp n off s x = do - let n' = min (n+n) chunkSize - fp' <- copy0 fp n n' - withForeignPtr fp' $ \p -> pokeByteOff p off x - loop n' (off+1) s fp' - trimUp fp off = B.PS fp 0 off - copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8) - copy0 !src !srcLen !destLen = -#if defined(ASSERTS) - assert (srcLen <= destLen) $ -#endif - do - dest <- mallocByteString destLen - withForeignPtr src $ \src' -> - withForeignPtr dest $ \dest' -> - memcpy dest' src' (fromIntegral srcLen) - return dest - --- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'. -unstream :: Stream Word8 -> ByteString -unstream = unstreamChunks defaultChunkSize - -decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8 - -> s -> Step s Char -decodeError func kind onErr mb i = - case onErr desc mb of - Nothing -> Skip i - Just c -> Yield c i - where desc = "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Invalid " ++ - kind ++ " stream" diff --git a/Data/Text/Internal/Lazy/Fusion.hs b/Data/Text/Internal/Lazy/Fusion.hs deleted file mode 100644 --- a/Data/Text/Internal/Lazy/Fusion.hs +++ /dev/null @@ -1,120 +0,0 @@ -{-# LANGUAGE BangPatterns #-} --- | --- Module : Data.Text.Lazy.Fusion --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Core stream fusion functionality for text. - -module Data.Text.Internal.Lazy.Fusion - ( - stream - , unstream - , unstreamChunks - , length - , unfoldrN - , index - , countChar - ) where - -import Prelude hiding (length) -import qualified Data.Text.Internal.Fusion.Common as S -import Control.Monad.ST (runST) -import Data.Text.Internal.Fusion.Types -import Data.Text.Internal.Fusion.Size (isEmpty, unknownSize) -import Data.Text.Internal.Lazy -import qualified Data.Text.Internal as I -import qualified Data.Text.Array as A -import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftL) -import Data.Text.Unsafe (Iter(..), iter) -import Data.Int (Int64) - -default(Int64) - --- | /O(n)/ Convert a 'Text' into a 'Stream Char'. -stream :: Text -> Stream Char -stream text = Stream next (text :*: 0) unknownSize - where - next (Empty :*: _) = Done - next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i) - | i >= len = next (ts :*: 0) - | otherwise = Yield c (txt :*: i+d) - where Iter c d = iter t i -{-# INLINE [0] stream #-} - --- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given --- chunk size. -unstreamChunks :: Int -> Stream Char -> Text -unstreamChunks !chunkSize (Stream next s0 len0) - | isEmpty len0 = Empty - | otherwise = outer s0 - where - outer so = {-# SCC "unstreamChunks/outer" #-} - case next so of - Done -> Empty - Skip s' -> outer s' - Yield x s' -> runST $ do - a <- A.new unknownLength - unsafeWrite a 0 x >>= inner a unknownLength s' - where unknownLength = 4 - where - inner marr !len s !i - | i + 1 >= chunkSize = finish marr i s - | i + 1 >= len = {-# SCC "unstreamChunks/resize" #-} do - let newLen = min (len `shiftL` 1) chunkSize - marr' <- A.new newLen - A.copyM marr' 0 marr 0 len - inner marr' newLen s i - | otherwise = - {-# SCC "unstreamChunks/inner" #-} - case next s of - Done -> finish marr i s - Skip s' -> inner marr len s' i - Yield x s' -> do d <- unsafeWrite marr i x - inner marr len s' (i+d) - finish marr len s' = do - arr <- A.unsafeFreeze marr - return (I.Text arr 0 len `Chunk` outer s') -{-# INLINE [0] unstreamChunks #-} - --- | /O(n)/ Convert a 'Stream Char' into a 'Text', using --- 'defaultChunkSize'. -unstream :: Stream Char -> Text -unstream = unstreamChunks defaultChunkSize -{-# INLINE [0] unstream #-} - --- | /O(n)/ Returns the number of characters in a text. -length :: Stream Char -> Int64 -length = S.lengthI -{-# INLINE[0] length #-} - -{-# RULES "LAZY STREAM stream/unstream fusion" forall s. - stream (unstream s) = s #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed --- value. However, the length of the result is limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the length of the result is known. -unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Stream Char -unfoldrN n = S.unfoldrNI n -{-# INLINE [0] unfoldrN #-} - --- | /O(n)/ stream index (subscript) operator, starting from 0. -index :: Stream Char -> Int64 -> Char -index = S.indexI -{-# INLINE [0] index #-} - --- | /O(n)/ The 'count' function returns the number of times the query --- element appears in the given stream. -countChar :: Char -> Stream Char -> Int64 -countChar = S.countCharI -{-# INLINE [0] countChar #-} diff --git a/Data/Text/Internal/Lazy/Search.hs b/Data/Text/Internal/Lazy/Search.hs deleted file mode 100644 --- a/Data/Text/Internal/Lazy/Search.hs +++ /dev/null @@ -1,134 +0,0 @@ -{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} - --- | --- Module : Data.Text.Lazy.Search --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fast substring search for lazy 'Text', based on work by Boyer, --- Moore, Horspool, Sunday, and Lundh. Adapted from the strict --- implementation. - -module Data.Text.Internal.Lazy.Search - ( - indices - ) where - -import qualified Data.Text.Array as A -import Data.Int (Int64) -import Data.Word (Word16, Word64) -import qualified Data.Text.Internal as T -import Data.Text.Internal.Fusion.Types (PairS(..)) -import Data.Text.Internal.Lazy (Text(..), foldlChunks) -import Data.Bits ((.|.), (.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftL) - --- | /O(n+m)/ Find the offsets of all non-overlapping indices of --- @needle@ within @haystack@. --- --- This function is strict in @needle@, and lazy (as far as possible) --- in the chunks of @haystack@. --- --- In (unlikely) bad cases, this algorithm's complexity degrades --- towards /O(n*m)/. -indices :: Text -- ^ Substring to search for (@needle@) - -> Text -- ^ Text to search in (@haystack@) - -> [Int64] -indices needle@(Chunk n ns) _haystack@(Chunk k ks) - | nlen <= 0 = [] - | nlen == 1 = indicesOne (nindex 0) 0 k ks - | otherwise = advance k ks 0 0 - where - advance x@(T.Text _ _ l) xs = scan - where - scan !g !i - | i >= m = case xs of - Empty -> [] - Chunk y ys -> advance y ys g (i-m) - | lackingHay (i + nlen) x xs = [] - | c == z && candidateMatch 0 = g : scan (g+nlen) (i+nlen) - | otherwise = scan (g+delta) (i+delta) - where - m = fromIntegral l - c = hindex (i + nlast) - delta | nextInPattern = nlen + 1 - | c == z = skip + 1 - | otherwise = 1 - nextInPattern = mask .&. swizzle (hindex (i+nlen)) == 0 - candidateMatch !j - | j >= nlast = True - | hindex (i+j) /= nindex j = False - | otherwise = candidateMatch (j+1) - hindex = index x xs - nlen = wordLength needle - nlast = nlen - 1 - nindex = index n ns - z = foldlChunks fin 0 needle - where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1) - (mask :: Word64) :*: skip = buildTable n ns 0 0 0 (nlen-2) - swizzle w = 1 `shiftL` (fromIntegral w .&. 0x3f) - buildTable (T.Text xarr xoff xlen) xs = go - where - go !(g::Int64) !i !msk !skp - | i >= xlast = case xs of - Empty -> (msk .|. swizzle z) :*: skp - Chunk y ys -> buildTable y ys g 0 msk' skp' - | otherwise = go (g+1) (i+1) msk' skp' - where c = A.unsafeIndex xarr (xoff+i) - msk' = msk .|. swizzle c - skp' | c == z = nlen - g - 2 - | otherwise = skp - xlast = xlen - 1 - -- | Check whether an attempt to index into the haystack at the - -- given offset would fail. - lackingHay q = go 0 - where - go p (T.Text _ _ l) ps = p' < q && case ps of - Empty -> True - Chunk r rs -> go p' r rs - where p' = p + fromIntegral l -indices _ _ = [] - --- | Fast index into a partly unpacked 'Text'. We take into account --- the possibility that the caller might try to access one element --- past the end. -index :: T.Text -> Text -> Int64 -> Word16 -index (T.Text arr off len) xs !i - | j < len = A.unsafeIndex arr (off+j) - | otherwise = case xs of - Empty - -- out of bounds, but legal - | j == len -> 0 - -- should never happen, due to lackingHay above - | otherwise -> emptyError "index" - Chunk c cs -> index c cs (i-fromIntegral len) - where j = fromIntegral i - --- | A variant of 'indices' that scans linearly for a single 'Word16'. -indicesOne :: Word16 -> Int64 -> T.Text -> Text -> [Int64] -indicesOne c = chunk - where - chunk !i (T.Text oarr ooff olen) os = go 0 - where - go h | h >= olen = case os of - Empty -> [] - Chunk y ys -> chunk (i+fromIntegral olen) y ys - | on == c = i + fromIntegral h : go (h+1) - | otherwise = go (h+1) - where on = A.unsafeIndex oarr (ooff+h) - --- | The number of 'Word16' values in a 'Text'. -wordLength :: Text -> Int64 -wordLength = foldlChunks sumLength 0 - where sumLength i (T.Text _ _ l) = i + fromIntegral l - -emptyError :: String -> a -emptyError fun = error ("Data.Text.Lazy.Search." ++ fun ++ ": empty input") diff --git a/Data/Text/Internal/Private.hs b/Data/Text/Internal/Private.hs deleted file mode 100644 --- a/Data/Text/Internal/Private.hs +++ /dev/null @@ -1,37 +0,0 @@ -{-# LANGUAGE BangPatterns, Rank2Types, UnboxedTuples #-} - --- | --- Module : Data.Text.Internal.Private --- Copyright : (c) 2011 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC - -module Data.Text.Internal.Private - ( - runText - , span_ - ) where - -import Control.Monad.ST (ST, runST) -import Data.Text.Internal (Text(..), text) -import Data.Text.Unsafe (Iter(..), iter) -import qualified Data.Text.Array as A - -span_ :: (Char -> Bool) -> Text -> (# Text, Text #) -span_ p t@(Text arr off len) = (# hd,tl #) - where hd = text arr off k - tl = text arr (off+k) (len-k) - !k = loop 0 - loop !i | i < len && p c = loop (i+d) - | otherwise = i - where Iter c d = iter t i -{-# INLINE span_ #-} - -runText :: (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text -runText act = runST (act $ \ !marr !len -> do - arr <- A.unsafeFreeze marr - return $! text arr 0 len) -{-# INLINE runText #-} diff --git a/Data/Text/Internal/Read.hs b/Data/Text/Internal/Read.hs deleted file mode 100644 --- a/Data/Text/Internal/Read.hs +++ /dev/null @@ -1,62 +0,0 @@ --- | --- Module : Data.Text.Internal.Read --- Copyright : (c) 2014 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Common internal functions for reading textual data. -module Data.Text.Internal.Read - ( - IReader - , IParser(..) - , T(..) - , digitToInt - , hexDigitToInt - , perhaps - ) where - -import Control.Applicative as App (Applicative(..)) -import Control.Arrow (first) -import Control.Monad (ap) -import Data.Char (ord) - -type IReader t a = t -> Either String (a,t) - -newtype IParser t a = P { - runP :: IReader t a - } - -instance Functor (IParser t) where - fmap f m = P $ fmap (first f) . runP m - -instance Applicative (IParser t) where - pure a = P $ \t -> Right (a,t) - {-# INLINE pure #-} - (<*>) = ap - -instance Monad (IParser t) where - return = App.pure - m >>= k = P $ \t -> case runP m t of - Left err -> Left err - Right (a,t') -> runP (k a) t' - {-# INLINE (>>=) #-} - fail msg = P $ \_ -> Left msg - -data T = T !Integer !Int - -perhaps :: a -> IParser t a -> IParser t a -perhaps def m = P $ \t -> case runP m t of - Left _ -> Right (def,t) - r@(Right _) -> r - -hexDigitToInt :: Char -> Int -hexDigitToInt c - | c >= '0' && c <= '9' = ord c - ord '0' - | c >= 'a' && c <= 'f' = ord c - (ord 'a' - 10) - | otherwise = ord c - (ord 'A' - 10) - -digitToInt :: Char -> Int -digitToInt c = ord c - ord '0' diff --git a/Data/Text/Internal/Search.hs b/Data/Text/Internal/Search.hs deleted file mode 100644 --- a/Data/Text/Internal/Search.hs +++ /dev/null @@ -1,89 +0,0 @@ -{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} - --- | --- Module : Data.Text.Internal.Search --- Copyright : (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Fast substring search for 'Text', based on work by Boyer, Moore, --- Horspool, Sunday, and Lundh. --- --- References: --- --- * R. S. Boyer, J. S. Moore: A Fast String Searching Algorithm. --- Communications of the ACM, 20, 10, 762-772 (1977) --- --- * R. N. Horspool: Practical Fast Searching in Strings. Software - --- Practice and Experience 10, 501-506 (1980) --- --- * D. M. Sunday: A Very Fast Substring Search Algorithm. --- Communications of the ACM, 33, 8, 132-142 (1990) --- --- * F. Lundh: The Fast Search Algorithm. --- (2006) - -module Data.Text.Internal.Search - ( - indices - ) where - -import qualified Data.Text.Array as A -import Data.Word (Word64) -import Data.Text.Internal (Text(..)) -import Data.Bits ((.|.), (.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftL) - -data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int - --- | /O(n+m)/ Find the offsets of all non-overlapping indices of --- @needle@ within @haystack@. The offsets returned represent --- uncorrected indices in the low-level \"needle\" array, to which its --- offset must be added. --- --- In (unlikely) bad cases, this algorithm's complexity degrades --- towards /O(n*m)/. -indices :: Text -- ^ Substring to search for (@needle@) - -> Text -- ^ Text to search in (@haystack@) - -> [Int] -indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen) - | nlen == 1 = scanOne (nindex 0) - | nlen <= 0 || ldiff < 0 = [] - | otherwise = scan 0 - where - ldiff = hlen - nlen - nlast = nlen - 1 - z = nindex nlast - nindex k = A.unsafeIndex narr (noff+k) - hindex k = A.unsafeIndex harr (hoff+k) - hindex' k | k == hlen = 0 - | otherwise = A.unsafeIndex harr (hoff+k) - buildTable !i !msk !skp - | i >= nlast = (msk .|. swizzle z) :* skp - | otherwise = buildTable (i+1) (msk .|. swizzle c) skp' - where c = nindex i - skp' | c == z = nlen - i - 2 - | otherwise = skp - swizzle k = 1 `shiftL` (fromIntegral k .&. 0x3f) - scan !i - | i > ldiff = [] - | c == z && candidateMatch 0 = i : scan (i + nlen) - | otherwise = scan (i + delta) - where c = hindex (i + nlast) - candidateMatch !j - | j >= nlast = True - | hindex (i+j) /= nindex j = False - | otherwise = candidateMatch (j+1) - delta | nextInPattern = nlen + 1 - | c == z = skip + 1 - | otherwise = 1 - where nextInPattern = mask .&. swizzle (hindex' (i+nlen)) == 0 - !(mask :* skip) = buildTable 0 0 (nlen-2) - scanOne c = loop 0 - where loop !i | i >= hlen = [] - | hindex i == c = i : loop (i+1) - | otherwise = loop (i+1) -{-# INLINE indices #-} diff --git a/Data/Text/Internal/Unsafe.hs b/Data/Text/Internal/Unsafe.hs deleted file mode 100644 --- a/Data/Text/Internal/Unsafe.hs +++ /dev/null @@ -1,56 +0,0 @@ -{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-} -{-# OPTIONS_HADDOCK not-home #-} - --- | --- Module : Data.Text.Internal.Unsafe --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- A module containing /unsafe/ operations, for /very very careful/ use --- in /heavily tested/ code. -module Data.Text.Internal.Unsafe - ( - inlineInterleaveST - , inlinePerformIO - ) where - -import GHC.ST (ST(..)) -#if defined(__GLASGOW_HASKELL__) -import GHC.IO (IO(IO)) -import GHC.Base (realWorld#) -#endif - - --- | Just like unsafePerformIO, but we inline it. Big performance gains as --- it exposes lots of things to further inlining. /Very unsafe/. In --- particular, you should do no memory allocation inside an --- 'inlinePerformIO' block. On Hugs this is just @unsafePerformIO@. --- -{-# INLINE inlinePerformIO #-} -inlinePerformIO :: IO a -> a -#if defined(__GLASGOW_HASKELL__) -inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r -#else -inlinePerformIO = unsafePerformIO -#endif - --- | Allow an 'ST' computation to be deferred lazily. When passed an --- action of type 'ST' @s@ @a@, the action will only be performed when --- the value of @a@ is demanded. --- --- This function is identical to the normal unsafeInterleaveST, but is --- inlined and hence faster. --- --- /Note/: This operation is highly unsafe, as it can introduce --- externally visible non-determinism into an 'ST' action. -inlineInterleaveST :: ST s a -> ST s a -inlineInterleaveST (ST m) = ST $ \ s -> - let r = case m s of (# _, res #) -> res in (# s, r #) -{-# INLINE inlineInterleaveST #-} diff --git a/Data/Text/Internal/Unsafe/Char.hs b/Data/Text/Internal/Unsafe/Char.hs deleted file mode 100644 --- a/Data/Text/Internal/Unsafe/Char.hs +++ /dev/null @@ -1,95 +0,0 @@ -{-# LANGUAGE CPP, MagicHash #-} - --- | --- Module : Data.Text.Internal.Unsafe.Char --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fast character manipulation functions. -module Data.Text.Internal.Unsafe.Char - ( - ord - , unsafeChr - , unsafeChr8 - , unsafeChr32 - , unsafeWrite - -- , unsafeWriteRev - ) where - -#ifdef ASSERTS -import Control.Exception (assert) -#endif -import Control.Monad.ST (ST) -import Data.Bits ((.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftR) -import GHC.Exts (Char(..), Int(..), chr#, ord#, word2Int#) -import GHC.Word (Word8(..), Word16(..), Word32(..)) -import qualified Data.Text.Array as A - -ord :: Char -> Int -ord (C# c#) = I# (ord# c#) -{-# INLINE ord #-} - -unsafeChr :: Word16 -> Char -unsafeChr (W16# w#) = C# (chr# (word2Int# w#)) -{-# INLINE unsafeChr #-} - -unsafeChr8 :: Word8 -> Char -unsafeChr8 (W8# w#) = C# (chr# (word2Int# w#)) -{-# INLINE unsafeChr8 #-} - -unsafeChr32 :: Word32 -> Char -unsafeChr32 (W32# w#) = C# (chr# (word2Int# w#)) -{-# INLINE unsafeChr32 #-} - --- | Write a character into the array at the given offset. Returns --- the number of 'Word16's written. -unsafeWrite :: A.MArray s -> Int -> Char -> ST s Int -unsafeWrite marr i c - | n < 0x10000 = do -#if defined(ASSERTS) - assert (i >= 0) . assert (i < A.length marr) $ return () -#endif - A.unsafeWrite marr i (fromIntegral n) - return 1 - | otherwise = do -#if defined(ASSERTS) - assert (i >= 0) . assert (i < A.length marr - 1) $ return () -#endif - A.unsafeWrite marr i lo - A.unsafeWrite marr (i+1) hi - return 2 - where n = ord c - m = n - 0x10000 - lo = fromIntegral $ (m `shiftR` 10) + 0xD800 - hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00 -{-# INLINE unsafeWrite #-} - -{- -unsafeWriteRev :: A.MArray s Word16 -> Int -> Char -> ST s Int -unsafeWriteRev marr i c - | n < 0x10000 = do - assert (i >= 0) . assert (i < A.length marr) $ - A.unsafeWrite marr i (fromIntegral n) - return (i-1) - | otherwise = do - assert (i >= 1) . assert (i < A.length marr) $ - A.unsafeWrite marr (i-1) lo - A.unsafeWrite marr i hi - return (i-2) - where n = ord c - m = n - 0x10000 - lo = fromIntegral $ (m `shiftR` 10) + 0xD800 - hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00 -{-# INLINE unsafeWriteRev #-} --} diff --git a/Data/Text/Internal/Unsafe/Shift.hs b/Data/Text/Internal/Unsafe/Shift.hs deleted file mode 100644 --- a/Data/Text/Internal/Unsafe/Shift.hs +++ /dev/null @@ -1,72 +0,0 @@ -{-# LANGUAGE MagicHash #-} - --- | --- Module : Data.Text.Internal.Unsafe.Shift --- Copyright : (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fast, unchecked bit shifting functions. - -module Data.Text.Internal.Unsafe.Shift - ( - UnsafeShift(..) - ) where - --- import qualified Data.Bits as Bits -import GHC.Base -import GHC.Word - --- | This is a workaround for poor optimisation in GHC 6.8.2. It --- fails to notice constant-width shifts, and adds a test and branch --- to every shift. This imposes about a 10% performance hit. --- --- These functions are undefined when the amount being shifted by is --- greater than the size in bits of a machine Int#. -class UnsafeShift a where - shiftL :: a -> Int -> a - shiftR :: a -> Int -> a - -instance UnsafeShift Word16 where - {-# INLINE shiftL #-} - shiftL (W16# x#) (I# i#) = W16# (narrow16Word# (x# `uncheckedShiftL#` i#)) - - {-# INLINE shiftR #-} - shiftR (W16# x#) (I# i#) = W16# (x# `uncheckedShiftRL#` i#) - -instance UnsafeShift Word32 where - {-# INLINE shiftL #-} - shiftL (W32# x#) (I# i#) = W32# (narrow32Word# (x# `uncheckedShiftL#` i#)) - - {-# INLINE shiftR #-} - shiftR (W32# x#) (I# i#) = W32# (x# `uncheckedShiftRL#` i#) - -instance UnsafeShift Word64 where - {-# INLINE shiftL #-} - shiftL (W64# x#) (I# i#) = W64# (x# `uncheckedShiftL64#` i#) - - {-# INLINE shiftR #-} - shiftR (W64# x#) (I# i#) = W64# (x# `uncheckedShiftRL64#` i#) - -instance UnsafeShift Int where - {-# INLINE shiftL #-} - shiftL (I# x#) (I# i#) = I# (x# `iShiftL#` i#) - - {-# INLINE shiftR #-} - shiftR (I# x#) (I# i#) = I# (x# `iShiftRA#` i#) - -{- -instance UnsafeShift Integer where - {-# INLINE shiftL #-} - shiftL = Bits.shiftL - - {-# INLINE shiftR #-} - shiftR = Bits.shiftR --} diff --git a/Data/Text/Lazy.hs b/Data/Text/Lazy.hs deleted file mode 100644 --- a/Data/Text/Lazy.hs +++ /dev/null @@ -1,1695 +0,0 @@ -{-# OPTIONS_GHC -fno-warn-orphans #-} -{-# LANGUAGE BangPatterns, MagicHash, CPP #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif -#if __GLASGOW_HASKELL__ >= 708 -{-# LANGUAGE TypeFamilies #-} -#endif - --- | --- Module : Data.Text.Lazy --- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- A time and space-efficient implementation of Unicode text using --- lists of packed arrays. --- --- /Note/: Read below the synopsis for important notes on the use of --- this module. --- --- The representation used by this module is suitable for high --- performance use and for streaming large quantities of data. It --- provides a means to manipulate a large body of text without --- requiring that the entire content be resident in memory. --- --- Some operations, such as 'concat', 'append', 'reverse' and 'cons', --- have better time complexity than their "Data.Text" equivalents, due --- to the underlying representation being a list of chunks. For other --- operations, lazy 'Text's are usually within a few percent of strict --- ones, but often with better heap usage if used in a streaming --- fashion. For data larger than available memory, or if you have --- tight memory constraints, this module will be the only option. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions. eg. --- --- > import qualified Data.Text.Lazy as L - -module Data.Text.Lazy - ( - -- * Fusion - -- $fusion - - -- * Acceptable data - -- $replacement - - -- * Types - Text - - -- * Creation and elimination - , pack - , unpack - , singleton - , empty - , fromChunks - , toChunks - , toStrict - , fromStrict - , foldrChunks - , foldlChunks - - -- * Basic interface - , cons - , snoc - , append - , uncons - , head - , last - , tail - , init - , null - , length - , compareLength - - -- * Transformations - , map - , intercalate - , intersperse - , transpose - , reverse - , replace - - -- ** Case conversion - -- $case - , toCaseFold - , toLower - , toUpper - , toTitle - - -- ** Justification - , justifyLeft - , justifyRight - , center - - -- * Folds - , foldl - , foldl' - , foldl1 - , foldl1' - , foldr - , foldr1 - - -- ** Special folds - , concat - , concatMap - , any - , all - , maximum - , minimum - - -- * Construction - - -- ** Scans - , scanl - , scanl1 - , scanr - , scanr1 - - -- ** Accumulating maps - , mapAccumL - , mapAccumR - - -- ** Generation and unfolding - , repeat - , replicate - , cycle - , iterate - , unfoldr - , unfoldrN - - -- * Substrings - - -- ** Breaking strings - , take - , takeEnd - , drop - , dropEnd - , takeWhile - , takeWhileEnd - , dropWhile - , dropWhileEnd - , dropAround - , strip - , stripStart - , stripEnd - , splitAt - , span - , breakOn - , breakOnEnd - , break - , group - , groupBy - , inits - , tails - - -- ** Breaking into many substrings - -- $split - , splitOn - , split - , chunksOf - -- , breakSubstring - - -- ** Breaking into lines and words - , lines - , words - , unlines - , unwords - - -- * Predicates - , isPrefixOf - , isSuffixOf - , isInfixOf - - -- ** View patterns - , stripPrefix - , stripSuffix - , commonPrefixes - - -- * Searching - , filter - , find - , breakOnAll - , partition - - -- , findSubstring - - -- * Indexing - , index - , count - - -- * Zipping and unzipping - , zip - , zipWith - - -- -* Ordered text - -- , sort - ) where - -import Prelude (Char, Bool(..), Maybe(..), String, - Eq(..), Ord(..), Ordering(..), Read(..), Show(..), - (&&), (||), (+), (-), (.), ($), (++), - error, flip, fmap, fromIntegral, not, otherwise, quot) -import qualified Prelude as P -#if defined(HAVE_DEEPSEQ) -import Control.DeepSeq (NFData(..)) -#endif -import Data.Int (Int64) -import qualified Data.List as L -import Data.Char (isSpace) -import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, - Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) -import Data.Binary (Binary(get, put)) -import Data.Monoid (Monoid(..)) -#if MIN_VERSION_base(4,9,0) -import Data.Semigroup (Semigroup(..)) -#endif -import Data.String (IsString(..)) -import qualified Data.Text as T -import qualified Data.Text.Internal as T -import qualified Data.Text.Internal.Fusion.Common as S -import qualified Data.Text.Unsafe as T -import qualified Data.Text.Internal.Lazy.Fusion as S -import Data.Text.Internal.Fusion.Types (PairS(..)) -import Data.Text.Internal.Lazy.Fusion (stream, unstream) -import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, - foldrChunks, smallChunkSize) -import Data.Text.Internal (firstf, safe, text) -import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8) -import qualified Data.Text.Internal.Functions as F -import Data.Text.Internal.Lazy.Search (indices) -#if __GLASGOW_HASKELL__ >= 702 -import qualified GHC.CString as GHC -#else -import qualified GHC.Base as GHC -#endif -#if __GLASGOW_HASKELL__ >= 708 -import qualified GHC.Exts as Exts -#endif -import GHC.Prim (Addr#) -#if MIN_VERSION_base(4,7,0) -import Text.Printf (PrintfArg, formatArg, formatString) -#endif - --- $fusion --- --- Most of the functions in this module are subject to /fusion/, --- meaning that a pipeline of such functions will usually allocate at --- most one 'Text' value. --- --- As an example, consider the following pipeline: --- --- > import Data.Text.Lazy as T --- > import Data.Text.Lazy.Encoding as E --- > import Data.ByteString.Lazy (ByteString) --- > --- > countChars :: ByteString -> Int --- > countChars = T.length . T.toUpper . E.decodeUtf8 --- --- From the type signatures involved, this looks like it should --- allocate one 'ByteString' value, and two 'Text' values. However, --- when a module is compiled with optimisation enabled under GHC, the --- two intermediate 'Text' values will be optimised away, and the --- function will be compiled down to a single loop over the source --- 'ByteString'. --- --- Functions that can be fused by the compiler are documented with the --- phrase \"Subject to fusion\". - --- $replacement --- --- A 'Text' value is a sequence of Unicode scalar values, as defined --- in §3.9, definition D76 of the Unicode 5.2 standard: --- . As --- such, a 'Text' cannot contain values in the range U+D800 to U+DFFF --- inclusive. Haskell implementations admit all Unicode code points --- (§3.4, definition D10) as 'Char' values, including code points --- from this invalid range. This means that there are some 'Char' --- values that are not valid Unicode scalar values, and the functions --- in this module must handle those cases. --- --- Within this module, many functions construct a 'Text' from one or --- more 'Char' values. Those functions will substitute 'Char' values --- that are not valid Unicode scalar values with the replacement --- character \"�\" (U+FFFD). Functions that perform this --- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". --- --- (One reason for this policy of replacement is that internally, a --- 'Text' value is represented as packed UTF-16 data. Values in the --- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate --- code points, and so cannot be represented. The functions replace --- invalid scalar values, instead of dropping them, as a security --- measure. For details, see Unicode Technical Report 36, §3.5: --- ) - -equal :: Text -> Text -> Bool -equal Empty Empty = True -equal Empty _ = False -equal _ Empty = False -equal (Chunk a as) (Chunk b bs) = - case compare lenA lenB of - LT -> a == (T.takeWord16 lenA b) && - as `equal` Chunk (T.dropWord16 lenA b) bs - EQ -> a == b && as `equal` bs - GT -> T.takeWord16 lenB a == b && - Chunk (T.dropWord16 lenB a) as `equal` bs - where lenA = T.lengthWord16 a - lenB = T.lengthWord16 b - -instance Eq Text where - (==) = equal - {-# INLINE (==) #-} - -instance Ord Text where - compare = compareText - -compareText :: Text -> Text -> Ordering -compareText Empty Empty = EQ -compareText Empty _ = LT -compareText _ Empty = GT -compareText (Chunk a0 as) (Chunk b0 bs) = outer a0 b0 - where - outer ta@(T.Text arrA offA lenA) tb@(T.Text arrB offB lenB) = go 0 0 - where - go !i !j - | i >= lenA = compareText as (chunk (T.Text arrB (offB+j) (lenB-j)) bs) - | j >= lenB = compareText (chunk (T.Text arrA (offA+i) (lenA-i)) as) bs - | a < b = LT - | a > b = GT - | otherwise = go (i+di) (j+dj) - where T.Iter a di = T.iter ta i - T.Iter b dj = T.iter tb j - -instance Show Text where - showsPrec p ps r = showsPrec p (unpack ps) r - -instance Read Text where - readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] - -#if MIN_VERSION_base(4,9,0) --- Semigroup orphan instances for older GHCs are provided by --- 'semigroups` package - -instance Semigroup Text where - (<>) = append -#endif - -instance Monoid Text where - mempty = empty -#if MIN_VERSION_base(4,9,0) - mappend = (<>) -- future-proof definition -#else - mappend = append -#endif - mconcat = concat - -instance IsString Text where - fromString = pack - -#if __GLASGOW_HASKELL__ >= 708 -instance Exts.IsList Text where - type Item Text = Char - fromList = pack - toList = unpack -#endif - -#if defined(HAVE_DEEPSEQ) -instance NFData Text where - rnf Empty = () - rnf (Chunk _ ts) = rnf ts -#endif - -instance Binary Text where - put t = put (encodeUtf8 t) - get = do - bs <- get - case decodeUtf8' bs of - P.Left exn -> P.fail (P.show exn) - P.Right a -> P.return a - --- | This instance preserves data abstraction at the cost of inefficiency. --- We omit reflection services for the sake of data abstraction. --- --- This instance was created by copying the updated behavior of --- @"Data.Text".@'Data.Text.Text' -instance Data Text where - gfoldl f z txt = z pack `f` (unpack txt) - toConstr _ = packConstr - gunfold k z c = case constrIndex c of - 1 -> k (z pack) - _ -> error "Data.Text.Lazy.Text.gunfold" - dataTypeOf _ = textDataType - -#if MIN_VERSION_base(4,7,0) --- | Only defined for @base-4.7.0.0@ and later -instance PrintfArg Text where - formatArg txt = formatString $ unpack txt -#endif - -packConstr :: Constr -packConstr = mkConstr textDataType "pack" [] Prefix - -textDataType :: DataType -textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] - --- | /O(n)/ Convert a 'String' into a 'Text'. --- --- Subject to fusion. Performs replacement on invalid scalar values. -pack :: String -> Text -pack = unstream . S.streamList . L.map safe -{-# INLINE [1] pack #-} - --- | /O(n)/ Convert a 'Text' into a 'String'. --- Subject to fusion. -unpack :: Text -> String -unpack t = S.unstreamList (stream t) -{-# INLINE [1] unpack #-} - --- | /O(n)/ Convert a literal string into a Text. -unpackCString# :: Addr# -> Text -unpackCString# addr# = unstream (S.streamCString# addr#) -{-# NOINLINE unpackCString# #-} - -{-# RULES "TEXT literal" forall a. - unstream (S.streamList (L.map safe (GHC.unpackCString# a))) - = unpackCString# a #-} - -{-# RULES "TEXT literal UTF8" forall a. - unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) - = unpackCString# a #-} - -{-# RULES "LAZY TEXT empty literal" - unstream (S.streamList (L.map safe [])) - = Empty #-} - -{-# RULES "LAZY TEXT empty literal" forall a. - unstream (S.streamList (L.map safe [a])) - = Chunk (T.singleton a) Empty #-} - --- | /O(1)/ Convert a character into a Text. Subject to fusion. --- Performs replacement on invalid scalar values. -singleton :: Char -> Text -singleton c = Chunk (T.singleton c) Empty -{-# INLINE [1] singleton #-} - -{-# RULES -"LAZY TEXT singleton -> fused" [~1] forall c. - singleton c = unstream (S.singleton c) -"LAZY TEXT singleton -> unfused" [1] forall c. - unstream (S.singleton c) = singleton c - #-} - --- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. -fromChunks :: [T.Text] -> Text -fromChunks cs = L.foldr chunk Empty cs - --- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. -toChunks :: Text -> [T.Text] -toChunks cs = foldrChunks (:) [] cs - --- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. -toStrict :: Text -> T.Text -toStrict t = T.concat (toChunks t) -{-# INLINE [1] toStrict #-} - --- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. -fromStrict :: T.Text -> Text -fromStrict t = chunk t Empty -{-# INLINE [1] fromStrict #-} - --- ----------------------------------------------------------------------------- --- * Basic functions - --- | /O(n)/ Adds a character to the front of a 'Text'. This function --- is more costly than its 'List' counterpart because it requires --- copying a new array. Subject to fusion. -cons :: Char -> Text -> Text -cons c t = Chunk (T.singleton c) t -{-# INLINE [1] cons #-} - -infixr 5 `cons` - -{-# RULES -"LAZY TEXT cons -> fused" [~1] forall c t. - cons c t = unstream (S.cons c (stream t)) -"LAZY TEXT cons -> unfused" [1] forall c t. - unstream (S.cons c (stream t)) = cons c t - #-} - --- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process, unless fused. Subject to fusion. -snoc :: Text -> Char -> Text -snoc t c = foldrChunks Chunk (singleton c) t -{-# INLINE [1] snoc #-} - -{-# RULES -"LAZY TEXT snoc -> fused" [~1] forall t c. - snoc t c = unstream (S.snoc (stream t) c) -"LAZY TEXT snoc -> unfused" [1] forall t c. - unstream (S.snoc (stream t) c) = snoc t c - #-} - --- | /O(n\/c)/ Appends one 'Text' to another. Subject to fusion. -append :: Text -> Text -> Text -append xs ys = foldrChunks Chunk ys xs -{-# INLINE [1] append #-} - -{-# RULES -"LAZY TEXT append -> fused" [~1] forall t1 t2. - append t1 t2 = unstream (S.append (stream t1) (stream t2)) -"LAZY TEXT append -> unfused" [1] forall t1 t2. - unstream (S.append (stream t1) (stream t2)) = append t1 t2 - #-} - --- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. Subject to fusion. -uncons :: Text -> Maybe (Char, Text) -uncons Empty = Nothing -uncons (Chunk t ts) = Just (T.unsafeHead t, ts') - where ts' | T.compareLength t 1 == EQ = ts - | otherwise = Chunk (T.unsafeTail t) ts -{-# INLINE uncons #-} - --- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. Subject to fusion. -head :: Text -> Char -head t = S.head (stream t) -{-# INLINE head #-} - --- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. Subject to fusion. -tail :: Text -> Text -tail (Chunk t ts) = chunk (T.tail t) ts -tail Empty = emptyError "tail" -{-# INLINE [1] tail #-} - -{-# RULES -"LAZY TEXT tail -> fused" [~1] forall t. - tail t = unstream (S.tail (stream t)) -"LAZY TEXT tail -> unfused" [1] forall t. - unstream (S.tail (stream t)) = tail t - #-} - --- | /O(n\/c)/ Returns all but the last character of a 'Text', which must --- be non-empty. Subject to fusion. -init :: Text -> Text -init (Chunk t0 ts0) = go t0 ts0 - where go t (Chunk t' ts) = Chunk t (go t' ts) - go t Empty = chunk (T.init t) Empty -init Empty = emptyError "init" -{-# INLINE [1] init #-} - -{-# RULES -"LAZY TEXT init -> fused" [~1] forall t. - init t = unstream (S.init (stream t)) -"LAZY TEXT init -> unfused" [1] forall t. - unstream (S.init (stream t)) = init t - #-} - --- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to --- fusion. -null :: Text -> Bool -null Empty = True -null _ = False -{-# INLINE [1] null #-} - -{-# RULES -"LAZY TEXT null -> fused" [~1] forall t. - null t = S.null (stream t) -"LAZY TEXT null -> unfused" [1] forall t. - S.null (stream t) = null t - #-} - --- | /O(1)/ Tests whether a 'Text' contains exactly one character. --- Subject to fusion. -isSingleton :: Text -> Bool -isSingleton = S.isSingleton . stream -{-# INLINE isSingleton #-} - --- | /O(n\/c)/ Returns the last character of a 'Text', which must be --- non-empty. Subject to fusion. -last :: Text -> Char -last Empty = emptyError "last" -last (Chunk t ts) = go t ts - where go _ (Chunk t' ts') = go t' ts' - go t' Empty = T.last t' -{-# INLINE [1] last #-} - -{-# RULES -"LAZY TEXT last -> fused" [~1] forall t. - last t = S.last (stream t) -"LAZY TEXT last -> unfused" [1] forall t. - S.last (stream t) = last t - #-} - --- | /O(n)/ Returns the number of characters in a 'Text'. --- Subject to fusion. -length :: Text -> Int64 -length = foldlChunks go 0 - where go l t = l + fromIntegral (T.length t) -{-# INLINE [1] length #-} - -{-# RULES -"LAZY TEXT length -> fused" [~1] forall t. - length t = S.length (stream t) -"LAZY TEXT length -> unfused" [1] forall t. - S.length (stream t) = length t - #-} - --- | /O(n)/ Compare the count of characters in a 'Text' to a number. --- Subject to fusion. --- --- This function gives the same answer as comparing against the result --- of 'length', but can short circuit if the count of characters is --- greater than the number, and hence be more efficient. -compareLength :: Text -> Int64 -> Ordering -compareLength t n = S.compareLengthI (stream t) n -{-# INLINE [1] compareLength #-} - --- We don't apply those otherwise appealing length-to-compareLength --- rewrite rules here, because they can change the strictness --- properties of code. - --- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Subject to fusion. Performs replacement on --- invalid scalar values. -map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map (safe . f) (stream t)) -{-# INLINE [1] map #-} - --- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of --- 'Text's and concatenates the list after interspersing the first --- argument between each element of the list. -intercalate :: Text -> [Text] -> Text -intercalate t = concat . (F.intersperse t) -{-# INLINE intercalate #-} - --- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Subject to fusion. Performs --- replacement on invalid scalar values. -intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse (safe c) (stream t)) -{-# INLINE intersperse #-} - --- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. Performs --- replacement on invalid scalar values. --- --- Examples: --- --- > justifyLeft 7 'x' "foo" == "fooxxxx" --- > justifyLeft 3 'x' "foobar" == "foobar" -justifyLeft :: Int64 -> Char -> Text -> Text -justifyLeft k c t - | len >= k = t - | otherwise = t `append` replicateChar (k-len) c - where len = length t -{-# INLINE [1] justifyLeft #-} - -{-# RULES -"LAZY TEXT justifyLeft -> fused" [~1] forall k c t. - justifyLeft k c t = unstream (S.justifyLeftI k c (stream t)) -"LAZY TEXT justifyLeft -> unfused" [1] forall k c t. - unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t - #-} - --- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Performs replacement on --- invalid scalar values. --- --- Examples: --- --- > justifyRight 7 'x' "bar" == "xxxxbar" --- > justifyRight 3 'x' "foobar" == "foobar" -justifyRight :: Int64 -> Char -> Text -> Text -justifyRight k c t - | len >= k = t - | otherwise = replicateChar (k-len) c `append` t - where len = length t -{-# INLINE justifyRight #-} - --- | /O(n)/ Center a string to the given length, using the specified --- fill character on either side. Performs replacement on invalid --- scalar values. --- --- Examples: --- --- > center 8 'x' "HS" = "xxxHSxxx" -center :: Int64 -> Char -> Text -> Text -center k c t - | len >= k = t - | otherwise = replicateChar l c `append` t `append` replicateChar r c - where len = length t - d = k - len - r = d `quot` 2 - l = d - r -{-# INLINE center #-} - --- | /O(n)/ The 'transpose' function transposes the rows and columns --- of its 'Text' argument. Note that this function uses 'pack', --- 'unpack', and the list version of transpose, and is thus not very --- efficient. -transpose :: [Text] -> [Text] -transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) - (L.transpose (L.map unpack ts)) --- TODO: make this fast - --- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. -reverse :: Text -> Text -reverse = rev Empty - where rev a Empty = a - rev a (Chunk t ts) = rev (Chunk (T.reverse t) a) ts - --- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in --- @haystack@ with @replacement@. --- --- This function behaves as though it was defined as follows: --- --- @ --- replace needle replacement haystack = --- 'intercalate' replacement ('splitOn' needle haystack) --- @ --- --- As this suggests, each occurrence is replaced exactly once. So if --- @needle@ occurs in @replacement@, that occurrence will /not/ itself --- be replaced recursively: --- --- > replace "oo" "foo" "oo" == "foo" --- --- In cases where several instances of @needle@ overlap, only the --- first one will be replaced: --- --- > replace "ofo" "bar" "ofofo" == "barfo" --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -replace :: Text - -- ^ @needle@ to search for. If this string is empty, an - -- error will occur. - -> Text - -- ^ @replacement@ to replace @needle@ with. - -> Text - -- ^ @haystack@ in which to search. - -> Text -replace s d = intercalate d . splitOn s -{-# INLINE replace #-} - --- ---------------------------------------------------------------------------- --- ** Case conversions (folds) - --- $case --- --- With Unicode text, it is incorrect to use combinators like @map --- toUpper@ to case convert each character of a string individually. --- Instead, use the whole-string case conversion functions from this --- module. For correctness in different writing systems, these --- functions may map one input character to two or three output --- characters. - --- | /O(n)/ Convert a string to folded case. Subject to fusion. --- --- This function is mainly useful for performing caseless (or case --- insensitive) string comparisons. --- --- A string @x@ is a caseless match for a string @y@ if and only if: --- --- @toCaseFold x == toCaseFold y@ --- --- The result string may be longer than the input string, and may --- differ from applying 'toLower' to the input string. For instance, --- the Armenian small ligature men now (U+FB13) is case folded to the --- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is --- case folded to the Greek small letter letter mu (U+03BC) instead of --- itself. -toCaseFold :: Text -> Text -toCaseFold t = unstream (S.toCaseFold (stream t)) -{-# INLINE [0] toCaseFold #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. Subject to fusion. --- --- The result string may be longer than the input string. For --- instance, the Latin capital letter I with dot above (U+0130) maps --- to the sequence Latin small letter i (U+0069) followed by combining --- dot above (U+0307). -toLower :: Text -> Text -toLower t = unstream (S.toLower (stream t)) -{-# INLINE toLower #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. Subject to fusion. --- --- The result string may be longer than the input string. For --- instance, the German eszett (U+00DF) maps to the two-letter --- sequence SS. -toUpper :: Text -> Text -toUpper t = unstream (S.toUpper (stream t)) -{-# INLINE toUpper #-} - - --- | /O(n)/ Convert a string to title case, using simple case --- conversion. Subject to fusion. --- --- The first letter of the input is converted to title case, as is --- every subsequent letter that immediately follows a non-letter. --- Every letter that immediately follows another letter is converted --- to lower case. --- --- The result string may be longer than the input string. For example, --- the Latin small ligature fl (U+FB02) is converted to the --- sequence Latin capital letter F (U+0046) followed by Latin small --- letter l (U+006C). --- --- /Note/: this function does not take language or culture specific --- rules into account. For instance, in English, different style --- guides disagree on whether the book name \"The Hill of the Red --- Fox\" is correctly title cased—but this function will --- capitalize /every/ word. -toTitle :: Text -> Text -toTitle t = unstream (S.toTitle (stream t)) -{-# INLINE toTitle #-} - --- | /O(n)/ 'foldl', applied to a binary operator, a starting value --- (typically the left-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from left to right. --- Subject to fusion. -foldl :: (a -> Char -> a) -> a -> Text -> a -foldl f z t = S.foldl f z (stream t) -{-# INLINE foldl #-} - --- | /O(n)/ A strict version of 'foldl'. --- Subject to fusion. -foldl' :: (a -> Char -> a) -> a -> Text -> a -foldl' f z t = S.foldl' f z (stream t) -{-# INLINE foldl' #-} - --- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to fusion. -foldl1 :: (Char -> Char -> Char) -> Text -> Char -foldl1 f t = S.foldl1 f (stream t) -{-# INLINE foldl1 #-} - --- | /O(n)/ A strict version of 'foldl1'. Subject to fusion. -foldl1' :: (Char -> Char -> Char) -> Text -> Char -foldl1' f t = S.foldl1' f (stream t) -{-# INLINE foldl1' #-} - --- | /O(n)/ 'foldr', applied to a binary operator, a starting value --- (typically the right-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from right to left. --- Subject to fusion. -foldr :: (Char -> a -> a) -> a -> Text -> a -foldr f z t = S.foldr f z (stream t) -{-# INLINE foldr #-} - --- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to --- fusion. -foldr1 :: (Char -> Char -> Char) -> Text -> Char -foldr1 f t = S.foldr1 f (stream t) -{-# INLINE foldr1 #-} - --- | /O(n)/ Concatenate a list of 'Text's. -concat :: [Text] -> Text -concat = to - where - go Empty css = to css - go (Chunk c cs) css = Chunk c (go cs css) - to [] = Empty - to (cs:css) = go cs css -{-# INLINE concat #-} - --- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and --- concatenate the results. -concatMap :: (Char -> Text) -> Text -> Text -concatMap f = concat . foldr ((:) . f) [] -{-# INLINE concatMap #-} - --- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. Subject to fusion. -any :: (Char -> Bool) -> Text -> Bool -any p t = S.any p (stream t) -{-# INLINE any #-} - --- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. Subject to fusion. -all :: (Char -> Bool) -> Text -> Bool -all p t = S.all p (stream t) -{-# INLINE all #-} - --- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. Subject to fusion. -maximum :: Text -> Char -maximum t = S.maximum (stream t) -{-# INLINE maximum #-} - --- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. Subject to fusion. -minimum :: Text -> Char -minimum t = S.minimum (stream t) -{-# INLINE minimum #-} - --- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. Subject to fusion. --- Performs replacement on invalid scalar values. --- --- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --- --- Note that --- --- > last (scanl f z xs) == foldl f z xs. -scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl g z (stream t)) - where g a b = safe (f a b) -{-# INLINE scanl #-} - --- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. Subject to fusion. Performs replacement on --- invalid scalar values. --- --- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] -scanl1 :: (Char -> Char -> Char) -> Text -> Text -scanl1 f t0 = case uncons t0 of - Nothing -> empty - Just (t,ts) -> scanl f t ts -{-# INLINE scanl1 #-} - --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs --- replacement on invalid scalar values. --- --- > scanr f v == reverse . scanl (flip f) v . reverse -scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f v = reverse . scanl g v . reverse - where g a b = safe (f b a) - --- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. Performs replacement on invalid scalar values. -scanr1 :: (Char -> Char -> Char) -> Text -> Text -scanr1 f t | null t = empty - | otherwise = scanr f (last t) (init t) - --- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a --- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. Performs --- replacement on invalid scalar values. -mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumL f = go - where - go z (Chunk c cs) = (z'', Chunk c' cs') - where (z', c') = T.mapAccumL f z c - (z'', cs') = go z' cs - go z Empty = (z, Empty) -{-# INLINE mapAccumL #-} - --- | The 'mapAccumR' function behaves like a combination of 'map' and --- a strict 'foldr'; it applies a function to each element of a --- 'Text', passing an accumulating parameter from right to left, and --- returning a final value of this accumulator together with the new --- 'Text'. Performs replacement on invalid scalar values. -mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumR f = go - where - go z (Chunk c cs) = (z'', Chunk c' cs') - where (z'', c') = T.mapAccumR f z' c - (z', cs') = go z cs - go z Empty = (z, Empty) -{-# INLINE mapAccumR #-} - --- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every --- element. -repeat :: Char -> Text -repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t - in t - --- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input --- @t@ repeated @n@ times. -replicate :: Int64 -> Text -> Text -replicate n t - | null t || n <= 0 = empty - | isSingleton t = replicateChar n (head t) - | otherwise = concat (rep 0) - where rep !i | i >= n = [] - | otherwise = t : rep (i+1) -{-# INLINE [1] replicate #-} - --- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or --- equivalently, the infinite repetition of the original 'Text'. -cycle :: Text -> Text -cycle Empty = emptyError "cycle" -cycle t = let t' = foldrChunks Chunk t' t - in t' - --- | @'iterate' f x@ returns an infinite 'Text' of repeated applications --- of @f@ to @x@: --- --- > iterate f x == [x, f x, f (f x), ...] -iterate :: (Char -> Char) -> Char -> Text -iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) - in t c - --- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the --- value of every element. Subject to fusion. -replicateChar :: Int64 -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n (safe c)) -{-# INLINE replicateChar #-} - -{-# RULES -"LAZY TEXT replicate/singleton -> replicateChar" [~1] forall n c. - replicate n (singleton c) = replicateChar n c - #-} - --- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' --- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a --- 'Text' from a seed value. The function takes the element and --- returns 'Nothing' if it is done producing the 'Text', otherwise --- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. Performs --- replacement on invalid scalar values. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) -{-# INLINE unfoldr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed --- value. However, the length of the result should be limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the maximum length of the result is known and --- correct, otherwise its performance is similar to 'unfoldr'. --- Performs replacement on invalid scalar values. -unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) -{-# INLINE unfoldrN #-} - --- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the --- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. Subject to fusion. -take :: Int64 -> Text -> Text -take i _ | i <= 0 = Empty -take i t0 = take' i t0 - where take' 0 _ = Empty - take' _ Empty = Empty - take' n (Chunk t ts) - | n < len = Chunk (T.take (fromIntegral n) t) Empty - | otherwise = Chunk t (take' (n - len) ts) - where len = fromIntegral (T.length t) -{-# INLINE [1] take #-} - -{-# RULES -"LAZY TEXT take -> fused" [~1] forall n t. - take n t = unstream (S.take n (stream t)) -"LAZY TEXT take -> unfused" [1] forall n t. - unstream (S.take n (stream t)) = take n t - #-} - --- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after --- taking @n@ characters from the end of @t@. --- --- Examples: --- --- > takeEnd 3 "foobar" == "bar" -takeEnd :: Int64 -> Text -> Text -takeEnd n t0 - | n <= 0 = empty - | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 - where takeChunk _ acc [] = acc - takeChunk i acc (t:ts) - | i <= l = chunk (T.takeEnd (fromIntegral i) t) acc - | otherwise = takeChunk (i-l) (Chunk t acc) ts - where l = fromIntegral (T.length t) - --- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the --- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. Subject to fusion. -drop :: Int64 -> Text -> Text -drop i t0 - | i <= 0 = t0 - | otherwise = drop' i t0 - where drop' 0 ts = ts - drop' _ Empty = Empty - drop' n (Chunk t ts) - | n < len = Chunk (T.drop (fromIntegral n) t) ts - | otherwise = drop' (n - len) ts - where len = fromIntegral (T.length t) -{-# INLINE [1] drop #-} - -{-# RULES -"LAZY TEXT drop -> fused" [~1] forall n t. - drop n t = unstream (S.drop n (stream t)) -"LAZY TEXT drop -> unfused" [1] forall n t. - unstream (S.drop n (stream t)) = drop n t - #-} - --- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after --- dropping @n@ characters from the end of @t@. --- --- Examples: --- --- > dropEnd 3 "foobar" == "foo" -dropEnd :: Int64 -> Text -> Text -dropEnd n t0 - | n <= 0 = t0 - | otherwise = dropChunk n . L.reverse . toChunks $ t0 - where dropChunk _ [] = empty - dropChunk m (t:ts) - | m >= l = dropChunk (m-l) ts - | otherwise = fromChunks . L.reverse $ - T.dropEnd (fromIntegral m) t : ts - where l = fromIntegral (T.length t) - --- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word16' --- values dropped, or the empty 'Text' if @n@ is greater than the --- number of 'Word16' values present. -dropWords :: Int64 -> Text -> Text -dropWords i t0 - | i <= 0 = t0 - | otherwise = drop' i t0 - where drop' 0 ts = ts - drop' _ Empty = Empty - drop' n (Chunk (T.Text arr off len) ts) - | n < len' = chunk (text arr (off+n') (len-n')) ts - | otherwise = drop' (n - len') ts - where len' = fromIntegral len - n' = fromIntegral n - --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', --- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. Subject to fusion. -takeWhile :: (Char -> Bool) -> Text -> Text -takeWhile p t0 = takeWhile' t0 - where takeWhile' Empty = Empty - takeWhile' (Chunk t ts) = - case T.findIndex (not . p) t of - Just n | n > 0 -> Chunk (T.take n t) Empty - | otherwise -> Empty - Nothing -> Chunk t (takeWhile' ts) -{-# INLINE [1] takeWhile #-} - -{-# RULES -"LAZY TEXT takeWhile -> fused" [~1] forall p t. - takeWhile p t = unstream (S.takeWhile p (stream t)) -"LAZY TEXT takeWhile -> unfused" [1] forall p t. - unstream (S.takeWhile p (stream t)) = takeWhile p t - #-} --- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', --- returns the longest suffix (possibly empty) of elements that --- satisfy @p@. --- Examples: --- --- > takeWhileEnd (=='o') "foo" == "oo" -takeWhileEnd :: (Char -> Bool) -> Text -> Text -takeWhileEnd p = takeChunk empty . L.reverse . toChunks - where takeChunk acc [] = acc - takeChunk acc (t:ts) = if T.length t' < T.length t - then (Chunk t' acc) - else takeChunk (Chunk t' acc) ts - where t' = T.takeWhileEnd p t -{-# INLINE takeWhileEnd #-} - --- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. Subject to fusion. -dropWhile :: (Char -> Bool) -> Text -> Text -dropWhile p t0 = dropWhile' t0 - where dropWhile' Empty = Empty - dropWhile' (Chunk t ts) = - case T.findIndex (not . p) t of - Just n -> Chunk (T.drop n t) ts - Nothing -> dropWhile' ts -{-# INLINE [1] dropWhile #-} - -{-# RULES -"LAZY TEXT dropWhile -> fused" [~1] forall p t. - dropWhile p t = unstream (S.dropWhile p (stream t)) -"LAZY TEXT dropWhile -> unfused" [1] forall p t. - unstream (S.dropWhile p (stream t)) = dropWhile p t - #-} - --- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after --- dropping characters that satisfy the predicate @p@ from the end of --- @t@. --- --- Examples: --- --- > dropWhileEnd (=='.') "foo..." == "foo" -dropWhileEnd :: (Char -> Bool) -> Text -> Text -dropWhileEnd p = go - where go Empty = Empty - go (Chunk t Empty) = if T.null t' - then Empty - else Chunk t' Empty - where t' = T.dropWhileEnd p t - go (Chunk t ts) = case go ts of - Empty -> go (Chunk t Empty) - ts' -> Chunk t ts' -{-# INLINE dropWhileEnd #-} - --- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after --- dropping characters that satisfy the predicate @p@ from both the --- beginning and end of @t@. Subject to fusion. -dropAround :: (Char -> Bool) -> Text -> Text -dropAround p = dropWhile p . dropWhileEnd p -{-# INLINE [1] dropAround #-} - --- | /O(n)/ Remove leading white space from a string. Equivalent to: --- --- > dropWhile isSpace -stripStart :: Text -> Text -stripStart = dropWhile isSpace -{-# INLINE [1] stripStart #-} - --- | /O(n)/ Remove trailing white space from a string. Equivalent to: --- --- > dropWhileEnd isSpace -stripEnd :: Text -> Text -stripEnd = dropWhileEnd isSpace -{-# INLINE [1] stripEnd #-} - --- | /O(n)/ Remove leading and trailing white space from a string. --- Equivalent to: --- --- > dropAround isSpace -strip :: Text -> Text -strip = dropAround isSpace -{-# INLINE [1] strip #-} - --- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a --- prefix of @t@ of length @n@, and whose second is the remainder of --- the string. It is equivalent to @('take' n t, 'drop' n t)@. -splitAt :: Int64 -> Text -> (Text, Text) -splitAt = loop - where loop _ Empty = (empty, empty) - loop n t | n <= 0 = (empty, t) - loop n (Chunk t ts) - | n < len = let (t',t'') = T.splitAt (fromIntegral n) t - in (Chunk t' Empty, Chunk t'' ts) - | otherwise = let (ts',ts'') = loop (n - len) ts - in (Chunk t ts', ts'') - where len = fromIntegral (T.length t) - --- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first --- element is a prefix of @t@ whose chunks contain @n@ 'Word16' --- values, and whose second is the remainder of the string. -splitAtWord :: Int64 -> Text -> PairS Text Text -splitAtWord _ Empty = empty :*: empty -splitAtWord x (Chunk c@(T.Text arr off len) cs) - | y >= len = let h :*: t = splitAtWord (x-fromIntegral len) cs - in Chunk c h :*: t - | otherwise = chunk (text arr off y) empty :*: - chunk (text arr (off+y) (len-y)) cs - where y = fromIntegral x - --- | /O(n+m)/ Find the first instance of @needle@ (which must be --- non-'null') in @haystack@. The first element of the returned tuple --- is the prefix of @haystack@ before @needle@ is matched. The second --- is the remainder of @haystack@, starting with the match. --- --- Examples: --- --- > breakOn "::" "a::b::c" ==> ("a", "::b::c") --- > breakOn "/" "foobar" ==> ("foobar", "") --- --- Laws: --- --- > append prefix match == haystack --- > where (prefix, match) = breakOn needle haystack --- --- If you need to break a string by a substring repeatedly (e.g. you --- want to break on every instance of a substring), use 'breakOnAll' --- instead, as it has lower startup overhead. --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -breakOn :: Text -> Text -> (Text, Text) -breakOn pat src - | null pat = emptyError "breakOn" - | otherwise = case indices pat src of - [] -> (src, empty) - (x:_) -> let h :*: t = splitAtWord x src - in (h, t) - --- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. --- --- The first element of the returned tuple is the prefix of @haystack@ --- up to and including the last match of @needle@. The second is the --- remainder of @haystack@, following the match. --- --- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") -breakOnEnd :: Text -> Text -> (Text, Text) -breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) - in (reverse b, reverse a) -{-# INLINE breakOnEnd #-} - --- | /O(n+m)/ Find all non-overlapping instances of @needle@ in --- @haystack@. Each element of the returned list consists of a pair: --- --- * The entire string prior to the /k/th match (i.e. the prefix) --- --- * The /k/th match, followed by the remainder of the string --- --- Examples: --- --- > breakOnAll "::" "" --- > ==> [] --- > breakOnAll "/" "a/b/c/" --- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. --- --- The @needle@ parameter may not be empty. -breakOnAll :: Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -breakOnAll pat src - | null pat = emptyError "breakOnAll" - | otherwise = go 0 empty src (indices pat src) - where - go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s - h' = append p h - in (h',t) : go x h' t xs - go _ _ _ _ = [] - --- | /O(n)/ 'break' is like 'span', but the prefix returned is over --- elements that fail the predicate @p@. -break :: (Char -> Bool) -> Text -> (Text, Text) -break p t0 = break' t0 - where break' Empty = (empty, empty) - break' c@(Chunk t ts) = - case T.findIndex p t of - Nothing -> let (ts', ts'') = break' ts - in (Chunk t ts', ts'') - Just n | n == 0 -> (Empty, c) - | otherwise -> let (a,b) = T.splitAt n t - in (Chunk a Empty, Chunk b ts) - --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns --- a pair whose first element is the longest prefix (possibly empty) --- of @t@ of elements that satisfy @p@, and whose second is the --- remainder of the list. -span :: (Char -> Bool) -> Text -> (Text, Text) -span p = break (not . p) -{-# INLINE span #-} - --- | The 'group' function takes a 'Text' and returns a list of 'Text's --- such that the concatenation of the result is equal to the argument. --- Moreover, each sublist in the result contains only equal elements. --- For example, --- --- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] --- --- It is a special case of 'groupBy', which allows the programmer to --- supply their own equality test. -group :: Text -> [Text] -group = groupBy (==) -{-# INLINE group #-} - --- | The 'groupBy' function is the non-overloaded version of 'group'. -groupBy :: (Char -> Char -> Bool) -> Text -> [Text] -groupBy _ Empty = [] -groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs - where (ys,zs) = span (eq x) xs - x = T.unsafeHead t - xs = chunk (T.unsafeTail t) ts - --- | /O(n)/ Return all initial segments of the given 'Text', --- shortest first. -inits :: Text -> [Text] -inits = (Empty :) . inits' - where inits' Empty = [] - inits' (Chunk t ts) = L.map (\t' -> Chunk t' Empty) (L.tail (T.inits t)) - ++ L.map (Chunk t) (inits' ts) - --- | /O(n)/ Return all final segments of the given 'Text', longest --- first. -tails :: Text -> [Text] -tails Empty = Empty : [] -tails ts@(Chunk t ts') - | T.length t == 1 = ts : tails ts' - | otherwise = ts : tails (Chunk (T.unsafeTail t) ts') - --- $split --- --- Splitting functions in this library do not perform character-wise --- copies to create substrings; they just construct new 'Text's that --- are slices of the original. - --- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' --- argument (which cannot be an empty string), consuming the --- delimiter. An empty delimiter is invalid, and will cause an error --- to be raised. --- --- Examples: --- --- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] --- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] --- > splitOn "x" "x" == ["",""] --- --- and --- --- > intercalate s . splitOn s == id --- > splitOn (singleton c) == split (==c) --- --- (Note: the string @s@ to split on above cannot be empty.) --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -splitOn :: Text - -- ^ String to split on. If this string is empty, an error - -- will occur. - -> Text - -- ^ Input text. - -> [Text] -splitOn pat src - | null pat = emptyError "splitOn" - | isSingleton pat = split (== head pat) src - | otherwise = go 0 (indices pat src) src - where - go _ [] cs = [cs] - go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs - in h : go (x+l) xs (dropWords l t) - l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat -{-# INLINE [1] splitOn #-} - -{-# RULES -"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. - splitOn (singleton c) t = split (==c) t - #-} - --- | /O(n)/ Splits a 'Text' into components delimited by separators, --- where the predicate returns True for a separator element. The --- resulting components do not contain the separators. Two adjacent --- separators result in an empty component in the output. eg. --- --- > split (=='a') "aabbaca" == ["","","bb","c",""] --- > split (=='a') [] == [""] -split :: (Char -> Bool) -> Text -> [Text] -split _ Empty = [Empty] -split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 - where comb acc (s:[]) Empty = revChunks (s:acc) : [] - comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts - comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts - comb _ [] _ = impossibleError "split" -{-# INLINE split #-} - --- | /O(n)/ Splits a 'Text' into components of length @k@. The last --- element may be shorter than the other chunks, depending on the --- length of the input. Examples: --- --- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] --- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] -chunksOf :: Int64 -> Text -> [Text] -chunksOf k = go - where - go t = case splitAt k t of - (a,b) | null a -> [] - | otherwise -> a : go b -{-# INLINE chunksOf #-} - --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at --- newline 'Char's. The resulting strings do not contain newlines. -lines :: Text -> [Text] -lines Empty = [] -lines t = let (l,t') = break ((==) '\n') t - in l : if null t' then [] - else lines (tail t') - --- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's --- representing white space. -words :: Text -> [Text] -words = L.filter (not . null) . split isSpace -{-# INLINE words #-} - --- | /O(n)/ Joins lines, after appending a terminating newline to --- each. -unlines :: [Text] -> Text -unlines = concat . L.map (`snoc` '\n') -{-# INLINE unlines #-} - --- | /O(n)/ Joins words using single space characters. -unwords :: [Text] -> Text -unwords = intercalate (singleton ' ') -{-# INLINE unwords #-} - --- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. Subject to fusion. -isPrefixOf :: Text -> Text -> Bool -isPrefixOf Empty _ = True -isPrefixOf _ Empty = False -isPrefixOf (Chunk x xs) (Chunk y ys) - | lx == ly = x == y && isPrefixOf xs ys - | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) - | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys - where (xh,xt) = T.splitAt ly x - (yh,yt) = T.splitAt lx y - lx = T.length x - ly = T.length y -{-# INLINE [1] isPrefixOf #-} - -{-# RULES -"LAZY TEXT isPrefixOf -> fused" [~1] forall s t. - isPrefixOf s t = S.isPrefixOf (stream s) (stream t) -"LAZY TEXT isPrefixOf -> unfused" [1] forall s t. - S.isPrefixOf (stream s) (stream t) = isPrefixOf s t - #-} - --- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns --- 'True' iff the first is a suffix of the second. -isSuffixOf :: Text -> Text -> Bool -isSuffixOf x y = reverse x `isPrefixOf` reverse y -{-# INLINE isSuffixOf #-} --- TODO: a better implementation - --- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns --- 'True' iff the first is contained, wholly and intact, anywhere --- within the second. --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -isInfixOf :: Text -> Text -> Bool -isInfixOf needle haystack - | null needle = True - | isSingleton needle = S.elem (head needle) . S.stream $ haystack - | otherwise = not . L.null . indices needle $ haystack -{-# INLINE [1] isInfixOf #-} - -{-# RULES -"LAZY TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. - isInfixOf (singleton n) h = S.elem n (S.stream h) - #-} - -------------------------------------------------------------------------------- --- * View patterns - --- | /O(n)/ Return the suffix of the second string if its prefix --- matches the entire first string. --- --- Examples: --- --- > stripPrefix "foo" "foobar" == Just "bar" --- > stripPrefix "" "baz" == Just "baz" --- > stripPrefix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text.Lazy as T --- > --- > fnordLength :: Text -> Int --- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf --- > fnordLength _ = -1 -stripPrefix :: Text -> Text -> Maybe Text -stripPrefix p t - | null p = Just t - | otherwise = case commonPrefixes p t of - Just (_,c,r) | null c -> Just r - _ -> Nothing - --- | /O(n)/ Find the longest non-empty common prefix of two strings --- and return it, along with the suffixes of each string at which they --- no longer match. --- --- If the strings do not have a common prefix or either one is empty, --- this function returns 'Nothing'. --- --- Examples: --- --- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") --- > commonPrefixes "veeble" "fetzer" == Nothing --- > commonPrefixes "" "baz" == Nothing -commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) -commonPrefixes Empty _ = Nothing -commonPrefixes _ Empty = Nothing -commonPrefixes a0 b0 = Just (go a0 b0 []) - where - go t0@(Chunk x xs) t1@(Chunk y ys) ps - = case T.commonPrefixes x y of - Just (p,a,b) - | T.null a -> go xs (chunk b ys) (p:ps) - | T.null b -> go (chunk a xs) ys (p:ps) - | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) - Nothing -> (fromChunks (L.reverse ps),t0,t1) - go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) - --- | /O(n)/ Return the prefix of the second string if its suffix --- matches the entire first string. --- --- Examples: --- --- > stripSuffix "bar" "foobar" == Just "foo" --- > stripSuffix "" "baz" == Just "baz" --- > stripSuffix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text.Lazy as T --- > --- > quuxLength :: Text -> Int --- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre --- > quuxLength _ = -1 -stripSuffix :: Text -> Text -> Maybe Text -stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) - --- | /O(n)/ 'filter', applied to a predicate and a 'Text', --- returns a 'Text' containing those characters that satisfy the --- predicate. -filter :: (Char -> Bool) -> Text -> Text -filter p t = unstream (S.filter p (stream t)) -{-# INLINE filter #-} - --- | /O(n)/ The 'find' function takes a predicate and a 'Text', and --- returns the first element in matching the predicate, or 'Nothing' --- if there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.findBy p (stream t) -{-# INLINE find #-} - --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', --- and returns the pair of 'Text's with elements which do and do not --- satisfy the predicate, respectively; i.e. --- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} - --- | /O(n)/ 'Text' index (subscript) operator, starting from 0. -index :: Text -> Int64 -> Char -index t n = S.index (stream t) n -{-# INLINE index #-} - --- | /O(n+m)/ The 'count' function returns the number of times the --- query string appears in the given 'Text'. An empty query string is --- invalid, and will cause an error to be raised. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -count :: Text -> Text -> Int64 -count pat src - | null pat = emptyError "count" - | otherwise = go 0 (indices pat src) - where go !n [] = n - go !n (_:xs) = go (n+1) xs -{-# INLINE [1] count #-} - -{-# RULES -"LAZY TEXT count/singleton -> countChar" [~1] forall c t. - count (singleton c) t = countChar c t - #-} - --- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. Subject to fusion. -countChar :: Char -> Text -> Int64 -countChar c t = S.countChar c (stream t) - --- | /O(n)/ 'zip' takes two 'Text's and returns a list of --- corresponding pairs of bytes. If one input 'Text' is short, --- excess elements of the longer 'Text' are discarded. This is --- equivalent to a pair of 'unpack' operations. -zip :: Text -> Text -> [(Char,Char)] -zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) -{-# INLINE [0] zip #-} - --- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function --- given as the first argument, instead of a tupling function. --- Performs replacement on invalid scalar values. -zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) - where g a b = safe (f a b) -{-# INLINE [0] zipWith #-} - -revChunks :: [T.Text] -> Text -revChunks = L.foldl' (flip chunk) Empty - -emptyError :: String -> a -emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") - -impossibleError :: String -> a -impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") diff --git a/Data/Text/Lazy/Builder.hs b/Data/Text/Lazy/Builder.hs deleted file mode 100644 --- a/Data/Text/Lazy/Builder.hs +++ /dev/null @@ -1,58 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, Rank2Types #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - ------------------------------------------------------------------------------ --- | --- Module : Data.Text.Lazy.Builder --- Copyright : (c) 2013 Bryan O'Sullivan --- (c) 2010 Johan Tibell --- License : BSD-style (see LICENSE) --- --- Maintainer : Johan Tibell --- Stability : experimental --- Portability : portable to Hugs and GHC --- --- Efficient construction of lazy @Text@ values. The principal --- operations on a @Builder@ are @singleton@, @fromText@, and --- @fromLazyText@, which construct new builders, and 'mappend', which --- concatenates two builders. --- --- To get maximum performance when building lazy @Text@ values using a --- builder, associate @mappend@ calls to the right. For example, --- prefer --- --- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') --- --- to --- --- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' --- --- as the latter associates @mappend@ to the left. Or, equivalently, --- prefer --- --- > singleton 'a' <> singleton 'b' <> singleton 'c' --- --- since the '<>' from recent versions of 'Data.Monoid' associates --- to the right. - ------------------------------------------------------------------------------ - -module Data.Text.Lazy.Builder - ( -- * The Builder type - Builder - , toLazyText - , toLazyTextWith - - -- * Constructing Builders - , singleton - , fromText - , fromLazyText - , fromString - - -- * Flushing the buffer state - , flush - ) where - -import Data.Text.Internal.Builder diff --git a/Data/Text/Lazy/Builder/Int.hs b/Data/Text/Lazy/Builder/Int.hs deleted file mode 100644 --- a/Data/Text/Lazy/Builder/Int.hs +++ /dev/null @@ -1,262 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, ScopedTypeVariables, - UnboxedTuples #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - --- Module: Data.Text.Lazy.Builder.Int --- Copyright: (c) 2013 Bryan O'Sullivan --- (c) 2011 MailRank, Inc. --- License: BSD-style --- Maintainer: Bryan O'Sullivan --- Stability: experimental --- Portability: portable --- --- Efficiently write an integral value to a 'Builder'. - -module Data.Text.Lazy.Builder.Int - ( - decimal - , hexadecimal - ) where - -import Data.Int (Int8, Int16, Int32, Int64) -import Data.Monoid (mempty) -import qualified Data.ByteString.Unsafe as B -import Data.Text.Internal.Builder.Functions ((<>), i2d) -import Data.Text.Internal.Builder -import Data.Text.Internal.Builder.Int.Digits (digits) -import Data.Text.Array -import Data.Word (Word, Word8, Word16, Word32, Word64) -import GHC.Base (quotInt, remInt) -import GHC.Num (quotRemInteger) -import GHC.Types (Int(..)) -import Control.Monad.ST - -#ifdef __GLASGOW_HASKELL__ -# if defined(INTEGER_GMP) -import GHC.Integer.GMP.Internals (Integer(S#)) -# elif defined(INTEGER_SIMPLE) -import GHC.Integer -# else -# error "You need to use either GMP or integer-simple." -# endif -#endif - -#if defined(INTEGER_GMP) || defined(INTEGER_SIMPLE) -# define PAIR(a,b) (# a,b #) -#else -# define PAIR(a,b) (a,b) -#endif - -decimal :: Integral a => a -> Builder -{-# RULES "decimal/Int8" decimal = boundedDecimal :: Int8 -> Builder #-} -{-# RULES "decimal/Int" decimal = boundedDecimal :: Int -> Builder #-} -{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-} -{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-} -{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-} -{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-} -{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-} -{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-} -{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-} -{-# RULES "decimal/Word64" decimal = positive :: Word64 -> Builder #-} -{-# RULES "decimal/Integer" decimal = integer 10 :: Integer -> Builder #-} -decimal i = decimal' (<= -128) i -{-# NOINLINE decimal #-} - -boundedDecimal :: (Integral a, Bounded a) => a -> Builder -{-# SPECIALIZE boundedDecimal :: Int -> Builder #-} -{-# SPECIALIZE boundedDecimal :: Int8 -> Builder #-} -{-# SPECIALIZE boundedDecimal :: Int16 -> Builder #-} -{-# SPECIALIZE boundedDecimal :: Int32 -> Builder #-} -{-# SPECIALIZE boundedDecimal :: Int64 -> Builder #-} -boundedDecimal i = decimal' (== minBound) i - -decimal' :: (Integral a) => (a -> Bool) -> a -> Builder -{-# INLINE decimal' #-} -decimal' p i - | i < 0 = if p i - then let (q, r) = i `quotRem` 10 - qq = -q - !n = countDigits qq - in writeN (n + 2) $ \marr off -> do - unsafeWrite marr off minus - posDecimal marr (off+1) n qq - unsafeWrite marr (off+n+1) (i2w (-r)) - else let j = -i - !n = countDigits j - in writeN (n + 1) $ \marr off -> - unsafeWrite marr off minus >> posDecimal marr (off+1) n j - | otherwise = positive i - -positive :: (Integral a) => a -> Builder -{-# SPECIALIZE positive :: Int -> Builder #-} -{-# SPECIALIZE positive :: Int8 -> Builder #-} -{-# SPECIALIZE positive :: Int16 -> Builder #-} -{-# SPECIALIZE positive :: Int32 -> Builder #-} -{-# SPECIALIZE positive :: Int64 -> Builder #-} -{-# SPECIALIZE positive :: Word -> Builder #-} -{-# SPECIALIZE positive :: Word8 -> Builder #-} -{-# SPECIALIZE positive :: Word16 -> Builder #-} -{-# SPECIALIZE positive :: Word32 -> Builder #-} -{-# SPECIALIZE positive :: Word64 -> Builder #-} -positive i - | i < 10 = writeN 1 $ \marr off -> unsafeWrite marr off (i2w i) - | otherwise = let !n = countDigits i - in writeN n $ \marr off -> posDecimal marr off n i - -posDecimal :: (Integral a) => - forall s. MArray s -> Int -> Int -> a -> ST s () -{-# INLINE posDecimal #-} -posDecimal marr off0 ds v0 = go (off0 + ds - 1) v0 - where go off v - | v >= 100 = do - let (q, r) = v `quotRem` 100 - write2 off r - go (off - 2) q - | v < 10 = unsafeWrite marr off (i2w v) - | otherwise = write2 off v - write2 off i0 = do - let i = fromIntegral i0; j = i + i - unsafeWrite marr off $ get (j + 1) - unsafeWrite marr (off - 1) $ get j - get = fromIntegral . B.unsafeIndex digits - -minus, zero :: Word16 -{-# INLINE minus #-} -{-# INLINE zero #-} -minus = 45 -zero = 48 - -i2w :: (Integral a) => a -> Word16 -{-# INLINE i2w #-} -i2w v = zero + fromIntegral v - -countDigits :: (Integral a) => a -> Int -{-# INLINE countDigits #-} -countDigits v0 - | fromIntegral v64 == v0 = go 1 v64 - | otherwise = goBig 1 (fromIntegral v0) - where v64 = fromIntegral v0 - goBig !k (v :: Integer) - | v > big = goBig (k + 19) (v `quot` big) - | otherwise = go k (fromIntegral v) - big = 10000000000000000000 - go !k (v :: Word64) - | v < 10 = k - | v < 100 = k + 1 - | v < 1000 = k + 2 - | v < 1000000000000 = - k + if v < 100000000 - then if v < 1000000 - then if v < 10000 - then 3 - else 4 + fin v 100000 - else 6 + fin v 10000000 - else if v < 10000000000 - then 8 + fin v 1000000000 - else 10 + fin v 100000000000 - | otherwise = go (k + 12) (v `quot` 1000000000000) - fin v n = if v >= n then 1 else 0 - -hexadecimal :: Integral a => a -> Builder -{-# SPECIALIZE hexadecimal :: Int -> Builder #-} -{-# SPECIALIZE hexadecimal :: Int8 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Int16 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Int32 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Int64 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Word -> Builder #-} -{-# SPECIALIZE hexadecimal :: Word8 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Word16 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Word32 -> Builder #-} -{-# SPECIALIZE hexadecimal :: Word64 -> Builder #-} -{-# RULES "hexadecimal/Integer" - hexadecimal = hexInteger :: Integer -> Builder #-} -hexadecimal i - | i < 0 = error hexErrMsg - | otherwise = go i - where - go n | n < 16 = hexDigit n - | otherwise = go (n `quot` 16) <> hexDigit (n `rem` 16) -{-# NOINLINE[0] hexadecimal #-} - -hexInteger :: Integer -> Builder -hexInteger i - | i < 0 = error hexErrMsg - | otherwise = integer 16 i - -hexErrMsg :: String -hexErrMsg = "Data.Text.Lazy.Builder.Int.hexadecimal: applied to negative number" - -hexDigit :: Integral a => a -> Builder -hexDigit n - | n <= 9 = singleton $! i2d (fromIntegral n) - | otherwise = singleton $! toEnum (fromIntegral n + 87) -{-# INLINE hexDigit #-} - -data T = T !Integer !Int - -integer :: Int -> Integer -> Builder -#ifdef INTEGER_GMP -integer 10 (S# i#) = decimal (I# i#) -integer 16 (S# i#) = hexadecimal (I# i#) -#endif -integer base i - | i < 0 = singleton '-' <> go (-i) - | otherwise = go i - where - go n | n < maxInt = int (fromInteger n) - | otherwise = putH (splitf (maxInt * maxInt) n) - - splitf p n - | p > n = [n] - | otherwise = splith p (splitf (p*p) n) - - splith p (n:ns) = case n `quotRemInteger` p of - PAIR(q,r) | q > 0 -> q : r : splitb p ns - | otherwise -> r : splitb p ns - splith _ _ = error "splith: the impossible happened." - - splitb p (n:ns) = case n `quotRemInteger` p of - PAIR(q,r) -> q : r : splitb p ns - splitb _ _ = [] - - T maxInt10 maxDigits10 = - until ((>mi) . (*10) . fstT) (\(T n d) -> T (n*10) (d+1)) (T 10 1) - where mi = fromIntegral (maxBound :: Int) - T maxInt16 maxDigits16 = - until ((>mi) . (*16) . fstT) (\(T n d) -> T (n*16) (d+1)) (T 16 1) - where mi = fromIntegral (maxBound :: Int) - - fstT (T a _) = a - - maxInt | base == 10 = maxInt10 - | otherwise = maxInt16 - maxDigits | base == 10 = maxDigits10 - | otherwise = maxDigits16 - - putH (n:ns) = case n `quotRemInteger` maxInt of - PAIR(x,y) - | q > 0 -> int q <> pblock r <> putB ns - | otherwise -> int r <> putB ns - where q = fromInteger x - r = fromInteger y - putH _ = error "putH: the impossible happened" - - putB (n:ns) = case n `quotRemInteger` maxInt of - PAIR(x,y) -> pblock q <> pblock r <> putB ns - where q = fromInteger x - r = fromInteger y - putB _ = Data.Monoid.mempty - - int :: Int -> Builder - int x | base == 10 = decimal x - | otherwise = hexadecimal x - - pblock = loop maxDigits - where - loop !d !n - | d == 1 = hexDigit n - | otherwise = loop (d-1) q <> hexDigit r - where q = n `quotInt` base - r = n `remInt` base diff --git a/Data/Text/Lazy/Builder/RealFloat.hs b/Data/Text/Lazy/Builder/RealFloat.hs deleted file mode 100644 --- a/Data/Text/Lazy/Builder/RealFloat.hs +++ /dev/null @@ -1,242 +0,0 @@ -{-# LANGUAGE CPP, OverloadedStrings #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - --- | --- Module: Data.Text.Lazy.Builder.RealFloat --- Copyright: (c) The University of Glasgow 1994-2002 --- License: see libraries/base/LICENSE --- --- Write a floating point value to a 'Builder'. - -module Data.Text.Lazy.Builder.RealFloat - ( - FPFormat(..) - , realFloat - , formatRealFloat - ) where - -import Data.Array.Base (unsafeAt) -import Data.Array.IArray -import Data.Text.Internal.Builder.Functions ((<>), i2d) -import Data.Text.Lazy.Builder.Int (decimal) -import Data.Text.Internal.Builder.RealFloat.Functions (roundTo) -import Data.Text.Lazy.Builder -import qualified Data.Text as T - --- | Control the rendering of floating point numbers. -data FPFormat = Exponent - -- ^ Scientific notation (e.g. @2.3e123@). - | Fixed - -- ^ Standard decimal notation. - | Generic - -- ^ Use decimal notation for values between @0.1@ and - -- @9,999,999@, and scientific notation otherwise. - deriving (Enum, Read, Show) - --- | Show a signed 'RealFloat' value to full precision, --- using standard decimal notation for arguments whose absolute value lies --- between @0.1@ and @9,999,999@, and scientific notation otherwise. -realFloat :: (RealFloat a) => a -> Builder -{-# SPECIALIZE realFloat :: Float -> Builder #-} -{-# SPECIALIZE realFloat :: Double -> Builder #-} -realFloat x = formatRealFloat Generic Nothing x - -formatRealFloat :: (RealFloat a) => - FPFormat - -> Maybe Int -- ^ Number of decimal places to render. - -> a - -> Builder -{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Float -> Builder #-} -{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Double -> Builder #-} -formatRealFloat fmt decs x - | isNaN x = "NaN" - | isInfinite x = if x < 0 then "-Infinity" else "Infinity" - | x < 0 || isNegativeZero x = singleton '-' <> doFmt fmt (floatToDigits (-x)) - | otherwise = doFmt fmt (floatToDigits x) - where - doFmt format (is, e) = - let ds = map i2d is in - case format of - Generic -> - doFmt (if e < 0 || e > 7 then Exponent else Fixed) - (is,e) - Exponent -> - case decs of - Nothing -> - let show_e' = decimal (e-1) in - case ds of - "0" -> "0.0e0" - [d] -> singleton d <> ".0e" <> show_e' - (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> show_e' - [] -> error "formatRealFloat/doFmt/Exponent: []" - Just dec -> - let dec' = max dec 1 in - case is of - [0] -> "0." <> fromText (T.replicate dec' "0") <> "e0" - _ -> - let - (ei,is') = roundTo (dec'+1) is - (d:ds') = map i2d (if ei > 0 then init is' else is') - in - singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> decimal (e-1+ei) - Fixed -> - let - mk0 ls = case ls of { "" -> "0" ; _ -> fromString ls} - in - case decs of - Nothing - | e <= 0 -> "0." <> fromText (T.replicate (-e) "0") <> fromString ds - | otherwise -> - let - f 0 s rs = mk0 (reverse s) <> singleton '.' <> mk0 rs - f n s "" = f (n-1) ('0':s) "" - f n s (r:rs) = f (n-1) (r:s) rs - in - f e "" ds - Just dec -> - let dec' = max dec 0 in - if e >= 0 then - let - (ei,is') = roundTo (dec' + e) is - (ls,rs) = splitAt (e+ei) (map i2d is') - in - mk0 ls <> (if null rs then "" else singleton '.' <> fromString rs) - else - let - (ei,is') = roundTo dec' (replicate (-e) 0 ++ is) - d:ds' = map i2d (if ei > 0 then is' else 0:is') - in - singleton d <> (if null ds' then "" else singleton '.' <> fromString ds') - - --- Based on "Printing Floating-Point Numbers Quickly and Accurately" --- by R.G. Burger and R.K. Dybvig in PLDI 96. --- This version uses a much slower logarithm estimator. It should be improved. - --- | 'floatToDigits' takes a base and a non-negative 'RealFloat' number, --- and returns a list of digits and an exponent. --- In particular, if @x>=0@, and --- --- > floatToDigits base x = ([d1,d2,...,dn], e) --- --- then --- --- (1) @n >= 1@ --- --- (2) @x = 0.d1d2...dn * (base**e)@ --- --- (3) @0 <= di <= base-1@ - -floatToDigits :: (RealFloat a) => a -> ([Int], Int) -{-# SPECIALIZE floatToDigits :: Float -> ([Int], Int) #-} -{-# SPECIALIZE floatToDigits :: Double -> ([Int], Int) #-} -floatToDigits 0 = ([0], 0) -floatToDigits x = - let - (f0, e0) = decodeFloat x - (minExp0, _) = floatRange x - p = floatDigits x - b = floatRadix x - minExp = minExp0 - p -- the real minimum exponent - -- Haskell requires that f be adjusted so denormalized numbers - -- will have an impossibly low exponent. Adjust for this. - (f, e) = - let n = minExp - e0 in - if n > 0 then (f0 `quot` (expt b n), e0+n) else (f0, e0) - (r, s, mUp, mDn) = - if e >= 0 then - let be = expt b e in - if f == expt b (p-1) then - (f*be*b*2, 2*b, be*b, be) -- according to Burger and Dybvig - else - (f*be*2, 2, be, be) - else - if e > minExp && f == expt b (p-1) then - (f*b*2, expt b (-e+1)*2, b, 1) - else - (f*2, expt b (-e)*2, 1, 1) - k :: Int - k = - let - k0 :: Int - k0 = - if b == 2 then - -- logBase 10 2 is very slightly larger than 8651/28738 - -- (about 5.3558e-10), so if log x >= 0, the approximation - -- k1 is too small, hence we add one and need one fixup step less. - -- If log x < 0, the approximation errs rather on the high side. - -- That is usually more than compensated for by ignoring the - -- fractional part of logBase 2 x, but when x is a power of 1/2 - -- or slightly larger and the exponent is a multiple of the - -- denominator of the rational approximation to logBase 10 2, - -- k1 is larger than logBase 10 x. If k1 > 1 + logBase 10 x, - -- we get a leading zero-digit we don't want. - -- With the approximation 3/10, this happened for - -- 0.5^1030, 0.5^1040, ..., 0.5^1070 and values close above. - -- The approximation 8651/28738 guarantees k1 < 1 + logBase 10 x - -- for IEEE-ish floating point types with exponent fields - -- <= 17 bits and mantissae of several thousand bits, earlier - -- convergents to logBase 10 2 would fail for long double. - -- Using quot instead of div is a little faster and requires - -- fewer fixup steps for negative lx. - let lx = p - 1 + e0 - k1 = (lx * 8651) `quot` 28738 - in if lx >= 0 then k1 + 1 else k1 - else - -- f :: Integer, log :: Float -> Float, - -- ceiling :: Float -> Int - ceiling ((log (fromInteger (f+1) :: Float) + - fromIntegral e * log (fromInteger b)) / - log 10) ---WAS: fromInt e * log (fromInteger b)) - - fixup n = - if n >= 0 then - if r + mUp <= expt 10 n * s then n else fixup (n+1) - else - if expt 10 (-n) * (r + mUp) <= s then n else fixup (n+1) - in - fixup k0 - - gen ds rn sN mUpN mDnN = - let - (dn, rn') = (rn * 10) `quotRem` sN - mUpN' = mUpN * 10 - mDnN' = mDnN * 10 - in - case (rn' < mDnN', rn' + mUpN' > sN) of - (True, False) -> dn : ds - (False, True) -> dn+1 : ds - (True, True) -> if rn' * 2 < sN then dn : ds else dn+1 : ds - (False, False) -> gen (dn:ds) rn' sN mUpN' mDnN' - - rds = - if k >= 0 then - gen [] r (s * expt 10 k) mUp mDn - else - let bk = expt 10 (-k) in - gen [] (r * bk) s (mUp * bk) (mDn * bk) - in - (map fromIntegral (reverse rds), k) - --- Exponentiation with a cache for the most common numbers. -minExpt, maxExpt :: Int -minExpt = 0 -maxExpt = 1100 - -expt :: Integer -> Int -> Integer -expt base n - | base == 2 && n >= minExpt && n <= maxExpt = expts `unsafeAt` n - | base == 10 && n <= maxExpt10 = expts10 `unsafeAt` n - | otherwise = base^n - -expts :: Array Int Integer -expts = array (minExpt,maxExpt) [(n,2^n) | n <- [minExpt .. maxExpt]] - -maxExpt10 :: Int -maxExpt10 = 324 - -expts10 :: Array Int Integer -expts10 = array (minExpt,maxExpt10) [(n,10^n) | n <- [minExpt .. maxExpt10]] diff --git a/Data/Text/Lazy/Encoding.hs b/Data/Text/Lazy/Encoding.hs deleted file mode 100644 --- a/Data/Text/Lazy/Encoding.hs +++ /dev/null @@ -1,240 +0,0 @@ -{-# LANGUAGE BangPatterns,CPP #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif --- | --- Module : Data.Text.Lazy.Encoding --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- Functions for converting lazy 'Text' values to and from lazy --- 'ByteString', using several standard encodings. --- --- To gain access to a much larger variety of encodings, use the --- @text-icu@ package: - -module Data.Text.Lazy.Encoding - ( - -- * Decoding ByteStrings to Text - -- $strict - decodeASCII - , decodeLatin1 - , decodeUtf8 - , decodeUtf16LE - , decodeUtf16BE - , decodeUtf32LE - , decodeUtf32BE - - -- ** Catchable failure - , decodeUtf8' - - -- ** Controllable error handling - , decodeUtf8With - , decodeUtf16LEWith - , decodeUtf16BEWith - , decodeUtf32LEWith - , decodeUtf32BEWith - - -- * Encoding Text to ByteStrings - , encodeUtf8 - , encodeUtf16LE - , encodeUtf16BE - , encodeUtf32LE - , encodeUtf32BE - - -- * Encoding Text using ByteString Builders - , encodeUtf8Builder - , encodeUtf8BuilderEscaped - ) where - -import Control.Exception (evaluate, try) -import Data.Monoid (Monoid(..)) -import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) -import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) -import Data.Word (Word8) -import qualified Data.ByteString as S -import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith) -import qualified Data.ByteString.Builder.Prim as BP -import qualified Data.ByteString.Lazy as B -import qualified Data.ByteString.Lazy.Internal as B -import qualified Data.ByteString.Unsafe as B -import qualified Data.Text as T -import qualified Data.Text.Encoding as TE -import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E -import qualified Data.Text.Internal.Lazy.Fusion as F -import Data.Text.Unsafe (unsafeDupablePerformIO) - --- $strict --- --- All of the single-parameter functions for decoding bytestrings --- encoded in one of the Unicode Transformation Formats (UTF) operate --- in a /strict/ mode: each will throw an exception if given invalid --- input. --- --- Each function has a variant, whose name is suffixed with -'With', --- that gives greater control over the handling of decoding errors. --- For instance, 'decodeUtf8' will throw an exception, but --- 'decodeUtf8With' allows the programmer to determine what to do on a --- decoding error. - --- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII --- encoded text. -decodeASCII :: B.ByteString -> Text -decodeASCII = decodeUtf8 -{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-} - --- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. -decodeLatin1 :: B.ByteString -> Text -decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks - --- | Decode a 'ByteString' containing UTF-8 encoded text. -decodeUtf8With :: OnDecodeError -> B.ByteString -> Text -decodeUtf8With onErr (B.Chunk b0 bs0) = - case TE.streamDecodeUtf8With onErr b0 of - TE.Some t l f -> chunk t (go f l bs0) - where - go f0 _ (B.Chunk b bs) = - case f0 b of - TE.Some t l f -> chunk t (go f l bs) - go _ l _ - | S.null l = empty - | otherwise = case onErr desc (Just (B.unsafeHead l)) of - Nothing -> empty - Just c -> Chunk (T.singleton c) Empty - desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream" -decodeUtf8With _ _ = empty - --- | Decode a 'ByteString' containing UTF-8 encoded text that is known --- to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown that cannot be caught in pure code. For more control over --- the handling of invalid data, use 'decodeUtf8'' or --- 'decodeUtf8With'. -decodeUtf8 :: B.ByteString -> Text -decodeUtf8 = decodeUtf8With strictDecode -{-# INLINE[0] decodeUtf8 #-} - --- This rule seems to cause performance loss. -{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1] - forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-} - --- | Decode a 'ByteString' containing UTF-8 encoded text.. --- --- If the input contains any invalid UTF-8 data, the relevant --- exception will be returned, otherwise the decoded text. --- --- /Note/: this function is /not/ lazy, as it must decode its entire --- input before it can return a result. If you need lazy (streaming) --- decoding, use 'decodeUtf8With' in lenient mode. -decodeUtf8' :: B.ByteString -> Either UnicodeException Text -decodeUtf8' bs = unsafeDupablePerformIO $ do - let t = decodeUtf8 bs - try (evaluate (rnf t `seq` t)) - where - rnf Empty = () - rnf (Chunk _ ts) = rnf ts -{-# INLINE decodeUtf8' #-} - -encodeUtf8 :: Text -> B.ByteString -encodeUtf8 Empty = B.empty -encodeUtf8 lt@(Chunk t _) = - B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt - where - -- To improve our small string performance, we use a strategy that - -- allocates a buffer that is guaranteed to be large enough for the - -- encoding of the first chunk, but not larger than the default - -- B.smallChunkSize. We clamp the firstChunkSize to ensure that we don't - -- generate too large buffers which hamper streaming. - firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1)) - strategy = B.safeStrategy firstChunkSize B.defaultChunkSize - -encodeUtf8Builder :: Text -> B.Builder -encodeUtf8Builder = - foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty - -{-# INLINE encodeUtf8BuilderEscaped #-} -encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder -encodeUtf8BuilderEscaped prim = - foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty - --- | Decode text from little endian UTF-16 encoding. -decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) -{-# INLINE decodeUtf16LEWith #-} - --- | Decode text from little endian UTF-16 encoding. --- --- If the input contains any invalid little endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16LEWith'. -decodeUtf16LE :: B.ByteString -> Text -decodeUtf16LE = decodeUtf16LEWith strictDecode -{-# INLINE decodeUtf16LE #-} - --- | Decode text from big endian UTF-16 encoding. -decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) -{-# INLINE decodeUtf16BEWith #-} - --- | Decode text from big endian UTF-16 encoding. --- --- If the input contains any invalid big endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16BEWith'. -decodeUtf16BE :: B.ByteString -> Text -decodeUtf16BE = decodeUtf16BEWith strictDecode -{-# INLINE decodeUtf16BE #-} - --- | Encode text using little endian UTF-16 encoding. -encodeUtf16LE :: Text -> B.ByteString -encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) -{-# INLINE encodeUtf16LE #-} - --- | Encode text using big endian UTF-16 encoding. -encodeUtf16BE :: Text -> B.ByteString -encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) -{-# INLINE encodeUtf16BE #-} - --- | Decode text from little endian UTF-32 encoding. -decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) -{-# INLINE decodeUtf32LEWith #-} - --- | Decode text from little endian UTF-32 encoding. --- --- If the input contains any invalid little endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32LEWith'. -decodeUtf32LE :: B.ByteString -> Text -decodeUtf32LE = decodeUtf32LEWith strictDecode -{-# INLINE decodeUtf32LE #-} - --- | Decode text from big endian UTF-32 encoding. -decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) -{-# INLINE decodeUtf32BEWith #-} - --- | Decode text from big endian UTF-32 encoding. --- --- If the input contains any invalid big endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32BEWith'. -decodeUtf32BE :: B.ByteString -> Text -decodeUtf32BE = decodeUtf32BEWith strictDecode -{-# INLINE decodeUtf32BE #-} - --- | Encode text using little endian UTF-32 encoding. -encodeUtf32LE :: Text -> B.ByteString -encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) -{-# INLINE encodeUtf32LE #-} - --- | Encode text using big endian UTF-32 encoding. -encodeUtf32BE :: Text -> B.ByteString -encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) -{-# INLINE encodeUtf32BE #-} diff --git a/Data/Text/Lazy/IO.hs b/Data/Text/Lazy/IO.hs deleted file mode 100644 --- a/Data/Text/Lazy/IO.hs +++ /dev/null @@ -1,196 +0,0 @@ -{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif --- | --- Module : Data.Text.Lazy.IO --- Copyright : (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Simon Marlow --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Efficient locale-sensitive support for lazy text I\/O. --- --- Skip past the synopsis for some important notes on performance and --- portability across different versions of GHC. - -module Data.Text.Lazy.IO - ( - -- * Performance - -- $performance - - -- * Locale support - -- $locale - -- * File-at-a-time operations - readFile - , writeFile - , appendFile - -- * Operations on handles - , hGetContents - , hGetLine - , hPutStr - , hPutStrLn - -- * Special cases for standard input and output - , interact - , getContents - , getLine - , putStr - , putStrLn - ) where - -import Data.Text.Lazy (Text) -import Prelude hiding (appendFile, getContents, getLine, interact, - putStr, putStrLn, readFile, writeFile) -import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout, - withFile) -import qualified Data.Text.IO as T -import qualified Data.Text.Lazy as L -import qualified Control.Exception as E -import Control.Monad (when) -import Data.IORef (readIORef) -import Data.Text.Internal.IO (hGetLineWith, readChunk) -import Data.Text.Internal.Lazy (chunk, empty) -import GHC.IO.Buffer (isEmptyBuffer) -import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException) -import GHC.IO.Handle.Internals (augmentIOError, hClose_help, - wantReadableHandle, withHandle) -import GHC.IO.Handle.Types (Handle__(..), HandleType(..)) -import System.IO (BufferMode(..), hGetBuffering, hSetBuffering) -import System.IO.Error (isEOFError) -import System.IO.Unsafe (unsafeInterleaveIO) - --- $performance --- --- The functions in this module obey the runtime system's locale, --- character set encoding, and line ending conversion settings. --- --- If you know in advance that you will be working with data that has --- a specific encoding (e.g. UTF-8), and your application is highly --- performance sensitive, you may find that it is faster to perform --- I\/O with bytestrings and to encode and decode yourself than to use --- the functions in this module. --- --- Whether this will hold depends on the version of GHC you are using, --- the platform you are working on, the data you are working with, and --- the encodings you are using, so be sure to test for yourself. - --- | Read a file and return its contents as a string. The file is --- read lazily, as with 'getContents'. -readFile :: FilePath -> IO Text -readFile name = openFile name ReadMode >>= hGetContents - --- | Write a string to a file. The file is truncated to zero length --- before writing begins. -writeFile :: FilePath -> Text -> IO () -writeFile p = withFile p WriteMode . flip hPutStr - --- | Write a string the end of a file. -appendFile :: FilePath -> Text -> IO () -appendFile p = withFile p AppendMode . flip hPutStr - --- | Lazily read the remaining contents of a 'Handle'. The 'Handle' --- will be closed after the read completes, or on error. -hGetContents :: Handle -> IO Text -hGetContents h = do - chooseGoodBuffering h - wantReadableHandle "hGetContents" h $ \hh -> do - ts <- lazyRead h - return (hh{haType=SemiClosedHandle}, ts) - --- | Use a more efficient buffer size if we're reading in --- block-buffered mode with the default buffer size. -chooseGoodBuffering :: Handle -> IO () -chooseGoodBuffering h = do - bufMode <- hGetBuffering h - when (bufMode == BlockBuffering Nothing) $ - hSetBuffering h (BlockBuffering (Just 16384)) - -lazyRead :: Handle -> IO Text -lazyRead h = unsafeInterleaveIO $ - withHandle "hGetContents" h $ \hh -> do - case haType hh of - ClosedHandle -> return (hh, L.empty) - SemiClosedHandle -> lazyReadBuffered h hh - _ -> ioException - (IOError (Just h) IllegalOperation "hGetContents" - "illegal handle type" Nothing Nothing) - -lazyReadBuffered :: Handle -> Handle__ -> IO (Handle__, Text) -lazyReadBuffered h hh@Handle__{..} = do - buf <- readIORef haCharBuffer - (do t <- readChunk hh buf - ts <- lazyRead h - return (hh, chunk t ts)) `E.catch` \e -> do - (hh', _) <- hClose_help hh - if isEOFError e - then return $ if isEmptyBuffer buf - then (hh', empty) - else (hh', L.singleton '\r') - else E.throwIO (augmentIOError e "hGetContents" h) - --- | Read a single line from a handle. -hGetLine :: Handle -> IO Text -hGetLine = hGetLineWith L.fromChunks - --- | Write a string to a handle. -hPutStr :: Handle -> Text -> IO () -hPutStr h = mapM_ (T.hPutStr h) . L.toChunks - --- | Write a string to a handle, followed by a newline. -hPutStrLn :: Handle -> Text -> IO () -hPutStrLn h t = hPutStr h t >> hPutChar h '\n' - --- | The 'interact' function takes a function of type @Text -> Text@ --- as its argument. The entire input from the standard input device is --- passed (lazily) to this function as its argument, and the resulting --- string is output on the standard output device. -interact :: (Text -> Text) -> IO () -interact f = putStr . f =<< getContents - --- | Lazily read all user input on 'stdin' as a single string. -getContents :: IO Text -getContents = hGetContents stdin - --- | Read a single line of user input from 'stdin'. -getLine :: IO Text -getLine = hGetLine stdin - --- | Write a string to 'stdout'. -putStr :: Text -> IO () -putStr = hPutStr stdout - --- | Write a string to 'stdout', followed by a newline. -putStrLn :: Text -> IO () -putStrLn = hPutStrLn stdout - --- $locale --- --- /Note/: The behaviour of functions in this module depends on the --- version of GHC you are using. --- --- Beginning with GHC 6.12, text I\/O is performed using the system or --- handle's current locale and line ending conventions. --- --- Under GHC 6.10 and earlier, the system I\/O libraries /do not --- support/ locale-sensitive I\/O or line ending conversion. On these --- versions of GHC, functions in this library all use UTF-8. What --- does this mean in practice? --- --- * All data that is read will be decoded as UTF-8. --- --- * Before data is written, it is first encoded as UTF-8. --- --- * On both reading and writing, the platform's native newline --- conversion is performed. --- --- If you must use a non-UTF-8 locale on an older version of GHC, you --- will have to perform the transcoding yourself, e.g. as follows: --- --- > import qualified Data.ByteString.Lazy as B --- > import Data.Text.Lazy (Text) --- > import Data.Text.Lazy.Encoding (encodeUtf16) --- > --- > putStr_Utf16LE :: Text -> IO () --- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t) diff --git a/Data/Text/Lazy/Internal.hs b/Data/Text/Lazy/Internal.hs deleted file mode 100644 --- a/Data/Text/Lazy/Internal.hs +++ /dev/null @@ -1,20 +0,0 @@ -{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} --- | --- Module : Data.Text.Lazy.Internal --- Copyright : (c) 2013 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- This module has been renamed to 'Data.Text.Internal.Lazy'. This --- name for the module will be removed in the next major release. - -module Data.Text.Lazy.Internal - {-# DEPRECATED "Use Data.Text.Internal.Lazy instead" #-} - ( - module Data.Text.Internal.Lazy - ) where - -import Data.Text.Internal.Lazy diff --git a/Data/Text/Lazy/Read.hs b/Data/Text/Lazy/Read.hs deleted file mode 100644 --- a/Data/Text/Lazy/Read.hs +++ /dev/null @@ -1,193 +0,0 @@ -{-# LANGUAGE OverloadedStrings, CPP #-} -#if __GLASGOW_HASKELL__ >= 704 -{-# LANGUAGE Safe #-} -#elif __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - --- | --- Module : Data.Text.Lazy.Read --- Copyright : (c) 2010, 2011 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Functions used frequently when reading textual data. -module Data.Text.Lazy.Read - ( - Reader - , decimal - , hexadecimal - , signed - , rational - , double - ) where - -import Control.Monad (liftM) -import Data.Char (isDigit, isHexDigit) -import Data.Int (Int8, Int16, Int32, Int64) -import Data.Ratio ((%)) -import Data.Text.Internal.Read -import Data.Text.Lazy as T -import Data.Word (Word, Word8, Word16, Word32, Word64) - --- | Read some text. If the read succeeds, return its value and the --- remaining text, otherwise an error message. -type Reader a = IReader Text a -type Parser = IParser Text - --- | Read a decimal integer. The input must begin with at least one --- decimal digit, and is consumed until a non-digit or end of string --- is reached. --- --- This function does not handle leading sign characters. If you need --- to handle signed input, use @'signed' 'decimal'@. --- --- /Note/: For fixed-width integer types, this function does not --- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. If you are worried about overflow, use --- 'Integer' for your result type. -decimal :: Integral a => Reader a -{-# SPECIALIZE decimal :: Reader Int #-} -{-# SPECIALIZE decimal :: Reader Int8 #-} -{-# SPECIALIZE decimal :: Reader Int16 #-} -{-# SPECIALIZE decimal :: Reader Int32 #-} -{-# SPECIALIZE decimal :: Reader Int64 #-} -{-# SPECIALIZE decimal :: Reader Integer #-} -{-# SPECIALIZE decimal :: Reader Data.Word.Word #-} -{-# SPECIALIZE decimal :: Reader Word8 #-} -{-# SPECIALIZE decimal :: Reader Word16 #-} -{-# SPECIALIZE decimal :: Reader Word32 #-} -{-# SPECIALIZE decimal :: Reader Word64 #-} -decimal txt - | T.null h = Left "input does not start with a digit" - | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.span isDigit txt - go n d = (n * 10 + fromIntegral (digitToInt d)) - --- | Read a hexadecimal integer, consisting of an optional leading --- @\"0x\"@ followed by at least one hexadecimal digit. Input is --- consumed until a non-hex-digit or end of string is reached. --- This function is case insensitive. --- --- This function does not handle leading sign characters. If you need --- to handle signed input, use @'signed' 'hexadecimal'@. --- --- /Note/: For fixed-width integer types, this function does not --- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. If you are worried about overflow, use --- 'Integer' for your result type. -hexadecimal :: Integral a => Reader a -{-# SPECIALIZE hexadecimal :: Reader Int #-} -{-# SPECIALIZE hexadecimal :: Reader Integer #-} -hexadecimal txt - | h == "0x" || h == "0X" = hex t - | otherwise = hex txt - where (h,t) = T.splitAt 2 txt - -hex :: Integral a => Reader a -{-# SPECIALIZE hexadecimal :: Reader Int #-} -{-# SPECIALIZE hexadecimal :: Reader Int8 #-} -{-# SPECIALIZE hexadecimal :: Reader Int16 #-} -{-# SPECIALIZE hexadecimal :: Reader Int32 #-} -{-# SPECIALIZE hexadecimal :: Reader Int64 #-} -{-# SPECIALIZE hexadecimal :: Reader Integer #-} -{-# SPECIALIZE hexadecimal :: Reader Word #-} -{-# SPECIALIZE hexadecimal :: Reader Word8 #-} -{-# SPECIALIZE hexadecimal :: Reader Word16 #-} -{-# SPECIALIZE hexadecimal :: Reader Word32 #-} -{-# SPECIALIZE hexadecimal :: Reader Word64 #-} -hex txt - | T.null h = Left "input does not start with a hexadecimal digit" - | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.span isHexDigit txt - go n d = (n * 16 + fromIntegral (hexDigitToInt d)) - --- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and --- apply it to the result of applying the given reader. -signed :: Num a => Reader a -> Reader a -{-# INLINE signed #-} -signed f = runP (signa (P f)) - --- | Read a rational number. --- --- This function accepts an optional leading sign character, followed --- by at least one decimal digit. The syntax similar to that accepted --- by the 'read' function, with the exception that a trailing @\'.\'@ --- or @\'e\'@ /not/ followed by a number is not consumed. --- --- Examples: --- --- >rational "3" == Right (3.0, "") --- >rational "3.1" == Right (3.1, "") --- >rational "3e4" == Right (30000.0, "") --- >rational "3.1e4" == Right (31000.0, "") --- >rational ".3" == Left "input does not start with a digit" --- >rational "e3" == Left "input does not start with a digit" --- --- Examples of differences from 'read': --- --- >rational "3.foo" == Right (3.0, ".foo") --- >rational "3e" == Right (3.0, "e") -rational :: Fractional a => Reader a -{-# SPECIALIZE rational :: Reader Double #-} -rational = floaty $ \real frac fracDenom -> fromRational $ - real % 1 + frac % fracDenom - --- | Read a rational number. --- --- The syntax accepted by this function is the same as for 'rational'. --- --- /Note/: This function is almost ten times faster than 'rational', --- but is slightly less accurate. --- --- The 'Double' type supports about 16 decimal places of accuracy. --- For 94.2% of numbers, this function and 'rational' give identical --- results, but for the remaining 5.8%, this function loses precision --- around the 15th decimal place. For 0.001% of numbers, this --- function will lose precision at the 13th or 14th decimal place. -double :: Reader Double -double = floaty $ \real frac fracDenom -> - fromIntegral real + - fromIntegral frac / fromIntegral fracDenom - -signa :: Num a => Parser a -> Parser a -{-# SPECIALIZE signa :: Parser Int -> Parser Int #-} -{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-} -{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-} -{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-} -{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} -{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} -signa p = do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - if sign == '+' then p else negate `liftM` p - -char :: (Char -> Bool) -> Parser Char -char p = P $ \t -> case T.uncons t of - Just (c,t') | p c -> Right (c,t') - _ -> Left "character does not match" - -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a -{-# INLINE floaty #-} -floaty f = runP $ do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - real <- P decimal - T fraction fracDigits <- perhaps (T 0 0) $ do - _ <- char (=='.') - digits <- P $ \t -> Right (fromIntegral . T.length $ T.takeWhile isDigit t, t) - n <- P decimal - return $ T n digits - let e c = c == 'e' || c == 'E' - power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int) - let n = if fracDigits == 0 - then if power == 0 - then fromIntegral real - else fromIntegral real * (10 ^^ power) - else if power == 0 - then f real fraction (10 ^ fracDigits) - else f real fraction (10 ^ fracDigits) * (10 ^^ power) - return $! if sign == '+' - then n - else -n diff --git a/Data/Text/Read.hs b/Data/Text/Read.hs deleted file mode 100644 --- a/Data/Text/Read.hs +++ /dev/null @@ -1,201 +0,0 @@ -{-# LANGUAGE OverloadedStrings, UnboxedTuples, CPP #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - --- | --- Module : Data.Text.Read --- Copyright : (c) 2010, 2011 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- Functions used frequently when reading textual data. -module Data.Text.Read - ( - Reader - , decimal - , hexadecimal - , signed - , rational - , double - ) where - -import Control.Monad (liftM) -import Data.Char (isDigit, isHexDigit) -import Data.Int (Int8, Int16, Int32, Int64) -import Data.Ratio ((%)) -import Data.Text as T -import Data.Text.Internal.Private (span_) -import Data.Text.Internal.Read -import Data.Word (Word, Word8, Word16, Word32, Word64) - --- | Read some text. If the read succeeds, return its value and the --- remaining text, otherwise an error message. -type Reader a = IReader Text a -type Parser a = IParser Text a - --- | Read a decimal integer. The input must begin with at least one --- decimal digit, and is consumed until a non-digit or end of string --- is reached. --- --- This function does not handle leading sign characters. If you need --- to handle signed input, use @'signed' 'decimal'@. --- --- /Note/: For fixed-width integer types, this function does not --- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. If you are worried about overflow, use --- 'Integer' for your result type. -decimal :: Integral a => Reader a -{-# SPECIALIZE decimal :: Reader Int #-} -{-# SPECIALIZE decimal :: Reader Int8 #-} -{-# SPECIALIZE decimal :: Reader Int16 #-} -{-# SPECIALIZE decimal :: Reader Int32 #-} -{-# SPECIALIZE decimal :: Reader Int64 #-} -{-# SPECIALIZE decimal :: Reader Integer #-} -{-# SPECIALIZE decimal :: Reader Data.Word.Word #-} -{-# SPECIALIZE decimal :: Reader Word8 #-} -{-# SPECIALIZE decimal :: Reader Word16 #-} -{-# SPECIALIZE decimal :: Reader Word32 #-} -{-# SPECIALIZE decimal :: Reader Word64 #-} -decimal txt - | T.null h = Left "input does not start with a digit" - | otherwise = Right (T.foldl' go 0 h, t) - where (# h,t #) = span_ isDigit txt - go n d = (n * 10 + fromIntegral (digitToInt d)) - --- | Read a hexadecimal integer, consisting of an optional leading --- @\"0x\"@ followed by at least one hexadecimal digit. Input is --- consumed until a non-hex-digit or end of string is reached. --- This function is case insensitive. --- --- This function does not handle leading sign characters. If you need --- to handle signed input, use @'signed' 'hexadecimal'@. --- --- /Note/: For fixed-width integer types, this function does not --- attempt to detect overflow, so a sufficiently long input may give --- incorrect results. If you are worried about overflow, use --- 'Integer' for your result type. -hexadecimal :: Integral a => Reader a -{-# SPECIALIZE hexadecimal :: Reader Int #-} -{-# SPECIALIZE hexadecimal :: Reader Int8 #-} -{-# SPECIALIZE hexadecimal :: Reader Int16 #-} -{-# SPECIALIZE hexadecimal :: Reader Int32 #-} -{-# SPECIALIZE hexadecimal :: Reader Int64 #-} -{-# SPECIALIZE hexadecimal :: Reader Integer #-} -{-# SPECIALIZE hexadecimal :: Reader Word #-} -{-# SPECIALIZE hexadecimal :: Reader Word8 #-} -{-# SPECIALIZE hexadecimal :: Reader Word16 #-} -{-# SPECIALIZE hexadecimal :: Reader Word32 #-} -{-# SPECIALIZE hexadecimal :: Reader Word64 #-} -hexadecimal txt - | h == "0x" || h == "0X" = hex t - | otherwise = hex txt - where (h,t) = T.splitAt 2 txt - -hex :: Integral a => Reader a -{-# SPECIALIZE hex :: Reader Int #-} -{-# SPECIALIZE hex :: Reader Int8 #-} -{-# SPECIALIZE hex :: Reader Int16 #-} -{-# SPECIALIZE hex :: Reader Int32 #-} -{-# SPECIALIZE hex :: Reader Int64 #-} -{-# SPECIALIZE hex :: Reader Integer #-} -{-# SPECIALIZE hex :: Reader Word #-} -{-# SPECIALIZE hex :: Reader Word8 #-} -{-# SPECIALIZE hex :: Reader Word16 #-} -{-# SPECIALIZE hex :: Reader Word32 #-} -{-# SPECIALIZE hex :: Reader Word64 #-} -hex txt - | T.null h = Left "input does not start with a hexadecimal digit" - | otherwise = Right (T.foldl' go 0 h, t) - where (# h,t #) = span_ isHexDigit txt - go n d = (n * 16 + fromIntegral (hexDigitToInt d)) - --- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and --- apply it to the result of applying the given reader. -signed :: Num a => Reader a -> Reader a -{-# INLINE signed #-} -signed f = runP (signa (P f)) - --- | Read a rational number. --- --- This function accepts an optional leading sign character, followed --- by at least one decimal digit. The syntax similar to that accepted --- by the 'read' function, with the exception that a trailing @\'.\'@ --- or @\'e\'@ /not/ followed by a number is not consumed. --- --- Examples (with behaviour identical to 'read'): --- --- >rational "3" == Right (3.0, "") --- >rational "3.1" == Right (3.1, "") --- >rational "3e4" == Right (30000.0, "") --- >rational "3.1e4" == Right (31000.0, "") --- >rational ".3" == Left "input does not start with a digit" --- >rational "e3" == Left "input does not start with a digit" --- --- Examples of differences from 'read': --- --- >rational "3.foo" == Right (3.0, ".foo") --- >rational "3e" == Right (3.0, "e") -rational :: Fractional a => Reader a -{-# SPECIALIZE rational :: Reader Double #-} -rational = floaty $ \real frac fracDenom -> fromRational $ - real % 1 + frac % fracDenom - --- | Read a rational number. --- --- The syntax accepted by this function is the same as for 'rational'. --- --- /Note/: This function is almost ten times faster than 'rational', --- but is slightly less accurate. --- --- The 'Double' type supports about 16 decimal places of accuracy. --- For 94.2% of numbers, this function and 'rational' give identical --- results, but for the remaining 5.8%, this function loses precision --- around the 15th decimal place. For 0.001% of numbers, this --- function will lose precision at the 13th or 14th decimal place. -double :: Reader Double -double = floaty $ \real frac fracDenom -> - fromIntegral real + - fromIntegral frac / fromIntegral fracDenom - -signa :: Num a => Parser a -> Parser a -{-# SPECIALIZE signa :: Parser Int -> Parser Int #-} -{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-} -{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-} -{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-} -{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} -{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} -signa p = do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - if sign == '+' then p else negate `liftM` p - -char :: (Char -> Bool) -> Parser Char -char p = P $ \t -> case T.uncons t of - Just (c,t') | p c -> Right (c,t') - _ -> Left "character does not match" - -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a -{-# INLINE floaty #-} -floaty f = runP $ do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - real <- P decimal - T fraction fracDigits <- perhaps (T 0 0) $ do - _ <- char (=='.') - digits <- P $ \t -> Right (T.length $ T.takeWhile isDigit t, t) - n <- P decimal - return $ T n digits - let e c = c == 'e' || c == 'E' - power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int) - let n = if fracDigits == 0 - then if power == 0 - then fromIntegral real - else fromIntegral real * (10 ^^ power) - else if power == 0 - then f real fraction (10 ^ fracDigits) - else f real fraction (10 ^ fracDigits) * (10 ^^ power) - return $! if sign == '+' - then n - else -n diff --git a/Data/Text/Show.hs b/Data/Text/Show.hs deleted file mode 100644 --- a/Data/Text/Show.hs +++ /dev/null @@ -1,89 +0,0 @@ -{-# LANGUAGE CPP, MagicHash #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} -#if __GLASGOW_HASKELL__ >= 702 -{-# LANGUAGE Trustworthy #-} -#endif - --- | --- Module : Data.Text.Show --- Copyright : (c) 2009-2015 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC - -module Data.Text.Show - ( - singleton - , unpack - , unpackCString# - ) where - -import Control.Monad.ST (ST) -import Data.Text.Internal (Text(..), empty_, safe) -import Data.Text.Internal.Fusion (stream, unstream) -import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import GHC.Prim (Addr#) -import qualified Data.Text.Array as A -import qualified Data.Text.Internal.Fusion.Common as S - -#if __GLASGOW_HASKELL__ >= 702 -import qualified GHC.CString as GHC -#else -import qualified GHC.Base as GHC -#endif - -instance Show Text where - showsPrec p ps r = showsPrec p (unpack ps) r - --- | /O(n)/ Convert a 'Text' into a 'String'. Subject to fusion. -unpack :: Text -> String -unpack = S.unstreamList . stream -{-# INLINE [1] unpack #-} - --- | /O(n)/ Convert a literal string into a 'Text'. Subject to --- fusion. --- --- This is exposed solely for people writing GHC rewrite rules. -unpackCString# :: Addr# -> Text -unpackCString# addr# = unstream (S.streamCString# addr#) -{-# NOINLINE unpackCString# #-} - -{-# RULES "TEXT literal" [1] forall a. - unstream (S.map safe (S.streamList (GHC.unpackCString# a))) - = unpackCString# a #-} - -{-# RULES "TEXT literal UTF8" [1] forall a. - unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a))) - = unpackCString# a #-} - -{-# RULES "TEXT empty literal" [1] - unstream (S.map safe (S.streamList [])) - = empty_ #-} - -{-# RULES "TEXT singleton literal" [1] forall a. - unstream (S.map safe (S.streamList [a])) - = singleton_ a #-} - --- | /O(1)/ Convert a character into a Text. Subject to fusion. --- Performs replacement on invalid scalar values. -singleton :: Char -> Text -singleton = unstream . S.singleton . safe -{-# INLINE [1] singleton #-} - -{-# RULES "TEXT singleton" forall a. - unstream (S.singleton (safe a)) - = singleton_ a #-} - --- This is intended to reduce inlining bloat. -singleton_ :: Char -> Text -singleton_ c = Text (A.run x) 0 len - where x :: ST s (A.MArray s) - x = do arr <- A.new len - _ <- unsafeWrite arr 0 d - return arr - len | d < '\x10000' = 1 - | otherwise = 2 - d = safe c -{-# NOINLINE singleton_ #-} diff --git a/Data/Text/Unsafe.hs b/Data/Text/Unsafe.hs deleted file mode 100644 --- a/Data/Text/Unsafe.hs +++ /dev/null @@ -1,123 +0,0 @@ -{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-} --- | --- Module : Data.Text.Unsafe --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : portable --- --- A module containing unsafe 'Text' operations, for very very careful --- use in heavily tested code. -module Data.Text.Unsafe - ( - inlineInterleaveST - , inlinePerformIO - , unsafeDupablePerformIO - , Iter(..) - , iter - , iter_ - , reverseIter - , reverseIter_ - , unsafeHead - , unsafeTail - , lengthWord16 - , takeWord16 - , dropWord16 - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.Text.Internal.Encoding.Utf16 (chr2) -import Data.Text.Internal (Text(..)) -import Data.Text.Internal.Unsafe (inlineInterleaveST, inlinePerformIO) -import Data.Text.Internal.Unsafe.Char (unsafeChr) -import qualified Data.Text.Array as A -import GHC.IO (unsafeDupablePerformIO) - --- | /O(1)/ A variant of 'head' for non-empty 'Text'. 'unsafeHead' --- omits the check for the empty case, so there is an obligation on --- the programmer to provide a proof that the 'Text' is non-empty. -unsafeHead :: Text -> Char -unsafeHead (Text arr off _len) - | m < 0xD800 || m > 0xDBFF = unsafeChr m - | otherwise = chr2 m n - where m = A.unsafeIndex arr off - n = A.unsafeIndex arr (off+1) -{-# INLINE unsafeHead #-} - --- | /O(1)/ A variant of 'tail' for non-empty 'Text'. 'unsafeTail' --- omits the check for the empty case, so there is an obligation on --- the programmer to provide a proof that the 'Text' is non-empty. -unsafeTail :: Text -> Text -unsafeTail t@(Text arr off len) = -#if defined(ASSERTS) - assert (d <= len) $ -#endif - Text arr (off+d) (len-d) - where d = iter_ t 0 -{-# INLINE unsafeTail #-} - -data Iter = Iter {-# UNPACK #-} !Char {-# UNPACK #-} !Int - --- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-16 --- array, returning the current character and the delta to add to give --- the next offset to iterate at. -iter :: Text -> Int -> Iter -iter (Text arr off _len) i - | m < 0xD800 || m > 0xDBFF = Iter (unsafeChr m) 1 - | otherwise = Iter (chr2 m n) 2 - where m = A.unsafeIndex arr j - n = A.unsafeIndex arr k - j = off + i - k = j + 1 -{-# INLINE iter #-} - --- | /O(1)/ Iterate one step through a UTF-16 array, returning the --- delta to add to give the next offset to iterate at. -iter_ :: Text -> Int -> Int -iter_ (Text arr off _len) i | m < 0xD800 || m > 0xDBFF = 1 - | otherwise = 2 - where m = A.unsafeIndex arr (off+i) -{-# INLINE iter_ #-} - --- | /O(1)/ Iterate one step backwards through a UTF-16 array, --- returning the current character and the delta to add (i.e. a --- negative number) to give the next offset to iterate at. -reverseIter :: Text -> Int -> (Char,Int) -reverseIter (Text arr off _len) i - | m < 0xDC00 || m > 0xDFFF = (unsafeChr m, -1) - | otherwise = (chr2 n m, -2) - where m = A.unsafeIndex arr j - n = A.unsafeIndex arr k - j = off + i - k = j - 1 -{-# INLINE reverseIter #-} - --- | /O(1)/ Iterate one step backwards through a UTF-16 array, --- returning the delta to add (i.e. a negative number) to give the --- next offset to iterate at. -reverseIter_ :: Text -> Int -> Int -reverseIter_ (Text arr off _len) i - | m < 0xDC00 || m > 0xDFFF = -1 - | otherwise = -2 - where m = A.unsafeIndex arr (off+i) -{-# INLINE reverseIter_ #-} - --- | /O(1)/ Return the length of a 'Text' in units of 'Word16'. This --- is useful for sizing a target array appropriately before using --- 'unsafeCopyToPtr'. -lengthWord16 :: Text -> Int -lengthWord16 (Text _arr _off len) = len -{-# INLINE lengthWord16 #-} - --- | /O(1)/ Unchecked take of 'k' 'Word16's from the front of a 'Text'. -takeWord16 :: Int -> Text -> Text -takeWord16 k (Text arr off _len) = Text arr off k -{-# INLINE takeWord16 #-} - --- | /O(1)/ Unchecked drop of 'k' 'Word16's from the front of a 'Text'. -dropWord16 :: Int -> Text -> Text -dropWord16 k (Text arr off len) = Text arr (off+k) (len-k) -{-# INLINE dropWord16 #-} diff --git a/README.markdown b/README.markdown deleted file mode 100644 --- a/README.markdown +++ /dev/null @@ -1,42 +0,0 @@ -# Text: Fast, packed Unicode strings, using stream fusion - -This package provides the Data.Text library, a library for the space- -and time-efficient manipulation of Unicode text in Haskell. - - -# Normalization, conversion, and collation, oh my! - -This library intentionally provides a simple API based on the -Haskell prelude's list manipulation functions. For more complicated -real-world tasks, such as Unicode normalization, conversion to and -from a larger variety of encodings, and collation, use the [text-icu -package](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/text-icu). - -That library uses the well-respected and liberally licensed ICU -library to provide these facilities. - - -# Get involved! - -Please report bugs via the -[github issue tracker](https://github.com/bos/text/issues). - -Master [git repository](https://github.com/bos/text): - -* `git clone git://github.com/bos/text.git` - -There's also a [Mercurial mirror](https://bitbucket.org/bos/text): - -* `hg clone https://bitbucket.org/bos/text` - -(You can create and contribute changes using either Mercurial or git.) - - -# Authors - -The base code for this library was originally written by Tom Harper, -based on the stream fusion framework developed by Roman Leshchinskiy, -Duncan Coutts, and Don Stewart. - -The core library was fleshed out, debugged, and tested by Bryan -O'Sullivan , and he is the current maintainer. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# text [![Hackage](http://img.shields.io/hackage/v/text.svg)](https://hackage.haskell.org/package/text) [![Stackage LTS](http://stackage.org/package/text/badge/lts)](http://stackage.org/lts/package/text) [![Stackage Nightly](http://stackage.org/package/text/badge/nightly)](http://stackage.org/nightly/package/text) + +Haskell library for space- and time-efficient operations over Unicode text. + +# Get involved! + +Please report bugs via the +[github issue tracker](https://github.com/haskell/text/issues). + +The main repo: + +```bash +git clone https://github.com/haskell/text +``` + +To run benchmarks please clone and unpack test files: + +```bash +cd text +git clone https://github.com/haskell/text-test-data benchmarks/text-test-data +make -Cbenchmarks/text-test-data +``` + +# Authors + +The base code for this library was originally written by Tom Harper, +based on the stream fusion framework developed by Roman Leshchinskiy, +Duncan Coutts, and Don Stewart. + +The core library was fleshed out, debugged, and tested by Bryan +O'Sullivan. Transition from UTF-16 to UTF-8 is by Andrew Lelechenko. diff --git a/benchmarks/Setup.hs b/benchmarks/Setup.hs deleted file mode 100644 --- a/benchmarks/Setup.hs +++ /dev/null @@ -1,2 +0,0 @@ -import Distribution.Simple -main = defaultMain diff --git a/benchmarks/cbits/time_iconv.c b/benchmarks/cbits/time_iconv.c deleted file mode 100644 --- a/benchmarks/cbits/time_iconv.c +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include -#include -#include - -int time_iconv(char *srcbuf, size_t srcbufsize) -{ - uint16_t *destbuf = NULL; - size_t destbufsize; - static uint16_t *origdestbuf; - static size_t origdestbufsize; - iconv_t ic = (iconv_t) -1; - int ret = 0; - - if (ic == (iconv_t) -1) { - ic = iconv_open("UTF-16LE", "UTF-8"); - if (ic == (iconv_t) -1) { - ret = -1; - goto done; - } - } - - destbufsize = srcbufsize * sizeof(uint16_t); - if (destbufsize > origdestbufsize) { - free(origdestbuf); - origdestbuf = destbuf = malloc(origdestbufsize = destbufsize); - } else { - destbuf = origdestbuf; - } - - iconv(ic, &srcbuf, &srcbufsize, (char**) &destbuf, &destbufsize); - - done: - return ret; -} diff --git a/benchmarks/haskell/Benchmarks.hs b/benchmarks/haskell/Benchmarks.hs --- a/benchmarks/haskell/Benchmarks.hs +++ b/benchmarks/haskell/Benchmarks.hs @@ -1,21 +1,29 @@ --- | Main module to run the micro benchmarks --- +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} + module Main ( main ) where -import Criterion.Main (Benchmark, defaultMain, bgroup) +import Test.Tasty.Bench (defaultMain, bgroup, env) import System.FilePath (()) -import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8) +import System.IO +#ifdef mingw32_HOST_OS +import System.IO.Temp (emptySystemTempFile) +import System.Directory (removeFile) +#endif + import qualified Benchmarks.Builder as Builder +import qualified Benchmarks.Concat as Concat import qualified Benchmarks.DecodeUtf8 as DecodeUtf8 import qualified Benchmarks.EncodeUtf8 as EncodeUtf8 import qualified Benchmarks.Equality as Equality import qualified Benchmarks.FileRead as FileRead +import qualified Benchmarks.FileWrite as FileWrite import qualified Benchmarks.FoldLines as FoldLines -import qualified Benchmarks.Mul as Mul +import qualified Benchmarks.Micro as Micro +import qualified Benchmarks.Multilang as Multilang import qualified Benchmarks.Pure as Pure import qualified Benchmarks.ReadNumbers as ReadNumbers import qualified Benchmarks.Replace as Replace @@ -30,50 +38,74 @@ import qualified Benchmarks.Programs.StripTags as Programs.StripTags import qualified Benchmarks.Programs.Throughput as Programs.Throughput -main :: IO () -main = benchmarks >>= defaultMain - -benchmarks :: IO [Benchmark] -benchmarks = do - sink <- openFile "/dev/null" WriteMode +mkSink :: IO (FilePath, Handle) +mkSink = do +#ifdef mingw32_HOST_OS + sinkFn <- emptySystemTempFile "dev.null" +#else + let sinkFn = "/dev/null" +#endif + sink <- openFile sinkFn WriteMode hSetEncoding sink utf8 + pure (sinkFn, sink) - -- Traditional benchmarks - bs <- sequence +rmSink :: FilePath -> IO () +#ifdef mingw32_HOST_OS +rmSink = removeFile +#else +rmSink _ = pure () +#endif + +main :: IO () +main = do + let tf = ("benchmarks/text-test-data" ) + -- Cannot use envWithCleanup, because there is no instance NFData Handle + (sinkFn, sink) <- mkSink + (fileWriteBenchmarks, fileWriteCleanup) <- FileWrite.mkFileWriteBenchmarks $ do + (fp, h) <- mkSink + return (h, rmSink fp) + defaultMain [ Builder.benchmark - , DecodeUtf8.benchmark "html" (tf "libya-chinese.html") - , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml") - , DecodeUtf8.benchmark "ascii" (tf "ascii.txt") - , DecodeUtf8.benchmark "russian" (tf "russian.txt") - , DecodeUtf8.benchmark "japanese" (tf "japanese.txt") - , EncodeUtf8.benchmark "επανάληψη 竺法蘭共譯" - , Equality.benchmark (tf "japanese.txt") + , Concat.benchmark + , Micro.benchmark + , bgroup "DecodeUtf8" + [ env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html") + , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml") + , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmark "ascii") + , env (DecodeUtf8.initEnv (tf "russian.txt")) (DecodeUtf8.benchmark "russian") + , env (DecodeUtf8.initEnv (tf "japanese.txt")) (DecodeUtf8.benchmark "japanese") + , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmarkASCII) + ] + , bgroup "EncodeUtf8" + [ EncodeUtf8.benchmark "non-ASCII" "επανάληψη 竺法蘭共譯" + , EncodeUtf8.benchmark "ASCII" "lorem ipsum" + ] + , env (Equality.initEnv (tf "japanese.txt")) Equality.benchmark , FileRead.benchmark (tf "russian.txt") + , fileWriteBenchmarks , FoldLines.benchmark (tf "russian.txt") - , Mul.benchmark - , Pure.benchmark "tiny" (tf "tiny.txt") - , Pure.benchmark "ascii" (tf "ascii-small.txt") - -- , Pure.benchmark "france" (tf "france.html") - , Pure.benchmark "russian" (tf "russian-small.txt") - , Pure.benchmark "japanese" (tf "japanese.txt") - , ReadNumbers.benchmark (tf "numbers.txt") - , Replace.benchmark (tf "russian.txt") "принимая" "своем" - , Search.benchmark (tf "russian.txt") "принимая" - , Stream.benchmark (tf "russian.txt") - , WordFrequencies.benchmark (tf "russian.txt") - ] - - -- Program-like benchmarks - ps <- bgroup "Programs" `fmap` sequence - [ Programs.BigTable.benchmark sink - , Programs.Cut.benchmark (tf "russian.txt") sink 20 40 - , Programs.Fold.benchmark (tf "russian.txt") sink - , Programs.Sort.benchmark (tf "russian.txt") sink - , Programs.StripTags.benchmark (tf "yiwiki.xml") sink - , Programs.Throughput.benchmark (tf "russian.txt") sink + , Multilang.benchmark + , bgroup "Pure" + [ env (Pure.initEnv (tf "tiny.txt")) (Pure.benchmark "tiny") + , env (Pure.initEnv (tf "ascii-small.txt")) (Pure.benchmark "ascii-small") + , env (Pure.initEnv (tf "ascii.txt")) (Pure.benchmark "ascii") + , env (Pure.initEnv (tf "english.txt")) (Pure.benchmark "english") + , env (Pure.initEnv (tf "russian-small.txt")) (Pure.benchmark "russian") + , env (Pure.initEnv (tf "japanese.txt")) (Pure.benchmark "japanese") + ] + , env (ReadNumbers.initEnv (tf "numbers.txt")) ReadNumbers.benchmark + , env (Replace.initEnv (tf "russian.txt")) (Replace.benchmark "принимая" "своем") + , env (Search.initEnv (tf "russian.txt")) (Search.benchmark "принимая") + , env (Stream.initEnv (tf "russian.txt")) Stream.benchmark + , env (WordFrequencies.initEnv (tf "russian.txt")) WordFrequencies.benchmark + , bgroup "Programs" + [ Programs.BigTable.benchmark sink + , Programs.Cut.benchmark (tf "russian.txt") sink 20 40 + , Programs.Fold.benchmark (tf "russian.txt") sink + , Programs.Sort.benchmark (tf "russian.txt") sink + , Programs.StripTags.benchmark (tf "yiwiki.xml") sink + , Programs.Throughput.benchmark (tf "russian.txt") sink + ] ] - - return $ bs ++ [ps] - where - -- Location of a test file - tf = ("../tests/text-test-data" ) + rmSink sinkFn + fileWriteCleanup diff --git a/benchmarks/haskell/Benchmarks/Builder.hs b/benchmarks/haskell/Benchmarks/Builder.hs --- a/benchmarks/haskell/Benchmarks/Builder.hs +++ b/benchmarks/haskell/Benchmarks/Builder.hs @@ -4,36 +4,24 @@ -- -- * Concatenating many small strings using a builder -- -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Builder ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, nf) -import Data.Binary.Builder as B +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) import Data.ByteString.Char8 () -import Data.Monoid (mconcat, mempty) -import qualified Blaze.ByteString.Builder as Blaze -import qualified Blaze.ByteString.Builder.Char.Utf8 as Blaze -import qualified Data.ByteString as SB -import qualified Data.ByteString.Lazy as LB import qualified Data.Text as T import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Builder as LTB import qualified Data.Text.Lazy.Builder.Int as Int import Data.Int (Int64) -benchmark :: IO Benchmark -benchmark = return $ bgroup "Builder" +benchmark :: Benchmark +benchmark = bgroup "Builder" [ bgroup "Comparison" [ bench "LazyText" $ nf (LT.length . LTB.toLazyText . mconcat . map LTB.fromText) texts - , bench "Binary" $ nf - (LB.length . B.toLazyByteString . mconcat . map B.fromByteString) - byteStrings - , bench "Blaze" $ nf - (LB.length . Blaze.toLazyByteString . mconcat . map Blaze.fromString) - strings ] , bgroup "Int" [ bgroup "Decimal" @@ -63,13 +51,3 @@ texts :: [T.Text] texts = take 200000 $ cycle ["foo", "λx", "由の"] {-# NOINLINE texts #-} - --- Note that the non-ascii characters will be chopped -byteStrings :: [SB.ByteString] -byteStrings = take 200000 $ cycle ["foo", "λx", "由の"] -{-# NOINLINE byteStrings #-} - --- Note that the non-ascii characters will be chopped -strings :: [String] -strings = take 200000 $ cycle ["foo", "λx", "由の"] -{-# NOINLINE strings #-} diff --git a/benchmarks/haskell/Benchmarks/Concat.hs b/benchmarks/haskell/Benchmarks/Concat.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/Concat.hs @@ -0,0 +1,25 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Benchmarks.Concat (benchmark) where + +import Control.Monad.Trans.Writer +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) +import Data.Text as T + +benchmark :: Benchmark +benchmark = bgroup "Concat" + [ bench "append" $ whnf (append4 "Text 1" "Text 2" "Text 3") "Text 4" + , bench "concat" $ whnf (concat4 "Text 1" "Text 2" "Text 3") "Text 4" + , bench "write" $ whnf (write4 "Text 1" "Text 2" "Text 3") "Text 4" + ] + +append4, concat4, write4 :: Text -> Text -> Text -> Text -> Text + +{-# NOINLINE append4 #-} +append4 x1 x2 x3 x4 = x1 `append` x2 `append` x3 `append` x4 + +{-# NOINLINE concat4 #-} +concat4 x1 x2 x3 x4 = T.concat [x1, x2, x3, x4] + +{-# NOINLINE write4 #-} +write4 x1 x2 x3 x4 = execWriter $ tell x1 >> tell x2 >> tell x3 >> tell x4 diff --git a/benchmarks/haskell/Benchmarks/DecodeUtf8.hs b/benchmarks/haskell/Benchmarks/DecodeUtf8.hs --- a/benchmarks/haskell/Benchmarks/DecodeUtf8.hs +++ b/benchmarks/haskell/Benchmarks/DecodeUtf8.hs @@ -15,18 +15,13 @@ -- The latter are used for testing stream fusion. -- module Benchmarks.DecodeUtf8 - ( benchmark + ( initEnv + , benchmark + , benchmarkASCII ) where -import Foreign.C.Types -import Data.ByteString.Internal (ByteString(..)) import Data.ByteString.Lazy.Internal (ByteString(..)) -import Foreign.Ptr (Ptr, plusPtr) -import Foreign.ForeignPtr (withForeignPtr) -import Data.Word (Word8) -import qualified Criterion as C -import Criterion (Benchmark, bgroup, nf, whnfIO) -import qualified Codec.Binary.UTF8.Generic as U8 +import Test.Tasty.Bench import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T @@ -34,34 +29,39 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: String -> FilePath -> IO Benchmark -benchmark kind fp = do +type Env = (B.ByteString, BL.ByteString) + +initEnv :: FilePath -> IO Env +initEnv fp = do bs <- B.readFile fp lbs <- BL.readFile fp - let bench name = C.bench (name ++ "+" ++ kind) - decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of + return (bs, lbs) + +benchmark :: String -> Env -> Benchmark +benchmark kind ~(bs, lbs) = + let decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of T.Some t0 _ f0 -> t0 : go f0 bs0 where go f (Chunk b bs1) = case f b of T.Some t1 _ f1 -> t1 : go f1 bs1 go _ _ = [] decodeStream _ = [] - return $ bgroup "DecodeUtf8" + in bgroup kind [ bench "Strict" $ nf T.decodeUtf8 bs , bench "Stream" $ nf decodeStream lbs - , bench "IConv" $ whnfIO $ iconv bs , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs , bench "Lazy" $ nf TL.decodeUtf8 lbs , bench "LazyLength" $ nf (TL.length . TL.decodeUtf8) lbs , bench "LazyInitLength" $ nf (TL.length . TL.init . TL.decodeUtf8) lbs - , bench "StrictStringUtf8" $ nf U8.toString bs - , bench "StrictStringUtf8Length" $ nf (length . U8.toString) bs - , bench "LazyStringUtf8" $ nf U8.toString lbs - , bench "LazyStringUtf8Length" $ nf (length . U8.toString) lbs ] -iconv :: B.ByteString -> IO CInt -iconv (PS fp off len) = withForeignPtr fp $ \ptr -> - time_iconv (ptr `plusPtr` off) (fromIntegral len) - -foreign import ccall unsafe time_iconv :: Ptr Word8 -> CSize -> IO CInt +benchmarkASCII :: Env -> Benchmark +benchmarkASCII ~(bs, lbs) = + bgroup "ascii" + [ bench "strict decodeUtf8" $ nf T.decodeUtf8 bs + , bench "strict decodeLatin1" $ nf T.decodeLatin1 bs + , bench "strict decodeASCII" $ nf T.decodeASCII bs + , bench "lazy decodeUtf8" $ nf TL.decodeUtf8 lbs + , bench "lazy decodeLatin1" $ nf TL.decodeLatin1 lbs + , bench "lazy decodeASCII" $ nf TL.decodeASCII lbs + ] diff --git a/benchmarks/haskell/Benchmarks/EncodeUtf8.hs b/benchmarks/haskell/Benchmarks/EncodeUtf8.hs --- a/benchmarks/haskell/Benchmarks/EncodeUtf8.hs +++ b/benchmarks/haskell/Benchmarks/EncodeUtf8.hs @@ -10,19 +10,23 @@ ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnf) +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf, whnf) import qualified Data.ByteString as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: String -> IO Benchmark -benchmark string = do - return $ bgroup "EncodeUtf8" +benchmark :: String -> String -> Benchmark +benchmark name string = + bgroup name [ bench "Text" $ whnf (B.length . T.encodeUtf8) text , bench "LazyText" $ whnf (BL.length . TL.encodeUtf8) lazyText + , bench "Text/encodeUtf8Builder" $ nf (B.toLazyByteString . T.encodeUtf8Builder) text + , bench "Text/encodeUtf8BuilderEscaped" $ nf (B.toLazyByteString . T.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) text ] where -- The string in different formats diff --git a/benchmarks/haskell/Benchmarks/Equality.hs b/benchmarks/haskell/Benchmarks/Equality.hs --- a/benchmarks/haskell/Benchmarks/Equality.hs +++ b/benchmarks/haskell/Benchmarks/Equality.hs @@ -6,10 +6,11 @@ -- * Comparison of strings (Eq instance) -- module Benchmarks.Equality - ( benchmark + ( initEnv + , benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnf) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as BL import qualified Data.Text as T @@ -17,22 +18,17 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: FilePath -> IO Benchmark -benchmark fp = do +type Env = (T.Text, TL.Text) + +initEnv :: FilePath -> IO Env +initEnv fp = do b <- B.readFile fp bl1 <- BL.readFile fp - -- A lazy bytestring is a list of chunks. When we do not explicitly create two - -- different lazy bytestrings at a different address, the bytestring library - -- will compare the chunk addresses instead of the chunk contents. This is why - -- we read the lazy bytestring twice here. - bl2 <- BL.readFile fp - l <- readFile fp - let t = T.decodeUtf8 b - tl = TL.decodeUtf8 bl1 - return $ bgroup "Equality" + return (T.decodeUtf8 b, TL.decodeUtf8 bl1) + +benchmark :: Env -> Benchmark +benchmark ~(t, tl) = + bgroup "Equality" [ bench "Text" $ whnf (== T.init t `T.snoc` '\xfffd') t , bench "LazyText" $ whnf (== TL.init tl `TL.snoc` '\xfffd') tl - , bench "ByteString" $ whnf (== B.init b `B.snoc` '\xfffd') b - , bench "LazyByteString" $ whnf (== BL.init bl2 `BL.snoc` '\xfffd') bl1 - , bench "String" $ whnf (== init l ++ "\xfffd") l ] diff --git a/benchmarks/haskell/Benchmarks/FileRead.hs b/benchmarks/haskell/Benchmarks/FileRead.hs --- a/benchmarks/haskell/Benchmarks/FileRead.hs +++ b/benchmarks/haskell/Benchmarks/FileRead.hs @@ -4,12 +4,14 @@ -- -- * Reading a file from the disk -- + +{-# LANGUAGE CPP #-} + module Benchmarks.FileRead ( benchmark ) where -import Control.Applicative ((<$>)) -import Criterion (Benchmark, bgroup, bench, whnfIO) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) import qualified Data.ByteString as SB import qualified Data.ByteString.Lazy as LB import qualified Data.Text as T @@ -19,12 +21,9 @@ import qualified Data.Text.Lazy.Encoding as LT import qualified Data.Text.Lazy.IO as LT -benchmark :: FilePath -> IO Benchmark -benchmark p = return $ bgroup "FileRead" - [ bench "String" $ whnfIO $ length <$> readFile p - , bench "ByteString" $ whnfIO $ SB.length <$> SB.readFile p - , bench "LazyByteString" $ whnfIO $ LB.length <$> LB.readFile p - , bench "Text" $ whnfIO $ T.length <$> T.readFile p +benchmark :: FilePath -> Benchmark +benchmark p = bgroup "FileRead" + [ bench "Text" $ whnfIO $ T.length <$> T.readFile p , bench "LazyText" $ whnfIO $ LT.length <$> LT.readFile p , bench "TextByteString" $ whnfIO $ (T.length . T.decodeUtf8) <$> SB.readFile p diff --git a/benchmarks/haskell/Benchmarks/FileWrite.hs b/benchmarks/haskell/Benchmarks/FileWrite.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/FileWrite.hs @@ -0,0 +1,135 @@ +-- | Benchmarks simple file writing +-- +-- Tested in this benchmark: +-- +-- * Writing a file to the disk +-- + +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP #-} + +module Benchmarks.FileWrite + ( mkFileWriteBenchmarks + ) where + +import Control.DeepSeq (NFData, deepseq) +import Data.Bifunctor (first) +import Data.List (intercalate, intersperse) +import Data.String (fromString) +import Data.Text (StrictText) +import Data.Text.Internal.Lazy (LazyText, defaultChunkSize) +import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(..), hSetBuffering, hSetNewlineMode) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfAppIO) +import qualified Data.Text as T +import qualified Data.Text.IO as T +import qualified Data.Text.IO.Utf8 as Utf8 +import qualified Data.Text.Lazy as L +import qualified Data.Text.Lazy.IO as L + +mkFileWriteBenchmarks :: IO (Handle, IO ()) -> IO (Benchmark, IO ()) +mkFileWriteBenchmarks mkSinkNRemove = do + let writeData = L.cycle $ fromString [minBound..maxBound] + +#ifdef ExtendedBenchmarks + lengths = [0..5] <> [10,20..100] <> [1000,3000,10000,100000] +#else + lengths = [0,1,100,3000,10000,100000] +#endif + + testGroup :: NFData text => (Handle -> text -> IO ()) -> ((String, StrictText -> text)) -> Newline -> BufferMode -> IO (Benchmark, IO ()) + testGroup hPutStr (textCharacteristics, select) nl mode = do + (h, removeFile) <- mkSinkNRemove + hSetBuffering h mode + hSetNewlineMode h $ NewlineMode nl nl + pure + ( bgroup (intercalate " " [textCharacteristics, show nl, show mode]) $ + lengths <&> \n -> let + t = select $ L.toStrict $ L.take n writeData + in bench ("length " <> show n) + $ deepseq t + $ whnfAppIO (hPutStr h) t + , removeFile + ) + + sequenceGroup "FileWrite hPutStr" +#ifdef ExtendedBenchmarks + [ testGroup T.hPutStr strict LF NoBuffering + , testGroup L.hPutStr lazy LF NoBuffering + + , testGroup T.hPutStr strict LF LineBuffering + , testGroup T.hPutStr strict CRLF LineBuffering + , testGroup T.hPutStr strictNewlines LF LineBuffering + , testGroup T.hPutStr strictNewlines CRLF LineBuffering + + , testGroup L.hPutStr lazy LF LineBuffering + , testGroup L.hPutStr lazy CRLF LineBuffering + , testGroup L.hPutStr lazySmallChunks LF LineBuffering + , testGroup L.hPutStr lazySmallChunks CRLF LineBuffering + , testGroup L.hPutStr lazyNewlines LF LineBuffering + , testGroup L.hPutStr lazyNewlines CRLF LineBuffering + , testGroup L.hPutStr lazySmallChunksNewlines LF LineBuffering + , testGroup L.hPutStr lazySmallChunksNewlines CRLF LineBuffering + + , testGroup T.hPutStr strict LF (BlockBuffering Nothing) + , testGroup T.hPutStr strict CRLF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines LF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing) + + , testGroup L.hPutStr lazy LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazy CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunks LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunks CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunksNewlines LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunksNewlines CRLF (BlockBuffering Nothing) + + , sequenceGroup "UTF-8" + [ testGroup Utf8.hPutStr strict LF NoBuffering + , testGroup Utf8.hPutStr strict LF LineBuffering + , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing) + ] + ] +#else + [ testGroup T.hPutStr strictNewlines LF LineBuffering + , testGroup T.hPutStr strictNewlines CRLF LineBuffering + + , testGroup T.hPutStr strict LF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing) + + , testGroup L.hPutStr lazyNewlines LF LineBuffering + , testGroup L.hPutStr lazyNewlines CRLF LineBuffering + + , testGroup L.hPutStr lazy LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing) + + , sequenceGroup "UTF-8" + [ testGroup Utf8.hPutStr strict LF LineBuffering + , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing) + ] + ] +#endif + + where + lazy, lazyNewlines :: (String, StrictText -> LazyText) + lazy = ("lazy", L.fromChunks . T.chunksOf defaultChunkSize) + lazyNewlines = ("lazy many newlines", snd lazy . snd strictNewlines) + +#ifdef ExtendedBenchmarks + lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText) + lazySmallChunks = ("lazy small chunks", L.fromChunks . T.chunksOf 10) + lazySmallChunksNewlines = ("lazy small chunks many newlines", snd lazySmallChunks . snd strictNewlines) +#endif + + strict, strictNewlines :: (String, StrictText -> StrictText) + strict = ("strict", id) + strictNewlines = ("strict many newlines", mconcat . intersperse "\n" . T.chunksOf 5) + + sequenceGroup groupName tgs + = first (bgroup groupName) + . foldr (\(b,r) (bs,rs) -> (b:bs,r>>rs)) ([], return ()) + <$> sequence tgs + +(<&>) :: Functor f => f a -> (a -> b) -> f b +(<&>) = flip fmap + diff --git a/benchmarks/haskell/Benchmarks/FoldLines.hs b/benchmarks/haskell/Benchmarks/FoldLines.hs --- a/benchmarks/haskell/Benchmarks/FoldLines.hs +++ b/benchmarks/haskell/Benchmarks/FoldLines.hs @@ -10,16 +10,14 @@ ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnfIO) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) import System.IO -import qualified Data.ByteString as B import qualified Data.Text as T import qualified Data.Text.IO as T -benchmark :: FilePath -> IO Benchmark -benchmark fp = return $ bgroup "ReadLines" +benchmark :: FilePath -> Benchmark +benchmark fp = bgroup "ReadLines" [ bench "Text" $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int) - , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int) ] where withHandle f = whnfIO $ do @@ -42,17 +40,3 @@ l <- T.hGetLine h let z' = f z l in go z' {-# INLINE foldLinesT #-} - --- | ByteString line fold --- -foldLinesB :: (a -> B.ByteString -> a) -> a -> Handle -> IO a -foldLinesB f z0 h = go z0 - where - go !z = do - eof <- hIsEOF h - if eof - then return z - else do - l <- B.hGetLine h - let z' = f z l in go z' -{-# INLINE foldLinesB #-} diff --git a/benchmarks/haskell/Benchmarks/Micro.hs b/benchmarks/haskell/Benchmarks/Micro.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/Micro.hs @@ -0,0 +1,33 @@ +-- | Benchmarks on artificial data. + +module Benchmarks.Micro (benchmark) where + +import qualified Data.List.NonEmpty as NE +import qualified Data.Text.Lazy as TL +import qualified Data.Text as T +import Test.Tasty.Bench (Benchmark, Benchmarkable, bgroup, bcompareWithin, bench, nf) + +benchmark :: Benchmark +benchmark = bgroup "Micro" + [ blinear "lazy-inits--last" 500000 2 0.1 $ \len -> + nf (NE.last . TL.initsNE) (chunks len) + , blinear "lazy-inits--map-take1" 500000 2 0.1 $ \len -> + nf (map (TL.take 1) . TL.inits) (chunks len) + ] + +chunks :: Int -> TL.Text +chunks n = TL.fromChunks (replicate n (T.pack "a")) + +-- Check that running an action with input length (m * baseLen) +-- runs m times slower than the same action with input length baseLen. +blinear :: String -- ^ Name (must be globally unique!) + -> Int -- ^ Base length + -> Int -- ^ Multiplier m + -> Double -- ^ Slack s + -> (Int -> Benchmarkable) -- ^ Action to measure, parameterized by input length + -> Benchmark +blinear name baseLen m s run = bgroup name + [ bench "baseline" $ run baseLen + , bcompareWithin (fromIntegral m * (1 - s)) (fromIntegral m * (1 + s)) (name ++ ".baseline") $ + bench ("x" ++ show m) $ run (m * baseLen) + ] diff --git a/benchmarks/haskell/Benchmarks/Mul.hs b/benchmarks/haskell/Benchmarks/Mul.hs deleted file mode 100644 --- a/benchmarks/haskell/Benchmarks/Mul.hs +++ /dev/null @@ -1,138 +0,0 @@ -module Benchmarks.Mul (benchmark) where - -import Control.Exception (evaluate) -import Criterion.Main -import Data.Int (Int32, Int64) -import Data.Text.Internal (mul32, mul64) -import qualified Data.Vector.Unboxed as U - -oldMul :: Int64 -> Int64 -> Int64 -oldMul m n - | n == 0 = 0 - | m <= maxBound `quot` n = m * n - | otherwise = error "overflow" - -benchmark :: IO Benchmark -benchmark = do - _ <- evaluate testVector32 - _ <- evaluate testVector64 - return $ bgroup "Mul" [ - bench "oldMul" $ whnf (U.map (uncurry oldMul)) testVector64 - , bench "mul64" $ whnf (U.map (uncurry mul64)) testVector64 - , bench "*64" $ whnf (U.map (uncurry (*))) testVector64 - , bench "mul32" $ whnf (U.map (uncurry mul32)) testVector32 - , bench "*32" $ whnf (U.map (uncurry (*))) testVector32 - ] - -testVector64 :: U.Vector (Int64,Int64) -testVector64 = U.fromList [ - (0,1248868987182846646),(169004623633872,24458),(482549039517835,7614), - (372,8157063115504364),(27,107095594861148252),(3,63249878517962420), - (4363,255694473572912),(86678474,1732634806),(1572453024,1800489338), - (9384523143,77053781),(49024709555,75095046),(7,43457620410239131), - (8,8201563008844571),(387719037,1520696708),(189869238220197,1423), - (46788016849611,23063),(503077742109974359,0),(104,1502010908706487), - (30478140346,207525518),(80961140129236192,14),(4283,368012829143675), - (1028719181728108146,6),(318904,5874863049591),(56724427166898,110794), - (234539368,31369110449),(2,251729663598178612),(103291548194451219,5), - (76013,5345328755566),(1769631,2980846129318),(40898,60598477385754), - (0,98931348893227155),(573555872156917492,3),(318821187115,4476566), - (11152874213584,243582),(40274276,16636653248),(127,4249988676030597), - (103543712111871836,5),(71,16954462148248238),(3963027173504,216570), - (13000,503523808916753),(17038308,20018685905),(0,510350226577891549), - (175898,3875698895405),(425299191292676,5651),(17223451323664536,50), - (61755131,14247665326),(0,1018195131697569303),(36433751497238985,20), - (3473607861601050,1837),(1392342328,1733971838),(225770297367,3249655), - (14,127545244155254102),(1751488975299136,2634),(3949208,504190668767), - (153329,831454434345),(1066212122928663658,2),(351224,2663633539556), - (344565,53388869217),(35825609350446863,54),(276011553660081475,10), - (1969754174790470349,3),(35,68088438338633),(506710,3247689556438), - (11099382291,327739909),(105787303549,32824363),(210366111,14759049409), - (688893241579,3102676),(8490,70047474429581),(152085,29923000251880), - (5046974599257095,400),(4183167795,263434071),(10089728,502781960687), - (44831977765,4725378),(91,8978094664238578),(30990165721,44053350), - (1772377,149651820860),(243420621763408572,4),(32,5790357453815138), - (27980806337993771,5),(47696295759774,20848),(1745874142313778,1098), - (46869334770121,1203),(886995283,1564424789),(40679396544,76002479), - (1,672849481568486995),(337656187205,3157069),(816980552858963,6003), - (2271434085804831543,1),(0,1934521023868747186),(6266220038281,15825), - (4160,107115946987394),(524,246808621791561),(0,1952519482439636339), - (128,2865935904539691),(1044,3211982069426297),(16000511542473,88922), - (1253596745404082,2226),(27041,56836278958002),(23201,49247489754471), - (175906590497,21252392),(185163584757182295,24),(34742225226802197,150), - (2363228,250824838408),(216327527109550,45),(24,81574076994520675), - (28559899906542,15356),(10890139774837133,511),(2293,707179303654492), - (2749366833,40703233),(0,4498229704622845986),(439,4962056468281937), - (662,1453820621089921),(16336770612459631,220),(24282989393,74239137), - (2724564648490195,3),(743672760,124992589),(4528103,704330948891), - (6050483122491561,250),(13322953,13594265152),(181794,22268101450214), - (25957941712,75384092),(43352,7322262295009),(32838,52609059549923), - (33003585202001564,2),(103019,68430142267402),(129918230800,8742978), - (0,2114347379589080688),(2548,905723041545274),(222745067962838382,0), - (1671683850790425181,1),(455,4836932776795684),(794227702827214,6620), - (212534135175874,1365),(96432431858,29784975),(466626763743380,3484), - (29793949,53041519613),(8359,309952753409844),(3908960585331901,26), - (45185288970365760,114),(10131829775,68110174),(58039242399640479,83), - (628092278238719399,6),(1,196469106875361889),(302336625,16347502444), - (148,3748088684181047),(1,1649096568849015456),(1019866864,2349753026), - (8211344830,569363306),(65647579546873,34753),(2340190,1692053129069), - (64263301,30758930355),(48681618072372209,110),(7074794736,47640197), - (249634721521,7991792),(1162917363807215,232),(7446433349,420634045), - (63398619383,60709817),(51359004508011,14200),(131788797028647,7072), - (52079887791430043,7),(7,136277667582599838),(28582879735696,50327), - (1404582800566278,833),(469164435,15017166943),(99567079957578263,49), - (1015285971,3625801566),(321504843,4104079293),(5196954,464515406632), - (114246832260876,7468),(8149664437,487119673),(12265299,378168974869), - (37711995764,30766513),(3971137243,710996152),(483120070302,603162), - (103009942,61645547145),(8476344625340,6987),(547948761229739,1446), - (42234,18624767306301),(13486714173011,58948),(4,198309153268019840), - (9913176974,325539248),(28246225540203,116822),(2882463945582154,18), - (959,25504987505398),(3,1504372236378217710),(13505229956793,374987), - (751661959,457611342),(27375926,36219151769),(482168869,5301952074), - (1,1577425863241520640),(714116235611821,1164),(904492524250310488,0), - (5983514941763398,68),(10759472423,23540686),(72539568471529,34919), - (4,176090672310337473),(938702842110356453,1),(673652445,3335287382), - (3111998893666122,917),(1568013,3168419765469)] - -testVector32 :: U.Vector (Int32,Int32) -testVector32 = U.fromList [ - (39242,410),(0,100077553),(2206,9538),(509400240,1),(38048,6368), - (1789,651480),(2399,157032),(701,170017),(5241456,14),(11212,70449), - (1,227804876),(749687254,1),(74559,2954),(1158,147957),(410604456,1), - (170851,1561),(92643422,1),(6192,180509),(7,24202210),(3440,241481), - (5753677,5),(294327,1622),(252,4454673),(127684121,11),(28315800,30), - (340370905,0),(1,667887987),(592782090,1),(49023,27641),(750,290387), - (72886,3847),(0,301047933),(3050276,473),(1,788366142),(59457,15813), - (637726933,1),(1135,344317),(853616,264),(696816,493),(7038,12046), - (125219574,4),(803694088,1),(107081726,1),(39294,21699),(16361,38191), - (132561123,12),(1760,23499),(847543,484),(175687349,1),(2963,252678), - (6248,224553),(27596,4606),(5422922,121),(1542,485890),(131,583035), - (59096,4925),(3637115,132),(0,947225435),(86854,6794),(2984745,339), - (760129569,1),(1,68260595),(380835652,2),(430575,2579),(54514,7211), - (15550606,3),(9,27367402),(3007053,207),(7060988,60),(28560,27130), - (1355,21087),(10880,53059),(14563646,4),(461886361,1),(2,169260724), - (241454126,2),(406797,1),(61631630,16),(44473,5943),(63869104,12), - (950300,1528),(2113,62333),(120817,9358),(100261456,1),(426764723,1), - (119,12723684),(3,53358711),(4448071,18),(1,230278091),(238,232102), - (8,57316440),(42437979,10),(6769,19555),(48590,22006),(11500585,79), - (2808,97638),(42,26952545),(11,32104194),(23954638,1),(785427272,0), - (513,81379),(31333960,37),(897772,1009),(4,25679692),(103027993,12), - (104972702,11),(546,443401),(7,65137092),(88574269,3),(872139069,0), - (2,97417121),(378802603,0),(141071401,4),(22613,10575),(2191743,118), - (470,116119),(7062,38166),(231056,1847),(43901963,9),(2400,70640), - (63553,1555),(34,11249573),(815174,1820),(997894011,0),(98881794,2), - (5448,43132),(27956,9),(904926,1357),(112608626,3),(124,613021), - (282086,1966),(99,10656881),(113799,1501),(433318,2085),(442,948171), - (165380,1043),(28,14372905),(14880,50462),(2386,219918),(229,1797565), - (1174961,298),(3925,41833),(3903515,299),(15690452,111),(360860521,3), - (7440846,81),(2541026,507),(0,492448477),(6869,82469),(245,8322939), - (3503496,253),(123495298,0),(150963,2299),(33,4408482),(1,200911107), - (305,252121),(13,123369189),(215846,8181),(2440,65387),(776764401,1), - (1241172,434),(8,15493155),(81953961,6),(17884993,5),(26,6893822), - (0,502035190),(1,582451018),(2,514870139),(227,3625619),(49,12720258), - (1456769,207),(94797661,10),(234407,893),(26843,5783),(15688,24547), - (4091,86268),(4339448,151),(21360,6294),(397046497,2),(1227,205936), - (9966,21959),(160046791,1),(0,159992224),(27,24974797),(19177,29334), - (4136148,42),(21179785,53),(61256583,31),(385,344176),(7,11934915), - (1,18992566),(3488065,5),(768021,224),(36288474,7),(8624,117561), - (8,20341439),(5903,261475),(561,1007618),(1738,392327),(633049,1708)] diff --git a/benchmarks/haskell/Benchmarks/Multilang.hs b/benchmarks/haskell/Benchmarks/Multilang.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/Multilang.hs @@ -0,0 +1,18 @@ +{-# LANGUAGE BangPatterns, OverloadedStrings, RankNTypes #-} + +module Benchmarks.Multilang (benchmark) where + +import qualified Data.ByteString as B +import qualified Data.Text as Text +import Data.Text.Encoding (decodeUtf8) +import Data.Text (Text) +import Test.Tasty.Bench (Benchmark, bgroup, bench, env, nf) + +readYiwiki :: IO Text +readYiwiki = decodeUtf8 `fmap` B.readFile "benchmarks/text-test-data/yiwiki.xml" + +benchmark :: Benchmark +benchmark = env readYiwiki $ \content -> bgroup "Multilang" + [ bench "find_first" $ nf (Text.isInfixOf "en:Benin") content + , bench "find_index" $ nf (Text.findIndex (=='c')) content + ] diff --git a/benchmarks/haskell/Benchmarks/Programs/BigTable.hs b/benchmarks/haskell/Benchmarks/Programs/BigTable.hs --- a/benchmarks/haskell/Benchmarks/Programs/BigTable.hs +++ b/benchmarks/haskell/Benchmarks/Programs/BigTable.hs @@ -6,20 +6,19 @@ -- -- * Writing to a handle -- -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.BigTable ( benchmark ) where -import Criterion (Benchmark, bench, whnfIO) -import Data.Monoid (mappend, mconcat) +import Test.Tasty.Bench (Benchmark, bench, whnfIO) import Data.Text.Lazy.Builder (Builder, fromText, toLazyText) import Data.Text.Lazy.IO (hPutStr) import System.IO (Handle) import qualified Data.Text as T -benchmark :: Handle -> IO Benchmark -benchmark sink = return $ bench "BigTable" $ whnfIO $ do +benchmark :: Handle -> Benchmark +benchmark sink = bench "BigTable" $ whnfIO $ do hPutStr sink "Content-Type: text/html\n\n" hPutStr sink . toLazyText . makeTable =<< rows hPutStr sink "
" diff --git a/benchmarks/haskell/Benchmarks/Programs/Cut.hs b/benchmarks/haskell/Benchmarks/Programs/Cut.hs --- a/benchmarks/haskell/Benchmarks/Programs/Cut.hs +++ b/benchmarks/haskell/Benchmarks/Programs/Cut.hs @@ -16,12 +16,10 @@ ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnfIO) -import System.IO (Handle, hPutStr) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) +import System.IO (Handle) import qualified Data.ByteString as B -import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Char8 as BLC import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T @@ -29,41 +27,15 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> Int -> Int -> IO Benchmark -benchmark p sink from to = return $ bgroup "Cut" - [ bench' "String" string - , bench' "ByteString" byteString - , bench' "LazyByteString" lazyByteString - , bench' "Text" text +benchmark :: FilePath -> Handle -> Int -> Int -> Benchmark +benchmark p sink from to = bgroup "Cut" + [ bench' "Text" text , bench' "LazyText" lazyText , bench' "TextByteString" textByteString , bench' "LazyTextByteString" lazyTextByteString ] where bench' n s = bench n $ whnfIO (s p sink from to) - -string :: FilePath -> Handle -> Int -> Int -> IO () -string fp sink from to = do - s <- readFile fp - hPutStr sink $ cut s - where - cut = unlines . map (take (to - from) . drop from) . lines - -byteString :: FilePath -> Handle -> Int -> Int -> IO () -byteString fp sink from to = do - bs <- B.readFile fp - B.hPutStr sink $ cut bs - where - cut = BC.unlines . map (B.take (to - from) . B.drop from) . BC.lines - -lazyByteString :: FilePath -> Handle -> Int -> Int -> IO () -lazyByteString fp sink from to = do - bs <- BL.readFile fp - BL.hPutStr sink $ cut bs - where - cut = BLC.unlines . map (BL.take (to' - from') . BL.drop from') . BLC.lines - from' = fromIntegral from - to' = fromIntegral to text :: FilePath -> Handle -> Int -> Int -> IO () text fp sink from to = do diff --git a/benchmarks/haskell/Benchmarks/Programs/Fold.hs b/benchmarks/haskell/Benchmarks/Programs/Fold.hs --- a/benchmarks/haskell/Benchmarks/Programs/Fold.hs +++ b/benchmarks/haskell/Benchmarks/Programs/Fold.hs @@ -12,25 +12,25 @@ -- -- * Writing back to a handle -- -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.Fold ( benchmark ) where -import Data.List (foldl') +import Data.Foldable (Foldable(..)) import Data.List (intersperse) -import Data.Monoid (mempty, mappend, mconcat) +import Prelude hiding (Foldable(..)) import System.IO (Handle) -import Criterion (Benchmark, bench, whnfIO) +import Test.Tasty.Bench (Benchmark, bench, whnfIO) import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy.Builder as TLB import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark -benchmark i o = return $ - bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80 +benchmark :: FilePath -> Handle -> Benchmark +benchmark i o = + bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . foldText 80 -- | We represent a paragraph by a word list -- @@ -38,8 +38,8 @@ -- | Fold a text -- -fold :: Int -> T.Text -> TL.Text -fold maxWidth = TLB.toLazyText . mconcat . +foldText :: Int -> T.Text -> TL.Text +foldText maxWidth = TLB.toLazyText . mconcat . intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs -- | Fold a paragraph diff --git a/benchmarks/haskell/Benchmarks/Programs/Sort.hs b/benchmarks/haskell/Benchmarks/Programs/Sort.hs --- a/benchmarks/haskell/Benchmarks/Programs/Sort.hs +++ b/benchmarks/haskell/Benchmarks/Programs/Sort.hs @@ -12,18 +12,15 @@ -- -- * Writing back to a handle -- -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.Sort ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnfIO) -import Data.Monoid (mconcat) -import System.IO (Handle, hPutStr) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) +import System.IO (Handle) import qualified Data.ByteString as B -import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Char8 as BLC import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Encoding as T @@ -33,13 +30,9 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark -benchmark i o = return $ bgroup "Sort" - [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string - , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString - , bench "LazyByteString" $ whnfIO $ - BL.readFile i >>= BL.hPutStr o . lazyByteString - , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text +benchmark :: FilePath -> Handle -> Benchmark +benchmark i o = bgroup "Sort" + [ bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text , bench "LazyText" $ whnfIO $ TL.readFile i >>= TL.hPutStr o . lazyText , bench "TextByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8 @@ -48,15 +41,6 @@ , bench "TextBuilder" $ whnfIO $ B.readFile i >>= BL.hPutStr o . TL.encodeUtf8 . textBuilder . T.decodeUtf8 ] - -string :: String -> String -string = unlines . L.sort . lines - -byteString :: B.ByteString -> B.ByteString -byteString = BC.unlines . L.sort . BC.lines - -lazyByteString :: BL.ByteString -> BL.ByteString -lazyByteString = BLC.unlines . L.sort . BLC.lines text :: T.Text -> T.Text text = T.unlines . L.sort . T.lines diff --git a/benchmarks/haskell/Benchmarks/Programs/StripTags.hs b/benchmarks/haskell/Benchmarks/Programs/StripTags.hs --- a/benchmarks/haskell/Benchmarks/Programs/StripTags.hs +++ b/benchmarks/haskell/Benchmarks/Programs/StripTags.hs @@ -15,32 +15,22 @@ ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnfIO) -import Data.List (mapAccumL) -import System.IO (Handle, hPutStr) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) +import System.IO (Handle) import qualified Data.ByteString as B -import qualified Data.ByteString.Char8 as BC import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T -benchmark :: FilePath -> Handle -> IO Benchmark -benchmark i o = return $ bgroup "StripTags" - [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string - , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString - , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text +benchmark :: FilePath -> Handle -> Benchmark +benchmark i o = bgroup "StripTags" + [ bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text , bench "TextByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8 ] -string :: String -> String -string = snd . mapAccumL step 0 - text :: T.Text -> T.Text text = snd . T.mapAccumL step 0 - -byteString :: B.ByteString -> B.ByteString -byteString = snd . BC.mapAccumL step 0 step :: Int -> Char -> (Int, Char) step d c diff --git a/benchmarks/haskell/Benchmarks/Programs/Throughput.hs b/benchmarks/haskell/Benchmarks/Programs/Throughput.hs --- a/benchmarks/haskell/Benchmarks/Programs/Throughput.hs +++ b/benchmarks/haskell/Benchmarks/Programs/Throughput.hs @@ -18,8 +18,8 @@ ( benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnfIO) -import System.IO (Handle, hPutStr) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) +import System.IO (Handle) import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL import qualified Data.Text.Encoding as T @@ -27,12 +27,9 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark -benchmark fp sink = return $ bgroup "Throughput" - [ bench "String" $ whnfIO $ readFile fp >>= hPutStr sink - , bench "ByteString" $ whnfIO $ B.readFile fp >>= B.hPutStr sink - , bench "LazyByteString" $ whnfIO $ BL.readFile fp >>= BL.hPutStr sink - , bench "Text" $ whnfIO $ T.readFile fp >>= T.hPutStr sink +benchmark :: FilePath -> Handle -> Benchmark +benchmark fp sink = bgroup "Throughput" + [ bench "Text" $ whnfIO $ T.readFile fp >>= T.hPutStr sink , bench "LazyText" $ whnfIO $ TL.readFile fp >>= TL.hPutStr sink , bench "TextByteString" $ whnfIO $ B.readFile fp >>= B.hPutStr sink . T.encodeUtf8 . T.decodeUtf8 diff --git a/benchmarks/haskell/Benchmarks/Pure.hs b/benchmarks/haskell/Benchmarks/Pure.hs --- a/benchmarks/haskell/Benchmarks/Pure.hs +++ b/benchmarks/haskell/Benchmarks/Pure.hs @@ -5,29 +5,49 @@ -- * Most pure functions defined the string types -- {-# LANGUAGE BangPatterns, CPP, GADTs, MagicHash #-} +{-# LANGUAGE DeriveGeneric, RecordWildCards #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Benchmarks.Pure - ( benchmark + ( initEnv + , benchmark ) where import Control.DeepSeq (NFData (..)) import Control.Exception (evaluate) -import Criterion (Benchmark, bgroup, bench, nf) -import Data.Char (toLower, toUpper) -import Data.Monoid (mappend, mempty) -import GHC.Base (Char (..), Int (..), chr#, ord#, (+#)) +import Data.Char (chr, ord) +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) +import GHC.Generics (Generic) +import GHC.Int (Int64) import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.Lazy.Char8 as BL -import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import qualified Data.Text.Lazy.Encoding as TL +import Data.Semigroup +import Data.List.NonEmpty (NonEmpty((:|))) -benchmark :: String -> FilePath -> IO Benchmark -benchmark kind fp = do +data Env = Env + { bsa :: !BS.ByteString + , ta :: !T.Text + , tb :: !T.Text + , tla :: !TL.Text + , tlb :: !TL.Text + , bla :: !BL.ByteString + , bsa_len :: !Int + , ta_len :: !Int + , bla_len :: !Int64 + , tla_len :: !Int64 + , tl :: [T.Text] + , tll :: [TL.Text] + } deriving (Generic) + +instance NFData Env + +initEnv :: FilePath -> IO Env +initEnv fp = do -- Evaluate stuff before actually running the benchmark, we don't want to -- count it here. @@ -40,64 +60,51 @@ tla <- evaluate $ TL.fromChunks (T.chunksOf 16376 ta) tlb <- evaluate $ TL.fromChunks (T.chunksOf 16376 tb) - -- ByteString B, LazyByteString A/B - bsb <- evaluate $ T.encodeUtf8 tb bla <- evaluate $ BL.fromChunks (chunksOf 16376 bsa) - blb <- evaluate $ BL.fromChunks (chunksOf 16376 bsb) - -- String A/B - sa <- evaluate $ UTF8.toString bsa - sb <- evaluate $ T.unpack tb - -- Lengths bsa_len <- evaluate $ BS.length bsa ta_len <- evaluate $ T.length ta bla_len <- evaluate $ BL.length bla tla_len <- evaluate $ TL.length tla - sa_len <- evaluate $ L.length sa -- Lines - bsl <- evaluate $ BS.lines bsa - bll <- evaluate $ BL.lines bla tl <- evaluate $ T.lines ta tll <- evaluate $ TL.lines tla - sl <- evaluate $ L.lines sa - return $ bgroup "Pure" + return Env{..} + +benchmark :: String -> Env -> Benchmark +benchmark kind ~Env{..} = + bgroup kind [ bgroup "append" [ benchT $ nf (T.append tb) ta , benchTL $ nf (TL.append tlb) tla - , benchBS $ nf (BS.append bsb) bsa - , benchBSL $ nf (BL.append blb) bla - , benchS $ nf ((++) sb) sa ] , bgroup "concat" [ benchT $ nf T.concat tl , benchTL $ nf TL.concat tll - , benchBS $ nf BS.concat bsl - , benchBSL $ nf BL.concat bll - , benchS $ nf L.concat sl ] + , bgroup "sconcat" + [ benchT $ nf sconcat (T.empty :| tl) + , benchTL $ nf sconcat (TL.empty :| tll) + ] + , bgroup "stimes" + [ benchT $ nf (stimes (10 :: Int)) ta + , benchTL $ nf (stimes (10 :: Int)) tla + ] , bgroup "cons" [ benchT $ nf (T.cons c) ta , benchTL $ nf (TL.cons c) tla - , benchBS $ nf (BS.cons c) bsa - , benchBSL $ nf (BL.cons c) bla - , benchS $ nf (c:) sa ] - , bgroup "concatMap" - [ benchT $ nf (T.concatMap (T.replicate 3 . T.singleton)) ta - , benchTL $ nf (TL.concatMap (TL.replicate 3 . TL.singleton)) tla - , benchBS $ nf (BS.concatMap (BS.replicate 3)) bsa - , benchBSL $ nf (BL.concatMap (BL.replicate 3)) bla - , benchS $ nf (L.concatMap (L.replicate 3 . (:[]))) sa - ] + -- concatMap exceeds 4G heap size on current test data + -- , bgroup "concatMap" + -- [ benchT $ nf (T.concatMap (T.replicate 3 . T.singleton)) ta + -- , benchTL $ nf (TL.concatMap (TL.replicate 3 . TL.singleton)) tla + -- ] , bgroup "decode" [ benchT $ nf T.decodeUtf8 bsa , benchTL $ nf TL.decodeUtf8 bla - , benchBS $ nf BS.unpack bsa - , benchBSL $ nf BL.unpack bla - , benchS $ nf UTF8.toString bsa ] , bgroup "decode'" [ benchT $ nf T.decodeUtf8' bsa @@ -106,342 +113,224 @@ , bgroup "drop" [ benchT $ nf (T.drop (ta_len `div` 3)) ta , benchTL $ nf (TL.drop (tla_len `div` 3)) tla - , benchBS $ nf (BS.drop (bsa_len `div` 3)) bsa - , benchBSL $ nf (BL.drop (bla_len `div` 3)) bla - , benchS $ nf (L.drop (sa_len `div` 3)) sa ] , bgroup "encode" [ benchT $ nf T.encodeUtf8 ta , benchTL $ nf TL.encodeUtf8 tla - , benchBS $ nf BS.pack sa - , benchBSL $ nf BL.pack sa - , benchS $ nf UTF8.fromString sa ] , bgroup "filter" [ benchT $ nf (T.filter p0) ta , benchTL $ nf (TL.filter p0) tla - , benchBS $ nf (BS.filter p0) bsa - , benchBSL $ nf (BL.filter p0) bla - , benchS $ nf (L.filter p0) sa ] , bgroup "filter.filter" [ benchT $ nf (T.filter p1 . T.filter p0) ta , benchTL $ nf (TL.filter p1 . TL.filter p0) tla - , benchBS $ nf (BS.filter p1 . BS.filter p0) bsa - , benchBSL $ nf (BL.filter p1 . BL.filter p0) bla - , benchS $ nf (L.filter p1 . L.filter p0) sa ] , bgroup "foldl'" [ benchT $ nf (T.foldl' len 0) ta , benchTL $ nf (TL.foldl' len 0) tla - , benchBS $ nf (BS.foldl' len 0) bsa - , benchBSL $ nf (BL.foldl' len 0) bla - , benchS $ nf (L.foldl' len 0) sa ] , bgroup "foldr" [ benchT $ nf (L.length . T.foldr (:) []) ta , benchTL $ nf (L.length . TL.foldr (:) []) tla - , benchBS $ nf (L.length . BS.foldr (:) []) bsa - , benchBSL $ nf (L.length . BL.foldr (:) []) bla - , benchS $ nf (L.length . L.foldr (:) []) sa ] , bgroup "head" [ benchT $ nf T.head ta , benchTL $ nf TL.head tla - , benchBS $ nf BS.head bsa - , benchBSL $ nf BL.head bla - , benchS $ nf L.head sa ] , bgroup "init" [ benchT $ nf T.init ta , benchTL $ nf TL.init tla - , benchBS $ nf BS.init bsa - , benchBSL $ nf BL.init bla - , benchS $ nf L.init sa ] , bgroup "intercalate" [ benchT $ nf (T.intercalate tsw) tl , benchTL $ nf (TL.intercalate tlw) tll - , benchBS $ nf (BS.intercalate bsw) bsl - , benchBSL $ nf (BL.intercalate blw) bll - , benchS $ nf (L.intercalate lw) sl ] , bgroup "intersperse" [ benchT $ nf (T.intersperse c) ta , benchTL $ nf (TL.intersperse c) tla - , benchBS $ nf (BS.intersperse c) bsa - , benchBSL $ nf (BL.intersperse c) bla - , benchS $ nf (L.intersperse c) sa ] , bgroup "isInfixOf" [ benchT $ nf (T.isInfixOf tsw) ta , benchTL $ nf (TL.isInfixOf tlw) tla - , benchBS $ nf (BS.isInfixOf bsw) bsa - -- no isInfixOf for lazy bytestrings - , benchS $ nf (L.isInfixOf lw) sa ] , bgroup "last" [ benchT $ nf T.last ta , benchTL $ nf TL.last tla - , benchBS $ nf BS.last bsa - , benchBSL $ nf BL.last bla - , benchS $ nf L.last sa ] , bgroup "map" [ benchT $ nf (T.map f) ta , benchTL $ nf (TL.map f) tla - , benchBS $ nf (BS.map f) bsa - , benchBSL $ nf (BL.map f) bla - , benchS $ nf (L.map f) sa ] , bgroup "mapAccumL" [ benchT $ nf (T.mapAccumL g 0) ta , benchTL $ nf (TL.mapAccumL g 0) tla - , benchBS $ nf (BS.mapAccumL g 0) bsa - , benchBSL $ nf (BL.mapAccumL g 0) bla - , benchS $ nf (L.mapAccumL g 0) sa ] , bgroup "mapAccumR" [ benchT $ nf (T.mapAccumR g 0) ta , benchTL $ nf (TL.mapAccumR g 0) tla - , benchBS $ nf (BS.mapAccumR g 0) bsa - , benchBSL $ nf (BL.mapAccumR g 0) bla - , benchS $ nf (L.mapAccumR g 0) sa ] , bgroup "map.map" [ benchT $ nf (T.map f . T.map f) ta , benchTL $ nf (TL.map f . TL.map f) tla - , benchBS $ nf (BS.map f . BS.map f) bsa - , benchBSL $ nf (BL.map f . BL.map f) bla - , benchS $ nf (L.map f . L.map f) sa ] , bgroup "replicate char" [ benchT $ nf (T.replicate bsa_len) (T.singleton c) , benchTL $ nf (TL.replicate (fromIntegral bsa_len)) (TL.singleton c) - , benchBS $ nf (BS.replicate bsa_len) c - , benchBSL $ nf (BL.replicate (fromIntegral bsa_len)) c - , benchS $ nf (L.replicate bsa_len) c ] , bgroup "replicate string" [ benchT $ nf (T.replicate (bsa_len `div` T.length tsw)) tsw , benchTL $ nf (TL.replicate (fromIntegral bsa_len `div` TL.length tlw)) tlw - , benchS $ nf (replicat (bsa_len `div` T.length tsw)) lw ] , bgroup "reverse" [ benchT $ nf T.reverse ta , benchTL $ nf TL.reverse tla - , benchBS $ nf BS.reverse bsa - , benchBSL $ nf BL.reverse bla - , benchS $ nf L.reverse sa ] , bgroup "take" [ benchT $ nf (T.take (ta_len `div` 3)) ta , benchTL $ nf (TL.take (tla_len `div` 3)) tla - , benchBS $ nf (BS.take (bsa_len `div` 3)) bsa - , benchBSL $ nf (BL.take (bla_len `div` 3)) bla - , benchS $ nf (L.take (sa_len `div` 3)) sa ] , bgroup "tail" [ benchT $ nf T.tail ta , benchTL $ nf TL.tail tla - , benchBS $ nf BS.tail bsa - , benchBSL $ nf BL.tail bla - , benchS $ nf L.tail sa ] , bgroup "toLower" [ benchT $ nf T.toLower ta , benchTL $ nf TL.toLower tla - , benchBS $ nf (BS.map toLower) bsa - , benchBSL $ nf (BL.map toLower) bla - , benchS $ nf (L.map toLower) sa ] , bgroup "toUpper" [ benchT $ nf T.toUpper ta , benchTL $ nf TL.toUpper tla - , benchBS $ nf (BS.map toUpper) bsa - , benchBSL $ nf (BL.map toUpper) bla - , benchS $ nf (L.map toUpper) sa ] + , bgroup "toTitle" + [ benchT $ nf T.toTitle ta + , benchTL $ nf TL.toTitle tla + ] , bgroup "uncons" [ benchT $ nf T.uncons ta , benchTL $ nf TL.uncons tla - , benchBS $ nf BS.uncons bsa - , benchBSL $ nf BL.uncons bla - , benchS $ nf L.uncons sa ] , bgroup "words" [ benchT $ nf T.words ta , benchTL $ nf TL.words tla - , benchBS $ nf BS.words bsa - , benchBSL $ nf BL.words bla - , benchS $ nf L.words sa ] , bgroup "zipWith" [ benchT $ nf (T.zipWith min tb) ta , benchTL $ nf (TL.zipWith min tlb) tla - , benchBS $ nf (BS.zipWith min bsb) bsa - , benchBSL $ nf (BL.zipWith min blb) bla - , benchS $ nf (L.zipWith min sb) sa ] + , bgroup "length . unpack" -- length should fuse with unpack + [ benchT $ nf (L.length . T.unpack) ta + , benchTL $ nf (L.length . TL.unpack) tla + ] + , bgroup "length . drop 1 . unpack" -- no list fusion because of drop 1 + [ benchT $ nf (L.length . L.drop 1 . T.unpack) ta + , benchTL $ nf (L.length . L.drop 1 . TL.unpack) tla + ] , bgroup "length" [ bgroup "cons" [ benchT $ nf (T.length . T.cons c) ta , benchTL $ nf (TL.length . TL.cons c) tla - , benchBS $ nf (BS.length . BS.cons c) bsa - , benchBSL $ nf (BL.length . BL.cons c) bla - , benchS $ nf (L.length . (:) c) sa ] , bgroup "decode" [ benchT $ nf (T.length . T.decodeUtf8) bsa , benchTL $ nf (TL.length . TL.decodeUtf8) bla - , benchBS $ nf (L.length . BS.unpack) bsa - , benchBSL $ nf (L.length . BL.unpack) bla - , bench "StringUTF8" $ nf (L.length . UTF8.toString) bsa ] , bgroup "drop" [ benchT $ nf (T.length . T.drop (ta_len `div` 3)) ta , benchTL $ nf (TL.length . TL.drop (tla_len `div` 3)) tla - , benchBS $ nf (BS.length . BS.drop (bsa_len `div` 3)) bsa - , benchBSL $ nf (BL.length . BL.drop (bla_len `div` 3)) bla - , benchS $ nf (L.length . L.drop (sa_len `div` 3)) sa ] , bgroup "filter" [ benchT $ nf (T.length . T.filter p0) ta , benchTL $ nf (TL.length . TL.filter p0) tla - , benchBS $ nf (BS.length . BS.filter p0) bsa - , benchBSL $ nf (BL.length . BL.filter p0) bla - , benchS $ nf (L.length . L.filter p0) sa ] , bgroup "filter.filter" [ benchT $ nf (T.length . T.filter p1 . T.filter p0) ta , benchTL $ nf (TL.length . TL.filter p1 . TL.filter p0) tla - , benchBS $ nf (BS.length . BS.filter p1 . BS.filter p0) bsa - , benchBSL $ nf (BL.length . BL.filter p1 . BL.filter p0) bla - , benchS $ nf (L.length . L.filter p1 . L.filter p0) sa ] , bgroup "init" [ benchT $ nf (T.length . T.init) ta , benchTL $ nf (TL.length . TL.init) tla - , benchBS $ nf (BS.length . BS.init) bsa - , benchBSL $ nf (BL.length . BL.init) bla - , benchS $ nf (L.length . L.init) sa ] , bgroup "intercalate" [ benchT $ nf (T.length . T.intercalate tsw) tl , benchTL $ nf (TL.length . TL.intercalate tlw) tll - , benchBS $ nf (BS.length . BS.intercalate bsw) bsl - , benchBSL $ nf (BL.length . BL.intercalate blw) bll - , benchS $ nf (L.length . L.intercalate lw) sl ] , bgroup "intersperse" [ benchT $ nf (T.length . T.intersperse c) ta , benchTL $ nf (TL.length . TL.intersperse c) tla - , benchBS $ nf (BS.length . BS.intersperse c) bsa - , benchBSL $ nf (BL.length . BL.intersperse c) bla - , benchS $ nf (L.length . L.intersperse c) sa ] , bgroup "map" [ benchT $ nf (T.length . T.map f) ta , benchTL $ nf (TL.length . TL.map f) tla - , benchBS $ nf (BS.length . BS.map f) bsa - , benchBSL $ nf (BL.length . BL.map f) bla - , benchS $ nf (L.length . L.map f) sa ] , bgroup "map.map" [ benchT $ nf (T.length . T.map f . T.map f) ta , benchTL $ nf (TL.length . TL.map f . TL.map f) tla - , benchBS $ nf (BS.length . BS.map f . BS.map f) bsa - , benchS $ nf (L.length . L.map f . L.map f) sa ] , bgroup "replicate char" [ benchT $ nf (T.length . T.replicate bsa_len) (T.singleton c) , benchTL $ nf (TL.length . TL.replicate (fromIntegral bsa_len)) (TL.singleton c) - , benchBS $ nf (BS.length . BS.replicate bsa_len) c - , benchBSL $ nf (BL.length . BL.replicate (fromIntegral bsa_len)) c - , benchS $ nf (L.length . L.replicate bsa_len) c ] , bgroup "replicate string" [ benchT $ nf (T.length . T.replicate (bsa_len `div` T.length tsw)) tsw , benchTL $ nf (TL.length . TL.replicate (fromIntegral bsa_len `div` TL.length tlw)) tlw - , benchS $ nf (L.length . replicat (bsa_len `div` T.length tsw)) lw ] , bgroup "take" [ benchT $ nf (T.length . T.take (ta_len `div` 3)) ta , benchTL $ nf (TL.length . TL.take (tla_len `div` 3)) tla - , benchBS $ nf (BS.length . BS.take (bsa_len `div` 3)) bsa - , benchBSL $ nf (BL.length . BL.take (bla_len `div` 3)) bla - , benchS $ nf (L.length . L.take (sa_len `div` 3)) sa ] , bgroup "tail" [ benchT $ nf (T.length . T.tail) ta , benchTL $ nf (TL.length . TL.tail) tla - , benchBS $ nf (BS.length . BS.tail) bsa - , benchBSL $ nf (BL.length . BL.tail) bla - , benchS $ nf (L.length . L.tail) sa ] , bgroup "toLower" [ benchT $ nf (T.length . T.toLower) ta , benchTL $ nf (TL.length . TL.toLower) tla - , benchBS $ nf (BS.length . BS.map toLower) bsa - , benchBSL $ nf (BL.length . BL.map toLower) bla - , benchS $ nf (L.length . L.map toLower) sa ] , bgroup "toUpper" [ benchT $ nf (T.length . T.toUpper) ta , benchTL $ nf (TL.length . TL.toUpper) tla - , benchBS $ nf (BS.length . BS.map toUpper) bsa - , benchBSL $ nf (BL.length . BL.map toUpper) bla - , benchS $ nf (L.length . L.map toUpper) sa ] + , bgroup "toTitle" + [ benchT $ nf (T.length . T.toTitle) ta + , benchTL $ nf (TL.length . TL.toTitle) tla + ] , bgroup "words" [ benchT $ nf (L.length . T.words) ta , benchTL $ nf (L.length . TL.words) tla - , benchBS $ nf (L.length . BS.words) bsa - , benchBSL $ nf (L.length . BL.words) bla - , benchS $ nf (L.length . L.words) sa ] , bgroup "zipWith" [ benchT $ nf (T.length . T.zipWith min tb) ta , benchTL $ nf (TL.length . TL.zipWith min tlb) tla - , benchBS $ nf (L.length . BS.zipWith min bsb) bsa - , benchBSL $ nf (L.length . BL.zipWith min blb) bla - , benchS $ nf (L.length . L.zipWith min sb) sa ] ] , bgroup "Builder" - [ bench "mappend char" $ nf (TL.length . TB.toLazyText . mappendNChar 'a') 10000 - , bench "mappend 8 char" $ nf (TL.length . TB.toLazyText . mappend8Char) 'a' - , bench "mappend text" $ nf (TL.length . TB.toLazyText . mappendNText short) 10000 + [ bench "mappend char" $ + nf (TL.length . TB.toLazyText . mappendNChar 'a') 10000 + , bench "mappend 8 char" $ + nf (TL.length . TB.toLazyText . mappend8Char) 'a' + , bench "mappend text" $ + nf (TL.length . TB.toLazyText . mappendNText short) 10000 ] ] where - benchS = bench ("String+" ++ kind) - benchT = bench ("Text+" ++ kind) - benchTL = bench ("LazyText+" ++ kind) - benchBS = bench ("ByteString+" ++ kind) - benchBSL = bench ("LazyByteString+" ++ kind) + benchT = bench "Text" + benchTL = bench "LazyText" c = 'й' p0 = (== c) p1 = (/= 'д') lw = "право" - bsw = UTF8.fromString lw - blw = BL.fromChunks [bsw] tsw = T.pack lw tlw = TL.fromChunks [tsw] - f (C# c#) = C# (chr# (ord# c# +# 1#)) - g (I# i#) (C# c#) = (I# (i# +# 1#), C# (chr# (ord# c# +# i#))) len l _ = l + (1::Int) - replicat n = concat . L.replicate n short = T.pack "short" -#if !MIN_VERSION_bytestring(0,10,0) -instance NFData BS.ByteString - -instance NFData BL.ByteString where - rnf BL.Empty = () - rnf (BL.Chunk _ ts) = rnf ts -#endif + -- Valid 'Char' are in range [0..0x10FFFF], otherwise 'chr' throws an 'error'. + -- 'Data.Text.Internal.safe' does not validate this, it assumes that inputs + -- has been already sanitized to belong to the range. + f !ch = chr (min 0x10FFFF (ord ch + 1)) + g !i !ch = (i + 1, chr (min 0x10FFFF (ord ch + i))) data B where B :: NFData a => a -> B @@ -467,7 +356,7 @@ | i < n = TB.singleton c `mappend` go (i+1) | otherwise = mempty --- | Gives more opportunity for inlining and elimination of unnecesary +-- | Gives more opportunity for inlining and elimination of unnecessary -- bounds checks. -- mappend8Char :: Char -> TB.Builder diff --git a/benchmarks/haskell/Benchmarks/ReadNumbers.hs b/benchmarks/haskell/Benchmarks/ReadNumbers.hs --- a/benchmarks/haskell/Benchmarks/ReadNumbers.hs +++ b/benchmarks/haskell/Benchmarks/ReadNumbers.hs @@ -17,15 +17,13 @@ -- * Lexing/parsing of different numerical types -- module Benchmarks.ReadNumbers - ( benchmark + ( initEnv + , benchmark ) where -import Criterion (Benchmark, bgroup, bench, whnf) -import Data.List (foldl') -import Numeric (readDec, readFloat, readHex) -import qualified Data.ByteString.Char8 as B -import qualified Data.ByteString.Lazy.Char8 as BL -import qualified Data.ByteString.Lex.Fractional as B +import Data.Foldable (Foldable(..)) +import Prelude hiding (Foldable(..)) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL @@ -33,21 +31,18 @@ import qualified Data.Text.Lazy.Read as TL import qualified Data.Text.Read as T -benchmark :: FilePath -> IO Benchmark -benchmark fp = do - -- Read all files into lines: string, text, lazy text, bytestring, lazy - -- bytestring - s <- lines `fmap` readFile fp +type Env = ([T.Text], [TL.Text]) + +initEnv :: FilePath -> IO Env +initEnv fp = do t <- T.lines `fmap` T.readFile fp tl <- TL.lines `fmap` TL.readFile fp - b <- B.lines `fmap` B.readFile fp - bl <- BL.lines `fmap` BL.readFile fp - return $ bgroup "ReadNumbers" - [ bench "DecimalString" $ whnf (int . string readDec) s - , bench "HexadecimalString" $ whnf (int . string readHex) s - , bench "DoubleString" $ whnf (double . string readFloat) s + return (t, tl) - , bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t +benchmark :: Env -> Benchmark +benchmark ~(t, tl) = + bgroup "ReadNumbers" + [ bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t , bench "HexadecimalText" $ whnf (int . text (T.signed T.hexadecimal)) t , bench "DoubleText" $ whnf (double . text T.double) t , bench "RationalText" $ whnf (double . text T.rational) t @@ -60,12 +55,6 @@ whnf (double . text TL.double) tl , bench "RationalLazyText" $ whnf (double . text TL.rational) tl - - , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b - , bench "DoubleByteString" $ whnf (double . byteString B.readDecimal) b - - , bench "DecimalLazyByteString" $ - whnf (int . byteString BL.readInt) bl ] where -- Used for fixing types @@ -74,20 +63,8 @@ double :: Double -> Double double = id -string :: (Ord a, Num a) => (t -> [(a, t)]) -> [t] -> a -string reader = foldl' go 1000000 - where - go z t = case reader t of [(n, _)] -> min n z - _ -> z - text :: (Ord a, Num a) => (t -> Either String (a,t)) -> [t] -> a text reader = foldl' go 1000000 where go z t = case reader t of Left _ -> z Right (n, _) -> min n z - -byteString :: (Ord a, Num a) => (t -> Maybe (a,t)) -> [t] -> a -byteString reader = foldl' go 1000000 - where - go z t = case reader t of Nothing -> z - Just (n, _) -> min n z diff --git a/benchmarks/haskell/Benchmarks/Replace.hs b/benchmarks/haskell/Benchmarks/Replace.hs --- a/benchmarks/haskell/Benchmarks/Replace.hs +++ b/benchmarks/haskell/Benchmarks/Replace.hs @@ -7,37 +7,30 @@ -- module Benchmarks.Replace ( benchmark + , initEnv ) where -import Criterion (Benchmark, bgroup, bench, nf) -import qualified Data.ByteString.Char8 as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Search as BL -import qualified Data.ByteString.Search as B +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) import qualified Data.Text as T -import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> String -> String -> IO Benchmark -benchmark fp pat sub = do +type Env = (T.Text, TL.Text) + +initEnv :: FilePath -> IO Env +initEnv fp = do tl <- TL.readFile fp - bl <- BL.readFile fp let !t = TL.toStrict tl - !b = T.encodeUtf8 t - return $ bgroup "Replace" [ + return (t, tl) + +benchmark :: String -> String -> Env -> Benchmark +benchmark pat sub ~(t, tl) = + bgroup "Replace" [ bench "Text" $ nf (T.length . T.replace tpat tsub) t - , bench "ByteString" $ nf (BL.length . B.replace bpat bsub) b , bench "LazyText" $ nf (TL.length . TL.replace tlpat tlsub) tl - , bench "LazyByteString" $ nf (BL.length . BL.replace blpat blsub) bl ] where tpat = T.pack pat tsub = T.pack sub tlpat = TL.pack pat tlsub = TL.pack sub - bpat = T.encodeUtf8 tpat - bsub = T.encodeUtf8 tsub - blpat = B.concat $ BL.toChunks $ TL.encodeUtf8 tlpat - blsub = B.concat $ BL.toChunks $ TL.encodeUtf8 tlsub diff --git a/benchmarks/haskell/Benchmarks/Search.hs b/benchmarks/haskell/Benchmarks/Search.hs --- a/benchmarks/haskell/Benchmarks/Search.hs +++ b/benchmarks/haskell/Benchmarks/Search.hs @@ -1,45 +1,36 @@ --- | Search for a pattern in a file, find the number of occurences +-- | Search for a pattern in a file, find the number of occurrences -- -- Tested in this benchmark: -- --- * Searching all occurences of a pattern using library routines +-- * Searching all occurrences of a pattern using library routines -- module Benchmarks.Search - ( benchmark + ( initEnv + , benchmark ) where -import Criterion (Benchmark, bench, bgroup, whnf) -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Search as BL -import qualified Data.ByteString.Search as B +import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Text as T -import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> T.Text -> IO Benchmark -benchmark fp needleT = do - b <- B.readFile fp - bl <- BL.readFile fp +type Env = (T.Text, TL.Text) + +initEnv :: FilePath -> IO Env +initEnv fp = do t <- T.readFile fp tl <- TL.readFile fp - return $ bgroup "FileIndices" - [ bench "ByteString" $ whnf (byteString needleB) b - , bench "LazyByteString" $ whnf (lazyByteString needleB) bl - , bench "Text" $ whnf (text needleT) t + return (t, tl) + +benchmark :: T.Text -> Env -> Benchmark +benchmark needleT ~(t, tl) = + bgroup "FileIndices" + [ bench "Text" $ whnf (text needleT) t , bench "LazyText" $ whnf (lazyText needleTL) tl ] where - needleB = T.encodeUtf8 needleT needleTL = TL.fromChunks [needleT] - -byteString :: B.ByteString -> B.ByteString -> Int -byteString needle = length . B.indices needle - -lazyByteString :: B.ByteString -> BL.ByteString -> Int -lazyByteString needle = length . BL.indices needle text :: T.Text -> T.Text -> Int text = T.count diff --git a/benchmarks/haskell/Benchmarks/Stream.hs b/benchmarks/haskell/Benchmarks/Stream.hs --- a/benchmarks/haskell/Benchmarks/Stream.hs +++ b/benchmarks/haskell/Benchmarks/Stream.hs @@ -6,24 +6,31 @@ -- * Most streaming functions -- {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE DeriveGeneric, RecordWildCards #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Benchmarks.Stream - ( benchmark + ( initEnv + , benchmark ) where import Control.DeepSeq (NFData (..)) -import Criterion (Benchmark, bgroup, bench, nf) +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) +import qualified Data.Text as T +import qualified Data.ByteString as B +import qualified Data.Text.Lazy as TL +import qualified Data.ByteString.Lazy as BL import Data.Text.Internal.Fusion.Types (Step (..), Stream (..)) import qualified Data.Text.Encoding as T import qualified Data.Text.Encoding.Error as E import qualified Data.Text.Internal.Encoding.Fusion as T import qualified Data.Text.Internal.Encoding.Fusion.Common as F -import qualified Data.Text.Internal.Fusion as F +import qualified Data.Text.Internal.Fusion as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL -import qualified Data.Text.Internal.Lazy.Fusion as FL +import qualified Data.Text.Internal.Lazy.Fusion as TL import qualified Data.Text.Lazy.IO as TL +import GHC.Generics (Generic) instance NFData a => NFData (Stream a) where -- Currently, this implementation does not force evaluation of the size hint @@ -34,8 +41,26 @@ Skip s' -> go s' Yield x s' -> rnf x `seq` go s' -benchmark :: FilePath -> IO Benchmark -benchmark fp = do +data Env = Env + { t :: !T.Text + , utf8 :: !B.ByteString + , utf16le :: !B.ByteString + , utf16be :: !B.ByteString + , utf32le :: !B.ByteString + , utf32be :: !B.ByteString + , tl :: !TL.Text + , utf8L :: !BL.ByteString + , utf16leL :: !BL.ByteString + , utf16beL :: !BL.ByteString + , utf32leL :: !BL.ByteString + , utf32beL :: !BL.ByteString + , s :: T.Stream Char + } deriving (Generic) + +instance NFData Env + +initEnv :: FilePath -> IO Env +initEnv fp = do -- Different formats t <- T.readFile fp let !utf8 = T.encodeUtf8 t @@ -53,25 +78,16 @@ !utf32beL = TL.encodeUtf32BE tl -- For the functions which operate on streams - let !s = F.stream t - - return $ bgroup "Stream" + let !s = T.stream t + return Env{..} +benchmark :: Env -> Benchmark +benchmark ~Env{..} = + bgroup "Stream" -- Fusion [ bgroup "stream" $ - [ bench "Text" $ nf F.stream t - , bench "LazyText" $ nf FL.stream tl - ] - -- must perform exactly the same as stream above due to - -- stream/unstream (i.e. stream after unstream) fusion - , bgroup "stream-fusion" $ - [ bench "Text" $ nf (F.stream . F.unstream . F.stream) t - , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl - ] - -- measure the overhead of unstream after stream - , bgroup "stream-unstream" $ - [ bench "Text" $ nf (F.unstream . F.stream) t - , bench "LazyText" $ nf (FL.unstream . FL.stream) tl + [ bench "Text" $ nf T.stream t + , bench "LazyText" $ nf TL.stream tl ] -- Encoding.Fusion diff --git a/benchmarks/haskell/Benchmarks/WordFrequencies.hs b/benchmarks/haskell/Benchmarks/WordFrequencies.hs --- a/benchmarks/haskell/Benchmarks/WordFrequencies.hs +++ b/benchmarks/haskell/Benchmarks/WordFrequencies.hs @@ -9,27 +9,29 @@ -- * Comparing: Eq/Ord instances -- module Benchmarks.WordFrequencies - ( benchmark + ( initEnv + , benchmark ) where -import Criterion (Benchmark, bench, bgroup, whnf) -import Data.Char (toLower) -import Data.List (foldl') +import Data.Foldable (Foldable(..)) import Data.Map (Map) -import qualified Data.ByteString.Char8 as B +import Prelude hiding (Foldable(..)) +import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T -benchmark :: FilePath -> IO Benchmark -benchmark fp = do - s <- readFile fp - b <- B.readFile fp +type Env = T.Text + +initEnv :: FilePath -> IO Env +initEnv fp = do t <- T.readFile fp - return $ bgroup "WordFrequencies" - [ bench "String" $ whnf (frequencies . words . map toLower) s - , bench "ByteString" $ whnf (frequencies . B.words . B.map toLower) b - , bench "Text" $ whnf (frequencies . T.words . T.toLower) t + return t + +benchmark :: Env -> Benchmark +benchmark ~t = + bgroup "WordFrequencies" + [ bench "Text" $ whnf (frequencies . T.words . T.toLower) t ] frequencies :: Ord a => [a] -> Map a Int diff --git a/benchmarks/haskell/Multilang.hs b/benchmarks/haskell/Multilang.hs deleted file mode 100644 --- a/benchmarks/haskell/Multilang.hs +++ /dev/null @@ -1,32 +0,0 @@ -{-# LANGUAGE BangPatterns, OverloadedStrings, RankNTypes #-} - -module Main ( - main - ) where - -import Control.Monad (forM_) -import qualified Data.ByteString as B -import qualified Data.Text as Text -import Data.Text.Encoding (decodeUtf8) -import Data.Text (Text) -import System.IO (hFlush, stdout) -import Timer (timer) - -type BM = Text -> () - -bm :: forall a. (Text -> a) -> BM -bm f t = f t `seq` () - -benchmarks :: [(String, Text.Text -> ())] -benchmarks = [ - ("find_first", bm $ Text.isInfixOf "en:Benin") - , ("find_index", bm $ Text.findIndex (=='c')) - ] - -main :: IO () -main = do - !contents <- decodeUtf8 `fmap` B.readFile "../tests/text-test-data/yiwiki.xml" - forM_ benchmarks $ \(name, bmark) -> do - putStr $ name ++ " " - hFlush stdout - putStrLn =<< (timer 100 contents bmark) diff --git a/benchmarks/haskell/Timer.hs b/benchmarks/haskell/Timer.hs deleted file mode 100644 --- a/benchmarks/haskell/Timer.hs +++ /dev/null @@ -1,30 +0,0 @@ -{-# LANGUAGE BangPatterns #-} - -module Timer (timer) where - -import Control.Exception (evaluate) -import Data.Time.Clock.POSIX (getPOSIXTime) -import GHC.Float (FFFormat(..), formatRealFloat) - -ickyRound :: Int -> Double -> String -ickyRound k = formatRealFloat FFFixed (Just k) - -timer :: Int -> a -> (a -> b) -> IO String -timer count a0 f = do - let loop !k !fastest - | k <= 0 = return fastest - | otherwise = do - start <- getPOSIXTime - let inner a i - | i <= 0 = return () - | otherwise = evaluate (f a) >> inner a (i-1) - inner a0 count - end <- getPOSIXTime - let elapsed = end - start - loop (k-1) (min fastest (elapsed / fromIntegral count)) - t <- loop (3::Int) 1e300 - let log10 x = log x / log 10 - ft = realToFrac t - prec = round (log10 (fromIntegral count) - log10 ft) - return $! ickyRound prec ft -{-# NOINLINE timer #-} diff --git a/benchmarks/python/cut.py b/benchmarks/python/cut.py deleted file mode 100644 --- a/benchmarks/python/cut.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python - -import utils, sys, codecs - -def cut(filename, l, r): - content = open(filename, encoding='utf-8') - for line in content: - print(line[l:r]) - -for f in sys.argv[1:]: - t = utils.benchmark(lambda: cut(f, 20, 40)) - sys.stderr.write('{0}: {1}\n'.format(f, t)) diff --git a/benchmarks/python/multilang.py b/benchmarks/python/multilang.py deleted file mode 100644 --- a/benchmarks/python/multilang.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import math -import sys -import time - -def find_first(): - cf = contents.find - return timer(lambda: cf("en:Benin")) - -def timer(f, count=100): - a = 1e300 - def g(): - return - for i in xrange(3): - start = time.time() - for j in xrange(count): - g() - a = min(a, (time.time() - start) / count) - - b = 1e300 - for i in xrange(3): - start = time.time() - for j in xrange(count): - f() - b = min(b, (time.time() - start) / count) - - return round(b - a, int(round(math.log(count, 10) - math.log(b - a, 10)))) - -contents = open('../../tests/text-test-data/yiwiki.xml', 'r').read() -contents = contents.decode('utf-8') - -benchmarks = ( - find_first, - ) - -to_run = sys.argv[1:] -bms = [] -if to_run: - for r in to_run: - for b in benchmarks: - if b.__name__.startswith(r): - bms.append(b) -else: - bms = benchmarks - -for b in bms: - sys.stdout.write(b.__name__ + ' ') - sys.stdout.flush() - print b() diff --git a/benchmarks/python/sort.py b/benchmarks/python/sort.py deleted file mode 100644 --- a/benchmarks/python/sort.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -import utils, sys, codecs - -def sort(filename): - content = open(filename, encoding='utf-8').read() - lines = content.splitlines() - lines.sort() - print('\n'.join(lines)) - -for f in sys.argv[1:]: - t = utils.benchmark(lambda: sort(f)) - sys.stderr.write('{0}: {1}\n'.format(f, t)) diff --git a/benchmarks/python/strip_tags.py b/benchmarks/python/strip_tags.py deleted file mode 100644 --- a/benchmarks/python/strip_tags.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -import utils, sys - -def strip_tags(filename): - string = open(filename, encoding='utf-8').read() - - d = 0 - out = [] - - for c in string: - if c == '<': d += 1 - - if d > 0: - out += ' ' - else: - out += c - - if c == '>': d -= 1 - - print(''.join(out)) - -for f in sys.argv[1:]: - t = utils.benchmark(lambda: strip_tags(f)) - sys.stderr.write('{0}: {1}\n'.format(f, t)) diff --git a/benchmarks/python/utils.py b/benchmarks/python/utils.py deleted file mode 100644 --- a/benchmarks/python/utils.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -import sys, time - -def benchmark_once(f): - start = time.time() - f() - end = time.time() - return end - start - -def benchmark(f): - runs = 100 - total = 0.0 - for i in range(runs): - result = benchmark_once(f) - sys.stderr.write('Run {0}: {1}\n'.format(i, result)) - total += result - return total / runs diff --git a/benchmarks/ruby/cut.rb b/benchmarks/ruby/cut.rb deleted file mode 100644 --- a/benchmarks/ruby/cut.rb +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env ruby - -require './utils.rb' - -def cut(filename, l, r) - File.open(filename, 'r:utf-8') do |file| - file.each_line do |line| - puts line[l, r - l] - end - end -end - -ARGV.each do |f| - t = benchmark { cut(f, 20, 40) } - STDERR.puts "#{f}: #{t}" -end diff --git a/benchmarks/ruby/fold.rb b/benchmarks/ruby/fold.rb deleted file mode 100644 --- a/benchmarks/ruby/fold.rb +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env ruby - -require './utils.rb' - -def fold(filename, max_width) - File.open(filename, 'r:utf-8') do |file| - # Words in this paragraph - paragraph = [] - - file.each_line do |line| - # If we encounter an empty line, we reformat and dump the current - # paragraph - if line.strip.empty? - puts fold_paragraph(paragraph, max_width) - puts - paragraph = [] - # Otherwise, we append the words found in the line to the paragraph - else - paragraph.concat line.split - end - end - - # Last paragraph - puts fold_paragraph(paragraph, max_width) unless paragraph.empty? - end -end - -# Fold a single paragraph to the desired width -def fold_paragraph(paragraph, max_width) - # Gradually build our output - str, *rest = paragraph - width = str.length - - rest.each do |word| - if width + word.length + 1 <= max_width - str << ' ' << word - width += word.length + 1 - else - str << "\n" << word - width = word.length - end - end - - str -end - -ARGV.each do |f| - t = benchmark { fold(f, 80) } - STDERR.puts "#{f}: #{t}" -end diff --git a/benchmarks/ruby/sort.rb b/benchmarks/ruby/sort.rb deleted file mode 100644 --- a/benchmarks/ruby/sort.rb +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env ruby - -require './utils.rb' - -def sort(filename) - File.open(filename, 'r:utf-8') do |file| - content = file.read - puts content.lines.sort.join - end -end - -ARGV.each do |f| - t = benchmark { sort(f) } - STDERR.puts "#{f}: #{t}" -end diff --git a/benchmarks/ruby/strip_tags.rb b/benchmarks/ruby/strip_tags.rb deleted file mode 100644 --- a/benchmarks/ruby/strip_tags.rb +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env ruby - -require './utils.rb' - -def strip_tags(filename) - File.open(filename, 'r:utf-8') do |file| - str = file.read - - d = 0 - - str.each_char do |c| - d += 1 if c == '<' - putc(if d > 0 then ' ' else c end) - d -= 1 if c == '>' - end - end -end - -ARGV.each do |f| - t = benchmark { strip_tags(f) } - STDERR.puts "#{f}: #{t}" -end diff --git a/benchmarks/ruby/utils.rb b/benchmarks/ruby/utils.rb deleted file mode 100644 --- a/benchmarks/ruby/utils.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'benchmark' - -def benchmark(&block) - runs = 100 - total = 0 - - runs.times do |i| - result = Benchmark.measure(&block).total - $stderr.puts "Run #{i}: #{result}" - total += result - end - - total / runs -end diff --git a/benchmarks/text-benchmarks.cabal b/benchmarks/text-benchmarks.cabal deleted file mode 100644 --- a/benchmarks/text-benchmarks.cabal +++ /dev/null @@ -1,66 +0,0 @@ -name: text-benchmarks -version: 0.0.0.0 -synopsis: Benchmarks for the text package -description: Benchmarks for the text package -homepage: https://bitbucket.org/bos/text -license: BSD2 -license-file: ../LICENSE -author: Jasper Van der Jeugt , - Bryan O'Sullivan , - Tom Harper , - Duncan Coutts -maintainer: jaspervdj@gmail.com -category: Text -build-type: Simple - -cabal-version: >=1.2 - -flag bytestring-builder - description: Depend on the bytestring-builder package for backwards compatibility. - default: False - manual: False - -flag llvm - description: use LLVM - default: False - manual: True - -executable text-benchmarks - hs-source-dirs: haskell .. - c-sources: ../cbits/cbits.c - cbits/time_iconv.c - include-dirs: ../include - main-is: Benchmarks.hs - ghc-options: -Wall -O2 -rtsopts - if flag(llvm) - ghc-options: -fllvm - cpp-options: -DHAVE_DEEPSEQ -DINTEGER_GMP - build-depends: base == 4.*, - binary, - blaze-builder, - bytestring-lexing >= 0.5.0, - containers, - criterion >= 0.10.0.0, - deepseq, - directory, - filepath, - ghc-prim, - integer-gmp, - stringsearch, - utf8-string, - vector - - if flag(bytestring-builder) - build-depends: bytestring >= 0.9 && < 0.10.4, - bytestring-builder >= 0.10.4 - else - build-depends: bytestring >= 0.10.4 - -executable text-multilang - hs-source-dirs: haskell - main-is: Multilang.hs - ghc-options: -Wall -O2 - build-depends: base == 4.*, - bytestring, - text, - time diff --git a/cbits/aarch64/measure_off.c b/cbits/aarch64/measure_off.c new file mode 100644 --- /dev/null +++ b/cbits/aarch64/measure_off.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include + +/* + measure_off_naive / measure_off_neon + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +static inline const ssize_t measure_off_neon(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + uint8x16_t w128[4] = {vld1q_u8(src), vld1q_u8(src + 16), vld1q_u8(src + 32), vld1q_u8(src + 48)}; + // Which bytes are either < 128 or >= 192? + uint8x16_t mask0 = vcgtq_s8((int8x16_t)w128[0], vdupq_n_s8(0xBF)); + uint8x16_t mask1 = vcgtq_s8((int8x16_t)w128[1], vdupq_n_s8(0xBF)); + uint8x16_t mask2 = vcgtq_s8((int8x16_t)w128[2], vdupq_n_s8(0xBF)); + uint8x16_t mask3 = vcgtq_s8((int8x16_t)w128[3], vdupq_n_s8(0xBF)); + + uint8x16_t mask01 = vaddq_u8(mask0, mask1); + uint8x16_t mask23 = vaddq_u8(mask2, mask3); + uint8x16_t mask = vaddq_u8(mask01, mask23); + + size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask)); + + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + while (src < srcend - 15){ + uint8x16_t w128 = vld1q_u8(src); + // Which bytes are either < 128 or >= 192? + uint8x16_t mask = vcgtq_s8((int8x16_t)w128, vdupq_n_s8(0xBF)); + size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask)); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } + + return measure_off_naive(src, srcend, cnt); +} + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { + ssize_t ret = measure_off_neon(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +} diff --git a/cbits/cbits.c b/cbits/cbits.c deleted file mode 100644 --- a/cbits/cbits.c +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright (c) 2011 Bryan O'Sullivan . - * - * Portions copyright (c) 2008-2010 Björn Höhrmann . - * - * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. - */ - -#include -#include -#include -#include "text_cbits.h" - -void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff, - size_t n) -{ - memcpy(dest + (doff<<1), src + (soff<<1), n<<1); -} - -int _hs_text_memcmp(const void *a, size_t aoff, const void *b, size_t boff, - size_t n) -{ - return memcmp(a + (aoff<<1), b + (boff<<1), n<<1); -} - -#define UTF8_ACCEPT 0 -#define UTF8_REJECT 12 - -static const uint8_t utf8d[] = { - /* - * The first part of the table maps bytes to character classes that - * to reduce the size of the transition table and create bitmasks. - */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - - /* - * The second part is a transition table that maps a combination of - * a state of the automaton and a character class to a state. - */ - 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, - 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, - 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, - 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, - 12,36,12,12,12,12,12,12,12,12,12,12, -}; - -static inline uint32_t -decode(uint32_t *state, uint32_t* codep, uint32_t byte) { - uint32_t type = utf8d[byte]; - - *codep = (*state != UTF8_ACCEPT) ? - (byte & 0x3fu) | (*codep << 6) : - (0xff >> type) & (byte); - - return *state = utf8d[256 + *state + type]; -} - -/* - * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode - * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to - * an UTF16 array - */ -void -_hs_text_decode_latin1(uint16_t *dest, const uint8_t *src, - const uint8_t *srcend) -{ - const uint8_t *p = src; - -#if defined(__i386__) || defined(__x86_64__) - /* This optimization works on a little-endian systems by using - (aligned) 32-bit loads instead of 8-bit loads - */ - - /* consume unaligned prefix */ - while (p != srcend && (uintptr_t)p & 0x3) - *dest++ = *p++; - - /* iterate over 32-bit aligned loads */ - while (p < srcend - 3) { - const uint32_t w = *((const uint32_t *)p); - - *dest++ = w & 0xff; - *dest++ = (w >> 8) & 0xff; - *dest++ = (w >> 16) & 0xff; - *dest++ = (w >> 24) & 0xff; - - p += 4; - } -#endif - - /* handle unaligned suffix */ - while (p != srcend) - *dest++ = *p++; -} - -/* - * A best-effort decoder. Runs until it hits either end of input or - * the start of an invalid byte sequence. - * - * At exit, we update *destoff with the next offset to write to, *src - * with the next source location past the last one successfully - * decoded, and return the next source location to read from. - * - * Moreover, we expose the internal decoder state (state0 and - * codepoint0), allowing one to restart the decoder after it - * terminates (say, due to a partial codepoint). - * - * In particular, there are a few possible outcomes, - * - * 1) We decoded the buffer entirely: - * In this case we return srcend - * state0 == UTF8_ACCEPT - * - * 2) We met an invalid encoding - * In this case we return the address of the first invalid byte - * state0 == UTF8_REJECT - * - * 3) We reached the end of the buffer while decoding a codepoint - * In this case we return a pointer to the first byte of the partial codepoint - * state0 != UTF8_ACCEPT, UTF8_REJECT - * - */ -#if defined(__GNUC__) || defined(__clang__) -static inline uint8_t const * -_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff, - const uint8_t **src, const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) - __attribute((always_inline)); -#endif - -static inline uint8_t const * -_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff, - const uint8_t **src, const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) -{ - uint16_t *d = dest + *destoff; - const uint8_t *s = *src, *last = *src; - uint32_t state = *state0; - uint32_t codepoint = *codepoint0; - - while (s < srcend) { -#if defined(__i386__) || defined(__x86_64__) - /* - * This code will only work on a little-endian system that - * supports unaligned loads. - * - * It gives a substantial speed win on data that is purely or - * partly ASCII (e.g. HTML), at only a slight cost on purely - * non-ASCII text. - */ - - if (state == UTF8_ACCEPT) { - while (s < srcend - 4) { - codepoint = *((uint32_t *) s); - if ((codepoint & 0x80808080) != 0) - break; - s += 4; - - /* - * Tried 32-bit stores here, but the extra bit-twiddling - * slowed the code down. - */ - - *d++ = (uint16_t) (codepoint & 0xff); - *d++ = (uint16_t) ((codepoint >> 8) & 0xff); - *d++ = (uint16_t) ((codepoint >> 16) & 0xff); - *d++ = (uint16_t) ((codepoint >> 24) & 0xff); - } - last = s; - } -#endif - - if (decode(&state, &codepoint, *s++) != UTF8_ACCEPT) { - if (state != UTF8_REJECT) - continue; - break; - } - - if (codepoint <= 0xffff) - *d++ = (uint16_t) codepoint; - else { - *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10)); - *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF)); - } - last = s; - } - - *destoff = d - dest; - *codepoint0 = codepoint; - *state0 = state; - *src = last; - - return s; -} - -uint8_t const * -_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff, - const uint8_t **src, - const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) -{ - uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend, - codepoint0, state0); - if (*state0 == UTF8_REJECT) - ret -=1; - return ret; -} - -/* - * Helper to decode buffer and discard final decoder state - */ -const uint8_t * -_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff, - const uint8_t *src, const uint8_t *const srcend) -{ - uint32_t codepoint; - uint32_t state = UTF8_ACCEPT; - uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, &src, srcend, - &codepoint, &state); - /* Back up if we have an incomplete or invalid encoding */ - if (state != UTF8_ACCEPT) - ret -= 1; - return ret; -} - -void -_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff, - size_t srclen) -{ - const uint16_t *srcend; - uint8_t *dest = *destp; - - src += srcoff; - srcend = src + srclen; - - ascii: -#if defined(__x86_64__) - while (srcend - src >= 4) { - uint64_t w = *((uint64_t *) src); - - if (w & 0xFF80FF80FF80FF80ULL) { - if (!(w & 0x000000000000FF80ULL)) { - *dest++ = w & 0xFFFF; - src++; - if (!(w & 0x00000000FF800000ULL)) { - *dest++ = (w >> 16) & 0xFFFF; - src++; - if (!(w & 0x0000FF8000000000ULL)) { - *dest++ = (w >> 32) & 0xFFFF; - src++; - } - } - } - break; - } - *dest++ = w & 0xFFFF; - *dest++ = (w >> 16) & 0xFFFF; - *dest++ = (w >> 32) & 0xFFFF; - *dest++ = w >> 48; - src += 4; - } -#endif - -#if defined(__i386__) - while (srcend - src >= 2) { - uint32_t w = *((uint32_t *) src); - - if (w & 0xFF80FF80) - break; - *dest++ = w & 0xFFFF; - *dest++ = w >> 16; - src += 2; - } -#endif - - while (src < srcend) { - uint16_t w = *src++; - - if (w <= 0x7F) { - *dest++ = w; - /* An ASCII byte is likely to begin a run of ASCII bytes. - Falling back into the fast path really helps performance. */ - goto ascii; - } - else if (w <= 0x7FF) { - *dest++ = (w >> 6) | 0xC0; - *dest++ = (w & 0x3f) | 0x80; - } - else if (w < 0xD800 || w > 0xDBFF) { - *dest++ = (w >> 12) | 0xE0; - *dest++ = ((w >> 6) & 0x3F) | 0x80; - *dest++ = (w & 0x3F) | 0x80; - } else { - uint32_t c = ((((uint32_t) w) - 0xD800) << 10) + - (((uint32_t) *src++) - 0xDC00) + 0x10000; - *dest++ = (c >> 18) | 0xF0; - *dest++ = ((c >> 12) & 0x3F) | 0x80; - *dest++ = ((c >> 6) & 0x3F) | 0x80; - *dest++ = (c & 0x3F) | 0x80; - } - } - - *destp = dest; -} diff --git a/cbits/is_ascii.c b/cbits/is_ascii.c new file mode 100644 --- /dev/null +++ b/cbits/is_ascii.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#ifdef __x86_64__ +#include +#include +#endif +#include + +/* + _hs_text_is_ascii takes a UTF-8 encoded buffer, + and returns the length of the ASCII-compatible prefix. +*/ +const size_t _hs_text_is_ascii(const uint8_t *src0, const uint8_t *srcend){ + const uint8_t *src = src0; + +#ifdef __x86_64__ + // I experimented with larger vector registers, + // but did not notice any measurable speed up, so let's keep it simple. + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are < 128? + uint16_t mask = _mm_movemask_epi8(w128); + if (mask) break; + src+= 16; + } +#endif + + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + if (w64 & 0x8080808080808080ULL) break; + src+= 8; + } + + while (src < srcend){ + uint8_t leadByte = *src; + if(leadByte >= 0x80) break; + src++; + } + + return src - src0; +} + +/* + _hs_text_is_ascii_offset is a helper for calling _hs_text_is_ascii on Texts. +*/ +const size_t _hs_text_is_ascii_offset(const uint8_t *arr, size_t off, size_t len){ + return _hs_text_is_ascii(arr + off, arr + off + len); +} diff --git a/cbits/measure_off.c b/cbits/measure_off.c new file mode 100644 --- /dev/null +++ b/cbits/measure_off.c @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#ifdef __x86_64__ +#include +#include +#include +#include +#endif +#include + +// stdatomic.h has been introduces in gcc 4.9 +#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__)) +#ifndef __STDC_NO_ATOMICS__ +#define __STDC_NO_ATOMICS__ +#endif +#endif + +#ifndef __STDC_NO_ATOMICS__ +#include +#endif + +/* + Clang-6 does not enable proper -march flags for assembly modules + which leads to "error: instruction requires: AVX-512 ISA" + at the assembler phase. + + Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6 + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions + and it's the latest available version on macOS 10.13. + + Disable AVX-512 instructions as they are most likely not supported + on the hardware running clang-6. +*/ +#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \ + || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__) +#define COMPILER_SUPPORTS_AVX512 +#endif + + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +bool has_avx512_vl_bw() { +#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__) + uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; + __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); + // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features + const bool has_avx512_bw = ebx & (1 << 30); + const bool has_avx512_vl = ebx & (1 << 31); + // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); + return has_avx512_bw && has_avx512_vl; +#else + return false; +#endif +} +#endif + +/* + measure_off_naive / measure_off_avx / measure_off_sse + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +__attribute__((target("avx512vl,avx512bw"))) +static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + __m512i w512 = _mm512_loadu_si512((__m512i *)src); + // Which bytes are either < 128 or >= 192? + uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); + size_t leads = __builtin_popcountll(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties + // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html + + if (src < srcend - 31){ + __m256i w256 = _mm256_loadu_si256((__m256i *)src); + uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 32; + } + } + + if (src < srcend - 15){ + __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 + uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 16; + } + } + + return measure_off_naive(src, srcend, cnt); +} +#endif + +/* Count the number of bits set in the argument + * + This is a temporary workaround for the fact that + the GHC RTS linker does not recognize the `__builtin_popcountll` + symbol. + + See https://gitlab.haskell.org/ghc/ghc/-/issues/21787 + https://gitlab.haskell.org/ghc/ghc/-/issues/19900 + https://github.com/haskell/text/issues/450 + + It can be removed and the usages of popcount64 replaced with + `__builtin_popcountll` once affected versions of the compiler + are no longer in widespread use. + + Once GHC learns to recognize the symbol, this can be gated + by CPP to call __builtin_popcountll when using the appropriate + version of GHC. +*/ +static inline const size_t popcount16(uint16_t x) { + + // Taken from https://en.wikipedia.org/wiki/Hamming_weight + const uint16_t m1 = 0x5555; //binary: 0101... + const uint16_t m2 = 0x3333; //binary: 00110011.. + const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ... + x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits + x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits + return (x >> 8) + (x & 0x00FF); +} + +static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ +#ifdef __x86_64__ + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are either < 128 or >= 192? + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); + size_t leads = popcount16(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } +#endif + + return measure_off_naive(src, srcend, cnt); +} + +typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { +#ifndef __STDC_NO_ATOMICS__ + static _Atomic measure_off_t s_impl = (measure_off_t)NULL; + measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); + if (!impl) { +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) + impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; +#else + impl = measure_off_sse; +#endif + atomic_store_explicit(&s_impl, impl, memory_order_relaxed); + } +#else + measure_off_t impl = measure_off_sse; +#endif + ssize_t ret = (*impl)(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +} diff --git a/cbits/reverse.c b/cbits/reverse.c new file mode 100644 --- /dev/null +++ b/cbits/reverse.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include + +/* + _hs_text_reverse takes a UTF-8 encoded buffer, specified by (src0, off, len), + and reverses it, writing output starting from dst0. + + The input buffer (src0, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +void _hs_text_reverse(uint8_t *dst0, const uint8_t *src0, size_t off, size_t len) +{ + const uint8_t *src = src0 + off; + const uint8_t *srcend = src + len; + uint8_t *dst = dst0 + len - 1; + + while (src < srcend){ + uint8_t leadByte = *src++; + if (leadByte < 0x80){ + *dst-- = leadByte; + } else if (leadByte < 0xe0){ + *(dst-1) = leadByte; + *dst = *src++; + dst-=2; + } else if (leadByte < 0xf0){ + *(dst-2) = leadByte; + *(dst-1) = *src++; + *dst = *src++; + dst-=3; + } else { + *(dst-3) = leadByte; + *(dst-2) = *src++; + *(dst-1) = *src++; + *dst = *src++; + dst-=4; + } + } +} diff --git a/cbits/utils.c b/cbits/utils.c new file mode 100644 --- /dev/null +++ b/cbits/utils.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#include + +/* Changed name to disambiguate from _hs_text_memcmp, + which could be present in system-wide headers from installed ghc package */ +int _hs_text_memcmp2(const void *arr1, size_t off1, const void *arr2, size_t off2, size_t len) +{ + return memcmp(arr1 + off1, arr2 + off2, len); +} + +ssize_t _hs_text_memchr(const void *arr, size_t off, size_t len, uint8_t byte) +{ + const void *ptr = memchr(arr + off, byte, len); + return ptr == NULL ? -1 : ptr - (arr + off); +} diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,181 +1,440 @@ -1.2.2.2 - -* The `toTitle` function now correctly handles letters that - immediately follow punctuation. Before, `"there's"` would turn into - `"There'S"`. Now, it becomes `"There's"`. - -* The implementation of unstreaming is faster, resulting in operations - such as `map` and `intersperse` speeding up by up to 30%, with - smaller code generated. - -* The optimised length comparison function is now more likely to be - used after some rewrite rule tweaking. - -* Bug fix: an off-by-one bug in `takeEnd` is fixed. - -* Bug fix: a logic error in `takeWord16` is fixed. - -1.2.2.1 - -* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. - The build flag has been renamed accordingly. Your army of diligent - maintainers apologizes for the churn. - -* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec - (updated from 7.0) - -* An STG lint error has been fixed - -1.2.2.0 - -* The `integer-simple` package, upon which this package optionally - depended, has been replaced with `integer-pure`. The build flag has - been renamed accordingly. - -* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a - `get`, the error is propagated via `fail` instead of an uncatchable - crash. - -* New function: `takeWhileEnd` - -* New instances for the `Text` types: - * if `base` >= 4.7: `PrintfArg` - * if `base` >= 4.9: `Semigroup` - -1.2.1.3 - -* Bug fix: As it turns out, moving the literal rewrite rules to simplifier - phase 2 does not prevent competition with the `unpack` rule, which is - also active in this phase. Unfortunately this was hidden due to a silly - test environment mistake. Moving literal rules back to phase 1 finally - fixes GHC Trac #10528 correctly. - -1.2.1.2 - -* Bug fix: Run literal rewrite rules in simplifier phase 2. - The behavior of the simplifier changed in GHC 7.10.2, - causing these rules to fail to fire, leading to poor code generation - and long compilation times. See - [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). - -1.2.1.1 - -* Expose unpackCString#, which you should never use. - -1.2.1.0 - -* Added Binary instances for both Text types. (If you have previously - been using the text-binary package to get a Binary instance, it is - now obsolete.) - -1.2.0.6 - -* Fixed a space leak in UTF-8 decoding - -1.2.0.5 - -* Feature parity: repeat, cycle, iterate are now implemented for lazy - Text, and the Data instance is more complete - -* Build speed: an inliner space explosion has been fixed with toCaseFold - -* Bug fix: encoding Int to a Builder would infinite-loop if the - integer-simple package was used - -* Deprecation: OnEncodeError and EncodeError are deprecated, as they - are never used - -* Internals: some types that are used internally in fusion-related - functions have moved around, been renamed, or been deleted (we don't - bump the major version if .Internal modules change) - -* Spec compliance: toCaseFold now follows the Unicode 7.0 spec - (updated from 6.3) - -1.2.0.4 - -* Fixed an incompatibility with base < 4.5 - -1.2.0.3 - -* Update formatRealFloat to correspond to the definition in versions - of base newer than 4.5 (https://github.com/bos/text/issues/105) - -1.2.0.2 - -* Bumped lower bound on deepseq to 1.4 for compatibility with the - upcoming GHC 7.10 - -1.2.0.1 - -* Fixed a buffer overflow in rendering of large Integers - (https://github.com/bos/text/issues/99) - -1.2.0.0 - -* Fixed an integer overflow in the replace function - (https://github.com/bos/text/issues/81) - -* Fixed a hang in lazy decodeUtf8With - (https://github.com/bos/text/issues/87) - -* Reduced codegen bloat caused by use of empty and single-character - literals - -* Added an instance of IsList for GHC 7.8 and above - -1.1.1.0 - -* The Data.Data instance now allows gunfold to work, via a virtual - pack constructor - -* dropEnd, takeEnd: new functions - -* Comparing the length of a Text against a number can now - short-circuit in more cases - -1.1.0.1 - -* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes - in single-byte chunks - -1.1.0.0 - -* encodeUtf8: Performance is improved by up to 4x. - -* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, - available only if bytestring >= 0.10.4.0 is installed, that allow - very fast and flexible encoding of a Text value to a bytestring - Builder. - - As an example of the performance gain to be had, the - encodeUtf8BuilderEscaped function helps to double the speed of JSON - encoding in the latest version of aeson! (Note: if all you need is a - plain ByteString, encodeUtf8 is still the faster way to go.) - -* All of the internal module hierarchy is now publicly exposed. If a - module is in the .Internal hierarchy, or is documented as internal, - use at your own risk - there are no API stability guarantees for - internal modules! - -1.0.0.1 - -* decodeUtf8: Fixed a regression that caused us to incorrectly - identify truncated UTF-8 as valid (gh-61) - -1.0.0.0 - -* Added support for Unicode 6.3.0 to case conversion functions - -* New function toTitle converts words in a string to title case - -* New functions peekCStringLen and withCStringLen simplify - interoperability with C functionns - -* Added support for decoding UTF-8 in stream-friendly fashion - -* Fixed a bug in mapAccumL - -* Added trusted Haskell support - -* Removed support for GHC 6.10 (released in 2008) and older +### 2.1.4 - 2026-01-27 + +* [Upgrade to Unicode 17.0](https://github.com/haskell/text/pull/658) + [Fix `CaseMapping` generation script to not depend on GHC's Unicode data](https://github.com/haskell/text/pull/687) + +* [simdutf: update to 8.0.0](https://github.com/haskell/text/pull/685) + +* [Add `decodeUtf8Lenient` for lazy `Text`](https://github.com/haskell/text/pull/690) + +* [`scanl`/`scanr` should replace invalid `Char` in the initial value](https://github.com/haskell/text/pull/669) + +* [Shave off redundant field of `Text.Internal.Buffer`](https://github.com/haskell/text/pull/659) + +* [Switch from template-haskell to template-haskell-lift](https://github.com/haskell/text/pull/661) + +#### Minor changes + +* [Avoid calling `length` on chunks in lazy `splitAt`](https://github.com/haskell/text/pull/676) + +* [Check for zero length in internal `isSingleton`](https://github.com/haskell/text/pull/675) + +* [Implement folds directly, without resorting to streaming framework](https://github.com/haskell/text/pull/667) + +* [Implement `cons`, `snoc`, `head`, `isSingleton`, `isPrefixOf` directly, without resorting to streaming framework](https://github.com/haskell/text/pull/666) + +* [Mark `caseConvert` (the underlying implementation of `toUpper` / `toLower` / `toTitle`) as `INLINABLE`, not `INLINE`](https://github.com/haskell/text/pull/664) + +* [Express `index` via `measureOff` instead of going through fusion framework](https://github.com/haskell/text/pull/663) + +* [Guard `#define __STDC_NO_ATOMICS__` by `#ifndef`](https://github.com/haskell/text/pull/657) + +* [Support QuickCheck-2.17](https://github.com/haskell/text/pull/662) + +* [Bump lower bound of binary to >= 0.8.3](https://github.com/haskell/text/pull/673) + +#### Documentation + +* [A bit more documentation for `Data.Text.Internal.Encoding.Utf8`](https://github.com/haskell/text/pull/691) + +* [Clarify documentation of `Data.Text.Foreign`](https://github.com/haskell/text/pull/681) + +* [Haddocks: Hyperlink some identifiers and modules](https://github.com/haskell/text/pull/677) + +* [`since` pragmas for type synonyms](https://github.com/haskell/text/pull/671) + +* [Improve documentation for `streamDecodeUtf8With`](https://github.com/haskell/text/pull/665) + +* [Add comprehensive documentation for `hGetChunk`](https://github.com/haskell/text/pull/655) + +### 2.1.3 - 2025-08-01 + +* [Fix CRLF handling in IO functions](https://github.com/haskell/text/pull/649) + +* [Change `utf8LengthByLeader` to a branching implementation](https://github.com/haskell/text/pull/635) + +* [Define `stimes 0` for lazy text](https://github.com/haskell/text/pull/641) + +* [Add implementation of `sconcat` and `stimes` for strict `Text`](https://github.com/haskell/text/pull/580) and [Fix `stimes` for strict text when size wraps around `Int`](https://github.com/haskell/text/pull/639) + +* [Allow list fusion for `unpack` over both strict and lazy `Text`](https://github.com/haskell/text/pull/629) + +### 2.1.2 + +* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618) + +* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581) + +* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558) + +* [Add `foldlM'`](https://github.com/haskell/text/pull/543) + +* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599) + +* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608) + +* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619) + +* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565) + +* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579) + +* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600) + +* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589) + +* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617) + +* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575) + +* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572) + +* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611) + +* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code. + +### 2.1.1 + +* Add pure Haskell implementations as an alternative to C-based ones, + suitable for JavaScript backend. + +* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547) + +* [Share empty `Text` values](https://github.com/haskell/text/pull/493) + +* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553) + +* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551) + +* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560) + +### 2.1 + +* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474) + +* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503) + +* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483) + +* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527) + +* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528) + +* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534) + +### 2.0.2 + +* [Add decoding functions in `Data.Text.Encoding` that allow + more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge! + * `decodeASCIIPrefix` + * `decodeUtf8Chunk` + * `decodeUtf8More` + * `Utf8ValidState` + * `startUtf8ValidState` + * `StrictBuilder` + * `strictBuilderToText` + * `textToStrictBuilder` + * `validateUtf8Chunk` + * `validateUtf8More` + +* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495) + +* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497) + +* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499) + +* Add internal module `Data.Text.Internal.StrictBuilder` + +* Add internal module `Data.Text.Internal.Encoding` + +* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module. + +* [Speed up case conversions](https://github.com/haskell/text/pull/460) + +* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468) + +* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485) + +### 2.0.1 + +* Improve portability of C and C++ code. +* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413) +* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402) +* [Add `fromPtr0`](https://github.com/haskell/text/pull/423) +* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436) +* [Add `withCString`](https://github.com/haskell/text/pull/431) +* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437) + +### 2.0 + +* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): + * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. + * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. + * Rename functions and types in `Data.Text.Foreign` to reflect switch + from `Word16` to `Word8`. + * Rename slicing functions in `Data.Text.Unsafe` to reflect switch + from `Word16` to `Word8`. + * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. + * Change semantics and order of arguments of `Data.Text.Array.copyI`: + pass length, not end offset. + * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. + * Extend interface of `Data.Text.Array` with more utility functions. + * Add `instance Show Data.Text.Unsafe.Iter`. + * Add `Data.Text.measureOff`. + * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. + * Export `Data.Text.Internal.Lazy.equal`. + * Export `Data.Text.Internal.append`. + * Add `Data.Text.Internal.Private.spanAscii_`. + * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. +* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) +* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) +* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) +* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) +* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) +* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) +* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) + +### 1.2.5.0 + +* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) +* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320) +* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274) +* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281) +* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325) +* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302) +* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334) +* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333) + +### 1.2.4.1 + +* Support `template-haskell-2.17.0.0` +* Support `bytestring-0.11` +* Add `take . drop` related RULE + +### 1.2.4.0 + +* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232) + +* Update Haddock documentation to better reflect fusion eligibility; improve fusion + rules for `takeWhileEnd` and `length` (gh-241, ghc-202) + +* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times, + call it only `O(log n)` times on chunks of increasing size. The total + asymptotic complexity remains `O(nm)`. (gh-209) + +* Support `base-4.13.0.0` + +### 1.2.3.1 + +* Make `decodeUtf8With` fail explicitly for unsupported non-BMP + replacement characters instead silent undefined behaviour (gh-213) + +* Fix termination condition for file reads via `Data.Text.IO` + operations (gh-223) + +* A serious correctness issue affecting uses of `take` and `drop` with + negative counts has been fixed (gh-227) + +* A bug in the case-mapping functions resulting in unreasonably large + allocations with large arguments has been fixed (gh-221) + +### 1.2.3.0 + +* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec + (updated from 8.0). + +* Bug fix: the lazy `takeWhileEnd` function violated the + [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51) + (gh-184). + +* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197). + +* New function: `unsnoc` (gh-173). + +* Reduce memory overhead in `encodeUTF8` (gh-194). + +* Improve UTF-8 decoder error-recovery (gh-182). + +* Minor documentation improvements (`@since` annotations, more + examples, clarifications). + +#### 1.2.2.2 + +* The `toTitle` function now correctly handles letters that + immediately follow punctuation. Before, `"there's"` would turn into + `"There'S"`. Now, it becomes `"There's"`. + +* The implementation of unstreaming is faster, resulting in operations + such as `map` and `intersperse` speeding up by up to 30%, with + smaller code generated. + +* The optimised length comparison function is now more likely to be + used after some rewrite rule tweaking. + +* Bug fix: an off-by-one bug in `takeEnd` is fixed. + +* Bug fix: a logic error in `takeWord16` is fixed. + +#### 1.2.2.1 + +* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. + The build flag has been renamed accordingly. Your army of diligent + maintainers apologizes for the churn. + +* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec + (updated from 7.0) + +* An STG lint error has been fixed + +### 1.2.2.0 + +* The `integer-simple` package, upon which this package optionally + depended, has been replaced with `integer-pure`. The build flag has + been renamed accordingly. + +* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a + `get`, the error is propagated via `fail` instead of an uncatchable + crash. + +* New function: `takeWhileEnd` + +* New instances for the `Text` types: + * if `base` >= 4.7: `PrintfArg` + * if `base` >= 4.9: `Semigroup` + +#### 1.2.1.3 + +* Bug fix: As it turns out, moving the literal rewrite rules to simplifier + phase 2 does not prevent competition with the `unpack` rule, which is + also active in this phase. Unfortunately this was hidden due to a silly + test environment mistake. Moving literal rules back to phase 1 finally + fixes GHC Trac #10528 correctly. + +#### 1.2.1.2 + +* Bug fix: Run literal rewrite rules in simplifier phase 2. + The behavior of the simplifier changed in GHC 7.10.2, + causing these rules to fail to fire, leading to poor code generation + and long compilation times. See + [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). + +#### 1.2.1.1 + +* Expose unpackCString#, which you should never use. + +### 1.2.1.0 + +* Added Binary instances for both Text types. (If you have previously + been using the text-binary package to get a Binary instance, it is + now obsolete.) + +#### 1.2.0.6 + +* Fixed a space leak in UTF-8 decoding + +#### 1.2.0.5 + +* Feature parity: repeat, cycle, iterate are now implemented for lazy + Text, and the Data instance is more complete + +* Build speed: an inliner space explosion has been fixed with toCaseFold + +* Bug fix: encoding Int to a Builder would infinite-loop if the + integer-simple package was used + +* Deprecation: OnEncodeError and EncodeError are deprecated, as they + are never used + +* Internals: some types that are used internally in fusion-related + functions have moved around, been renamed, or been deleted (we don't + bump the major version if .Internal modules change) + +* Spec compliance: toCaseFold now follows the Unicode 7.0 spec + (updated from 6.3) + +#### 1.2.0.4 + +* Fixed an incompatibility with base < 4.5 + +#### 1.2.0.3 + +* Update formatRealFloat to correspond to the definition in versions + of base newer than 4.5 (https://github.com/bos/text/issues/105) + +#### 1.2.0.2 + +* Bumped lower bound on deepseq to 1.4 for compatibility with the + upcoming GHC 7.10 + +#### 1.2.0.1 + +* Fixed a buffer overflow in rendering of large Integers + (https://github.com/bos/text/issues/99) + +## 1.2.0.0 + +* Fixed an integer overflow in the replace function + (https://github.com/bos/text/issues/81) + +* Fixed a hang in lazy decodeUtf8With + (https://github.com/bos/text/issues/87) + +* Reduced codegen bloat caused by use of empty and single-character + literals + +* Added an instance of IsList for GHC 7.8 and above + +### 1.1.1.0 + +* The Data.Data instance now allows gunfold to work, via a virtual + pack constructor + +* dropEnd, takeEnd: new functions + +* Comparing the length of a Text against a number can now + short-circuit in more cases + +#### 1.1.0.1 + +* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes + in single-byte chunks + +## 1.1.0.0 + +* encodeUtf8: Performance is improved by up to 4x. + +* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, + available only if bytestring >= 0.10.4.0 is installed, that allow + very fast and flexible encoding of a Text value to a bytestring + Builder. + + As an example of the performance gain to be had, the + encodeUtf8BuilderEscaped function helps to double the speed of JSON + encoding in the latest version of aeson! (Note: if all you need is a + plain ByteString, encodeUtf8 is still the faster way to go.) + +* All of the internal module hierarchy is now publicly exposed. If a + module is in the .Internal hierarchy, or is documented as internal, + use at your own risk - there are no API stability guarantees for + internal modules! + +#### 1.0.0.1 + +* decodeUtf8: Fixed a regression that caused us to incorrectly + identify truncated UTF-8 as valid (gh-61) + +# 1.0.0.0 + +* Added support for Unicode 6.3.0 to case conversion functions + +* New function toTitle converts words in a string to title case + +* New functions peekCStringLen and withCStringLen simplify + interoperability with C functions + +* Added support for decoding UTF-8 in stream-friendly fashion + +* Fixed a bug in mapAccumL + +* Added trusted Haskell support + +* Removed support for GHC 6.10 (released in 2008) and older diff --git a/include/text_cbits.h b/include/text_cbits.h deleted file mode 100644 --- a/include/text_cbits.h +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2013 Bryan O'Sullivan . - */ - -#ifndef _text_cbits_h -#define _text_cbits_h - -#define UTF8_ACCEPT 0 -#define UTF8_REJECT 12 - -#endif diff --git a/scripts/Arsec.hs b/scripts/Arsec.hs --- a/scripts/Arsec.hs +++ b/scripts/Arsec.hs @@ -1,44 +1,48 @@ -module Arsec - ( - Comment - , comment - , semi - , showC - , unichar - , unichars - , module Control.Applicative - , module Control.Monad - , module Data.Char - , module Text.ParserCombinators.Parsec.Char - , module Text.ParserCombinators.Parsec.Combinator - , module Text.ParserCombinators.Parsec.Error - , module Text.ParserCombinators.Parsec.Prim - ) where - -import Control.Monad -import Control.Applicative -import Data.Char -import Numeric -import Text.ParserCombinators.Parsec.Char hiding (lower, upper) -import Text.ParserCombinators.Parsec.Combinator hiding (optional) -import Text.ParserCombinators.Parsec.Error -import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) - -type Comment = String - -unichar :: Parser Char -unichar = chr . fst . head . readHex <$> many1 hexDigit - -unichars :: Parser [Char] -unichars = manyTill (unichar <* spaces) semi - -semi :: Parser () -semi = char ';' *> spaces *> pure () - -comment :: Parser Comment -comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" - -showC :: Char -> String -showC c = "'\\x" ++ d ++ "'" - where h = showHex (ord c) "" - d = replicate (4 - length h) '0' ++ h +module Arsec + ( + Comment + , comment + , semi + , showC + , unichar + , unichars + , module Control.Applicative + , module Control.Monad + , module Text.ParserCombinators.Parsec.Char + , module Text.ParserCombinators.Parsec.Combinator + , module Text.ParserCombinators.Parsec.Error + , module Text.ParserCombinators.Parsec.Prim + ) where + +import Prelude hiding (head, tail) +import Control.Monad +import Control.Applicative +import Data.Char +import Numeric (readHex, showHex) +import Text.ParserCombinators.Parsec.Char hiding (lower, upper) +import Text.ParserCombinators.Parsec.Combinator hiding (optional) +import Text.ParserCombinators.Parsec.Error +import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) + +type Comment = String + +unichar :: Parser Char +unichar = do + digits <- many1 hexDigit + case readHex digits of + [] -> error "unichar: cannot parse hex digits" + (hd, _) : _ -> pure $ chr hd + +unichars :: Parser [Char] +unichars = manyTill (unichar <* spaces) semi + +semi :: Parser () +semi = char ';' *> spaces *> pure () + +comment :: Parser Comment +comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" + +showC :: Char -> String +showC c = "'\\x" ++ d ++ "'" + where h = showHex (ord c) "" + d = replicate (4 - length h) '0' ++ h diff --git a/scripts/CaseFolding.hs b/scripts/CaseFolding.hs --- a/scripts/CaseFolding.hs +++ b/scripts/CaseFolding.hs @@ -1,46 +1,47 @@ --- This script processes the following source file: --- --- http://unicode.org/Public/UNIDATA/CaseFolding.txt - -module CaseFolding - ( - CaseFolding(..) - , Fold(..) - , parseCF - , mapCF - ) where - -import Arsec - -data Fold = Fold { - code :: Char - , status :: Char - , mapping :: [Char] - , name :: String - } deriving (Eq, Ord, Show) - -data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } - deriving (Show) - -entries :: Parser CaseFolding -entries = CF <$> many comment <*> many (entry <* many comment) - where - entry = Fold <$> unichar <* semi - <*> oneOf "CFST" <* semi - <*> unichars - <*> (string "# " *> manyTill anyToken (char '\n')) - -parseCF :: FilePath -> IO (Either ParseError CaseFolding) -parseCF name = parse entries name <$> readFile name - -mapCF :: CaseFolding -> [String] -mapCF (CF _ ms) = typ ++ (map nice . filter p $ ms) ++ [last] - where - typ = ["foldMapping :: forall s. Char -> s -> Step (CC s) Char" - ,"{-# NOINLINE foldMapping #-}"] - last = "foldMapping c s = Yield (toLower c) (CC s '\\0' '\\0')" - nice c = "-- " ++ name c ++ "\n" ++ - "foldMapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")" - where [x,y,z] = (map showC . take 3) (mapping c ++ repeat '\0') - p f = status f `elem` "CF" && - mapping f /= [toLower (code f)] +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/CaseFolding.txt + +module CaseFolding + ( + CaseFolding(..) + , Fold(..) + , parseCF + , mapCF + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data Fold = Fold { + code :: Char + , status :: Char + , mapping :: [Char] + , name :: String + } deriving (Eq, Ord, Show) + +data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } + deriving (Show) + +entries :: Parser CaseFolding +entries = CF <$> many comment <*> many (entry <* many comment) + where + entry = Fold <$> unichar <* semi + <*> oneOf "CFST" <* semi + <*> unichars + <*> (string "# " *> manyTill anyToken (char '\n')) + +parseCF :: FilePath -> IO (Either ParseError CaseFolding) +parseCF name = parse entries name <$> readFile name + +mapCF :: CaseFolding -> [String] +mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last] + where + typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE foldMapping #-}" + ,"foldMapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = mapping c ++ repeat '\0' diff --git a/scripts/CaseMapping.hs b/scripts/CaseMapping.hs --- a/scripts/CaseMapping.hs +++ b/scripts/CaseMapping.hs @@ -1,38 +1,66 @@ -import System.Environment -import System.IO - -import Arsec -import CaseFolding -import SpecialCasing - -main = do - args <- getArgs - let oname = case args of - [] -> "../Data/Text/Internal/Fusion/CaseMapping.hs" - [o] -> o - psc <- parseSC "SpecialCasing.txt" - pcf <- parseCF "CaseFolding.txt" - scs <- case psc of - Left err -> print err >> return undefined - Right ms -> return ms - cfs <- case pcf of - Left err -> print err >> return undefined - Right ms -> return ms - h <- openFile oname WriteMode - let comments = map ("--" ++) $ - take 2 (cfComments cfs) ++ take 2 (scComments scs) - mapM_ (hPutStrLn h) $ - ["{-# LANGUAGE Rank2Types #-}" - ,"-- AUTOMATICALLY GENERATED - DO NOT EDIT" - ,"-- Generated by scripts/CaseMapping.hs"] ++ - comments ++ - ["" - ,"module Data.Text.Internal.Fusion.CaseMapping where" - ,"import Data.Char" - ,"import Data.Text.Internal.Fusion.Types" - ,""] - mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs) - mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs) - mapM_ (hPutStrLn h) (mapSC "title" title toTitle scs) - mapM_ (hPutStrLn h) (mapCF cfs) - hClose h +import Data.Char (isDigit) +import Data.Foldable (toList) +import Data.List (stripPrefix) +import Data.Maybe (fromJust) +import System.Environment +import System.IO + +import Arsec +import CaseFolding +import SpecialCasing +import UnicodeData + +main = do + args <- getArgs + let oname = case args of + [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs" + [o] -> o + psc <- parseSC "SpecialCasing.txt" + pcf <- parseCF "CaseFolding.txt" + ud <- parseUD "UnicodeData.txt" + scs <- case psc of + Left err -> print err >> return undefined + Right ms -> return ms + cfs <- case pcf of + Left err -> print err >> return undefined + Right ms -> return ms + ud <- case ud of + Left err -> print err >> return undefined + Right ms -> return ms + h <- openFile oname WriteMode + let comments = map ("--" ++) $ + take 2 (cfComments cfs) ++ take 2 (scComments scs) + version = parseVersion (cfComments cfs) + mapM_ (hPutStrLn h) $ + ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ,"-- Generated by scripts/CaseMapping.hs"] ++ + comments ++ + ["" + ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" + ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" + ,"module Data.Text.Internal.Fusion.CaseMapping where" + ,"import GHC.Int" + ,"import GHC.Exts" + ,"import Data.Version (Version, makeVersion)" + ,"unicodeVersion :: Version" + ,"unicodeVersion = makeVersion " ++ version + ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" + ,"unI64 (I64# n) = n" + ,""] + let get f = [(k, d) | c <- toList ud, Just d <- [f c], let k = charUD c, k /= d] + mapM_ (hPutStrLn h) (mapSC "upper" upper (get toUpperUD) scs) + mapM_ (hPutStrLn h) (mapSC "lower" lower (get toLowerUD) scs) + mapM_ (hPutStrLn h) (mapSC "title" title (get toTitleUD) scs) + mapM_ (hPutStrLn h) (mapCF cfs) + hClose h + +-- Parse version from CaseFolding comments +-- and render it as a list (an argument of makeVersion) +parseVersion :: [String] -> String +parseVersion comments = fromJust $ do + line' : _ <- pure comments + line'' <- stripPrefix " CaseFolding-" line' + let (v1, line1) = span isDigit line'' + (v2, line2) = span isDigit (drop 1 line1) + (v3, _) = span isDigit (drop 1 line2) + pure $ "[" ++ v1 ++ ", " ++ v2 ++ ", " ++ v3 ++ "]" diff --git a/scripts/SpecialCasing.hs b/scripts/SpecialCasing.hs --- a/scripts/SpecialCasing.hs +++ b/scripts/SpecialCasing.hs @@ -1,56 +1,61 @@ --- This script processes the following source file: --- --- http://unicode.org/Public/UNIDATA/SpecialCasing.txt - -module SpecialCasing - ( - SpecialCasing(..) - , Case(..) - , parseSC - , mapSC - ) where - -import Arsec - -data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } - deriving (Show) - -data Case = Case { - code :: Char - , lower :: [Char] - , title :: [Char] - , upper :: [Char] - , conditions :: String - , name :: String - } deriving (Eq, Ord, Show) - -entries :: Parser SpecialCasing -entries = SC <$> many comment <*> many (entry <* many comment) - where - entry = Case <$> unichar <* semi - <*> unichars - <*> unichars - <*> unichars - <*> manyTill anyToken (string "# ") - <*> manyTill anyToken (char '\n') - -parseSC :: FilePath -> IO (Either ParseError SpecialCasing) -parseSC name = parse entries name <$> readFile name - -mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing - -> [String] -mapSC which access twiddle (SC _ ms) = - typ ++ (map nice . filter p $ ms) ++ [last] - where - typ = [which ++ "Mapping :: forall s. Char -> s -> Step (CC s) Char" - ,"{-# NOINLINE " ++ which ++ "Mapping #-}"] - last = which ++ "Mapping c s = Yield (to" ++ ucFirst which ++ " c) (CC s '\\0' '\\0')" - nice c = "-- " ++ name c ++ "\n" ++ - which ++ "Mapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")" - where [x,y,z] = (map showC . take 3) (access c ++ repeat '\0') - p c = [k] /= a && a /= [twiddle k] && null (conditions c) - where a = access c - k = code c - -ucFirst (c:cs) = toUpper c : cs -ucFirst [] = [] +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/SpecialCasing.txt + +module SpecialCasing + ( + SpecialCasing(..) + , Case(..) + , parseSC + , mapSC + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } + deriving (Show) + +data Case = Case { + code :: Char + , lower :: [Char] + , title :: [Char] + , upper :: [Char] + , conditions :: String + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser SpecialCasing +entries = SC <$> many comment <*> many (entry <* many comment) + where + entry = Case <$> unichar <* semi + <*> unichars + <*> unichars + <*> unichars + <*> manyTill anyToken (string "# ") + <*> manyTill anyToken (char '\n') + +parseSC :: FilePath -> IO (Either ParseError SpecialCasing) +parseSC name = parse entries name <$> readFile name + +mapSC :: String -> (Case -> String) -> [(Char, Char)] -> SpecialCasing + -> [String] +mapSC which access twiddle (SC _ ms) = + typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] + where + ms' = filter p ms + p c = [k] /= a && null (conditions c) + where a = access c + k = code c + unusual = map code ms' + usual = filter (\(c, _) -> c `notElem` unusual) twiddle + + typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE " ++ which ++ "Mapping #-}" + ,which ++ "Mapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = access c ++ repeat '\0' + printUsual (c, c') = " " ++ showC c ++ "# -> unI64 " ++ show (ord c') diff --git a/scripts/UnicodeData.hs b/scripts/UnicodeData.hs new file mode 100644 --- /dev/null +++ b/scripts/UnicodeData.hs @@ -0,0 +1,51 @@ +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/UnicodeData.txt +-- +-- Format description: https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt + +module UnicodeData + ( UnicodeData + , Data(..) + , toTitleUD + , parseUD + ) where + +import Debug.Trace +import Arsec hiding (semi) +import Data.Array +import Data.Functor (void) +import Data.List (sort) +import Data.Maybe (fromMaybe) + +type UnicodeData = Array Int Data + +-- "Simple_Titlecase_Mapping: If this field is null, then the Simple_Titlecase_Mapping +-- is the same as the Simple_Uppercase_Mapping for this character." +-- -- https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt +toTitleUD :: Data -> Maybe Char +toTitleUD d = toTitleUD_ d <|> toUpperUD d + +data Data = Data { + charUD :: {-# UNPACK #-} !Char + , toUpperUD :: {-# UNPACK #-} !(Maybe Char) + , toLowerUD :: {-# UNPACK #-} !(Maybe Char) + , toTitleUD_ :: {-# UNPACK #-} !(Maybe Char) + } deriving (Eq, Ord, Show) + +-- I'm pretty sure UnicodeData.txt is sorted but still sort it to be 100% certain. +entries :: Parser UnicodeData +entries = (\xs -> listArray (0, length xs - 1) xs) <$> many entry <* eof + where + entry = Data <$> unichar <* semi + <* replicateM_ 11 (ignoreField <* semi) + <*> optional unichar <* semi + <*> optional unichar <* semi + <*> optional unichar <* char '\n' + semi = char ';' + +ignoreField :: Parser () +ignoreField = void (many (satisfy (\c -> c /= ';'))) + +parseUD :: FilePath -> IO (Either ParseError UnicodeData) +parseUD name = parse entries name <$> readFile name diff --git a/simdutf/LICENSE-APACHE b/simdutf/LICENSE-APACHE new file mode 100644 --- /dev/null +++ b/simdutf/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 The simdutf authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/simdutf/LICENSE-MIT b/simdutf/LICENSE-MIT new file mode 100644 --- /dev/null +++ b/simdutf/LICENSE-MIT @@ -0,0 +1,18 @@ +Copyright 2021 The simdutf authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/simdutf/hs_simdutf.c b/simdutf/hs_simdutf.c new file mode 100644 --- /dev/null +++ b/simdutf/hs_simdutf.c @@ -0,0 +1,9 @@ +#include "simdutf_c.h" + +int _hs_text_is_valid_utf8(const char *buf, size_t len) { + return simdutf_validate_utf8(buf, len); +} + +int _hs_text_is_valid_utf8_offset(const char *buf, size_t off, size_t len) { + return simdutf_validate_utf8(buf + off, len); +} diff --git a/simdutf/simdutf.cpp b/simdutf/simdutf.cpp new file mode 100644 # file too large to diff: simdutf/simdutf.cpp diff --git a/simdutf/simdutf.h b/simdutf/simdutf.h new file mode 100644 --- /dev/null +++ b/simdutf/simdutf.h @@ -0,0 +1,13573 @@ +/* auto-generated on 2026-01-13 09:03:21 +0100. Do not edit! */ +/* begin file include/simdutf.h */ +#ifndef SIMDUTF_H +#define SIMDUTF_H +#include + +/* begin file include/simdutf/compiler_check.h */ +#ifndef SIMDUTF_COMPILER_CHECK_H +#define SIMDUTF_COMPILER_CHECK_H + +#ifndef __cplusplus + #error simdutf requires a C++ compiler +#endif + +#ifndef SIMDUTF_CPLUSPLUS + #if defined(_MSVC_LANG) && !defined(__clang__) + #define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) + #else + #define SIMDUTF_CPLUSPLUS __cplusplus + #endif +#endif + +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + +// C++ 23 +#if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) + #define SIMDUTF_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDUTF_CPLUSPLUS20) && (SIMDUTF_CPLUSPLUS >= 202002L) + #define SIMDUTF_CPLUSPLUS20 1 +#endif + +// C++ 17 +#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) + #define SIMDUTF_CPLUSPLUS17 1 +#endif + +// C++ 14 +#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) + #define SIMDUTF_CPLUSPLUS14 1 +#endif + +// C++ 11 +#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) + #define SIMDUTF_CPLUSPLUS11 1 +#endif + +#ifndef SIMDUTF_CPLUSPLUS11 + #error simdutf requires a compiler compliant with the C++11 standard +#endif + +#endif // SIMDUTF_COMPILER_CHECK_H +/* end file include/simdutf/compiler_check.h */ +/* begin file include/simdutf/common_defs.h */ +#ifndef SIMDUTF_COMMON_DEFS_H +#define SIMDUTF_COMMON_DEFS_H + +/* begin file include/simdutf/portability.h */ +#ifndef SIMDUTF_PORTABILITY_H +#define SIMDUTF_PORTABILITY_H + + +#include +#include +#include +#include +#ifndef _WIN32 + // strcasecmp, strncasecmp + #include +#endif + +#if defined(__apple_build_version__) + #if __apple_build_version__ < 14000000 + #define SIMDUTF_SPAN_DISABLED \ + 1 // apple-clang/13 doesn't support std::convertible_to + #endif +#endif + +#if SIMDUTF_CPLUSPLUS20 + #include + #if __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L && \ + !defined(SIMDUTF_SPAN_DISABLED) + #define SIMDUTF_SPAN 1 + #endif // __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L + #if __cpp_lib_atomic_ref >= 201806L + #define SIMDUTF_ATOMIC_REF 1 + #endif // __cpp_lib_atomic_ref + #if __has_cpp_attribute(maybe_unused) >= 201603L + #define SIMDUTF_MAYBE_UNUSED_AVAILABLE 1 + #endif // __has_cpp_attribute(maybe_unused) >= 201603L +#endif + +/** + * We want to check that it is actually a little endian system at + * compile-time. + */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) + #define SIMDUTF_IS_BIG_ENDIAN 0 +#else + #if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ + #include + #elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) + #include + #else // defined(__APPLE__) || defined(__FreeBSD__) + + #ifdef __has_include + #if __has_include() + #include + #endif //__has_include() + #endif //__has_include + + #endif // defined(__APPLE__) || defined(__FreeBSD__) + + #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN 0 + #endif + + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 0 + #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 1 + #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ + +/** + * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined. + */ + +#ifdef _MSC_VER + #define SIMDUTF_VISUAL_STUDIO 1 + /** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + * + * Under clang for Windows, we enable: + * * target pragmas so that part and only part of the + * code gets compiled for advanced instructions. + * + */ + #ifdef __clang__ + // clang under visual studio + #define SIMDUTF_CLANG_VISUAL_STUDIO 1 + #else + // just regular visual studio (best guess) + #define SIMDUTF_REGULAR_VISUAL_STUDIO 1 + #endif // __clang__ +#endif // _MSC_VER + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + // https://en.wikipedia.org/wiki/C_alternative_tokens + // This header should have no effect, except maybe + // under Visual Studio. + #include +#endif + +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) + #define SIMDUTF_IS_X86_64 1 +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + #define SIMDUTF_IS_ARM64 1 +#elif defined(__PPC64__) || defined(_M_PPC64) + #if defined(__VEC__) && defined(__ALTIVEC__) + #define SIMDUTF_IS_PPC64 1 + #endif +#elif defined(__s390__) +// s390 IBM system. Big endian. +#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64 + // RISC-V 64-bit + #define SIMDUTF_IS_RISCV64 1 + + // #if __riscv_v_intrinsic >= 1000000 + // #define SIMDUTF_HAS_RVV_INTRINSICS 1 + // #define SIMDUTF_HAS_RVV_TARGET_REGION 1 + // #elif ... + // Check for special compiler versions that implement pre v1.0 intrinsics + #if __riscv_v_intrinsic >= 11000 + #define SIMDUTF_HAS_RVV_INTRINSICS 1 + #endif + + #define SIMDUTF_HAS_ZVBB_INTRINSICS \ + 0 // there is currently no way to detect this + + #if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector && \ + __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 + // RISC-V V extension + #define SIMDUTF_IS_RVV 1 + #if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000 + // RISC-V Vector Basic Bit-manipulation + #define SIMDUTF_IS_ZVBB 1 + #endif + #endif + +#elif defined(__loongarch_lp64) + #if defined(__loongarch_sx) && defined(__loongarch_asx) + #define SIMDUTF_IS_LSX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both + #elif defined(__loongarch_sx) + #define SIMDUTF_IS_LSX 1 + #endif +#else + // The simdutf library is designed + // for 64-bit processors and it seems that you are not + // compiling for a known 64-bit platform. Please + // use a 64-bit target such as x64 or 64-bit ARM for best performance. + #define SIMDUTF_IS_32BITS 1 + + // We do not support 32-bit platforms, but it can be + // handy to identify them. + #if defined(_M_IX86) || defined(__i386__) + #define SIMDUTF_IS_X86_32BITS 1 + #elif defined(__arm__) || defined(_M_ARM) + #define SIMDUTF_IS_ARM_32BITS 1 + #elif defined(__PPC__) || defined(_M_PPC) + #define SIMDUTF_IS_PPC_32BITS 1 + #endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) + +#ifdef SIMDUTF_IS_32BITS + #ifndef SIMDUTF_NO_PORTABILITY_WARNING + // In the future, we may want to warn users of 32-bit systems that + // the simdutf does not support accelerated kernels for such systems. + #endif // SIMDUTF_NO_PORTABILITY_WARNING +#endif // SIMDUTF_IS_32BITS + +// this is almost standard? +#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a +#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The simdutf users should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select +// SIMDUTF_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + #ifdef __clang__ + // clang does not have GCC push pop + // warning: clang attribute push can't be used within a namespace in clang + // up til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be + // *outside* of a namespace. + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma(SIMDUTF_STRINGIFY(clang attribute push( \ + __attribute__((target(T))), apply_to = function))) + #define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") + #elif defined(__GNUC__) + // GCC is easier + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T))) + #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") + #endif // clang then gcc + +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + +// Default target region macros don't do anything. +#ifndef SIMDUTF_TARGET_REGION + #define SIMDUTF_TARGET_REGION(T) + #define SIMDUTF_UNTARGET_REGION +#endif + +// Is threading enabled? +#if defined(_REENTRANT) || defined(_MT) + #ifndef SIMDUTF_THREADS_ENABLED + #define SIMDUTF_THREADS_ENABLED + #endif +#endif + +// workaround for large stack sizes under -O0. +// https://github.com/simdutf/simdutf/issues/691 +#ifdef __APPLE__ + #ifndef __OPTIMIZE__ + // Apple systems have small stack sizes in secondary threads. + // Lack of compiler optimization may generate high stack usage. + // Users may want to disable threads for safety, but only when + // in debug mode which we detect by the fact that the __OPTIMIZE__ + // macro is not defined. + #undef SIMDUTF_THREADS_ENABLED + #endif +#endif + +#ifdef SIMDUTF_VISUAL_STUDIO + // This is one case where we do not distinguish between + // regular visual studio and clang under visual studio. + // clang under Windows has _stricmp (like visual studio) but not strcasecmp + // (as clang normally has) + #define simdutf_strcasecmp _stricmp + #define simdutf_strncasecmp _strnicmp +#else + // The strcasecmp, strncasecmp, and strcasestr functions do not work with + // multibyte strings (e.g. UTF-8). So they are only useful for ASCII in our + // context. + // https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings + #define simdutf_strcasecmp strcasecmp + #define simdutf_strncasecmp strncasecmp +#endif + +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ >= 11 + #define SIMDUTF_GCC11ORMORE 1 + #endif // __GNUC__ >= 11 + #if __GNUC__ == 10 + #define SIMDUTF_GCC10 1 + #endif // __GNUC__ == 10 + #if __GNUC__ < 10 + #define SIMDUTF_GCC9OROLDER 1 + #endif // __GNUC__ == 10 +#endif // defined(__GNUC__) && !defined(__clang__) + +#endif // SIMDUTF_PORTABILITY_H +/* end file include/simdutf/portability.h */ +/* begin file include/simdutf/avx512.h */ +#ifndef SIMDUTF_AVX512_H_ +#define SIMDUTF_AVX512_H_ + +/* + It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS. + + All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`, + where a feature is a code name for extensions. + + Please see the listing below to find which are supported. +*/ + +#ifndef SIMDUTF_HAS_AVX512F + #if defined(__AVX512F__) && __AVX512F__ == 1 + #define SIMDUTF_HAS_AVX512F 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512DQ + #if defined(__AVX512DQ__) && __AVX512DQ__ == 1 + #define SIMDUTF_HAS_AVX512DQ 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512IFMA + #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1 + #define SIMDUTF_HAS_AVX512IFMA 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512CD + #if defined(__AVX512CD__) && __AVX512CD__ == 1 + #define SIMDUTF_HAS_AVX512CD 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BW + #if defined(__AVX512BW__) && __AVX512BW__ == 1 + #define SIMDUTF_HAS_AVX512BW 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VL + #if defined(__AVX512VL__) && __AVX512VL__ == 1 + #define SIMDUTF_HAS_AVX512VL 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI + #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1 + #define SIMDUTF_HAS_AVX512VBMI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI2 + #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1 + #define SIMDUTF_HAS_AVX512VBMI2 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VNNI + #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1 + #define SIMDUTF_HAS_AVX512VNNI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BITALG + #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1 + #define SIMDUTF_HAS_AVX512BITALG 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ + #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1 + #define SIMDUTF_HAS_AVX512VPOPCNTDQ 1 + #endif +#endif + +#endif // SIMDUTF_AVX512_H_ +/* end file include/simdutf/avx512.h */ + +// Sometimes logging is useful, but we want it disabled by default +// and free of any logging code in release builds. +#ifdef SIMDUTF_LOGGING + #include + #define simdutf_log(msg) \ + std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; + #define simdutf_log_assert(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cerr << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; \ + std::abort(); \ + } \ + } while (0) +#else + #define simdutf_log(msg) + #define simdutf_log_assert(cond, msg) +#endif + +#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) + #define SIMDUTF_DEPRECATED __declspec(deprecated) + + #define simdutf_really_inline __forceinline // really inline in release mode + #define simdutf_always_inline __forceinline // always inline, no matter what + #define simdutf_never_inline __declspec(noinline) + + #define simdutf_unused + #define simdutf_warn_unused + + #ifndef simdutf_likely + #define simdutf_likely(x) x + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) x + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning(push)) + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0)) + #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) \ + __pragma(warning(disable : WARNING_NUMBER)) + // Get rid of Intellisense-only warnings (Code Analysis) + // Though __has_include is C++17, it is supported in Visual Studio 2017 or + // better (_MSC_VER>=1910). + #ifdef __has_include + #if __has_include() + #include + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) + #endif + #endif + + #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING + #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning(pop)) + #define SIMDUTF_DISABLE_UNUSED_WARNING +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + #if defined(__OPTIMIZE__) || defined(NDEBUG) + #define simdutf_really_inline inline __attribute__((always_inline)) + #else + #define simdutf_really_inline inline + #endif + #define simdutf_always_inline \ + inline __attribute__((always_inline)) // always inline, no matter what + #define SIMDUTF_DEPRECATED __attribute__((deprecated)) + #define simdutf_never_inline inline __attribute__((noinline)) + + #define simdutf_unused __attribute__((unused)) + #define simdutf_warn_unused __attribute__((warn_unused_result)) + + #ifndef simdutf_likely + #define simdutf_likely(x) __builtin_expect(!!(x), 1) + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) + #endif + // clang-format off + #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") + // gcc doesn't seem to disable all warnings with all and extra, add warnings + // here as necessary + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) + #define SIMDUTF_PRAGMA(P) _Pragma(#P) + #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) \ + SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) + #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) + #else + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + #define SIMDUTF_DISABLE_DEPRECATED_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) + #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") + #define SIMDUTF_DISABLE_UNUSED_WARNING \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-function) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-const-variable) + // clang-format on + +#endif // MSC_VER + +// Conditional constexpr macro: expands to constexpr for C++17+, empty otherwise +#if SIMDUTF_CPLUSPLUS17 + #define simdutf_constexpr constexpr +#else + #define simdutf_constexpr +#endif + +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + +#ifndef SIMDUTF_DLLIMPORTEXPORT + #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio + /** + * Windows users need to do some extra work when building + * or using a dynamic library (DLL). When building, we need + * to set SIMDUTF_DLLIMPORTEXPORT to __declspec(dllexport). + * When *using* the DLL, the user needs to set + * SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport). + * + * Static libraries not need require such work. + * + * It does not matter here whether you are using + * the regular visual studio or clang under visual + * studio, you still need to handle these issues. + * + * Non-Windows systems do not have this complexity. + */ + #if SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY + + // We set SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL + // under Windows. It should never happen that both + // SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY and + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY are set. + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) + #elif SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY + // Windows user who call a dynamic library should set + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY to 1. + + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) + #else + // We assume by default static linkage + #define SIMDUTF_DLLIMPORTEXPORT + #endif + #else // defined(SIMDUTF_VISUAL_STUDIO) + // Non-Windows systems do not have this complexity. + #define SIMDUTF_DLLIMPORTEXPORT + #endif // defined(SIMDUTF_VISUAL_STUDIO) +#endif + +#if SIMDUTF_MAYBE_UNUSED_AVAILABLE + #define simdutf_maybe_unused [[maybe_unused]] +#else + #define simdutf_maybe_unused +#endif + +#endif // SIMDUTF_COMMON_DEFS_H +/* end file include/simdutf/common_defs.h */ +/* begin file include/simdutf/encoding_types.h */ +#ifndef SIMDUTF_ENCODING_TYPES_H +#define SIMDUTF_ENCODING_TYPES_H +#include + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include +#endif + +namespace simdutf { + +enum encoding_type { + UTF8 = 1, // BOM 0xef 0xbb 0xbf + UTF16_LE = 2, // BOM 0xff 0xfe + UTF16_BE = 4, // BOM 0xfe 0xff + UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00 + UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff + Latin1 = 32, + + unspecified = 0 +}; + +#ifndef SIMDUTF_IS_BIG_ENDIAN + #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." +#endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = +#if SIMDUTF_IS_BIG_ENDIAN + BIG +#else + LITTLE +#endif +}; + +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; +} + +simdutf_warn_unused std::string to_string(encoding_type bom); + +// Note that BOM for UTF8 is discouraged. +namespace BOM { + +/** + * Checks for a BOM. If not, returns unspecified + * @param input the string to process + * @param length the length of the string in code units + * @return the corresponding encoding + */ + +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); +/** + * Returns the size, in bytes, of the BOM for a given encoding type. + * Note that UTF8 BOM are discouraged. + * @param bom the encoding type + * @return the size in bytes of the corresponding BOM + */ +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); + +} // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + +} // namespace simdutf +#endif +/* end file include/simdutf/encoding_types.h */ +/* begin file include/simdutf/error.h */ +#ifndef SIMDUTF_ERROR_H +#define SIMDUTF_ERROR_H +namespace simdutf { + +enum error_code { + SUCCESS = 0, + HEADER_BITS, // Any byte must have fewer than 5 header bits. + TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, + // where N is the UTF-8 character length This is also the error + // when the input is truncated. + TOO_LONG, // We either have too many consecutive continuation bytes or the + // string starts with a continuation byte. + OVERLONG, // The decoded character must be above U+7F for two-byte characters, + // U+7FF for three-byte characters, and U+FFFF for four-byte + // characters. + TOO_LARGE, // The decoded character must be less than or equal to + // U+10FFFF,less than or equal than U+7F for ASCII OR less than + // equal than U+FF for Latin1 + SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or + // UTF-32) + // OR + // a high surrogate must be followed by a low surrogate + // and a low surrogate must be preceded by a high surrogate + // (UTF-16) + // OR + // there must be no surrogate at all and one is + // found (Latin1 functions) + // OR + // *specifically* for the function + // utf8_length_from_utf16_with_replacement, a surrogate (whether + // in error or not) has been found (I.e., whether we are in the + // Basic Multilingual Plane or not). + INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid + // base64 string. This may include a misplaced + // padding character ('='). + BASE64_INPUT_REMAINDER, // The base64 input terminates with a single + // character, excluding padding (=). It is also used + // in strict mode when padding is not adequate. + BASE64_EXTRA_BITS, // The base64 input terminates with non-zero + // padding bits. + OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. + OTHER // Not related to validation/transcoding. +}; +#if SIMDUTF_CPLUSPLUS17 +inline std::string_view error_to_string(error_code code) noexcept { + switch (code) { + case SUCCESS: + return "SUCCESS"; + case HEADER_BITS: + return "HEADER_BITS"; + case TOO_SHORT: + return "TOO_SHORT"; + case TOO_LONG: + return "TOO_LONG"; + case OVERLONG: + return "OVERLONG"; + case TOO_LARGE: + return "TOO_LARGE"; + case SURROGATE: + return "SURROGATE"; + case INVALID_BASE64_CHARACTER: + return "INVALID_BASE64_CHARACTER"; + case BASE64_INPUT_REMAINDER: + return "BASE64_INPUT_REMAINDER"; + case BASE64_EXTRA_BITS: + return "BASE64_EXTRA_BITS"; + case OUTPUT_BUFFER_TOO_SMALL: + return "OUTPUT_BUFFER_TOO_SMALL"; + default: + return "OTHER"; + } +} +#endif + +struct result { + error_code error; + size_t count; // In case of error, indicates the position of the error. In + // case of success, indicates the number of code units + // validated/written. + + simdutf_really_inline simdutf_constexpr23 result() noexcept + : error{error_code::SUCCESS}, count{0} {} + + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept + : error{err}, count{pos} {} + + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { + return error == error_code::SUCCESS; + } + + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { + return error != error_code::SUCCESS; + } +}; + +struct full_result { + error_code error; + size_t input_count; + size_t output_count; + bool padding_error = false; // true if the error is due to padding, only + // meaningful when error is not SUCCESS + + simdutf_really_inline simdutf_constexpr23 full_result() noexcept + : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} + + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out} {} + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out}, + padding_error{padding_err} {} + + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { + if (error == error_code::SUCCESS) { + return result{error, output_count}; + } else { + return result{error, input_count}; + } + } +}; + +} // namespace simdutf +#endif +/* end file include/simdutf/error.h */ + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + +// Public API +/* begin file include/simdutf/simdutf_version.h */ +// /include/simdutf/simdutf_version.h automatically generated by release.py, +// do not change by hand +#ifndef SIMDUTF_SIMDUTF_VERSION_H +#define SIMDUTF_SIMDUTF_VERSION_H + +/** The version of simdutf being used (major.minor.revision) */ +#define SIMDUTF_VERSION "8.0.0" + +namespace simdutf { +enum { + /** + * The major version (MAJOR.minor.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MAJOR = 8, + /** + * The minor version (major.MINOR.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MINOR = 0, + /** + * The revision (major.minor.REVISION) of simdutf being used. + */ + SIMDUTF_VERSION_REVISION = 0 +}; +} // namespace simdutf + +#endif // SIMDUTF_SIMDUTF_VERSION_H +/* end file include/simdutf/simdutf_version.h */ +/* begin file include/simdutf/implementation.h */ +#ifndef SIMDUTF_IMPLEMENTATION_H +#define SIMDUTF_IMPLEMENTATION_H +#if !defined(SIMDUTF_NO_THREADS) + #include +#endif +#include +#ifdef SIMDUTF_INTERNAL_TESTS + #include +#endif +/* begin file include/simdutf/internal/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDutf_INTERNAL_ISADETECTION_H +#define SIMDutf_INTERNAL_ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) + #include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + #include +#endif + + +// RISC-V ISA detection utilities +#if SIMDUTF_IS_RISCV64 && defined(__linux__) + #include // for syscall +// We define these ourselves, for backwards compatibility +struct simdutf_riscv_hwprobe { + int64_t key; + uint64_t value; +}; + #define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__) + #define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4 + #define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2) + #define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17) +#endif // SIMDUTF_IS_RISCV64 && defined(__linux__) + +#if defined(__loongarch__) && defined(__linux__) + #include +// bits/hwcap.h +// #define HWCAP_LOONGARCH_LSX (1 << 4) +// #define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +namespace simdutf { +namespace internal { + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40, + ALTIVEC = 0x80, + AVX512F = 0x100, + AVX512DQ = 0x200, + AVX512IFMA = 0x400, + AVX512PF = 0x800, + AVX512ER = 0x1000, + AVX512CD = 0x2000, + AVX512BW = 0x4000, + AVX512VL = 0x8000, + AVX512VBMI2 = 0x10000, + AVX512VPOPCNTDQ = 0x2000, + RVV = 0x4000, + ZVBB = 0x8000, + LSX = 0x40000, + LASX = 0x80000, +}; + +#if defined(__PPC64__) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::ALTIVEC; +} + +#elif SIMDUTF_IS_RISCV64 + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if SIMDUTF_IS_RVV + host_isa |= instruction_set::RVV; + #endif + #if SIMDUTF_IS_ZVBB + host_isa |= instruction_set::ZVBB; + #endif + #if defined(__linux__) + simdutf_riscv_hwprobe probes[] = {{SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0}}; + long ret = simdutf_riscv_hwprobe(&probes, sizeof probes / sizeof *probes, 0, + nullptr, 0); + if (ret == 0) { + uint64_t extensions = probes[0].value; + if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V) + host_isa |= instruction_set::RVV; + if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB) + host_isa |= instruction_set::ZVBB; + } + #endif + #if defined(RUN_IN_SPIKE_SIMULATOR) + // Proxy Kernel does not implement yet hwprobe syscall + host_isa |= instruction_set::RVV; + #endif + return host_isa; +} + +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#elif defined(__x86_64__) || defined(_M_AMD64) // x64 + +namespace { +namespace cpuid_bit { +// Can be found on Intel ISA Reference for CPUID + +// EAX = 0x01 +constexpr uint32_t pclmulqdq = uint32_t(1) + << 1; ///< @private bit 1 of ECX for EAX=0x1 +constexpr uint32_t sse42 = uint32_t(1) + << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t osxsave = + (uint32_t(1) << 26) | + (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + +// EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf) +// See: "Table 3-8. Information Returned by CPUID Instruction" +namespace ebx { +constexpr uint32_t bmi1 = uint32_t(1) << 3; +constexpr uint32_t avx2 = uint32_t(1) << 5; +constexpr uint32_t bmi2 = uint32_t(1) << 8; +constexpr uint32_t avx512f = uint32_t(1) << 16; +constexpr uint32_t avx512dq = uint32_t(1) << 17; +constexpr uint32_t avx512ifma = uint32_t(1) << 21; +constexpr uint32_t avx512cd = uint32_t(1) << 28; +constexpr uint32_t avx512bw = uint32_t(1) << 30; +constexpr uint32_t avx512vl = uint32_t(1) << 31; +} // namespace ebx + +namespace ecx { +constexpr uint32_t avx512vbmi = uint32_t(1) << 1; +constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6; +constexpr uint32_t avx512vnni = uint32_t(1) << 11; +constexpr uint32_t avx512bitalg = uint32_t(1) << 12; +constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14; +} // namespace ecx +namespace edx { +constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8; +} +namespace xcr0_bit { +constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX +constexpr uint64_t avx512_saved = + uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM +} // namespace xcr0_bit +} // namespace cpuid_bit +} // namespace + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { + #if defined(_MSC_VER) + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; + #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); + #else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; + #endif +} + +static inline uint64_t xgetbv() { + #if defined(_MSC_VER) + return _xgetbv(0); + #else + uint32_t xcr0_lo, xcr0_hi; + asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); + #endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax; + uint32_t ebx = 0; + uint32_t ecx = 0; + uint32_t edx = 0; + uint32_t host_isa = 0x0; + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_bit::sse42) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_bit::pclmulqdq) { + host_isa |= instruction_set::PCLMULQDQ; + } + + if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) { + return host_isa; + } + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; // Sub-leaf = 0 + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_bit::ebx::avx2) { + host_isa |= instruction_set::AVX2; + } + if (ebx & cpuid_bit::ebx::bmi1) { + host_isa |= instruction_set::BMI1; + } + if (ebx & cpuid_bit::ebx::bmi2) { + host_isa |= instruction_set::BMI2; + } + if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == + cpuid_bit::xcr0_bit::avx512_saved)) { + return host_isa; + } + if (ebx & cpuid_bit::ebx::avx512f) { + host_isa |= instruction_set::AVX512F; + } + if (ebx & cpuid_bit::ebx::avx512bw) { + host_isa |= instruction_set::AVX512BW; + } + if (ebx & cpuid_bit::ebx::avx512cd) { + host_isa |= instruction_set::AVX512CD; + } + if (ebx & cpuid_bit::ebx::avx512dq) { + host_isa |= instruction_set::AVX512DQ; + } + if (ebx & cpuid_bit::ebx::avx512vl) { + host_isa |= instruction_set::AVX512VL; + } + if (ecx & cpuid_bit::ecx::avx512vbmi2) { + host_isa |= instruction_set::AVX512VBMI2; + } + if (ecx & cpuid_bit::ecx::avx512vpopcnt) { + host_isa |= instruction_set::AVX512VPOPCNTDQ; + } + return host_isa; +} +#elif defined(__loongarch__) + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if defined(__linux__) + uint64_t hwcap = 0; + hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_LOONGARCH_LSX) { + host_isa |= instruction_set::LSX; + } + if (hwcap & HWCAP_LOONGARCH_LASX) { + host_isa |= instruction_set::LASX; + } + #endif + return host_isa; +} +#else // fallback + +// includes 32-bit ARM. +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif // end SIMD extension detection code + +} // namespace internal +} // namespace simdutf + +#endif // SIMDutf_INTERNAL_ISADETECTION_H +/* end file include/simdutf/internal/isadetection.h */ + +#if SIMDUTF_SPAN + #include + #include + #include + #include +#endif +#if SIMDUTF_CPLUSPLUS17 + #include +#endif +// The following defines are conditionally enabled/disabled during amalgamation. +// By default all features are enabled, regular code shouldn't check them. Only +// when user code really relies of a selected subset, it's good to verify these +// flags, like: +// +// #if !SIMDUTF_FEATURE_UTF16 +// # error("Please amalgamate simdutf with UTF-16 support") +// #endif +// +#define SIMDUTF_FEATURE_DETECT_ENCODING 1 +#define SIMDUTF_FEATURE_ASCII 1 +#define SIMDUTF_FEATURE_LATIN1 1 +#define SIMDUTF_FEATURE_UTF8 1 +#define SIMDUTF_FEATURE_UTF16 1 +#define SIMDUTF_FEATURE_UTF32 1 +#define SIMDUTF_FEATURE_BASE64 1 + +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include + +namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; + + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template +constexpr constexpr_ptr constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy operator*() const { + return constexpr_write_ptr_proxy{p}; + } + + constexpr constexpr_write_ptr_proxy + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif + +#if SIMDUTF_SPAN +/// helpers placed in namespace detail are not a part of the public API +namespace simdutf { +namespace detail { +/** + * matches a byte, in the many ways C++ allows. note that these + * are all distinct types. + */ +template +concept byte_like = std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v; + +template +concept is_byte_like = byte_like>; + +template +concept is_pointer = std::is_pointer_v; + +/** + * matches anything that behaves like std::span and points to character-like + * data such as: std::byte, char, unsigned char, signed char, std::int8_t, + * std::uint8_t + */ +template +concept input_span_of_byte_like = requires(const T &t) { + { t.size() } noexcept -> std::convertible_to; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; +}; + +template +concept is_mutable = !std::is_const_v>; + +/** + * like span_of_byte_like, but for an output span (intended to be written to) + */ +template +concept output_span_of_byte_like = requires(T &t) { + { t.size() } noexcept -> std::convertible_to; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; + { *t.data() } noexcept -> is_mutable; +}; + +/** + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array + * invalid examples: int*, std::array + */ +template +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t{} } -> simdutf::detail::byte_like; +}; +template +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +template +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; + +template +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; + +/** + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* + */ +template +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN + +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); +} + +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 +} + +namespace utf32 { +template constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; +} +} // namespace utf32 + +namespace utf16 { +template constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; +} +} // namespace utf16 + +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(src) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast(const_cast(src)); + const auto dst_value = + std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(dst) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter++; // ASCII + counter += static_cast( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template +simdutf_constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template +simdutf_constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +simdutf_really_inline constexpr bool low_surrogate(char16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +template +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate(c); + low_surrogate = is_low_surrogate(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if simdutf_constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl(reinterpret_cast(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16 +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors(buf, len, utf8_output, 0); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast(p[i] > 0x7F); // two-byte + counter += static_cast(p[i] > 0x7FF); // three-byte + counter += static_cast(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; + return res; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::indexes_into_byte_like) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + +#if SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const char *input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const uint8_t *input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. can be a anything span-like that has a + * data() and size() that points to character data: std::string, + * std::string_view, std::vector, std::span etc. + * @return the detected encoding type + */ +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding( + const detail::input_span_of_byte_like auto &input) noexcept { + return autodetect_encoding(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN + +/** + * Autodetect the possible encodings of the input in one pass. + * E.g., if the input might be UTF-16LE or UTF-8, this function returns + * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). + * + * Overridden by each implementation. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused int detect_encodings(const char *input, + size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const uint8_t *input, size_t length) noexcept { + return detect_encodings(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { + return detect_encodings(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-8 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf8_with_errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool +validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8(reinterpret_cast(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 +/** + * Validate the UTF-8 string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf8_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +validate_utf8_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8_with_errors( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii(reinterpret_cast(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string and stop on error. It might be faster than + * validate_utf8 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_ascii_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_ascii_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii_with_errors( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string as a UTF-16 sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16le_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Validate the UTF-16 string. + * This function may be best when you expect the input to be almost always + * valid. Otherwise, consider using validate_utf16_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-16LE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ +simdutf_warn_unused bool validate_utf16le(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool +validate_utf16le(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 +/** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ +simdutf_warn_unused bool validate_utf16be(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), input.size()); + } else + #endif + { + return validate_utf16be(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Validate the UTF-16 string and stop on error. + * It might be faster than validate_utf16 when an error is expected to occur + * early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16le_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16le_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16be_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16le(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16le(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16be(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16be(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. If input and output points to different + * memory areas, the procedure copies string, and it's expected that output + * memory is at least as big as the input. It's also possible to set input equal + * output, that makes replacements an in-place operation. + * + * @param input the UTF-16 string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-32 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf32_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ +simdutf_warn_unused bool validate_utf32(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf32(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 +/** + * Validate the UTF-32 string and stop on error. It might be faster than + * validate_utf32 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf32_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input, + size_t length, + char *utf8_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8( + const detail::input_span_of_byte_like auto &latin1_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert( + detail::constexpr_cast_ptr(latin1_input.data()), + latin1_input.size(), + detail::constexpr_cast_writeptr(utf8_output.data())); + } else + #endif + { + return convert_latin1_to_utf8( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-8 string with output limit. + * + * This function is suitable to work with inputs from untrusted sources. + * + * We write as many characters as possible. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t +convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert_safe_constexpr( + input.data(), input.size(), utf8_output.data(), utf8_output.size()); + } else + #endif + { + return convert_latin1_to_utf8_safe( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16le( + const detail::input_span_of_byte_like auto &latin1_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + latin1_input.data(), latin1_input.size(), utf16_output.data()); + } else + #endif + { + return convert_latin1_to_utf16le( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16be( + reinterpret_cast(input.data()), input.size(), + output.data()); + } +} + #endif // SIMDUTF_SPAN +/** + * Compute the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in Latin1 code units (char) required to + * encode the UTF-16 string as Latin1 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf16(size_t length) noexcept { + return length; +} + +/** + * Compute the number of code units that this Latin1 string would require in + * UTF-16 format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 2-byte code units (char16_t) required to + * encode the Latin1 string as UTF-16 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf32( + const detail::input_span_of_byte_like auto &latin1_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf32::convert( + latin1_input.data(), latin1_input.size(), utf32_output.data()); + } else + #endif + { + return convert_latin1_to_utf32( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input, + size_t length, + char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_latin1( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert(input.data(), input.size(), + output.data()); + } else + #endif + { + return convert_utf8_to_latin1(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_utf8_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format even when the UTF-16LE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +utf8_length_from_utf16le_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format even when the UTF-16BE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16be_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert a Latin1 string into a UTF-16 string. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t. + */ +simdutf_warn_unused size_t convert_latin1_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_latin1_with_errors( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_latin1_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_with_errors( + utf8_input.data(), utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf8_to_latin1_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16le_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16be_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf32_with_errors( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf32_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_valid_utf8_to_latin1( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_latin1( + const detail::input_span_of_byte_like auto &valid_utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_valid_utf8_to_latin1( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), latin1_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert valid UTF-8 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16le( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16le( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16be( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16be( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf32( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf32( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_latin1(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_latin1( + const detail::input_span_of_byte_like auto &latin1_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(), + latin1_input.size()); + } else + #endif + { + return utf8_length_from_latin1( + reinterpret_cast(latin1_input.data()), + latin1_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ +simdutf_warn_unused size_t latin1_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return latin1_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Compute the number of 2-byte code units that this UTF-8 string would require + * in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 string + * as UTF-16LE + */ +simdutf_warn_unused size_t utf16_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf16_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of 4-byte code units that this UTF-8 string would require + * in UTF-32 format. + * + * This function is equivalent to count_utf8 + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 string + * as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf32_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string with output limit. + * + * We write as many characters as possible into the output buffer, + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 16-bit code units (char16_t) + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input, + size_t length, + char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_safe( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + const full_result r = + scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data(), + utf8_output.size()); + if (r.error != error_code::SUCCESS && + r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + return 0; + } + return r.output_count; + } else + #endif + { + return convert_utf16_to_utf8_safe( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert UTF-16 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16LE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16le_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16le_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16BE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16be_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16be_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-32 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into + * UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf32_with_errors(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf32_with_errors( + std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf32_with_errors( + std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; compute the number of bytes that this UTF-16 + * string would require in UTF-8 format even when the UTF-16LE content contains + * mismatched surrogates that have to be replaced by the replacement character + * (0xFFFD). + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf8( + std::span utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf8_with_errors( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf8_with_errors( + std::span utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf8( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf8( + std::span valid_utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf8( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-32 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16le(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_latin1( + std::span utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_latin1_with_errors( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_latin1_with_errors( + std::span utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_with_errors( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf32_to_latin1( + std::span valid_utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_valid( + detail::constexpr_cast_ptr(valid_utf32_input.data()), + valid_utf32_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } + #endif + { + return convert_valid_utf32_to_latin1( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +latin1_length_from_utf32(size_t length) noexcept { + return length; +} + +/** + * Compute the number of bytes that this Latin1 string would require in UTF-32 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 4-byte code units (char32_t) required to + * encode the Latin1 string as UTF-32 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +utf32_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16be(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-32 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16_with_errors(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16le_with_errors( + std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16be_with_errors( + std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-32 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16(std::span valid_utf32_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16le(std::span valid_utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16le(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16be(std::span valid_utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16be(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ +void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(std::span utf16_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::change_endianness_utf16( + utf16_input.data(), utf16_input.size(), utf16_output.data()); + } else + #endif + { + return change_endianness_utf16(utf16_input.data(), utf16_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf32(std::span valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ +simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(std::span valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-32 format. + * + * This function is equivalent to count_utf16. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16le( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16be( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). It is acceptable to pass invalid UTF-16 strings but in such + * cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ +simdutf_warn_unused size_t count_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return count_utf8(reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-8 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so + * that the short UTF-8 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-8, but possibly + * truncated. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes + */ +simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return trim_partial_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Given a valid UTF-16BE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16BE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16BE, but possibly + * truncated. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16LE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16LE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16LE, but possibly + * truncated. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16, but possibly + * truncated. We use the native endianness. + * + * @param input the UTF-16 string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || \ + SIMDUTF_FEATURE_DETECT_ENCODING + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif +#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_BASE64 +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include +#include +#include +#include +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template simdutf_constexpr23 bool is_eight_byte(char_type c) { + if simdutf_constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast(x & 0xFF); + *dst++ = static_cast((x >> 8) & 0xFF); + *dst++ = static_cast((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + *dst++ = static_cast(triple & 0xFF); + } +} + +template +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if simdutf_constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if simdutf_constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src, + size_t srclen, + base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +template +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen +} + +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(base64_options options) { + switch (options) { + case base64_default: + return "base64_default"; + case base64_url: + return "base64_url"; + case base64_reverse_padding: + return "base64_reverse_padding"; + case base64_url_with_padding: + return "base64_url_with_padding"; + case base64_default_accept_garbage: + return "base64_default_accept_garbage"; + case base64_url_accept_garbage: + return "base64_url_accept_garbage"; + case base64_default_or_url: + return "base64_default_or_url"; + case base64_default_or_url_accept_garbage: + return "base64_default_or_url_accept_garbage"; + } + return ""; +} + #endif // SIMDUTF_CPLUSPLUS17 + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(last_chunk_handling_options options) { + switch (options) { + case loose: + return "loose"; + case strict: + return "strict"; + case stop_before_partial: + return "stop_before_partial"; + case only_full_chunks: + return "only_full_chunks"; + } + return ""; +} + #endif + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximum number of binary bytes + */ +simdutf_warn_unused size_t +maximal_binary_length_from_base64(const char *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ +simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, usually base64_default or + * base64_url, and base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if successful. + */ +simdutf_warn_unused result base64_to_binary( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} + +/** + * Provide the base64 length in bytes given the length of a binary input, + * taking into account line breaks. + * + * @param length the length of the input in bytes + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( + size_t length, base64_options options = base64_default, + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} + +/** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a binary input to a base64 output with line breaks. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options) + */ +size_t +binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), line_length, options); + } +} + #endif // SIMDUTF_SPAN + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a binary input to a base64 output, using atomic accesses. + * This function comes with a potentially significant performance + * penalty, but it may be useful in some cases where the input + * buffers are shared between threads, to avoid undefined + * behavior in case of data races. + * + * The function is for advanced users. Its main use case is when + * to silence sanitizer warnings. We have no documented use case + * where this function is actually necessary in terms of practical correctness. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @brief atomic_binary_to_base64 + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t +atomic_binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused size_t +atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + return atomic_binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail + * to provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of bytes written if successful. + */ +simdutf_warn_unused result +base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Check if a character is an ignorable base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is an ignorable base64 character, false + * otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} + +/** + * Check if a character is a valid base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * Note that padding characters are not considered valid base64 characters in + * this context, nor are spaces. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} + +/** + * Check if a character is a valid base64 character or the padding character + * ('='). Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are three possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer + * is too small (OUTPUT_BUFFER_TOO_SMALL). + * + * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written + * and the number of units processed, see description of the parameters and + * returned value. + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected + * to discard the output unless the parameter decode_up_to_bad_char is set to + * true. In that case, the function will decode up to the first invalid + * character. Extra padding characters ('=') are considered invalid characters. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 8-bit + * or 16-bit units + * @param length the length of the string in 8-bit or 16-bit units. + * @param output the pointer to a buffer that can hold the conversion + * result. + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of units processed if successful. + */ +simdutf_warn_unused result +base64_to_binary_safe(const char *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +// the span overload has moved to the bottom of the file + +simdutf_warn_unused result +base64_to_binary_safe(const char16_t *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; + // span overload moved to bottom of file + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a base64 input to a binary output with a size limit and using atomic + * operations. + * + * Like `base64_to_binary_safe` but using atomic operations, this function is + * thread-safe for concurrent memory access, allowing the output + * buffers to be shared between threads without undefined behavior in case of + * data races. + * + * This function comes with a potentially significant performance penalty, but + * is useful when thread safety is needed during base64 decoding. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @param input the base64 input to decode + * @param length the length of the input in bytes + * @param output the pointer to buffer that can hold the conversion + * result + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use (default, url, etc.) + * @param last_chunk_options the last chunk handling options (loose, strict, + * stop_before_partial) + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result struct with an error code and count indicating error + * position or success + */ +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char16_t *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept; + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline simdutf_warn_unused std::tuple +atomic_base64_to_binary_safe( + const detail::input_span_of_byte_like auto &binary_input, + detail::output_span_of_byte_like auto &&output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = output.size(); + auto ret = atomic_base64_to_binary_safe( + reinterpret_cast(binary_input.data()), binary_input.size(), + reinterpret_cast(output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_warn_unused std::tuple +atomic_base64_to_binary_safe( + std::span base64_input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + auto ret = atomic_base64_to_binary_safe( + base64_input.data(), base64_input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +#endif // SIMDUTF_FEATURE_BASE64 + +/** + * An implementation of simdutf for a particular CPU architecture. + * + * Also used to maintain the currently active implementation. The active + * implementation is automatically initialized on first use to the most advanced + * implementation supported by the host. + */ +class implementation { +public: + /** + * The name of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string name() const { return std::string(_name); } + + /** + * The description of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string description() const { return std::string(_description); } + + /** + * The instruction sets this implementation is compiled against + * and the current CPU match. This function may poll the current CPU/system + * and should therefore not be called too often if performance is a concern. + * + * + * @return true if the implementation can be safely used on the current system + * (determined at runtime) + */ + bool supported_by_runtime_system() const; + +#if SIMDUTF_FEATURE_DETECT_ENCODING + /** + * This function will try to detect the encoding + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual encoding_type autodetect_encoding(const char *input, + size_t length) const noexcept; + + /** + * This function will try to detect the possible encodings in one pass + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual int detect_encodings(const char *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + + /** + * @private For internal implementation use + * + * The instruction sets this implementation is compiled against. + * + * @return a mask of all required `internal::instruction_set::` values + */ + virtual uint32_t required_instruction_sets() const { + return _required_instruction_sets; + } + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ + simdutf_warn_unused virtual bool validate_utf8(const char *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 + /** + * Validate the UTF-8 string and stop on errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_ascii(const char *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-16LE string.This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ + simdutf_warn_unused virtual bool + validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 + /** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ + simdutf_warn_unused virtual bool + validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16le_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + + /** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16be_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + /** + * Copies the UTF-16LE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; + /** + * Copies the UTF-16BE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-32 string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ + simdutf_warn_unused virtual bool + validate_utf32(const char32_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 + /** + * Validate the UTF-32 string and stop on error. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf32_with_errors(const char32_t *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf8(const char *input, size_t length, + char *utf8_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_latin1_with_errors(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format even when the UTF-16LE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format even when the UTF-16BE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_utf32_with_errors(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of 2-byte code units that this UTF-8 string would + * require in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 + * string as UTF-16LE + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of 4-byte code units that this UTF-8 string would + * require in UTF-32 format. + * + * This function is equivalent to count_utf8. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function does not validate the input. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 + * string as UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-L16LE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF16-BE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32BE string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_latin1_with_errors(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_utf8_with_errors(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ + virtual void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_latin1(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf32(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-16LE/BE string would require in + * Latin1 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf16(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16le(const char16_t *input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16be(const char16_t *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_BASE64 + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char *input, size_t length) const noexcept; + + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) const noexcept; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if + * successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you + * fail to provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process, in ASCII stored as + * 16-bit units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the + * number of bytes written if successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of base64 bytes + */ + simdutf_warn_unused size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) const noexcept; + + /** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ + virtual size_t + binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) const noexcept = 0; + + /** + * Convert a binary input to a base64 output with lines of given length. + * Lines are separated by a single linefeed character. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of each line, values smaller than 4 are + * interpreted as 4 + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options, line_length) + */ + virtual size_t binary_to_base64_with_lines( + const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) const noexcept = 0; + + /** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ + virtual const char *find(const char *start, const char *end, + char character) const noexcept = 0; + virtual const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept = 0; +#endif // SIMDUTF_FEATURE_BASE64 + +#ifdef SIMDUTF_INTERNAL_TESTS + // This method is exported only in developer mode, its purpose + // is to expose some internal test procedures from the given + // implementation and then use them through our standard test + // framework. + // + // Regular users should not use it, the tests of the public + // API are enough. + + struct TestProcedure { + // display name + std::string name; + + // procedure should return whether given test pass or not + void (*procedure)(const implementation &); + }; + + virtual std::vector internal_tests() const; +#endif + +protected: + /** @private Construct an implementation with the given name and description. + * For subclasses. */ + simdutf_really_inline implementation(const char *name, + const char *description, + uint32_t required_instruction_sets) + : _name(name), _description(description), + _required_instruction_sets(required_instruction_sets) {} + +protected: + ~implementation() = default; + +private: + /** + * The name of this implementation. + */ + const char *_name; + + /** + * The description of this implementation. + */ + const char *_description; + + /** + * Instruction sets required for this implementation. + */ + const uint32_t _required_instruction_sets; +}; + +/** @private */ +namespace internal { + +/** + * The list of available implementations compiled into simdutf. + */ +class available_implementation_list { +public: + /** Get the list of available implementations compiled into simdutf */ + simdutf_really_inline available_implementation_list() {} + /** Number of implementations */ + size_t size() const noexcept; + /** STL const begin() iterator */ + const implementation *const *begin() const noexcept; + /** STL const end() iterator */ + const implementation *const *end() const noexcept; + + /** + * Get the implementation with the given name. + * + * Case sensitive. + * + * const implementation *impl = + * simdutf::available_implementations["westmere"]; if (!impl) { exit(1); } if + * (!imp->supported_by_runtime_system()) { exit(1); } + * simdutf::active_implementation = impl; + * + * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" + * @return the implementation, or nullptr if the parse failed. + */ + const implementation *operator[](const std::string &name) const noexcept { + for (const implementation *impl : *this) { + if (impl->name() == name) { + return impl; + } + } + return nullptr; + } + + /** + * Detect the most advanced implementation supported by the current host. + * + * This is used to initialize the implementation on startup. + * + * const implementation *impl = + * simdutf::available_implementation::detect_best_supported(); + * simdutf::active_implementation = impl; + * + * @return the most advanced supported implementation for the current host, or + * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no + * supported implementation. Will never return nullptr. + */ + const implementation *detect_best_supported() const noexcept; +}; + +template class atomic_ptr { +public: + atomic_ptr(T *_ptr) : ptr{_ptr} {} + +#if defined(SIMDUTF_NO_THREADS) + operator const T *() const { return ptr; } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr; } + + operator T *() { return ptr; } + T &operator*() { return *ptr; } + T *operator->() { return ptr; } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#else + operator const T *() const { return ptr.load(); } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr.load(); } + + operator T *() { return ptr.load(); } + T &operator*() { return *ptr; } + T *operator->() { return ptr.load(); } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#endif + +private: +#if defined(SIMDUTF_NO_THREADS) + T *ptr; +#else + std::atomic ptr; +#endif +}; + +class detect_best_supported_implementation_on_first_use; + +} // namespace internal + +/** + * The list of available implementations compiled into simdutf. + */ +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & +get_available_implementations(); + +/** + * The active implementation. + * + * Automatically initialized on first use to the most advanced implementation + * supported by this hardware. + */ +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr & +get_active_implementation(); + +} // namespace simdutf + +#if SIMDUTF_FEATURE_BASE64 + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +namespace simdutf { + +template +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same::value || + std::is_same::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = (std::min)( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple + base64_to_binary_safe( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#endif // SIMDUTF_FEATURE_BASE64 + +#endif // SIMDUTF_IMPLEMENTATION_H +/* end file include/simdutf/implementation.h */ + +// Implementation-internal files (must be included before the implementations +// themselves, to keep amalgamation working--otherwise, the first time a file is +// included, it might be put inside the #ifdef +// SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other +// implementations can't compile unless that implementation is turned on). + +SIMDUTF_POP_DISABLE_WARNINGS + +#endif // SIMDUTF_H +/* end file include/simdutf.h */ diff --git a/simdutf/simdutf_c.h b/simdutf/simdutf_c.h new file mode 100644 --- /dev/null +++ b/simdutf/simdutf_c.h @@ -0,0 +1,339 @@ +/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include() +#else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { +#endif + +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */ diff --git a/src/Data/Text.hs b/src/Data/Text.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text.hs @@ -0,0 +1,2276 @@ +{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text +-- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text. +-- Suitable for performance critical use, both in terms of large data +-- quantities and high speed. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text as T +-- +-- To use an extended and very rich family of functions for working +-- with Unicode text (including normalization, regular expressions, +-- non-standard encodings, text breaking, and locales), see the +-- . +-- + +module Data.Text + ( + -- * Strict vs lazy types + -- $strict + + -- * Acceptable data + -- $replacement + + -- * Definition of character + -- $character_definition + + -- * Fusion + -- $fusion + + -- * Types + Text + , StrictText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr' + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , replicate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , breakOn + , breakOnEnd + , break + , span + , spanM + , spanEndM + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + + -- ** Breaking into lines and words + , lines + --, lines' + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , breakOnAll + , find + , elem + , partition + + -- , findSubstring + + -- * Indexing + -- $index + , index + , findIndex + , count + + -- * Zipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + + -- * Low level operations + , copy + , unpackCString# + , unpackCStringAscii# + + , measureOff + ) where + +import Prelude (Char, Bool(..), Int, Maybe(..), String, + Eq, (==), (/=), Ord(..), Ordering(..), (++), + Monad(..), pure, Read(..), Show, + (&&), (||), (+), (-), (.), ($), ($!), (>>), + not, return, otherwise, quot) +import Control.DeepSeq (NFData(rnf)) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits ((.&.)) +import qualified Data.Char as Char +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Control.Monad (foldM) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import qualified Data.List as L hiding (head, tail) +import qualified Data.List.NonEmpty as NonEmptyList +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import Data.Text.Internal.ArrayUtils (memchr) +import Data.Text.Internal.IsAscii (isAscii) +import Data.Text.Internal.Reverse (reverse) +import Data.Text.Internal.Measure (measure_off) +import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, unstream) +import Data.Text.Internal.Private (span_) +import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, + reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray) +import Data.Text.Internal.Search (indices) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_) +#if defined(__HADDOCK__) +import Data.ByteString (ByteString) +import qualified Data.Text.Lazy as L +#endif +import Data.Word (Word8) +import Foreign.C.Types +import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) +import qualified GHC.Exts as Exts +import GHC.Int (Int8) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Lib as TH +import qualified Language.Haskell.TH.Syntax as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) +import System.Posix.Types (CSsize(..)) + +#if __GLASGOW_HASKELL__ >= 810 +import Data.Text.Foreign (asForeignPtr) +import System.IO.Unsafe (unsafePerformIO) +#endif + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +-- $character_definition +-- +-- This package uses the term /character/ to denote Unicode /code points/. +-- +-- Note that this is not the same thing as a grapheme (e.g. a +-- composition of code points that form one visual symbol). For +-- instance, consider the grapheme \"ä\". This symbol has two +-- Unicode representations: a single code-point representation +-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point), +-- and a two code point representation @U+0061@ (the \"@A@\" code +-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point). + +-- $strict +-- +-- This package provides both strict and lazy 'Text' types. The +-- strict type is provided by the "Data.Text" module, while the lazy +-- type is provided by the "Data.Text.Lazy" module. Internally, the +-- lazy @Text@ type consists of a list of strict chunks. +-- +-- The strict 'Text' type requires that an entire string fit into +-- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of +-- streaming strings that are larger than memory using a small memory +-- footprint. In many cases, the overhead of chunked streaming makes +-- the lazy 'Data.Text.Lazy.Text' type slower than its strict +-- counterpart, but this is not always the case. Sometimes, the time +-- complexity of a function in one module may be different from the +-- other, due to their differing internal structures. +-- +-- Each module provides an almost identical API, with the main +-- difference being that the strict module uses 'Int' values for +-- lengths and counts, while the lazy module uses 'Data.Int.Int64' +-- lengths. + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- . +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- () +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- .) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +instance Eq Text where + Text arrA offA lenA == Text arrB offB lenB + | lenA == lenB = A.equal arrA offA arrB offB lenA + | otherwise = False + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +-- +-- Beware: @stimes@ will crash if the given number does not fit into +-- an @Int@. +instance Semigroup Text where + (<>) = append + + stimes howManyTimes + | howManyTimes < 0 = P.error "Data.Text.stimes: given number is negative!" + | otherwise = + let howManyTimesInt = P.fromIntegral howManyTimes :: Int + in if P.fromIntegral howManyTimesInt == howManyTimes && howManyTimesInt >= 0 + then replicate howManyTimesInt + else P.error "Data.Text.stimes: given number does not fit into an Int!" + + sconcat = concat . NonEmptyList.toList + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where rnf !_ = () + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (lengthWord8 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you +-- feel a mistake has been made, please feel free to submit +-- improvements. +-- +-- The original discussion is archived here: +-- +-- +-- The followup discussion that changed the behavior of 'Data.Set.Set' +-- and 'Data.Map.Map' is archived here: +-- + +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> P.error "gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 914 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> [| empty |] + _ -> + let + bytesQ = TH.liftAddrCompat ptr 0 (P.fromIntegral len) + lenQ = TH.liftIntCompat (P.fromIntegral len) + in [| unpackCStringLen# $bytesQ $lenQ |] +#elif __GLASGOW_HASKELL__ >= 810 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> TH.varE 'empty + _ -> + let + bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len) + lenQ = liftInt (P.fromIntegral len) + liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n))) + in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ +#else + lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +#if __GLASGOW_HASKELL__ >= 810 +unpackCStringLen# :: Exts.Addr# -> Int -> Text +unpackCStringLen# addr# l = Text ba 0 l + where + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Exts.Ptr addr#) l + A.unsafeFreeze marr +{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Text" [packConstr] + +-- | /O(n)/ Compare two 'Text' values lexicographically. +compareText :: Text -> Text -> Ordering +compareText (Text arrA offA lenA) (Text arrB offB lenB) = + A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(n)/ Adds a character to the front of a 'Text'. This function +-- is more costly than its 'List' counterpart because it requires +-- copying a new array. Performs replacement on +-- invalid scalar values. +cons :: Char -> Text -> Text +cons c (Text srcArr srcOff srcLen) = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = chLen + srcLen + marr <- A.new totalLen + _ <- unsafeWrite marr 0 ch + A.copyI srcLen marr chLen srcArr srcOff + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +-- Performs replacement on invalid scalar values. +snoc :: Text -> Char -> Text +snoc (Text srcArr srcOff srcLen) c = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = srcLen + chLen + marr <- A.new totalLen + A.copyI srcLen marr 0 srcArr srcOff + _ <- unsafeWrite marr srcLen ch + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE snoc #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t + | null t = emptyError "head" + | otherwise = let Iter c _ = iter t 0 in c +{-# INLINE head #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons t@(Text arr off len) + | len <= 0 = Nothing + | otherwise = Just $ let !(Iter c d) = iter t 0 + in (c, text arr (off+d) (len-d)) +{-# INLINE [1] uncons #-} + +-- | /O(1)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last t@(Text _ _ len) + | null t = emptyError "last" + | otherwise = let Iter c _ = reverseIter t (len - 1) in c +{-# INLINE [1] last #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail t@(Text arr off len) + | null t = emptyError "tail" + | otherwise = text arr (off+d) (len-d) + where d = iter_ t 0 +{-# INLINE [1] tail #-} + +-- | /O(1)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init t@(Text arr off len) + | null t = emptyError "init" + | otherwise = text arr off (len + reverseIter_ t (len - 1)) +{-# INLINE [1] init #-} + +-- | /O(1)/ Returns all but the last character and the last character of a +-- 'Text', or 'Nothing' if empty. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc t@(Text arr off len) + | null t = Nothing + | otherwise = Just (text arr off (len + d), c) + where + Iter c d = reverseIter t (len - 1) +{-# INLINE [1] unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null (Text _arr _off len) = +#if defined(ASSERTS) + assert (len >= 0) $ +#endif + len <= 0 +{-# INLINE [1] null #-} + +{-# RULES + "TEXT null/empty -> True" null empty = True +#-} + +-- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/), +-- to be used together with '(:<)' or '(:>)'. +-- +-- @since 2.1.2 +pattern Empty :: Text +pattern Empty <- (null -> True) where + Empty = empty + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton (Text arr off len) = + len /= 0 && len == utf8LengthByLeader (A.unsafeIndex arr off) +{-# INLINE isSingleton #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Int +length = P.negate . measureOff P.maxBound +{-# INLINE [1] length #-} +-- length needs to be phased after the compareN/length rules otherwise +-- it may inline before the rules have an opportunity to fire. + +{-# RULES +"TEXT length/filter -> S.length/S.filter" forall p t. + length (filter p t) = S.length (S.filter p (stream t)) +"TEXT length/unstream -> S.length" forall t. + length (unstream t) = S.length t +"TEXT length/pack -> P.length" forall t. + length (pack t) = P.length t +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = mul (max 0 n) (length t) +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (mul 2 (length t) - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) +"TEXT length/empty -> 0" + length empty = 0 + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +{-# RULES +"TEXT compareN/length -> compareLength" [~1] forall t n. + compare (length t) n = compareLength t n + #-} + +{-# RULES +"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. + eqInt (length t) n = compareLength t n == EQ + #-} + +{-# RULES +"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. + neInt (length t) n = compareLength t n /= EQ + #-} + +{-# RULES +"TEXT compareLength/==LT" [~1] forall t n. + ltInt (length t) n = compareLength t n == LT + #-} + +{-# RULES +"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. + leInt (length t) n = compareLength t n /= GT + #-} + +{-# RULES +"TEXT >N/length -> compareLength/==GT" [~1] forall t n. + gtInt (length t) n = compareLength t n == GT + #-} + +{-# RULES +"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. + geInt (length t) n = compareLength t n /= LT + #-} + +-- ----------------------------------------------------------------------------- +-- * Transformations +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- +-- Example: +-- +-- >>> let message = pack "I am not angry. Not at all." +-- >>> T.map (\c -> if c == '.' then '!' else c) message +-- "I am not angry! Not at all!" +-- +-- Performs replacement on invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = \t -> if null t then empty else mapNonEmpty f t +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +-- +-- Example: +-- +-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] +-- "WeNI!seekNI!theNI!HolyNI!Grail" +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. +-- +-- Example: +-- +-- >>> T.intersperse '.' "SHIELD" +-- "S.H.I.E.L.D" +-- +-- Performs replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t@(Text src o l) = if null t then empty else runST $ do + let !cLen = utf8Length c + dstLen = l + length t P.* cLen + + dst <- A.new dstLen + + let writeSep = case cLen of + 1 -> \dstOff -> + A.unsafeWrite dst dstOff (ord8 c) + 2 -> let (c0, c1) = ord2 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + A.unsafeWrite dst (dstOff + 3) c3 + let go !srcOff !dstOff = if srcOff >= o + l then return () else do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff m0 + writeSep (dstOff + 1) + go (srcOff + 1) (dstOff + 1 + cLen) + 2 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + writeSep (dstOff + 2) + go (srcOff + 2) (dstOff + 2 + cLen) + 3 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + writeSep (dstOff + 3) + go (srcOff + 3) (dstOff + 3 + cLen) + _ -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + writeSep (dstOff + 4) + go (srcOff + 4) (dstOff + 4 + cLen) + + go o 0 + arr <- A.unsafeFreeze dst + return (Text arr 0 (dstLen - cLen)) +{-# INLINE [1] intersperse #-} + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- >>> replace "oo" "foo" "oo" +-- "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- >>> replace "ofo" "bar" "ofofo" +-- "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace needle@(Text _ _ neeLen) + (Text repArr repOff repLen) + haystack@(Text hayArr hayOff hayLen) + | neeLen == 0 = emptyError "replace" + | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it + | L.null ixs = haystack + | otherwise = Text (A.run x) 0 len + where + ixs = indices needle haystack + len = hayLen - (neeLen - repLen) `mul` L.length ixs + x :: ST s (A.MArray s) + x = do + marr <- A.new len + let loop (i:is) o d = do + let d0 = d + i - o + d1 = d0 + repLen + A.copyI (i - o) marr d hayArr (hayOff+o) + A.copyI repLen marr d0 repArr repOff + loop is (i + neeLen) d1 + loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) + loop ixs 0 0 + return marr + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- When case converting 'Text' values, do not use combinators like +-- @map toUpper@ to case convert each character of a string +-- individually, as this gives incorrect results according to the +-- rules of some writing systems. The whole-string case conversion +-- functions from this module, such as @toUpper@, obey the correct +-- case conversion rules. As a result, these functions may map one +-- input character to two or three output characters. For examples, +-- see the documentation of each function. +-- +-- /Note/: In some languages, case conversion is a locale- and +-- context-dependent operation. The case conversion functions in this +-- module are /not/ locale sensitive. Programs that require locale +-- sensitivity should use appropriate versions of the +-- . + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (also known +-- as case insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case +-- folded to the sequence \"մ\" (men, U+0574) followed by +-- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, +-- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) +-- instead of itself. +toCaseFold :: Text -> Text +toCaseFold = \t -> + if null t then empty + else toCaseFoldNonEmpty t +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, \"İ\" (Latin capital letter I with dot above, +-- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) +-- followed by \" ̇\" (combining dot above, U+0307). +toLower :: Text -> Text +toLower = \t -> + if null t then empty + else toLowerNonEmpty t +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German \"ß\" (eszett, U+00DF) maps to the +-- two-letter sequence \"SS\". +toUpper :: Text -> Text +toUpper = \t -> + if null t then empty + else toUpperNonEmpty t +{-# INLINE toUpper #-} + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = \t -> + if null t then empty + else toTitleNonEmpty t +{-# INLINE toTitle #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. +-- Performs replacement on invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyLeft 7 'x' "foo" +-- "fooxxxx" +-- +-- >>> justifyLeft 3 'x' "foobar" +-- "foobar" +justifyLeft :: Int -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChar (k-len) c + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyRight 7 'x' "bar" +-- "xxxxbar" +-- +-- >>> justifyRight 3 'x' "foobar" +-- "foobar" +justifyRight :: Int -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChar (k-len) c `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- >>> center 8 'x' "HS" +-- "xxxHSxxx" +center :: Int -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChar l c `append` t `append` replicateChar r c + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +-- +-- Examples: +-- +-- >>> transpose ["green","orange"] +-- ["go","rr","ea","en","ng","e"] +-- +-- >>> transpose ["blue","red"] +-- ["br","le","ud","e"] +transpose :: [Text] -> [Text] +transpose ts = P.map pack (L.transpose (P.map unpack ts)) + +-- ----------------------------------------------------------------------------- +-- * Reducing 'Text's (folds) + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z (Text arr off len) = go (off + len - 1) + where + go !i + | i < off = z + | otherwise = let !(Iter c l) = reverseIterArray arr i in f (go (i + l)) c +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) (f acc c) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = case uncons t of + Nothing -> emptyError "foldl" + Just (c, t') -> foldl f c t' +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = case uncons t of + Nothing -> emptyError "foldl'" + Just (c, t') -> foldl' f c t' +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = pure acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) P.=<< f acc c +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- If the binary operator is strict in its second argument, use 'foldr'' +-- instead. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +-- +-- Searches from left to right with short-circuiting behavior can +-- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem'). +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = case unsnoc t of + Nothing -> emptyError "foldr1" + Just (t', c) -> foldr f c t' +{-# INLINE foldr1 #-} + +-- | /O(n)/ A strict version of 'foldr'. +-- +-- 'foldr'' evaluates as a right-to-left traversal using constant stack space. +-- +-- @since 2.0.1 +foldr' :: (Char -> a -> a) -> a -> Text -> a +foldr' f z (Text arr off len) = go (off + len - 1) z + where + go !i !acc + | i < off = acc + | otherwise = let !(Iter c l) = reverseIterArray arr i in go (i + l) (f c acc) +{-# INLINE foldr' #-} + +-- ----------------------------------------------------------------------------- +-- ** Special folds + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat ts = case ts of + [] -> empty + [t] -> t + _ | len == 0 -> empty + | otherwise -> Text (A.run go) 0 len + where + len = sumP "concat" $ L.map lengthWord8 ts + go :: ST s (A.MArray s) + go = do + arr <- A.new len + let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) + foldM step 0 ts >> return arr + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p = foldr (\c acc -> p c || acc) False +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p = foldr (\c acc -> p c && acc) True +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum = foldl1' max +-- This could be implemented faster: look for the longest +-- and largest UTF-8 sequence, then decode it to Char only once, +-- instead of decoding all characters, but I doubt anyone cares +-- about the performance of 'maximum' much. +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum = foldl1' min +-- This could be implemented faster, see the comment for 'maximum' above. +{-# INLINE minimum #-} + +-- ----------------------------------------------------------------------------- +-- * Building 'Text's +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + marr <- A.new l' + d' <- unsafeWrite marr 0 c0' + outer marr l' o d' c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Char -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !c + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + pure $ Text arr 0 dstOff + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff c + | otherwise = do + let !(Iter c' d) = iterArray src srcOff + c'' = safe $ f c c' + d' <- unsafeWrite dst dstOff c'' + inner (srcOff + d) (dstOff + d') c'' + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t | null t = empty + | otherwise = scanl f (unsafeHead t) (unsafeTail t) +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + !d' = utf8Length c0' + marr <- A.new l' + _ <- unsafeWrite marr (l' - d') c0' + outer marr (l + o - 1) (l' - d' - 1) c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Char -> ST s Text + outer !dst = inner + where + inner !srcOff !dstOff !c + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + pure $ Text arr (dstOff + 1) (dstLen - dstOff - 1) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) c + | otherwise = do + let !(Iter c' d) = reverseIterArray src srcOff + c'' = safe $ f c' c + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' c'' + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) +{-# INLINE scanr1 #-} + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumL f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !z + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (z, Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff z + | otherwise = do + let !(Iter c d) = iterArray src srcOff + (z', c') = f z c + d' <- unsafeWrite dst dstOff (safe c') + inner (srcOff + d) (dstOff + d') z' + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. +-- Performs replacement on invalid scalar values. +mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumR f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + o - 1) (l + 4 - 1) z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) + outer !dst = inner + where + inner !srcOff !dstOff !z + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) z + | otherwise = do + let !(Iter c d) = reverseIterArray src srcOff + (z', c') = f z c + c'' = safe c' + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' z' + +-- ----------------------------------------------------------------------------- +-- ** Generating and unfolding 'Text's + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int -> Text -> Text +replicate n t@(Text a o l) + | n <= 0 || l <= 0 = empty + | n == 1 = t + | isSingleton t = replicateChar n (unsafeHead t) + | otherwise = runST $ do + let totalLen = n `mul` l + marr <- A.new totalLen + A.copyI l marr 0 a o + A.tile marr l + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen +{-# INLINE [1] replicate #-} + +{-# RULES +"TEXT replicate/singleton -> replicateChar" [~1] forall n c. + replicate n (singleton c) = replicateChar n c + #-} + +-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the +-- value of every element. +replicateChar :: Int -> Char -> Text +replicateChar !len !c' + | len <= 0 = empty + | Char.isAscii c = runST $ do + marr <- A.newFilled len (Char.ord c) + arr <- A.unsafeFreeze marr + return $ Text arr 0 len + | otherwise = runST $ do + let cLen = utf8Length c + totalLen = cLen P.* len + marr <- A.new totalLen + _ <- unsafeWrite marr 0 c + A.tile marr cLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen + where + c = safe c' +{-# INLINE replicateChar #-} + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- ----------------------------------------------------------------------------- +-- * Substrings + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int -> Text -> Text +take n t@(Text arr off len) + | n <= 0 = empty + | n >= len || m >= len || m < 0 = t + | otherwise = Text arr off m + where + m = measureOff n t +{-# INLINE [1] take #-} + +-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ +-- returns a non-negative number, measuring their size in 'Word8'. Otherwise, +-- if @t@ is shorter, return a non-positive number, which is a negated total count +-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. +-- +-- This function is used to implement 'take', 'drop', 'splitAt' and 'length' +-- and is useful on its own in streaming and parsing libraries. +-- +-- @since 2.0 +measureOff :: Int -> Text -> Int +measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else + cSsizeToInt $ + measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> takeEnd 3 "foobar" +-- "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int -> Text -> Text +takeEnd n t@(Text arr off len) + | n <= 0 = empty + | n >= len = t + | otherwise = text arr (off+i) (len-i) + where i = iterNEnd n t + +iterNEnd :: Int -> Text -> Int +iterNEnd n t@(Text _arr _off len) = loop (len-1) n + where loop i !m + | m <= 0 = i+1 + | i <= 0 = 0 + | otherwise = loop (i+d) (m-1) + where d = reverseIter_ t i + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int -> Text -> Text +drop n t@(Text arr off len) + | n <= 0 = t + | n >= len || m >= len || m < 0 = empty + | otherwise = Text arr (off+m) (len-m) + where m = measureOff n t +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> dropEnd 3 "foobar" +-- "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int -> Text -> Text +dropEnd n t@(Text arr off len) + | n <= 0 = t + | n >= len = empty + | otherwise = text arr off (iterNEnd n t) + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t@(Text arr off len) = loop 0 + where loop !i | i >= len = t + | p c = loop (i+d) + | otherwise = text arr off i + where Iter c d = iter t i +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- >>> takeWhileEnd (=='o') "foo" +-- "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = t + | p c = loop (i+d) (l+d) + | otherwise = text arr (off+l) (len-l) + where Iter c d = reverseIter t i +{-# INLINE [1] takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t@(Text arr off len) = loop 0 0 + where loop !i !l | l >= len = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr (off+i) (len-l) + where Iter c d = iter t i +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- >>> dropWhileEnd (=='.') "foo..." +-- "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr off l + where Iter c d = reverseIter t i +{-# INLINE [1] dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile Char.isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd Char.isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround Char.isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int -> Text -> (Text, Text) +splitAt n t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len || m < 0 = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m = measureOff n t + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p t = case span_ p t of + (# hd,tl #) -> (hd,tl) +{-# INLINE span #-} + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is +-- over elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p = span (not . p) +{-# INLINE break #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t@(Text arr off len) = go 0 + where + go !i | i < len = case iterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off i, text arr (off+i) (len-i)) + go _ = pure (t, empty) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t@(Text arr off len) = go (len-1) + where + go !i | 0 <= i = case reverseIterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off (i+1), text arr (off+i+1) (len-i-1)) + go _ = pure (empty, t) +{-# INLINE spanEndM #-} + +-- | /O(n)/ Group characters in a string according to a predicate. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy p = loop + where + loop t@(Text arr off len) + | null t = [] + | otherwise = text arr off n : loop (text arr (off+n) (len-n)) + where Iter c d = iter t 0 + n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) + +-- | Returns the /array/ index (in units of 'Word8') at which a +-- character may be found. This is /not/ the same as the logical +-- index returned by e.g. 'findIndex'. +findAIndexOrEnd :: (Char -> Bool) -> Text -> Int +findAIndexOrEnd q t@(Text _arr _off len) = go 0 + where go !i | i >= len || q c = i + | otherwise = go (i+d) + where Iter c d = iter t i + +-- | /O(n)/ Group characters in a string by equality. +group :: Text -> [Text] +group = groupBy (==) + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +inits :: Text -> [Text] +inits = (NonEmptyList.toList $!) . initsNE + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmptyList.NonEmpty Text +initsNE t = empty NonEmptyList.:| case t of + Text arr off len -> + let loop i + | i >= len = [] + | otherwise = let !j = i + iter_ t i in Text arr off j : loop j + in loop 0 + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NonEmptyList.toList $!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmptyList.NonEmpty Text +tailsNE t + | null t = empty NonEmptyList.:| [] + | otherwise = t NonEmptyList.:| tails (unsafeTail t) + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be empty), consuming the delimiter. An empty +-- delimiter is invalid, and will cause an error to be raised. +-- +-- Examples: +-- +-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne" +-- ["a","b","d","e"] +-- +-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa" +-- ["","X","X","X",""] +-- +-- >>> splitOn "x" "x" +-- ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "splitOn" + | isSingleton pat = split (== unsafeHead pat) src + | otherwise = go 0 (indices pat src) + where + go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs + go s _ = [text arr (s+off) (len-s)] +{-# INLINE [1] splitOn #-} + +{-# RULES +"TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- >>> split (=='a') "aabbaca" +-- ["","","bb","c",""] +-- +-- >>> split (=='a') "" +-- [""] +split :: (Char -> Bool) -> Text -> [Text] +split p t + | null t = [empty] + | otherwise = loop t + where loop s | null s' = [l] + | otherwise = l : loop (unsafeTail s') + where (# l, s' #) = span_ (not . p) s +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- >>> chunksOf 3 "foobarbaz" +-- ["foo","bar","baz"] +-- +-- >>> chunksOf 4 "haskell.org" +-- ["hask","ell.","org"] +chunksOf :: Int -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- ---------------------------------------------------------------------------- +-- * Searching + +------------------------------------------------------------------------------- +-- ** Searching with a predicate + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem = any . (==) +-- TODO This can be implemented much faster: there is no need to decode +-- any UTF-8 sequences at all. +{-# INLINE elem #-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element matching the predicate, or 'Nothing' if +-- there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p = foldr (\c acc -> if p c then Just c else acc) Nothing +{-# INLINE find #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = filter_ text p +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- >>> breakOn "::" "a::b::c" +-- ("a","::b::c") +-- +-- >>> breakOn "/" "foobar" +-- ("foobar","") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src@(Text arr off len) + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> (text arr off x, text arr (off+x) (len-x)) +{-# INLINE breakOn #-} + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the +-- string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- >>> breakOnEnd "::" "a::b::c" +-- ("a::b::","c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = (reverse b, reverse a) + where (a,b) = breakOn (reverse pat) (reverse src) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- >>> breakOnAll "::" "" +-- [] +-- +-- >>> breakOnAll "/" "a/b/c/" +-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")] +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src@(Text arr off slen) + | null pat = emptyError "breakOnAll" + | otherwise = L.map step (indices pat src) + where + step x = (chunk 0 x, chunk x (slen-x)) + chunk !n !l = text arr (n+off) l +{-# INLINE breakOnAll #-} + +------------------------------------------------------------------------------- +-- ** Indexing 'Text's + +-- $index +-- +-- If you think of a 'Text' value as an array of 'Char' values (which +-- it is not), you run the risk of writing inefficient code. +-- +-- An idiom that is common in some languages is to find the numeric +-- offset of a character or substring, then use that number to split +-- or trim the searched string. With a 'Text' value, this approach +-- would require two /O(n)/ operations: one to perform the search, and +-- one to operate from wherever the search ended. +-- +-- For example, suppose you have a string that you want to split on +-- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of +-- searching for the index of @\"::\"@ and taking the substrings +-- before and after that index, you would instead use @breakOnAll \"::\"@. + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int -> Char +index t@(Text _ _ lenInBytes) ix + | ix < 0 + = P.error $ "Data.Text.index: negative index " ++ P.show ix + | off < 0 || off == lenInBytes + = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + | otherwise = ch + where + off = measureOff ix t + Iter ch _ = iter t off +{-# INLINE index #-} + +-- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' +-- and returns the index of the first element in the 'Text' satisfying +-- the predicate. +findIndex :: (Char -> Bool) -> Text -> Maybe Int +findIndex p t = S.findIndex p (stream t) +{-# INLINE findIndex #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int +count pat + | null pat = emptyError "count" + | isSingleton pat = countChar (unsafeHead pat) + | otherwise = L.length . indices pat +{-# INLINE [1] count #-} + +{-# RULES +"TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int +countChar c = foldl' (\acc c' -> if c == c' then acc + 1 else acc) 0 +{-# INLINE countChar #-} + +------------------------------------------------------------------------------- +-- * Zipping + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [1] zipWith #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words (Text arr off len) = loop 0 0 + where + loop !start !n + | n >= len = if start == n + then [] + else [Text arr (start + off) (n - start)] + -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 + | isAsciiSpace w0 = + if start == n + then loop (n + 1) (n + 1) + else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) + | w0 < 0x80 = loop start (n + 1) + -- or 2 bytes for 0xA0 + | w0 == 0xC2, w1 == 0xA0 = + if start == n + then loop (n + 2) (n + 2) + else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) + | w0 < 0xE0 = loop start (n + 2) + -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 + | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 + || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) + || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = + if start == n + then loop (n + 3) (n + 3) + else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) + | otherwise = loop start (n + utf8LengthByLeader w0) + where + w0 = A.unsafeIndex arr (off + n) + w1 = A.unsafeIndex arr (off + n + 1) + w2 = A.unsafeIndex arr (off + n + 2) +{-# INLINE words #-} + +-- Adapted from Data.ByteString.Internal.isSpaceWord8 +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) +{-# INLINE isAsciiSpace #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines (Text arr@(A.ByteArray arr#) off len) = go off + where + go !n + | n >= len + off = [] + | delta < 0 = [Text arr n (len + off - n)] + | otherwise = Text arr n delta : go (n + delta + 1) + where + delta = memchr arr# n (len + off - n) 0x0A +{-# INLINE lines #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr bOff aLen +{-# INLINE [1] isPrefixOf #-} + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr (bOff+d) aLen +{-# INLINE isSuffixOf #-} + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripPrefix "foo" "foobar" +-- Just "bar" +-- +-- >>> stripPrefix "" "baz" +-- Just "baz" +-- +-- >>> stripPrefix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p@(Text _arr _off plen) t@(Text arr off len) + | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) + | otherwise = Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- >>> commonPrefixes "foobar" "fooquux" +-- Just ("foo","bar","quux") +-- +-- >>> commonPrefixes "veeble" "fetzer" +-- Nothing +-- +-- >>> commonPrefixes "" "baz" +-- Nothing +commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) +commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) + | len0 == 0 = Nothing + | len1 == 0 = Nothing + | otherwise = go 0 0 + where + go !i !j + | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) + | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) + | a == b = go (i + 1) k + | k > 0 = Just (Text arr0 off0 k, + Text arr0 (off0 + k) (len0 - k), + Text arr1 (off1 + k) (len1 - k)) + | otherwise = Nothing + where + a = A.unsafeIndex arr0 (off0 + i) + b = A.unsafeIndex arr1 (off1 + i) + isLeader = word8ToInt8 a >= -64 + k = if isLeader then i else j +{-# INLINE commonPrefixes #-} + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripSuffix "bar" "foobar" +-- Just "foo" +-- +-- >>> stripSuffix "" "baz" +-- Just "baz" +-- +-- >>> stripSuffix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p@(Text _arr _off plen) t@(Text arr off len) + | p `isSuffixOf` t = Just $! text arr off (len-plen) + | otherwise = Nothing + +-- | Add a list of non-negative numbers. Errors out on overflow. +sumP :: String -> [Int] -> Int +sumP fun = L.foldl' add 0 + where add a x + | ax >= 0 = ax + | otherwise = overflowError fun + where ax = a + x +{-# INLINE sumP #-} -- Use foldl' and inline for fusion. + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" + +overflowError :: HasCallStack => String -> a +overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" + +-- | Convert a value to 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +-- | /O(n)/ Make a distinct copy of the given string, sharing no +-- storage with the original string. +-- +-- As an example, suppose you read a large string, of which you need +-- only a small portion. If you do not use 'copy', the entire original +-- array will be kept alive in memory by the smaller string. Making a +-- copy \"breaks the link\" to the original array, allowing it to be +-- garbage collected if there are no other live references to it. +copy :: Text -> Text +copy t@(Text arr off len) + | null t = empty + | otherwise = Text (A.run go) 0 len + where + go :: ST s (A.MArray s) + go = do + marr <- A.new len + A.copyI len marr 0 arr off + return marr + +ord8 :: Char -> Word8 +ord8 = P.fromIntegral . Char.ord + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = P.fromIntegral + +word8ToInt8 :: Word8 -> Int8 +word8ToInt8 = P.fromIntegral + +------------------------------------------------- +-- NOTE: the named chunk below used by doctest; +-- verify the doctests via `doctest -fobject-code Data/Text.hs` + +-- $setup +-- >>> :set -XOverloadedStrings +-- >>> import qualified Data.Text as T diff --git a/src/Data/Text/Array.hs b/src/Data/Text/Array.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Array.hs @@ -0,0 +1,361 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-unused-matches #-} +-- | +-- Module : Data.Text.Array +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Packed, unboxed, heap-resident arrays. Suitable for performance +-- critical use, both in terms of large data quantities and high +-- speed. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text.Array as A +-- +-- The names in this module resemble those in the 'Data.Array' family +-- of modules, but are shorter due to the assumption of qualified +-- naming. +module Data.Text.Array + ( + -- * Types + Array + , pattern ByteArray + , MArray + , pattern MutableByteArray + -- * Functions + , resizeM + , shrinkM + , copyM + , copyI + , copyFromPointer + , copyToPointer + , empty + , equal + , compare + , run + , run2 + , toList + , unsafeFreeze + , unsafeIndex + , new + , newPinned + , newFilled + , unsafeWrite + , tile + , getSizeofMArray + ) where + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif +#if !MIN_VERSION_base(4,11,0) +import Foreign.C.Types (CInt(..)) +#endif +import GHC.Exts hiding (toList) +import GHC.ST (ST(..), runST) +import GHC.Word (Word8(..)) +import qualified Prelude +import Prelude hiding (length, read, compare) +import Data.Array.Byte (ByteArray(..), MutableByteArray(..)) + +-- | Immutable array type. +type Array = ByteArray + +-- | Mutable array type, for use in the ST monad. +type MArray = MutableByteArray + +-- | Create an uninitialized mutable array. +new :: forall s. Int -> ST s (MArray s) +new (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.new: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE new #-} + +-- | Create an uninitialized mutable pinned array. +-- +-- @since 2.0 +newPinned :: forall s. Int -> ST s (MArray s) +newPinned (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newPinnedByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE newPinned #-} + +-- | @since 2.0 +newFilled :: Int -> Int -> ST s (MArray s) +newFilled (I# len#) (I# c#) = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of + s3# -> (# s3#, MutableByteArray marr# #) +{-# INLINE newFilled #-} + +-- | @since 2.0 +tile :: MArray s -> Int -> ST s () +tile marr tileLen = do + totalLen <- getSizeofMArray marr + let go l + | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) + | otherwise = copyM marr l marr 0 l >> go (2 * l) + go tileLen +{-# INLINE tile #-} + +-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! +unsafeFreeze :: MArray s -> ST s Array +unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> + case unsafeFreezeByteArray# marr s1# of + (# s2#, ba# #) -> (# s2#, ByteArray ba# #) +{-# INLINE unsafeFreeze #-} + +-- | Unchecked read of an immutable array. May return garbage or +-- crash on an out-of-bounds access. +unsafeIndex :: +#if defined(ASSERTS) + HasCallStack => +#endif + Array -> Int -> Word8 +unsafeIndex (ByteArray arr) i@(I# i#) = +#if defined(ASSERTS) + let word8len = I# (sizeofByteArray# arr) in + if i < 0 || i >= word8len + then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) + else +#endif + case indexWord8Array# arr i# of r# -> (W8# r#) +{-# INLINE unsafeIndex #-} + +-- | @since 2.0 +getSizeofMArray :: MArray s -> ST s Int +getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> + -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is + -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. + case getSizeofMutableByteArray# marr s0# of + (# s1#, word8len# #) -> (# s1#, I# word8len# #) + +#if defined(ASSERTS) +checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () +checkBoundsM ma i elSize = do + len <- getSizeofMArray ma + if i < 0 || i + elSize > len + then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) + else return () +#endif + +-- | Unchecked write of a mutable array. May return garbage or crash +-- on an out-of-bounds access. +unsafeWrite :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> Word8 -> ST s () +unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = +#if defined(ASSERTS) + checkBoundsM ma i 1 >> +#endif + (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of + s2# -> (# s2#, () #)) +{-# INLINE unsafeWrite #-} + +-- | Convert an immutable array to a list. +toList :: Array -> Int -> Int -> [Word8] +toList ary off len = loop 0 + where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) + | otherwise = [] + +-- | An empty immutable array. +empty :: Array +empty = runST (new 0 >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result. +run :: (forall s. ST s (MArray s)) -> Array +run k = runST (k >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result paired with whatever else the action returns. +run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) +run2 k = runST (do + (marr,b) <- k + arr <- unsafeFreeze marr + return (arr,b)) +{-# INLINE run2 #-} + +-- | @since 2.0 +resizeM :: MArray s -> Int -> ST s (MArray s) +resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> + case resizeMutableByteArray# ma i# s1# of + (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) +{-# INLINE resizeM #-} + +-- | @since 2.0 +shrinkM :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> ST s () +shrinkM (MutableByteArray marr) i@(I# newSize) = do +#if defined(ASSERTS) + oldSize <- getSizeofMArray (MutableByteArray marr) + if I# newSize > oldSize + then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) + else return () +#endif + ST $ \s1# -> + case shrinkMutableByteArray# marr newSize s1# of + s2# -> (# s2#, () #) +{-# INLINE shrinkM #-} + +-- | Copy some elements of a mutable array. +copyM :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> MArray s -- ^ Source + -> Int -- ^ Source offset + -> Int -- ^ Count + -> ST s () +copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyM: count must be >= 0, but got " ++ show count +#endif + | otherwise = do +#if defined(ASSERTS) + srcLen <- getSizeofMArray src + dstLen <- getSizeofMArray dst + if srcOff + count > srcLen + then error "copyM: source is too short" + else return () + if dstOff + count > dstLen + then error "copyM: destination is too short" + else return () +#endif + ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyM #-} + +-- | Copy some elements of an immutable array. +copyI :: Int -- ^ Count + -> MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Array -- ^ Source + -> Int -- ^ Source offset + -> ST s () +copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyI: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyI #-} + +-- | Copy from pointer. +-- +-- @since 2.0 +copyFromPointer + :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Ptr Word8 -- ^ Source + -> Int -- ^ Count + -> ST s () +copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyFromPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyAddrToByteArray# src# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyFromPointer #-} + +-- | Copy to pointer. +-- +-- @since 2.0 +copyToPointer + :: Array -- ^ Source + -> Int -- ^ Source offset + -> Ptr Word8 -- ^ Destination + -> Int -- ^ Count + -> ST s () +copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyToPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArrayToAddr# src# srcOff# dst# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyToPointer #-} + +-- | Compare portions of two arrays for equality. No bounds checking +-- is performed. +equal + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Bool +equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 +{-# INLINE equal #-} + +-- | Compare portions of two arrays. No bounds checking is performed. +-- +-- @since 2.0 +compare + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Ordering +compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 +{-# INLINE compare #-} + +compareInternal + :: Array -- ^ First + -> Int -- ^ Offset into first + -> Array -- ^ Second + -> Int -- ^ Offset into second + -> Int -- ^ Count + -> Int +compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i + where +#if MIN_VERSION_base(4,11,0) + i = I# (compareByteArrays# src1# off1# src2# off2# count#) +#else + i = fromIntegral (memcmp src1# off1# src2# off2# count#) + +foreign import ccall unsafe "_hs_text_memcmp2" memcmp + :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt +#endif +{-# INLINE compareInternal #-} diff --git a/src/Data/Text/Encoding.hs b/src/Data/Text/Encoding.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Encoding.hs @@ -0,0 +1,588 @@ +{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, + UnliftedFFITypes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +-- | +-- Module : Data.Text.Encoding +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting 'Text' values to and from 'ByteString', +-- using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- . + +module Data.Text.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeASCIIPrefix + , decodeUtf8Lenient + , decodeUtf8' + , decodeASCII' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- *** Incremental UTF-8 decoding + -- $incremental + , decodeUtf8Chunk + , decodeUtf8More + , Utf8State + , startUtf8State + , StrictBuilder + , StrictTextBuilder + , strictBuilderToText + , textToStrictBuilder + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- ** Stream oriented decoding + -- $stream + , streamDecodeUtf8 + , streamDecodeUtf8With + , Decoding(..) + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + + -- * ByteString validation + -- $validation + , validateUtf8Chunk + , validateUtf8More + ) where + +import Control.Exception (evaluate, try) +import Data.Word (Word8) +import GHC.Exts (byteArrayContents#, unsafeCoerce#) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) +import Data.ByteString (ByteString) +#if defined(PURE_HASKELL) +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Char8 (unpack) +import Data.Text.Internal (pack) +import Foreign.Ptr (minusPtr, plusPtr) +import Foreign.Storable (poke) +#else +import Control.Monad.ST (runST) +import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) +import Data.Bits (shiftR, (.&.)) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Foreign.C.Types (CSize(..)) +import Foreign.Ptr (Ptr, minusPtr, plusPtr) +import Foreign.Storable (poke, peekByteOff) +#endif +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Encoding +import Data.Text.Internal.IsAscii (asciiPrefixLength) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Show () +import qualified Data.ByteString as B +import qualified Data.ByteString.Internal as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Builder.Prim.Internal as BP +import qualified Data.ByteString.Short.Internal as SBS +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Encoding.Fusion as E +import qualified Data.Text.Internal.Fusion as F +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +-- $validation +-- These functions are for validating 'ByteString's as encoded text. + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing ASCII text. +-- +-- This is a total function which returns a pair of the longest ASCII prefix +-- as 'Text', and the remaining suffix as 'ByteString'. +-- +-- Important note: the pair is lazy. This lets you check for errors by testing +-- whether the second component is empty, without forcing the first component +-- (which does a copy). +-- To drop references to the input bytestring, force the prefix +-- (using 'seq' or @BangPatterns@) and drop references to the suffix. +-- +-- === Properties +-- +-- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@. +-- - Either @suffix@ is empty, or @'B.head' suffix > 127@. +-- +-- @since 2.0.2 +decodeASCIIPrefix :: ByteString -> (Text, ByteString) +decodeASCIIPrefix bs = if B.null bs + then (empty, B.empty) + else + let len = asciiPrefixLength bs + prefix = + let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in + Text (A.ByteArray arr) 0 len + suffix = B.drop len bs in + (prefix, suffix) +{-# INLINE decodeASCIIPrefix #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a total function which returns either the 'ByteString' converted to a +-- 'Text' containing ASCII text, or 'Nothing'. +-- +-- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid +-- input instead of discarding it. +-- +-- @since 2.0.2 +decodeASCII' :: ByteString -> Maybe Text +decodeASCII' bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + if B.null suffix then Just prefix else Nothing +{-# INLINE decodeASCII' #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a partial function: it checks that input does not contain +-- anything except ASCII and copies buffer or throws an error otherwise. +decodeASCII :: ByteString -> Text +decodeASCII bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + case B.uncons suffix of + Nothing -> prefix + Just (word, _) -> + let !errPos = B.length bs - B.length suffix in + error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +-- +-- 'decodeLatin1' is semantically equivalent to +-- @'Data.Text.pack' . 'Data.ByteString.Char8.unpack'@ +-- +-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) +-- characters to UTf-8 requires actual work and is not just buffer copying. +-- +decodeLatin1 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Text +#if defined(PURE_HASKELL) +decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs) +#else +decodeLatin1 bs = withBS bs $ \fp len -> runST $ do + dst <- A.new (2 * len) + let inner srcOff dstOff = if srcOff >= len then return dstOff else do + asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) + if asciiPrefixLen == 0 + then do + byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff + A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) + A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) + inner (srcOff + 1) (dstOff + 2) + else do + unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen + inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) + actualLen <- inner 0 0 + dst' <- A.resizeM dst actualLen + arr <- A.unsafeFreeze dst' + return $ Text arr 0 actualLen +#endif + +#if !defined(PURE_HASKELL) +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize +#endif + +-- $stream +-- +-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept +-- a strict 'ByteString' that represents a possibly incomplete input (e.g. a +-- packet from a network stream) that may not end on a UTF-8 boundary +-- and return 'Decoding', which consists of: +-- +-- * The maximal prefix of 'Text' that could be decoded from the +-- given input. +-- +-- * The suffix of the 'ByteString' that could not be decoded due to +-- insufficient input. +-- +-- * A function that accepts another 'ByteString'. That string will +-- be assumed to directly follow the string that was passed as +-- input to the original function, and it will in turn be decoded. +-- +-- To help understand the use of these functions, consider the Unicode +-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi +-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. +-- +-- Now suppose that we receive this encoded string as 3 packets that +-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", +-- \"\\x83\"]@. We cannot decode the entire Unicode string until we +-- have received all three packets, but we would like to make progress +-- as we receive each one. +-- +-- @ +-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" +-- ghci> s0 +-- 'Some' \"hi \" \"\\xe2\" _ +-- @ +-- +-- We use the continuation @f0@ to decode our second packet. +-- +-- @ +-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" +-- ghci> s1 +-- 'Some' \"\" \"\\xe2\\x98\" +-- @ +-- +-- We could not give @f0@ enough input to decode anything, so it +-- returned an empty string. Once we feed our second continuation @f1@ +-- the last byte of input, it will make progress. +-- +-- @ +-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" +-- ghci> s2 +-- 'Some' \"\\x2603\" \"\" _ +-- @ +-- +-- If given invalid input, an exception will be thrown by the function +-- or continuation where it is encountered. + +-- | A stream-oriented decoding result (see 'streamDecodeUtf8' and 'streamDecodeUtf8With'). +-- +-- @since 1.0.0.0 +data Decoding = Some + !Text + -- ^ The maximal prefix that could be decoded from the given input. + !ByteString + -- ^ The remaining suffix of the input that could not be decoded + -- (usually because the input breaks in the middle of UTF-8 character) + (ByteString -> Decoding) + -- ^ The continuation call which should be fed with the next + -- chunk of the input. + +instance Show Decoding where + showsPrec d (Some t bs _) = showParen (d > prec) $ + showString "Some " . showsPrec prec' t . + showChar ' ' . showsPrec prec' bs . + showString " _" + where prec = 10; prec' = prec + 1 + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data that is known to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown (either by this function or a continuation) that cannot be +-- caught in pure code. For more control over the handling of invalid +-- data, use 'streamDecodeUtf8With'. +-- +-- @since 1.0.0.0 +streamDecodeUtf8 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Decoding +streamDecodeUtf8 = streamDecodeUtf8With strictDecode + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data. +-- +-- @since 1.0.0.0 +streamDecodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Decoding +streamDecodeUtf8With onErr = loop startUtf8State + where + loop s chunk = + let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk + in Some (strictBuilderToText builder) undecoded (loop s') + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Surrogate code points in replacement character returned by 'OnDecodeError' +-- will be automatically remapped to the replacement char @U+FFFD@. +decodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Text +decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg + +invalidUtf8Msg :: String +invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +-- +-- This is a partial function: it checks that input is a well-formed +-- UTF-8 sequence and copies buffer or throws an error otherwise. +-- +decodeUtf8 :: ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +decodeUtf8' :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Either UnicodeException Text +decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode +{-# INLINE decodeUtf8' #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.0 +decodeUtf8Lenient :: ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (step txt) + where + step txt@(Text arr off len) !k br@(B.BufferRange op ope) + -- Ensure that the common case is not recursive and therefore yields + -- better code. + | op' <= ope = do + unsafeSTToIO $ A.copyToPointer arr off op len + k (B.BufferRange op' ope) + | otherwise = textCopyStep txt k br + where + op' = op `plusPtr` len +{-# INLINE encodeUtf8Builder #-} + +textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a +textCopyStep (Text arr off len) k = + go off (off + len) + where + go !ip !ipe (B.BufferRange op ope) + | inpRemaining <= outRemaining = do + unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining + let !br = B.BufferRange (op `plusPtr` inpRemaining) ope + k br + | otherwise = do + unsafeSTToIO $ A.copyToPointer arr ip op outRemaining + let !ip' = ip + outRemaining + return $ B.bufferFull 1 ope (go ip' ipe) + where + outRemaining = ope `minusPtr` op + inpRemaining = ipe - ip + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +-- TODO: Extend documentation with references to source code in @blaze-html@ +-- or @aeson@ that uses this function. +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped be = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (mkBuildstep txt) + where + bound = max 4 $ BP.sizeBound be + + mkBuildstep (Text arr off len) !k = + outerLoop off + where + iend = off + len + + outerLoop !i0 !br@(B.BufferRange op0 ope) + | i0 >= iend = k br + | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) + -- TODO: Use a loop with an integrated bound's check if outRemaining + -- is smaller than 8, as this will save on divisions. + | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) + where + outRemaining = (ope `minusPtr` op0) `quot` bound + inpRemaining = iend - i0 + + goPartial !iendTmp = go i0 op0 + where + go !i !op + | i < iendTmp = do + let w = A.unsafeIndex arr i + if w < 0x80 + then BP.runB be w op >>= go (i + 1) + else poke op w >> go (i + 1) (op `plusPtr` 1) + | otherwise = outerLoop i (B.BufferRange op ope) + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> ByteString +encodeUtf8 (Text arr off len) + | len == 0 = B.empty + -- It would be easier to use Data.ByteString.Short.fromShort and slice later, + -- but this is undesirable when len is significantly smaller than length arr. + | otherwise = unsafeDupablePerformIO $ do + marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len + unsafeSTToIO $ A.copyI len marr 0 arr off + let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) + (PlainPtr mba) + pure $ B.fromForeignPtr fp 0 len + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> ByteString +encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> ByteString +encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> ByteString +encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> ByteString +encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) +{-# INLINE encodeUtf32BE #-} + +-- $incremental +-- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more +-- control for error-handling and streaming. +-- +-- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error. +-- For example this lets you insert or delete arbitrary text, or do some +-- stateful operations before resuming, such as keeping track of error locations. +-- In contrast, the older stream-oriented interface only lets you substitute +-- a single fixed 'Char' for each invalid byte in 'OnDecodeError'. +-- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks +-- before doing the copying work to construct a 'Text', or you can +-- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'. +-- +-- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'. diff --git a/src/Data/Text/Encoding/Error.hs b/src/Data/Text/Encoding/Error.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Encoding/Error.hs @@ -0,0 +1,119 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE Safe #-} +-- | +-- Module : Data.Text.Encoding.Error +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Types and functions for dealing with encoding and decoding errors +-- in Unicode text. +-- +-- The standard functions for encoding and decoding text are strict, +-- which is to say that they throw exceptions on invalid input. This +-- is often unhelpful on real world input, so alternative functions +-- exist that accept custom handlers for dealing with invalid inputs. +-- These 'OnError' handlers are normal Haskell functions. You can use +-- one of the presupplied functions in this module, or you can write a +-- custom handler of your own. + +module Data.Text.Encoding.Error + ( + -- * Error handling types + UnicodeException(..) + , OnError + , OnDecodeError + , OnEncodeError + -- * Useful error handling functions + , lenientDecode + , strictDecode + , strictEncode + , ignore + , replace + ) where + +import Control.DeepSeq (NFData (..)) +import Control.Exception (Exception, throw) +import Data.Word (Word8) +import Numeric (showHex) + +-- | Function type for handling a coding error. It is supplied with +-- two inputs: +-- +-- * A 'String' that describes the error. +-- +-- * The input value that caused the error. If the error arose +-- because the end of input was reached or could not be identified +-- precisely, this value will be 'Nothing'. +-- +-- If the handler returns a value wrapped with 'Just', that value will +-- be used in the output as the replacement for the invalid input. If +-- it returns 'Nothing', no value will be used in the output. +-- +-- Should the handler need to abort processing, it should use 'error' +-- or 'throw' an exception (preferably a 'UnicodeException'). It may +-- use the description provided to construct a more helpful error +-- report. +type OnError a b = String -> Maybe a -> Maybe b + +-- | A handler for a decoding error. +type OnDecodeError = OnError Word8 Char + +-- | A handler for an encoding error. +{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} +type OnEncodeError = OnError Char Word8 + +-- | An exception type for representing Unicode encoding errors. +data UnicodeException = + DecodeError String (Maybe Word8) + -- ^ Could not decode a byte sequence because it was invalid under + -- the given encoding, or ran out of input in mid-decode. + | EncodeError String (Maybe Char) + -- ^ Tried to encode a character that could not be represented + -- under the given encoding, or ran out of input in mid-encode. + deriving (Eq) + +{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} + +showUnicodeException :: UnicodeException -> String +showUnicodeException (DecodeError desc (Just w)) + = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) +showUnicodeException (DecodeError desc Nothing) + = "Cannot decode input: " ++ desc +showUnicodeException (EncodeError desc (Just c)) + = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) +showUnicodeException (EncodeError desc Nothing) + = "Cannot encode input: " ++ desc + +instance Show UnicodeException where + show = showUnicodeException + +instance Exception UnicodeException + +instance NFData UnicodeException where + rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () + rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () + +-- | Throw a 'UnicodeException' if decoding fails. +strictDecode :: OnDecodeError +strictDecode desc c = throw (DecodeError desc c) + +-- | Replace an invalid input byte with the Unicode replacement +-- character U+FFFD. +lenientDecode :: OnDecodeError +lenientDecode _ _ = Just '\xfffd' + +-- | Throw a 'UnicodeException' if encoding fails. +{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} +strictEncode :: OnEncodeError +strictEncode desc c = throw (EncodeError desc c) + +-- | Ignore an invalid input, substituting nothing in the output. +ignore :: OnError a b +ignore _ _ = Nothing + +-- | Replace an invalid input with a valid output. +replace :: b -> OnError a b +replace c _ _ = Just c diff --git a/src/Data/Text/Foreign.hs b/src/Data/Text/Foreign.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Foreign.hs @@ -0,0 +1,212 @@ +{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-} +-- | +-- Module : Data.Text.Foreign +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Support for using 'Text' data with native code via the Haskell +-- foreign function interface. + +module Data.Text.Foreign + ( + -- * Interoperability with native code + -- $interop + I8 + -- * Pointer conversion functions + , fromPtr + , fromPtr0 + , useAsPtr + , asForeignPtr + -- ** Encoding as UTF-8 + , peekCString + , withCString + , peekCStringLen + , withCStringLen + -- * Low-level manipulation + -- $lowlevel + , dropWord8 + , takeWord8 + , lengthWord8 + , unsafeCopyToPtr + ) where + +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Show (addrLen) +import Data.Text.Unsafe (lengthWord8) +import Data.Word (Word8) +import Foreign.C.String (CString, CStringLen) +import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) +import Foreign.Marshal.Alloc (allocaBytes) +import Foreign.Ptr (Ptr, castPtr) +import Foreign.Storable (pokeByteOff) +import GHC.Exts (Ptr(..)) +import qualified Data.Text.Array as A + +-- $interop +-- +-- The 'Text' type is implemented using arrays that are not guaranteed +-- to have a fixed address in the Haskell heap. All communication with +-- native code must thus occur by copying data back and forth. +-- +-- The 'Text' type's internal representation is UTF-8. +-- To interoperate with native libraries that use different +-- internal representations, such as UTF-16 or UTF-32, consider using +-- the functions in the 'Data.Text.Encoding' module. + +-- | A type representing a number of UTF-8 code units. +-- +-- @since 2.0 +newtype I8 = I8 Int + deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the array. +-- +-- __This function is unsafe.__ The source array must contain a valid +-- UTF-8 string of the given length. There are no guarantees about what +-- happens otherwise. +fromPtr :: Ptr Word8 -- ^ source array + -> I8 -- ^ length of source array (in 'Word8' units) + -> IO Text +fromPtr _ (I8 0) = pure empty +fromPtr ptr (I8 len) = unsafeSTToIO $ do + dst <- A.new len + A.copyFromPointer dst 0 ptr len + arr <- A.unsafeFreeze dst + return $! Text arr 0 len + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the NUL-terminated array. +-- +-- __This function is unsafe.__ The source array must contain a NULL-terminated +-- valid UTF-8 string. There are no guarantees about what happens otherwise. +-- +-- @since 2.0.1 +fromPtr0 :: Ptr Word8 -- ^ source array + -> IO Text +fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#)) + +-- $lowlevel +-- +-- Foreign functions that use UTF-8 internally may return indices in +-- units of 'Word8' instead of characters. These functions may +-- safely be used with such indices, as they will adjust offsets if +-- necessary to preserve the validity of a Unicode string. + +-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in +-- length. +-- +-- If @n@ would cause the 'Text' to end inside a code point, the +-- end of the prefix will be advanced by several additional 'Word8' units +-- to maintain its validity. +-- +-- @since 2.0 +takeWord8 :: I8 -> Text -> Text +takeWord8 = (fst .) . splitAtWord8 + +-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units +-- dropped from its beginning. +-- +-- If @n@ would cause the 'Text' to begin inside a code point, the +-- beginning of the suffix will be advanced by several additional 'Word8' +-- unit to maintain its validity. +-- +-- @since 2.0 +dropWord8 :: I8 -> Text -> Text +dropWord8 = (snd .) . splitAtWord8 + +splitAtWord8 :: I8 -> Text -> (Text, Text) +splitAtWord8 (I8 n) t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m | w0 < 0x80 = n -- last char is ASCII + | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence + | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence + | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence + | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence + | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence + | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence + | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence + | otherwise = n -- pre-pre-last char starts 3-byte sequence + w0 = A.unsafeIndex arr (off+n-1) + w1 = A.unsafeIndex arr (off+n-2) + w2 = A.unsafeIndex arr (off+n-3) + +-- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big +-- enough to hold the contents of the entire 'Text'. +unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () +unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len + +-- | /O(n)/ Perform an action on a temporary, mutable copy of a +-- 'Text'. The copy is freed as soon as the action returns. +useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a +useAsPtr t@(Text _arr _off len) action = + allocaBytes len $ \buf -> do + unsafeCopyToPtr t buf + action (castPtr buf) (I8 len) + +-- | /O(n)/ Make a mutable copy of a 'Text'. +asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) +asForeignPtr t@(Text _arr _off len) = do + fp <- mallocForeignPtrArray len + unsafeWithForeignPtr fp $ unsafeCopyToPtr t + return (fp, I8 len) + +-- | Marshal a 'Text' into a C string with a trailing NUL byte, +-- encoded as UTF-8 in temporary storage. +-- +-- The 'Text' itself must not contain any NUL bytes, this precondition +-- is not checked. Cf. 'withCStringLen'. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 2.0.1 +withCString :: Text -> (CString -> IO a) -> IO a +withCString t@(Text _arr _off len) action = + allocaBytes (len + 1) $ \buf -> do + unsafeCopyToPtr t buf + pokeByteOff buf len (0 :: Word8) + action (castPtr buf) + +-- | /O(n)/ Decode a C string with explicit length, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 1.0.0.0 +peekCStringLen :: CStringLen -> IO Text +peekCStringLen cs = do + bs <- unsafePackCStringLen cs + return $! decodeUtf8 bs + +-- | /O(n)/ Decode a null-terminated C string, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 2.1.2 +peekCString :: CString -> IO Text +peekCString cs = do + bs <- unsafePackCString cs + return $! decodeUtf8 bs + +-- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary +-- storage, with explicit length information. The encoded string may +-- contain NUL bytes, and is not followed by a trailing NUL byte. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 1.0.0.0 +withCStringLen :: Text -> (CStringLen -> IO a) -> IO a +withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act diff --git a/src/Data/Text/IO.hs b/src/Data/Text/IO.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/IO.hs @@ -0,0 +1,211 @@ +{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +-- | +-- Module : Data.Text.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Efficient locale-sensitive support for text I\/O. +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you want to do I\/O using the UTF-8 encoding, use "Data.Text.IO.Utf8", +-- which is faster than this module. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding, and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. + +module Data.Text.IO + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetChunk + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Data.Text (Text) +import Prelude hiding (appendFile, getContents, getLine, interact, + putStr, putStrLn, readFile, writeFile) +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, + withFile) +import qualified Control.Exception as E +import Control.Monad (liftM2, when) +import Data.IORef (readIORef) +import qualified Data.Text as T +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn) +import GHC.IO.Buffer (CharBuffer, isEmptyBuffer) +import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) +import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle) +import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..)) +import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) +import System.IO.Error (isEOFError) + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' +-- is a much faster and safer alternative. +readFile :: FilePath -> IO Text +readFile name = openFile name ReadMode >>= hGetContents + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile p = withFile p WriteMode . flip hPutStr + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile p = withFile p AppendMode . flip hPutStr + +catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool) +catchError caller h Handle__{..} err + | isEOFError err = do + buf <- readIORef haCharBuffer + return $ if isEmptyBuffer buf + then (T.empty, True) + else (T.singleton '\r', True) + | otherwise = E.throwIO (augmentIOError err caller h) + +-- | Wrap readChunk and return a value indicating if we're reached the EOF. +-- This is needed because unpack_nl is unable to discern the difference +-- between a buffer with just \r due to EOF or because not enough data was left +-- for decoding. e.g. the final character decoded from the byte buffer was \r. +readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool) +readChunkEof hh buf = do t <- readChunk hh buf + return (t, False) + +-- | Read a single chunk of strict text from a +-- 'Handle'. The size of the chunk depends on the amount of input +-- currently buffered. +-- +-- This function blocks only if there is no data available, and EOF +-- has not yet been reached. Once EOF is reached, this function +-- returns an empty string instead of throwing an exception. +-- +-- === Behavior +-- +-- Unlike byte-oriented functions, 'hGetChunk' operates on complete UTF-8 +-- characters. Since UTF-8 characters can occupy 1 to 4 bytes, this function +-- cannot guarantee reading an exact number of bytes. Instead, it reads +-- complete characters up to the handle's internal buffer limit. +-- +-- === Buffer Size +-- +-- The maximum chunk size is determined by the handle's internal character +-- buffer, which is set to 8192 bytes (2048 characters) by the GHC runtime +-- constant @dEFAULT_CHAR_BUFFER_SIZE@. This buffer size cannot be modified +-- through any public API. +-- +-- === UTF-8 Considerations +-- +-- When working with UTF-8 encoded text: +-- +-- * The function will never return a partial character +-- * The actual number of bytes read may vary depending on the character +-- encoding (ASCII characters = 1 byte, other Unicode characters = 2-4 bytes) +hGetChunk :: Handle -> IO Text +hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk + where + readSingleChunk hh@Handle__{..} = do + buf <- readIORef haCharBuffer + (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh + return (hh, t) + +-- | Read the remaining contents of a 'Handle' as a string. The +-- 'Handle' is closed once the contents have been read, or if an +-- exception is thrown. +-- +-- Internally, this function reads a chunk at a time from the +-- lower-level buffering abstraction, and concatenates the chunks into +-- a single string once the entire file has been read. +-- +-- As a result, it requires approximately twice as much memory as its +-- result to construct its result. For files more than a half of +-- available RAM in size, this may result in memory exhaustion. +hGetContents :: Handle -> IO Text +hGetContents h = do + chooseGoodBuffering h + wantReadableHandle "hGetContents" h readAll + where + readAll hh@Handle__{..} = do + let readChunks = do + buf <- readIORef haCharBuffer + (t, eof) <- readChunkEof hh buf + `E.catch` catchError "hGetContents" h hh + if eof + then return [t] + else (t:) `fmap` readChunks + ts <- readChunks + (hh', _) <- hClose_help hh + return (hh'{haType=ClosedHandle}, T.concat ts) + +-- | Use a more efficient buffer size if we're reading in +-- block-buffered mode with the default buffer size. When we can +-- determine the size of the handle we're reading, set the buffer size +-- to that, so that we can read the entire file in one chunk. +-- Otherwise, use a buffer size of at least 16KB. +chooseGoodBuffering :: Handle -> IO () +chooseGoodBuffering h = do + bufMode <- hGetBuffering h + case bufMode of + BlockBuffering Nothing -> do + d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> + if ioe_type e == InappropriateType + then return 16384 -- faster than the 2KB default + else E.throwIO e + when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d + _ -> return () + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = hGetLineWith T.concat + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = hGetContents stdin + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = hGetLine stdin + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = hPutStr stdout + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = hPutStrLn stdout diff --git a/src/Data/Text/IO/Utf8.hs b/src/Data/Text/IO/Utf8.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/IO/Utf8.hs @@ -0,0 +1,94 @@ +-- | +-- Module : Data.Text.IO.Utf8 +-- License : BSD-style +-- Portability : GHC +-- +-- Efficient UTF-8 support for text I\/O. +-- Unlike "Data.Text.IO", these functions do not depend on the locale +-- and do not do line ending conversion. +module Data.Text.IO.Utf8 + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Prelude () +import Control.Exception (evaluate) +import Control.Monad ((<=<), (=<<)) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Function ((.)) +import Data.Text (Text) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import GHC.IO.Handle (Handle) +import System.IO (IO, FilePath) + +decodeUtf8IO :: ByteString -> IO Text +decodeUtf8IO = evaluate . decodeUtf8 + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +readFile :: FilePath -> IO Text +readFile = decodeUtf8IO <=< B.readFile + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile fp = B.writeFile fp . encodeUtf8 + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile fp = B.appendFile fp . encodeUtf8 + +-- | Read the remaining contents of a 'Handle' as a string. +hGetContents :: Handle -> IO Text +hGetContents = decodeUtf8IO <=< B.hGetContents + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = decodeUtf8IO <=< B.hGetLine + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h = B.hPutStr h . encodeUtf8 + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h = B.hPutStrLn h . encodeUtf8 + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = decodeUtf8IO =<< B.getContents + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = decodeUtf8IO =<< B.getLine + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = B.putStr . encodeUtf8 + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = B.putStrLn . encodeUtf8 diff --git a/src/Data/Text/Internal.hs b/src/Data/Text/Internal.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal.hs @@ -0,0 +1,274 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE UnboxedTuples #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. +-- +-- You should not use this module unless you are determined to monkey +-- with the internals, as the functions here do just about nothing to +-- preserve data invariants. You have been warned! + +module Data.Text.Internal + ( + -- * Types + -- $internals + Text(..) + , StrictText + -- * Construction + , text + , textP + -- * Safety + , safe + -- * Code that must be here for accessibility + , empty + , append + -- * Utilities + , firstf + -- * Checked multiplication + , mul + , mul32 + , mul64 + -- * Debugging + , showText + -- * Conversions + , pack + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Control.Monad.ST (ST, runST) +import Data.Bits +import Data.Int (Int32, Int64) +import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) +import qualified Data.Text.Array as A + +-- | A space efficient, packed, unboxed Unicode text type. +data Text = Text + {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8 + {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + +-- | Type synonym for the strict flavour of 'Text'. +-- +-- @since 2.1.1 +type StrictText = Text + +-- | Smart constructor. +text_ :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text_ arr off len = +#if defined(ASSERTS) + let c = A.unsafeIndex arr off + in assert (len >= 0) . + assert (off >= 0) . + assert (len == 0 || c < 0x80 || c >= 0xC0) $ +#endif + Text arr off len +{-# INLINE text_ #-} + +-- | /O(1)/ The empty 'Text'. +empty :: Text +empty = Text A.empty 0 0 +{-# NOINLINE empty #-} + +-- | /O(n)/ Appends one 'Text' to the other by copying both of them +-- into a new 'Text'. +append :: Text -> Text -> Text +append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) + | len1 == 0 = b + | len2 == 0 = a + | len > 0 = Text (A.run x) 0 len + | otherwise = error $ "Data.Text.append: size overflow" + where + len = len1+len2 + x :: ST s (A.MArray s) + x = do + arr <- A.new len + A.copyI len1 arr 0 arr1 off1 + A.copyI len2 arr len1 arr2 off2 + return arr +{-# NOINLINE append #-} + +-- | Construct a 'Text' without invisibly pinning its byte array in +-- memory if its length has dwindled to zero. +-- It ensures that empty 'Text' values are shared. +text :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text arr off len | len == 0 = empty + | otherwise = text_ arr off len +{-# INLINE [0] text #-} + +textP :: A.Array -> Int -> Int -> Text +{-# DEPRECATED textP "Use text instead" #-} +textP = text + +-- | A useful 'show'-like function for debugging purposes. +showText :: Text -> String +showText (Text arr off len) = + "Text " ++ show (A.toList arr off len) ++ ' ' : + show off ++ ' ' : show len + +-- | Map a 'Char' to a 'Text'-safe value. +-- +-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode +-- scalar values, but are unfortunately admitted as valid 'Char' +-- values by Haskell. They cannot be represented in a 'Text'. This +-- function remaps those code points to the Unicode replacement +-- character (U+FFFD, \'�\'), and leaves other code points +-- unchanged. +safe :: Char -> Char +safe c + | ord c .&. 0x1ff800 /= 0xd800 = c + | otherwise = '\xfffd' +{-# INLINE [0] safe #-} + +-- | Apply a function to the first element of an optional pair. +firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) +firstf f (Just (a, b)) = Just (f a, b) +firstf _ Nothing = Nothing + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul :: Int -> Int -> Int +mul a b + | finiteBitSize (0 :: Word) == 64 + = int64ToInt $ intToInt64 a `mul64` intToInt64 b + | otherwise + = int32ToInt $ intToInt32 a `mul32` intToInt32 b +{-# INLINE mul #-} +infixl 7 `mul` + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul64 :: Int64 -> Int64 -> Int64 +mul64 a b + | a >= 0 && b >= 0 = mul64_ a b + | a >= 0 = -mul64_ a (-b) + | b >= 0 = -mul64_ (-a) b + | otherwise = mul64_ (-a) (-b) +{-# INLINE mul64 #-} +infixl 7 `mul64` + +mul64_ :: Int64 -> Int64 -> Int64 +mul64_ a b + | ahi > 0 && bhi > 0 = error "overflow" + | top > 0x7fffffff = error "overflow" + | total < 0 = error "overflow" + | otherwise = total + where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) + (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) + top = ahi * blo + alo * bhi + total = (top `shiftL` 32) + alo * blo +{-# INLINE mul64_ #-} + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul32 :: Int32 -> Int32 -> Int32 +mul32 a b = case int32ToInt64 a * int32ToInt64 b of + ab | ab < min32 || ab > max32 -> error "overflow" + | otherwise -> int64ToInt32 ab + where min32 = -0x80000000 :: Int64 + max32 = 0x7fffffff +{-# INLINE mul32 #-} +infixl 7 `mul32` + +intToInt64 :: Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Int +int64ToInt = fromIntegral + +intToInt32 :: Int -> Int32 +intToInt32 = fromIntegral + +int32ToInt :: Int32 -> Int +int32ToInt = fromIntegral + +int32ToInt64 :: Int32 -> Int64 +int32ToInt64 = fromIntegral + +int64ToInt32 :: Int64 -> Int32 +int64ToInt32 = fromIntegral + +-- $internals +-- +-- Internally, the 'Text' type is represented as an array of 'Word8' +-- UTF-8 code units. The offset and length fields in the constructor +-- are in these units, /not/ units of 'Char'. +-- +-- Invariants that all functions must maintain: +-- +-- * Since the 'Text' type uses UTF-8 internally, it cannot represent +-- characters in the reserved surrogate code point range U+D800 to +-- U+DFFF. To maintain this invariant, the 'safe' function maps +-- 'Char' values in this range to the replacement character (U+FFFD, +-- \'�\'). +-- +-- * Offset and length must point to a valid UTF-8 sequence of bytes. +-- Violation of this may cause memory access violation and divergence. + +-- ----------------------------------------------------------------------------- +-- * Conversion to/from 'Text' + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.unpack (pack "\55555") +-- "\65533" +pack :: String -> Text +pack [] = empty +pack xs = runST $ do + -- It's tempting to allocate a buffer of 4 * length xs bytes, + -- but not only it's wasteful for predominantly ASCII arguments, + -- the computation of length xs would force allocation of the entire xs at once. + let dstLen = 64 + dst <- A.new dstLen + outer dst dstLen 0 xs + where + outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text + outer !dst !dstLen = inner + where + inner !dstOff [] = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + inner !dstOff ccs@(c : cs) + -- Each 'Char' takes up to 4 bytes + | dstOff + 4 > dstLen = do + -- Double size of the buffer + let !dstLen' = dstLen * 2 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' dstOff ccs + | otherwise = do + d <- unsafeWrite dst dstOff (safe c) + inner (dstOff + d) cs +{-# NOINLINE [0] pack #-} diff --git a/src/Data/Text/Internal/ArrayUtils.hs b/src/Data/Text/Internal/ArrayUtils.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/ArrayUtils.hs @@ -0,0 +1,33 @@ +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE CPP #-} + +module Data.Text.Internal.ArrayUtils (memchr) where + +#if defined(PURE_HASKELL) +import qualified Data.Text.Array as A +import Data.List (elemIndex) +#else +import Foreign.C.Types +import System.Posix.Types (CSsize(..)) +#endif +import GHC.Exts (ByteArray#) +import Data.Word (Word8) + +memchr :: ByteArray# -> Int -> Int -> Word8 -> Int +#if defined(PURE_HASKELL) +memchr arr# off len w = + let tempBa = A.ByteArray arr# + in case elemIndex w (A.toList tempBa off len) of + Nothing -> -1 + Just i -> i +#else +memchr arr# off len w = fromIntegral $ c_memchr arr# (intToCSize off) (intToCSize len) w + +intToCSize :: Int -> CSize +intToCSize = fromIntegral + + +foreign import ccall unsafe "_hs_text_memchr" c_memchr + :: ByteArray# -> CSize -> CSize -> Word8 -> CSsize +#endif diff --git a/src/Data/Text/Internal/Builder.hs b/src/Data/Text/Internal/Builder.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Builder.hs @@ -0,0 +1,350 @@ +{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} +{-# OPTIONS_HADDOCK not-home #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Data.Text.Internal.Builder +-- Copyright : (c) 2013 Bryan O'Sullivan +-- (c) 2010 Johan Tibell +-- License : BSD-style (see LICENSE) +-- +-- Maintainer : Johan Tibell +-- Stability : experimental +-- Portability : portable to Hugs and GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. +-- +-- To get maximum performance when building lazy @Text@ values using a +-- builder, associate @mappend@ calls to the right. For example, +-- prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. +-- +----------------------------------------------------------------------------- + +module Data.Text.Internal.Builder + ( -- * Public API + -- ** The Builder type + Builder + , LazyTextBuilder + , toLazyText + , toLazyTextWith + + -- ** Constructing Builders + , singleton + , fromText + , fromLazyText + , fromString + + -- ** Flushing the buffer state + , flush + + -- * Internal functions + , append' + , ensureFree + , writeN + ) where + +import Control.Monad.ST (ST, runST) +import Data.Monoid (Monoid(..)) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Lazy (smallChunkSize) +import Data.Text.Unsafe (inlineInterleaveST) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Prelude hiding (map, putChar) + +import qualified Data.String as String +import qualified Data.Text as S +import qualified Data.Text.Array as A +import qualified Data.Text.Lazy as L + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +------------------------------------------------------------------------ + +-- | A @Builder@ is an efficient way to build lazy @Text@ values. +-- There are several functions for constructing builders, but only one +-- to inspect them: to extract any data, you have to turn them into +-- lazy @Text@ values using @toLazyText@. +-- +-- Internally, a builder constructs a lazy @Text@ by filling arrays +-- piece by piece. As each buffer is filled, it is \'popped\' off, to +-- become a new chunk of the resulting lazy @Text@. All this is +-- hidden from the user of the @Builder@. +newtype Builder = Builder { + -- Invariant (from Data.Text.Lazy): + -- The lists include no null Texts. + runBuilder :: forall s. (Buffer s -> ST s [S.Text]) + -> Buffer s + -> ST s [S.Text] + } + +-- | @since 2.1.2 +type LazyTextBuilder = Builder + +instance Semigroup Builder where + (<>) = append + {-# INLINE (<>) #-} + +instance Monoid Builder where + mempty = empty + {-# INLINE mempty #-} + mappend = (<>) + {-# INLINE mappend #-} + mconcat = foldr mappend Data.Monoid.mempty + {-# INLINE mconcat #-} + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Builder +-- "\65533" +instance String.IsString Builder where + fromString = fromString + {-# INLINE fromString #-} + +instance Show Builder where + show = show . toLazyText + +instance Eq Builder where + a == b = toLazyText a == toLazyText b + +instance Ord Builder where + a <= b = toLazyText a <= toLazyText b + +------------------------------------------------------------------------ + +-- | /O(1)./ The empty @Builder@, satisfying +-- +-- * @'toLazyText' 'empty' = 'L.empty'@ +-- +empty :: Builder +empty = Builder (\ k buf -> k buf) +{-# INLINE empty #-} + +-- | /O(1)./ A @Builder@ taking a single character, satisfying +-- +-- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ +-- +singleton :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> Builder +singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) +{-# INLINE singleton #-} + +------------------------------------------------------------------------ + +-- | /O(1)./ The concatenation of two builders, an associative +-- operation with identity 'empty', satisfying +-- +-- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ +-- +append :: Builder -> Builder -> Builder +append (Builder f) (Builder g) = Builder (f . g) +{-# INLINE [0] append #-} + +-- TODO: Experiment to find the right threshold. +copyLimit :: Int +copyLimit = 128 + +-- This function attempts to merge small @Text@ values instead of +-- treating each value as its own chunk. We may not always want this. + +-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying +-- +-- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ +-- +fromText :: S.Text -> Builder +fromText t@(Text arr off l) + | S.null t = empty + | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off + | otherwise = flush `append` mapBuilder (t :) +{-# INLINE [1] fromText #-} + +{-# RULES +"fromText/pack" forall s . + fromText (S.pack s) = fromString s + #-} + +-- | /O(1)./ A Builder taking a @String@, satisfying +-- +-- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ +-- +-- Performs replacement on invalid scalar values: +-- +-- >>> fromString "\55555" +-- "\65533" +-- +-- @since 1.2.0.0 +fromString :: String -> Builder +fromString str = Builder $ \k (Buffer p0 o0 u0) -> do + len <- A.getSizeofMArray p0 + -- `end` is 3 bytes before the actual end of `marr` + -- to make sure there's room for a 4-byte UTF-8 code point. + let loop !marr !o !u !_ [] = k (Buffer marr o u) + loop marr o u end s@(c:cs) + | u >= end = do + A.shrinkM marr (o + u) + arr <- A.unsafeFreeze marr + let !t = Text arr o u + marr' <- A.new chunkSize + ts <- inlineInterleaveST (loop marr' 0 0 (chunkSize - 3) s) + return $ t : ts + | otherwise = do + n <- unsafeWrite marr (o+u) (safe c) + loop marr o (u+n) end cs + loop p0 o0 u0 (len - o0 - 3) str + where + chunkSize = smallChunkSize +{-# INLINEABLE fromString #-} + +-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying +-- +-- * @'toLazyText' ('fromLazyText' t) = t@ +-- +fromLazyText :: L.Text -> Builder +fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) +{-# INLINE fromLazyText #-} + +------------------------------------------------------------------------ + +-- Our internal buffer type +data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) + {-# UNPACK #-} !Int -- offset + {-# UNPACK #-} !Int -- used units + +------------------------------------------------------------------------ + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default +-- buffer size. The construction work takes place if and when the +-- relevant part of the lazy @Text@ is demanded. +toLazyText :: Builder -> L.Text +toLazyText = toLazyTextWith smallChunkSize + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given +-- size for the initial buffer. The construction work takes place if +-- and when the relevant part of the lazy @Text@ is demanded. +-- +-- If the initial buffer is too small to hold all data, subsequent +-- buffers will be the default buffer size. +toLazyTextWith :: Int -> Builder -> L.Text +toLazyTextWith chunkSize m = L.fromChunks (runST $ + newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) + +-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, +-- yielding a new chunk in the result lazy @Text@. +flush :: Builder +flush = Builder $ \ k buf@(Buffer p o u) -> + if u == 0 + then k buf + else do arr <- A.unsafeFreeze p + let !b = Buffer p (o+u) 0 + !t = Text arr o u + ts <- inlineInterleaveST (k b) + return $! t : ts +{-# INLINE [1] flush #-} +-- defer inlining so that flush/flush rule may fire. + +------------------------------------------------------------------------ + +-- | Sequence an ST operation on the buffer +withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder +withBuffer f = Builder $ \k buf -> f buf >>= k +{-# INLINE withBuffer #-} + +-- | Get the size of the buffer +withSize :: (Int -> Builder) -> Builder +withSize f = Builder $ \ k buf@(Buffer arr offset used) -> do + len <- A.getSizeofMArray arr + runBuilder (f (len - offset - used)) k buf +{-# INLINE withSize #-} + +-- | Map the resulting list of texts. +mapBuilder :: ([S.Text] -> [S.Text]) -> Builder +mapBuilder f = Builder (fmap f .) + +------------------------------------------------------------------------ + +-- | Ensure that there are at least @n@ many elements available. +ensureFree :: Int -> Builder +ensureFree !n = withSize $ \ l -> + if n <= l + then empty + else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) +{-# INLINE [0] ensureFree #-} + +writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder +writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) +{-# INLINE [0] writeAtMost #-} + +-- | Ensure that @n@ many elements are available, and then use @f@ to +-- write some elements into the memory. +writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder +writeN n f = writeAtMost n (\ p o -> f p o >> return n) +{-# INLINE writeN #-} + +writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) +writeBuffer f (Buffer p o u) = do + n <- f p (o+u) + return $! Buffer p o (u+n) +{-# INLINE writeBuffer #-} + +newBuffer :: Int -> ST s (Buffer s) +newBuffer size = do + arr <- A.new size + return $! Buffer arr 0 0 +{-# INLINE newBuffer #-} + +------------------------------------------------------------------------ +-- Some nice rules for Builder + +-- This function makes GHC understand that 'writeN' and 'ensureFree' +-- are *not* recursive in the presence of the rewrite rules below. +-- This is not needed with GHC 7+. +append' :: Builder -> Builder -> Builder +append' (Builder f) (Builder g) = Builder (f . g) +{-# INLINE append' #-} + +{-# RULES + +"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int) ws. + append (writeAtMost a f) (append (writeAtMost b g) ws) = + append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s)) ws + +"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int). + append (writeAtMost a f) (writeAtMost b g) = + writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s) + +"ensureFree/ensureFree" forall a b . + append (ensureFree a) (ensureFree b) = ensureFree (max a b) + +"flush/flush" + append flush flush = flush + + #-} diff --git a/src/Data/Text/Internal/Builder/Functions.hs b/src/Data/Text/Internal/Builder/Functions.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Builder/Functions.hs @@ -0,0 +1,40 @@ +{-# LANGUAGE MagicHash #-} + +-- | +-- Module : Data.Text.Internal.Builder.Functions +-- Copyright : (c) 2011 MailRank, Inc. +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Useful functions and combinators. + +module Data.Text.Internal.Builder.Functions + ( + (<>) + , i2d + ) where + +import Data.Monoid (mappend) +import Data.Text.Lazy.Builder (Builder) +import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#)) +import Prelude () + +-- | Unsafe conversion for decimal digits. +{-# INLINE i2d #-} +i2d :: Int -> Char +i2d (I# i#) = C# (chr# (ord# '0'# +# i#)) + +-- | The normal 'mappend' function with right associativity instead of +-- left. +(<>) :: Builder -> Builder -> Builder +(<>) = mappend +{-# INLINE (<>) #-} + +infixr 4 <> diff --git a/src/Data/Text/Internal/Builder/Int/Digits.hs b/src/Data/Text/Internal/Builder/Int/Digits.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Builder/Int/Digits.hs @@ -0,0 +1,25 @@ +-- Module: Data.Text.Internal.Builder.Int.Digits +-- Copyright: (c) 2013 Bryan O'Sullivan +-- License: BSD-style +-- Maintainer: Bryan O'Sullivan +-- Stability: experimental +-- Portability: portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- This module exists because the C preprocessor does things that we +-- shall not speak of when confronted with Haskell multiline strings. + +module Data.Text.Internal.Builder.Int.Digits (digits) where + +import Data.ByteString.Char8 (ByteString, pack) + +digits :: ByteString +digits = pack + "0001020304050607080910111213141516171819\ + \2021222324252627282930313233343536373839\ + \4041424344454647484950515253545556575859\ + \6061626364656667686970717273747576777879\ + \8081828384858687888990919293949596979899" diff --git a/src/Data/Text/Internal/Builder/RealFloat/Functions.hs b/src/Data/Text/Internal/Builder/RealFloat/Functions.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Builder/RealFloat/Functions.hs @@ -0,0 +1,35 @@ +{-# LANGUAGE CPP #-} + +-- | +-- Module: Data.Text.Internal.Builder.RealFloat.Functions +-- Copyright: (c) The University of Glasgow 1994-2002 +-- License: see libraries/base/LICENSE +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! + +module Data.Text.Internal.Builder.RealFloat.Functions + ( + roundTo + ) where + +roundTo :: Int -> [Int] -> (Int,[Int]) +roundTo d is = + case f d True is of + x@(0,_) -> x + (1,xs) -> (1, 1:xs) + _ -> error "roundTo: bad Value" + where + b2 = base `quot` 2 + + f n _ [] = (0, replicate n 0) + f 0 e (x:xs) | x == b2 && e && all (== 0) xs = (0, []) -- Round to even when at exactly half the base + | otherwise = (if x >= b2 then 1 else 0, []) + f n _ (i:xs) + | i' == base = (1,0:ds) + | otherwise = (0,i':ds) + where + (c,ds) = f (n-1) (even i) xs + i' = c + i + base = 10 diff --git a/src/Data/Text/Internal/ByteStringCompat.hs b/src/Data/Text/Internal/ByteStringCompat.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/ByteStringCompat.hs @@ -0,0 +1,28 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +module Data.Text.Internal.ByteStringCompat (mkBS, withBS) where + +import Data.ByteString.Internal (ByteString (..)) +import Data.Word (Word8) +import Foreign.ForeignPtr (ForeignPtr) + +#if !MIN_VERSION_bytestring(0,11,0) +import GHC.ForeignPtr (plusForeignPtr) +#endif + +mkBS :: ForeignPtr Word8 -> Int -> ByteString +#if MIN_VERSION_bytestring(0,11,0) +mkBS dfp n = BS dfp n +#else +mkBS dfp n = PS dfp 0 n +#endif +{-# INLINE mkBS #-} + +withBS :: ByteString -> (ForeignPtr Word8 -> Int -> r) -> r +#if MIN_VERSION_bytestring(0,11,0) +withBS (BS !sfp !slen) kont = kont sfp slen +#else +withBS (PS !sfp !soff !slen) kont = kont (plusForeignPtr sfp soff) slen +#endif +{-# INLINE withBS #-} diff --git a/src/Data/Text/Internal/Encoding.hs b/src/Data/Text/Internal/Encoding.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding.hs @@ -0,0 +1,535 @@ +{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, + UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Internal.Builder +-- License : BSD-style (see LICENSE) +-- Stability : experimental +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Internals of "Data.Text.Encoding". +-- +-- @since 2.0.2 +module Data.Text.Internal.Encoding + ( validateUtf8Chunk + , validateUtf8More + , decodeUtf8Chunk + , decodeUtf8More + , decodeUtf8With1 + , decodeUtf8With2 + , Utf8State + , startUtf8State + , StrictTextBuilder() + , StrictBuilder + , strictBuilderToText + , textToStrictBuilder + + -- * Internal + , skipIncomplete + , getCompleteLen + , getPartialUtf8 + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits ((.&.), shiftL, shiftR) +import Data.ByteString (ByteString) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Word (Word32, Word8) +import Foreign.Storable (pokeElemOff) +import Data.Text.Encoding.Error (OnDecodeError) +import Data.Text.Internal (Text(..)) +import Data.Text.Internal.Encoding.Utf8 + (DecoderState, utf8AcceptState, utf8RejectState, updateDecoderState) +import Data.Text.Internal.StrictBuilder (StrictBuilder, StrictTextBuilder) +import qualified Data.ByteString as B +import qualified Data.ByteString.Internal as BI +import qualified Data.ByteString.Short.Internal as SBS +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.StrictBuilder as SB +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +#ifdef SIMDUTF +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Foreign.C.Types (CSize(..)) +import Foreign.C.Types (CInt(..)) +import Foreign.Ptr (Ptr) +#endif + +-- | Use 'StrictBuilder' to build 'Text'. +-- +-- @since 2.0.2 +strictBuilderToText :: StrictTextBuilder -> Text +strictBuilderToText = SB.toText + +-- | Copy 'Text' in a 'StrictBuilder' +-- +-- @since 2.0.2 +textToStrictBuilder :: Text -> StrictTextBuilder +textToStrictBuilder = SB.fromText + +-- | State of decoding a 'ByteString' in UTF-8. +-- Enables incremental decoding ('validateUtf8Chunk', 'validateUtf8More', +-- 'decodeUtf8Chunk', 'decodeUtf8More'). +-- +-- @since 2.0.2 + +-- Internal invariant: +-- the first component is the initial state if and only if +-- the second component is empty. +-- +-- @ +-- 'utf9CodePointState' s = 'utf8StartState' +-- <=> +-- 'partialUtf8CodePoint' s = 'PartialUtf8CodePoint' 0 +-- @ +data Utf8State = Utf8State + { -- | State of the UTF-8 state machine. + utf8CodePointState :: {-# UNPACK #-} !DecoderState + -- | Bytes of the currently incomplete code point (if any). + , partialUtf8CodePoint :: {-# UNPACK #-} !PartialUtf8CodePoint + } + deriving (Eq, Show) + +-- | Initial 'Utf8State'. +-- +-- @since 2.0.2 +startUtf8State :: Utf8State +startUtf8State = Utf8State utf8AcceptState partUtf8Empty + +-- | Prefix of a UTF-8 code point encoded in 4 bytes, +-- possibly empty. +-- +-- - The most significant byte contains the number of bytes, +-- between 0 and 3. +-- - The remaining bytes hold the incomplete code point. +-- - Unused bytes must be 0. +-- +-- All of operations available on it are the functions below. +-- The constructor should never be used outside of those. +-- +-- @since 2.0.2 +newtype PartialUtf8CodePoint = PartialUtf8CodePoint Word32 + deriving (Eq, Show) + +-- | Empty prefix. +partUtf8Empty :: PartialUtf8CodePoint +partUtf8Empty = PartialUtf8CodePoint 0 + +-- | Length of the partial code point, stored in the most significant byte. +partUtf8Len :: PartialUtf8CodePoint -> Int +partUtf8Len (PartialUtf8CodePoint w) = fromIntegral $ w `shiftR` 24 + +-- | Length of the code point once completed (it is known in the first byte). +-- 0 if empty. +partUtf8CompleteLen :: PartialUtf8CodePoint -> Int +partUtf8CompleteLen c@(PartialUtf8CodePoint w) + | partUtf8Len c == 0 = 0 + | 0xf0 <= firstByte = 4 + | 0xe0 <= firstByte = 3 + | 0xc2 <= firstByte = 2 + | otherwise = 0 + where + firstByte = (w `shiftR` 16) .&. 255 + +-- | Get the @n@-th byte, assuming it is within bounds: @0 <= n < partUtf8Len c@. +-- +-- Unsafe: no bounds checking. +partUtf8UnsafeIndex :: +#if defined(ASSERTS) + HasCallStack => +#endif + PartialUtf8CodePoint -> Int -> Word8 +partUtf8UnsafeIndex _c@(PartialUtf8CodePoint w) n = +#if defined(ASSERTS) + assert (0 <= n && n < partUtf8Len _c) $ +#endif + fromIntegral $ w `shiftR` (16 - 8 * n) + +-- | Append some bytes. +-- +-- Unsafe: no bounds checking. +partUtf8UnsafeAppend :: +#if defined(ASSERTS) + HasCallStack => +#endif + PartialUtf8CodePoint -> ByteString -> PartialUtf8CodePoint +partUtf8UnsafeAppend c@(PartialUtf8CodePoint word) bs = +#if defined(ASSERTS) + assert (lenc + lenbs <= 3) $ +#endif + PartialUtf8CodePoint $ + tryPush 0 $ tryPush 1 $ tryPush 2 $ word + (fromIntegral lenbs `shiftL` 24) + where + lenc = partUtf8Len c + lenbs = B.length bs + tryPush i w = + if i < lenbs + then w + (fromIntegral (B.index bs i) `shiftL` fromIntegral (16 - 8 * (lenc + i))) + else w + +-- | Fold a 'PartialUtf8CodePoint'. This avoids recursion so it can unfold to straightline code. +{-# INLINE partUtf8Foldr #-} +partUtf8Foldr :: (Word8 -> a -> a) -> a -> PartialUtf8CodePoint -> a +partUtf8Foldr f x0 c = case partUtf8Len c of + 0 -> x0 + 1 -> build 0 x0 + 2 -> build 0 (build 1 x0) + _ -> build 0 (build 1 (build 2 x0)) + where + build i x = f (partUtf8UnsafeIndex c i) x + +-- | Convert 'PartialUtf8CodePoint' to 'ByteString'. +partUtf8ToByteString :: PartialUtf8CodePoint -> B.ByteString +partUtf8ToByteString c = BI.unsafeCreate (partUtf8Len c) $ \ptr -> + partUtf8Foldr (\w k i -> pokeElemOff ptr i w >> k (i+1)) (\_ -> pure ()) c 0 + +-- | Exported for testing. +getCompleteLen :: Utf8State -> Int +getCompleteLen = partUtf8CompleteLen . partialUtf8CodePoint + +-- | Exported for testing. +getPartialUtf8 :: Utf8State -> B.ByteString +getPartialUtf8 = partUtf8ToByteString . partialUtf8CodePoint + +#ifdef SIMDUTF +foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8 + :: Ptr Word8 -> CSize -> IO CInt +#endif + +-- | Validate a 'ByteString' as UTF-8-encoded text. To be continued using 'validateUtf8More'. +-- +-- See also 'validateUtf8More' for details on the result of this function. +-- +-- @ +-- 'validateUtf8Chunk' = 'validateUtf8More' 'startUtf8State' +-- @ +-- +-- @since 2.0.2 +-- +-- === Properties +-- +-- Given: +-- +-- @ +-- 'validateUtf8Chunk' chunk = (n, ms) +-- @ +-- +-- - The prefix is valid UTF-8. In particular, it should be accepted +-- by this validation: +-- +-- @ +-- 'validateUtf8Chunk' ('Data.ByteString.take' n chunk) = (n, Just 'startUtf8State') +-- @ +validateUtf8Chunk :: ByteString -> (Int, Maybe Utf8State) +validateUtf8Chunk bs = validateUtf8ChunkFrom 0 bs (,) + +-- Assume bytes up to offset @ofs@ have been validated already. +-- +-- Using CPS lets us inline the continuation and avoid allocating a @Maybe@ +-- in the @decode...@ functions. +{-# INLINE validateUtf8ChunkFrom #-} +validateUtf8ChunkFrom :: forall r. Int -> ByteString -> (Int -> Maybe Utf8State -> r) -> r +validateUtf8ChunkFrom ofs bs k + -- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0. + -- MIN_VERSION_bytestring does not allow us to differentiate + -- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+. +#if defined(SIMDUTF) || MIN_VERSION_bytestring(0,12,1) + | guessUtf8Boundary > 0 && + -- the rest of the bytestring is valid utf-8 up to the boundary + ( +#ifdef SIMDUTF + withBS (B.drop ofs bs) $ \ fp _ -> unsafeDupablePerformIO $ + unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> + c_is_valid_utf8 ptr (fromIntegral guessUtf8Boundary) +#else + B.isValidUtf8 $ B.take guessUtf8Boundary (B.drop ofs bs) +#endif + ) = slowValidateUtf8ChunkFrom (ofs + guessUtf8Boundary) + -- No + | otherwise = slowValidateUtf8ChunkFrom ofs + where + len = B.length bs - ofs + isBoundary n p = len >= n && p (B.index bs (ofs + len - n)) + guessUtf8Boundary + | isBoundary 1 (<= 0x80) = len -- last char is ASCII (common short-circuit) + | isBoundary 1 (0xc2 <=) = len - 1 -- last char starts a two-(or more-)byte code point + | isBoundary 2 (0xe0 <=) = len - 2 -- pre-last char starts a three-or-four-byte code point + | isBoundary 3 (0xf0 <=) = len - 3 -- third to last char starts a four-byte code point + | otherwise = len +#else + = slowValidateUtf8ChunkFrom ofs + where +#endif + -- A pure Haskell implementation of validateUtf8More. + -- Ideally the primitives 'B.isValidUtf8' or 'c_is_valid_utf8' should give us + -- indices to let us avoid this function. + slowValidateUtf8ChunkFrom :: Int -> r + slowValidateUtf8ChunkFrom ofs1 = slowLoop ofs1 ofs1 utf8AcceptState + + slowLoop !utf8End i s + | i < B.length bs = + case updateDecoderState (B.index bs i) s of + s' | s' == utf8RejectState -> k utf8End Nothing + | s' == utf8AcceptState -> slowLoop (i + 1) (i + 1) s' + | otherwise -> slowLoop utf8End (i + 1) s' + | otherwise = k utf8End (Just (Utf8State s (partUtf8UnsafeAppend partUtf8Empty (B.drop utf8End bs)))) + +-- | Validate another 'ByteString' chunk in an ongoing stream of UTF-8-encoded text. +-- +-- Returns a pair: +-- +-- 1. The first component @n@ is the end position, relative to the current +-- chunk, of the longest prefix of the accumulated bytestring which is valid UTF-8. +-- @n@ may be negative: that happens when an incomplete code point started in +-- a previous chunk and is not completed by the current chunk (either +-- that code point is still incomplete, or it is broken by an invalid byte). +-- +-- 2. The second component @ms@ indicates the following: +-- +-- - if @ms = Nothing@, the remainder of the chunk contains an invalid byte, +-- within four bytes from position @n@; +-- - if @ms = Just s'@, you can carry on validating another chunk +-- by calling 'validateUtf8More' with the new state @s'@. +-- +-- @since 2.0.2 +-- +-- === Properties +-- +-- Given: +-- +-- @ +-- 'validateUtf8More' s chunk = (n, ms) +-- @ +-- +-- - If the chunk is invalid, it cannot be extended to be valid. +-- +-- @ +-- ms = Nothing +-- ==> 'validateUtf8More' s (chunk '<>' more) = (n, Nothing) +-- @ +-- +-- - Validating two chunks sequentially is the same as validating them +-- together at once: +-- +-- @ +-- ms = Just s' +-- ==> 'validateUtf8More' s (chunk '<>' more) = 'Data.Bifunctor.first' ('Data.ByteString.length' chunk '+') ('validateUtf8More' s' more) +-- @ +validateUtf8More :: Utf8State -> ByteString -> (Int, Maybe Utf8State) +validateUtf8More st bs = validateUtf8MoreCont st bs (,) + +-- CPS: inlining the continuation lets us make more tail calls and avoid +-- allocating a @Maybe@ in @decodeWith1/2@. +{-# INLINE validateUtf8MoreCont #-} +validateUtf8MoreCont :: Utf8State -> ByteString -> (Int -> Maybe Utf8State -> r) -> r +validateUtf8MoreCont st@(Utf8State s0 part) bs k + | len > 0 = loop 0 s0 + | otherwise = k (- partUtf8Len part) (Just st) + where + len = B.length bs + -- Complete an incomplete code point (if there is one) + -- and then jump to validateUtf8ChunkFrom + loop !i s + | s == utf8AcceptState = validateUtf8ChunkFrom i bs k + | i < len = + case updateDecoderState (B.index bs i) s of + s' | s' == utf8RejectState -> k (- partUtf8Len part) Nothing + | otherwise -> loop (i + 1) s' + | otherwise = k (- partUtf8Len part) (Just (Utf8State s (partUtf8UnsafeAppend part bs))) + +-- Eta-expanded to inline partUtf8Foldr +partUtf8ToStrictBuilder :: PartialUtf8CodePoint -> StrictTextBuilder +partUtf8ToStrictBuilder c = + partUtf8Foldr ((<>) . SB.unsafeFromWord8) mempty c + +utf8StateToStrictBuilder :: Utf8State -> StrictTextBuilder +utf8StateToStrictBuilder = partUtf8ToStrictBuilder . partialUtf8CodePoint + +-- | Decode another chunk in an ongoing UTF-8 stream. +-- +-- Returns a triple: +-- +-- 1. A 'StrictBuilder' for the decoded chunk of text. You can accumulate +-- chunks with @('<>')@ or output them with 'SB.toText'. +-- 2. The undecoded remainder of the given chunk, for diagnosing errors +-- and resuming (presumably after skipping some bytes). +-- 3. 'Just' the new state, or 'Nothing' if an invalid byte was encountered +-- (it will be within the first 4 bytes of the undecoded remainder). +-- +-- @since 2.0.2 +-- +-- === Properties +-- +-- Given: +-- +-- @ +-- (pre, suf, ms) = 'decodeUtf8More' s chunk +-- @ +-- +-- 1. If the output @pre@ is nonempty (alternatively, if @length chunk > length suf@) +-- +-- @ +-- s2b pre \`'Data.ByteString.append'\` suf = p2b s \`'Data.ByteString.append'\` chunk +-- @ +-- +-- where +-- +-- @ +-- s2b = 'Data.Text.Encoding.encodeUtf8' . 'Data.Text.Encoding.toText' +-- p2b = 'Data.Text.Internal.Encoding.partUtf8ToByteString' +-- @ +-- +-- 2. If the output @pre@ is empty (alternatively, if @length chunk = length suf@) +-- +-- @suf = chunk@ +-- +-- 3. Decoding chunks separately is equivalent to decoding their concatenation. +-- +-- Given: +-- +-- @ +-- (pre1, suf1, Just s1) = 'decodeUtf8More' s chunk1 +-- (pre2, suf2, ms2) = 'decodeUtf8More' s1 chunk2 +-- (pre3, suf3, ms3) = 'decodeUtf8More' s (chunk1 \`B.append\` chunk2) +-- @ +-- +-- we have: +-- +-- @ +-- s2b (pre1 '<>' pre2) = s2b pre3 +-- ms2 = ms3 +-- @ +decodeUtf8More :: Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State) +decodeUtf8More s bs = + validateUtf8MoreCont s bs $ \len ms -> + let builder | len <= 0 = mempty + | otherwise = utf8StateToStrictBuilder s + <> SB.unsafeFromByteString (B.take len bs) + in (builder, B.drop len bs, ms) + +-- | Decode a chunk of UTF-8 text. To be continued with 'decodeUtf8More'. +-- +-- See 'decodeUtf8More' for details on the result. +-- +-- @since 2.0.2 +-- +-- === Properties +-- +-- @ +-- 'decodeUtf8Chunk' = 'decodeUtf8More' 'startUtf8State' +-- @ +-- +-- Given: +-- +-- @ +-- 'decodeUtf8Chunk' chunk = (builder, rest, ms) +-- @ +-- +-- @builder@ is a prefix and @rest@ is a suffix of @chunk@. +-- +-- @ +-- 'Data.Text.Encoding.encodeUtf8' ('Data.Text.Encoding.strictBuilderToText' builder) '<>' rest = chunk +-- @ +decodeUtf8Chunk :: ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State) +decodeUtf8Chunk = decodeUtf8More startUtf8State + +-- | Call the error handler on each byte of the partial code point stored in +-- 'Utf8State' and append the results. +-- +-- Exported for use in lazy 'Data.Text.Lazy.Encoding.decodeUtf8With'. +-- +-- @since 2.0.2 +{-# INLINE skipIncomplete #-} +skipIncomplete :: OnDecodeError -> String -> Utf8State -> StrictTextBuilder +skipIncomplete onErr msg s = + partUtf8Foldr + ((<>) . handleUtf8Error onErr msg) + mempty (partialUtf8CodePoint s) + +{-# INLINE handleUtf8Error #-} +handleUtf8Error :: OnDecodeError -> String -> Word8 -> StrictTextBuilder +handleUtf8Error onErr msg w = case onErr msg (Just w) of + Just c -> SB.fromChar c + Nothing -> mempty + +-- | Helper for 'Data.Text.Encoding.decodeUtf8With'. +-- +-- @since 2.0.2 + +-- This could be shorter by calling 'decodeUtf8With2' directly, but we make the +-- first call validateUtf8Chunk directly to return even faster in successful +-- cases. +decodeUtf8With1 :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> String -> ByteString -> Text +decodeUtf8With1 onErr msg bs = validateUtf8ChunkFrom 0 bs $ \len ms -> case ms of + Just s + | len == B.length bs -> + let !(SBS.SBS arr) = SBS.toShort bs in + Text (A.ByteArray arr) 0 len + | otherwise -> SB.toText $ + SB.unsafeFromByteString (B.take len bs) <> skipIncomplete onErr msg s + Nothing -> + let (builder, _, s) = decodeUtf8With2 onErr msg startUtf8State (B.drop (len + 1) bs) in + SB.toText $ + SB.unsafeFromByteString (B.take len bs) <> + handleUtf8Error onErr msg (B.index bs len) <> + builder <> + skipIncomplete onErr msg s + +-- | Helper for 'Data.Text.Encoding.decodeUtf8With', +-- 'Data.Text.Encoding.streamDecodeUtf8With', and lazy +-- 'Data.Text.Lazy.Encoding.decodeUtf8With', +-- which use an 'OnDecodeError' to process bad bytes. +-- +-- See 'decodeUtf8Chunk' for a more flexible alternative. +-- +-- @since 2.0.2 +decodeUtf8With2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> String -> Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Utf8State) +decodeUtf8With2 onErr msg s0 bs = loop s0 0 mempty + where + loop s i !builder = + let nonEmptyPrefix len = builder + <> utf8StateToStrictBuilder s + <> SB.unsafeFromByteString (B.take len (B.drop i bs)) + in validateUtf8MoreCont s (B.drop i bs) $ \len ms -> case ms of + Nothing -> + if len < 0 + then + -- If the first byte cannot complete the partial code point in s, + -- retry from startUtf8State. + let builder' = builder <> skipIncomplete onErr msg s + -- Note: loop is strict on builder, so if onErr raises an error it will + -- be forced here, short-circuiting the loop as desired. + in loop startUtf8State i builder' + else + let builder' = nonEmptyPrefix len + <> handleUtf8Error onErr msg (B.index bs (i + len)) + in loop startUtf8State (i + len + 1) builder' + Just s' -> + let builder' = if len <= 0 then builder else nonEmptyPrefix len + undecoded = if B.length bs >= partUtf8Len (partialUtf8CodePoint s') + then B.drop (i + len) bs -- Reuse bs if possible + else partUtf8ToByteString (partialUtf8CodePoint s') + in (builder', undecoded, s') diff --git a/src/Data/Text/Internal/Encoding/Fusion.hs b/src/Data/Text/Internal/Encoding/Fusion.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding/Fusion.hs @@ -0,0 +1,217 @@ +{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Fusion +-- Copyright : (c) Tom Harper 2008-2009, +-- (c) Bryan O'Sullivan 2009, +-- (c) Duncan Coutts 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Fusible 'Stream'-oriented functions for converting between 'Text' +-- and several common encodings. + +module Data.Text.Internal.Encoding.Fusion + ( + -- * Streaming + streamASCII + , streamUtf8 + , streamUtf16LE + , streamUtf16BE + , streamUtf32LE + , streamUtf32BE + + -- * Unstreaming + , unstream + + , module Data.Text.Internal.Encoding.Fusion.Common + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits (shiftL, shiftR) +import Data.ByteString.Internal (ByteString(..), mallocByteString) +import Data.Text.Internal.Fusion (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size +import Data.Text.Encoding.Error +import Data.Text.Internal.Encoding.Fusion.Common +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Word (Word8, Word16, Word32) +import Foreign.ForeignPtr (ForeignPtr) +import Foreign.Marshal.Utils (copyBytes) +import Foreign.Storable (pokeByteOff) +import qualified Data.ByteString as B +import qualified Data.ByteString.Unsafe as B +import qualified Data.Text.Internal.Encoding.Utf8 as U8 +import qualified Data.Text.Internal.Encoding.Utf16 as U16 +import qualified Data.Text.Internal.Encoding.Utf32 as U32 +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Internal.ByteStringCompat + +streamASCII :: ByteString -> Stream Char +streamASCII bs = Stream next 0 (maxSize l) + where + l = B.length bs + {-# INLINE next #-} + next i + | i >= l = Done + | otherwise = Yield (unsafeChr8 x1) (i+1) + where + x1 = B.unsafeIndex bs i +{-# DEPRECATED streamASCII "Do not use this function" #-} +{-# INLINE [0] streamASCII #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8 +-- encoding. +streamUtf8 :: OnDecodeError -> ByteString -> Stream Char +streamUtf8 onErr bs = Stream next 0 (maxSize l) + where + l = B.length bs + next i + | i >= l = Done + | U8.validate1 x1 = Yield (unsafeChr8 x1) (i+1) + | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (i+2) + | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (i+3) + | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (i+4) + | otherwise = decodeError "streamUtf8" "UTF-8" onErr (Just x1) (i+1) + where + x1 = idx i + x2 = idx (i + 1) + x3 = idx (i + 2) + x4 = idx (i + 3) + idx = B.unsafeIndex bs +{-# INLINE [0] streamUtf8 #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little +-- endian UTF-16 encoding. +streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char +streamUtf16LE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) + where + l = B.length bs + {-# INLINE next #-} + next i + | i >= l = Done + | i+1 < l && U16.validate1 x1 = Yield (unsafeChr16 x1) (i+2) + | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) + | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1) + where + x1 = idx i + (idx (i + 1) `shiftL` 8) + x2 = idx (i + 2) + (idx (i + 3) `shiftL` 8) + idx = word8ToWord16 . B.unsafeIndex bs :: Int -> Word16 +{-# INLINE [0] streamUtf16LE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big +-- endian UTF-16 encoding. +streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char +streamUtf16BE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) + where + l = B.length bs + {-# INLINE next #-} + next i + | i >= l = Done + | i+1 < l && U16.validate1 x1 = Yield (unsafeChr16 x1) (i+2) + | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) + | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1) + where + x1 = (idx i `shiftL` 8) + idx (i + 1) + x2 = (idx (i + 2) `shiftL` 8) + idx (i + 3) + idx = word8ToWord16 . B.unsafeIndex bs :: Int -> Word16 +{-# INLINE [0] streamUtf16BE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big +-- endian UTF-32 encoding. +streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char +streamUtf32BE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) + where + l = B.length bs + {-# INLINE next #-} + next i + | i >= l = Done + | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4) + | otherwise = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (i+1) + where + x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 + x1 = idx i + x2 = idx (i+1) + x3 = idx (i+2) + x4 = idx (i+3) + idx = word8ToWord32 . B.unsafeIndex bs :: Int -> Word32 +{-# INLINE [0] streamUtf32BE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little +-- endian UTF-32 encoding. +streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char +streamUtf32LE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) + where + l = B.length bs + {-# INLINE next #-} + next i + | i >= l = Done + | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4) + | otherwise = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (i+1) + where + x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 + x1 = idx i + x2 = idx $ i+1 + x3 = idx $ i+2 + x4 = idx $ i+3 + idx = word8ToWord32 . B.unsafeIndex bs :: Int -> Word32 +{-# INLINE [0] streamUtf32LE #-} + +-- | /O(n)/ Convert a 'Stream' 'Word8' to a 'ByteString'. +unstream :: Stream Word8 -> ByteString +unstream (Stream next s0 len) = unsafeDupablePerformIO $ do + let mlen = upperBound 4 len + mallocByteString mlen >>= loop mlen 0 s0 + where + loop !n !off !s fp = case next s of + Done -> trimUp fp n off + Skip s' -> loop n off s' fp + Yield x s' + | off == n -> realloc fp n off s' x + | otherwise -> do + unsafeWithForeignPtr fp $ \p -> pokeByteOff p off x + loop n (off+1) s' fp + {-# NOINLINE realloc #-} + realloc fp n off s x = do + let n' = n+n + fp' <- copy0 fp n n' + unsafeWithForeignPtr fp' $ \p -> pokeByteOff p off x + loop n' (off+1) s fp' + {-# NOINLINE trimUp #-} + trimUp fp _ off = return $! mkBS fp off + copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8) + copy0 !src !srcLen !destLen = +#if defined(ASSERTS) + assert (srcLen <= destLen) $ +#endif + do + dest <- mallocByteString destLen + unsafeWithForeignPtr src $ \src' -> + unsafeWithForeignPtr dest $ \dest' -> + copyBytes dest' src' srcLen + return dest + +decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8 + -> s -> Step s Char +decodeError func kind onErr mb i = + case onErr desc mb of + Nothing -> Skip i + Just c -> Yield c i + where desc = "Data.Text.Internal.Encoding.Fusion." ++ func ++ ": Invalid " ++ + kind ++ " stream" + +word8ToWord16 :: Word8 -> Word16 +word8ToWord16 = fromIntegral + +word8ToWord32 :: Word8 -> Word32 +word8ToWord32 = fromIntegral diff --git a/src/Data/Text/Internal/Encoding/Fusion/Common.hs b/src/Data/Text/Internal/Encoding/Fusion/Common.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding/Fusion/Common.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Fusion.Common +-- Copyright : (c) Tom Harper 2008-2009, +-- (c) Bryan O'Sullivan 2009, +-- (c) Duncan Coutts 2009, +-- (c) Jasper Van der Jeugt 2011 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Use at your own risk! +-- +-- Fusible 'Stream'-oriented functions for converting between 'Text' +-- and several common encodings. + +module Data.Text.Internal.Encoding.Fusion.Common + ( + -- * Restreaming + -- Restreaming is the act of converting from one 'Stream' + -- representation to another. + restreamUtf16LE + , restreamUtf16BE + , restreamUtf32LE + , restreamUtf32BE + ) where + +import Data.Bits ((.&.), shiftR) +import Data.Text.Internal.Fusion (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Types (RS(..)) +import Data.Text.Internal.Unsafe.Char (ord) +import Data.Word (Word8) + +restreamUtf16BE :: Stream Char -> Stream Word8 +restreamUtf16BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) + where + next (RS0 s) = case next0 s of + Done -> Done + Skip s' -> Skip (RS0 s') + Yield x s' + | n < 0x10000 -> Yield (intToWord8 $ n `shiftR` 8) $ + RS1 s' (intToWord8 n) + | otherwise -> Yield c1 $ RS3 s' c2 c3 c4 + where + n = ord x + n1 = n - 0x10000 + c1 = intToWord8 (n1 `shiftR` 18 + 0xD8) + c2 = intToWord8 (n1 `shiftR` 10) + n2 = n1 .&. 0x3FF + c3 = intToWord8 (n2 `shiftR` 8 + 0xDC) + c4 = intToWord8 n2 + next (RS1 s x2) = Yield x2 (RS0 s) + next (RS2 s x2 x3) = Yield x2 (RS1 s x3) + next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) + {-# INLINE next #-} +{-# INLINE restreamUtf16BE #-} + +restreamUtf16LE :: Stream Char -> Stream Word8 +restreamUtf16LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) + where + next (RS0 s) = case next0 s of + Done -> Done + Skip s' -> Skip (RS0 s') + Yield x s' + | n < 0x10000 -> Yield (intToWord8 n) $ + RS1 s' (intToWord8 $ shiftR n 8) + | otherwise -> Yield c1 $ RS3 s' c2 c3 c4 + where + n = ord x + n1 = n - 0x10000 + c2 = intToWord8 (shiftR n1 18 + 0xD8) + c1 = intToWord8 (shiftR n1 10) + n2 = n1 .&. 0x3FF + c4 = intToWord8 (shiftR n2 8 + 0xDC) + c3 = intToWord8 n2 + next (RS1 s x2) = Yield x2 (RS0 s) + next (RS2 s x2 x3) = Yield x2 (RS1 s x3) + next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) + {-# INLINE next #-} +{-# INLINE restreamUtf16LE #-} + +restreamUtf32BE :: Stream Char -> Stream Word8 +restreamUtf32BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) + where + next (RS0 s) = case next0 s of + Done -> Done + Skip s' -> Skip (RS0 s') + Yield x s' -> Yield c1 (RS3 s' c2 c3 c4) + where + n = ord x + c1 = intToWord8 $ shiftR n 24 + c2 = intToWord8 $ shiftR n 16 + c3 = intToWord8 $ shiftR n 8 + c4 = intToWord8 n + next (RS1 s x2) = Yield x2 (RS0 s) + next (RS2 s x2 x3) = Yield x2 (RS1 s x3) + next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) + {-# INLINE next #-} +{-# INLINE restreamUtf32BE #-} + +restreamUtf32LE :: Stream Char -> Stream Word8 +restreamUtf32LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2) + where + next (RS0 s) = case next0 s of + Done -> Done + Skip s' -> Skip (RS0 s') + Yield x s' -> Yield c1 (RS3 s' c2 c3 c4) + where + n = ord x + c4 = intToWord8 $ shiftR n 24 + c3 = intToWord8 $ shiftR n 16 + c2 = intToWord8 $ shiftR n 8 + c1 = intToWord8 n + next (RS1 s x2) = Yield x2 (RS0 s) + next (RS2 s x2 x3) = Yield x2 (RS1 s x3) + next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4) + {-# INLINE next #-} +{-# INLINE restreamUtf32LE #-} + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral diff --git a/src/Data/Text/Internal/Encoding/Utf16.hs b/src/Data/Text/Internal/Encoding/Utf16.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding/Utf16.hs @@ -0,0 +1,52 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Utf16 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-16 validation and character manipulation. +module Data.Text.Internal.Encoding.Utf16 + ( + chr2 + , validate1 + , validate2 + ) where + +import GHC.Exts +import GHC.Word (Word16(..)) + +#if !MIN_VERSION_base(4,16,0) +-- harmless to import, except for warnings that it is unused. +import Data.Text.Internal.PrimCompat ( word16ToWord# ) +#endif + +chr2 :: Word16 -> Word16 -> Char +chr2 (W16# a#) (W16# b#) = C# (chr# (upper# +# lower# +# 0x10000#)) + where + !x# = word2Int# (word16ToWord# a#) + !y# = word2Int# (word16ToWord# b#) + !upper# = uncheckedIShiftL# (x# -# 0xD800#) 10# + !lower# = y# -# 0xDC00# +{-# INLINE chr2 #-} + +validate1 :: Word16 -> Bool +validate1 x1 = x1 < 0xD800 || x1 > 0xDFFF +{-# INLINE validate1 #-} + +validate2 :: Word16 -> Word16 -> Bool +validate2 x1 x2 = x1 >= 0xD800 && x1 <= 0xDBFF && + x2 >= 0xDC00 && x2 <= 0xDFFF +{-# INLINE validate2 #-} diff --git a/src/Data/Text/Internal/Encoding/Utf32.hs b/src/Data/Text/Internal/Encoding/Utf32.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding/Utf32.hs @@ -0,0 +1,26 @@ +-- | +-- Module : Data.Text.Internal.Encoding.Utf32 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-32 validation. +module Data.Text.Internal.Encoding.Utf32 + ( + validate + ) where + +import Data.Word (Word32) + +validate :: Word32 -> Bool +validate x1 = x1 < 0xD800 || (x1 > 0xDFFF && x1 <= 0x10FFFF) +{-# INLINE validate #-} diff --git a/src/Data/Text/Internal/Encoding/Utf8.hs b/src/Data/Text/Internal/Encoding/Utf8.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Encoding/Utf8.hs @@ -0,0 +1,318 @@ +{-# LANGUAGE CPP, MagicHash, BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Utf8 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-8 validation and character manipulation. +module Data.Text.Internal.Encoding.Utf8 + ( utf8Length + , utf8LengthByLeader + -- Decomposition + , ord2 + , ord3 + , ord4 + -- Construction + , chr2 + , chr3 + , chr4 + -- * Validation + , validate1 + , validate2 + , validate3 + , validate4 + -- * Naive decoding + , DecoderState(..) + , utf8AcceptState + , utf8RejectState + , updateDecoderState + , DecoderResult(..) + , CodePoint(..) + , utf8DecodeStart + , utf8DecodeContinue + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Bits (Bits(..)) +import Data.Char (ord, chr) +import GHC.Exts +import GHC.Word (Word8(..)) + +#if !MIN_VERSION_base(4,16,0) +-- harmless to import, except for warnings that it is unused. +import Data.Text.Internal.PrimCompat (word8ToWord#) +#endif + +default(Int) + +between :: Word8 -- ^ byte to check + -> Word8 -- ^ lower bound + -> Word8 -- ^ upper bound + -> Bool +between x y z = x >= y && x <= z +{-# INLINE between #-} + +-- This is a branchless version of +-- utf8Length c +-- | ord c < 0x80 = 1 +-- | ord c < 0x800 = 2 +-- | ord c < 0x10000 = 3 +-- | otherwise = 4 +-- Implementation suggested by Alex Mason. + +-- | Measure byte length of UTF-8 encoding for a given character. +-- +-- @since 2.0 +utf8Length :: Char -> Int +utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) +{-# INLINE utf8Length #-} + +-- | Measure byte length of UTF-8 encoding for characters, +-- starting with a given byte. +-- +-- @since 2.0 +utf8LengthByLeader :: Word8 -> Int +utf8LengthByLeader w + | w < 0x80 = 1 + | w < 0xE0 = 2 + | w < 0xF0 = 3 + | otherwise = 4 +{-# INLINE utf8LengthByLeader #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 2 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8) +ord2 c = +#if defined(ASSERTS) + assert (n >= 0x80 && n <= 0x07ff) +#endif + (x1,x2) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 + x2 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord2 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 3 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord3 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8) +ord3 c = +#if defined(ASSERTS) + assert (n >= 0x0800 && n <= 0xffff) +#endif + (x1,x2,x3) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 + x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x3 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord3 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 4 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord4 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8,Word8) +ord4 c = +#if defined(ASSERTS) + assert (n >= 0x10000) +#endif + (x1,x2,x3,x4) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 18) + 0xF0 + x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 + x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x4 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord4 #-} + +-- | @since 1.1.0.0 +chr2 :: Word8 -> Word8 -> Char +chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# + !z2# = y2# -# 0x80# +{-# INLINE chr2 #-} + +-- | @since 1.1.0.0 +chr3 :: Word8 -> Word8 -> Word8 -> Char +chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# + !z3# = y3# -# 0x80# +{-# INLINE chr3 #-} + +-- | @since 1.1.0.0 +chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char +chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = + C# (chr# (z1# +# z2# +# z3# +# z4#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !y4# = word2Int# (word8ToWord# x4#) + !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# + !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# + !z4# = y4# -# 0x80# +{-# INLINE chr4 #-} + +-- | @since 1.1.0.0 +validate1 :: Word8 -> Bool +validate1 x1 = x1 <= 0x7F +{-# INLINE validate1 #-} + +-- | @since 1.1.0.0 +validate2 :: Word8 -> Word8 -> Bool +validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF +{-# INLINE validate2 #-} + +-- | @since 1.1.0.0 +validate3 :: Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate3 #-} +validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 + where + validate3_1 = (x1 == 0xE0) && + between x2 0xA0 0xBF && + between x3 0x80 0xBF + validate3_2 = between x1 0xE1 0xEC && + between x2 0x80 0xBF && + between x3 0x80 0xBF + validate3_3 = x1 == 0xED && + between x2 0x80 0x9F && + between x3 0x80 0xBF + validate3_4 = between x1 0xEE 0xEF && + between x2 0x80 0xBF && + between x3 0x80 0xBF + +-- | @since 1.1.0.0 +validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate4 #-} +validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 + where + validate4_1 = x1 == 0xF0 && + between x2 0x90 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_2 = between x1 0xF1 0xF3 && + between x2 0x80 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_3 = x1 == 0xF4 && + between x2 0x80 0x8F && + between x3 0x80 0xBF && + between x4 0x80 0xBF + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +------------------------------------------------------------------------------- +-- Naive UTF8 decoder. +-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. + +newtype ByteClass = ByteClass Word8 + +byteToClass :: Word8 -> ByteClass +byteToClass n = ByteClass (W8# el#) + where + !(I# n#) = word8ToInt n + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# + +-- | @since 2.0 +newtype DecoderState = DecoderState Word8 + deriving (Eq, Show) + +-- | @since 2.0.2 +utf8AcceptState :: DecoderState +utf8AcceptState = DecoderState 0 + +-- | @since 2.0.2 +utf8RejectState :: DecoderState +utf8RejectState = DecoderState 12 + +updateState :: ByteClass -> DecoderState -> DecoderState +updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) + where + !(I# n#) = word8ToInt (c + s) + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# + +-- | @since 2.0.2 +updateDecoderState :: Word8 -> DecoderState -> DecoderState +updateDecoderState b s = updateState (byteToClass b) s + +-- | @since 2.0 +newtype CodePoint = CodePoint Int + +-- | @since 2.0 +data DecoderResult + = Accept !Char + | Incomplete !DecoderState !CodePoint + | Reject + +-- | @since 2.0 +utf8DecodeStart :: Word8 -> DecoderResult +utf8DecodeStart !w + | st == utf8AcceptState = Accept (chr (word8ToInt w)) + | st == utf8RejectState = Reject + | otherwise = Incomplete st (CodePoint cp) + where + cl@(ByteClass cl') = byteToClass w + st = updateState cl utf8AcceptState + cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w + +-- | @since 2.0 +utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult +utf8DecodeContinue !w !st (CodePoint !cp) + | st' == utf8AcceptState = Accept (chr cp') + | st' == utf8RejectState = Reject + | otherwise = Incomplete st' (CodePoint cp') + where + cl = byteToClass w + st' = updateState cl st + cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f) diff --git a/src/Data/Text/Internal/Fusion.hs b/src/Data/Text/Internal/Fusion.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Fusion.hs @@ -0,0 +1,326 @@ +{-# LANGUAGE BangPatterns, CPP, MagicHash #-} + +-- | +-- Module : Data.Text.Internal.Fusion +-- Copyright : (c) Tom Harper 2008-2009, +-- (c) Bryan O'Sullivan 2009-2010, +-- (c) Duncan Coutts 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Text manipulation functions represented as fusible operations over +-- streams. +module Data.Text.Internal.Fusion + ( + -- * Types + Stream(..) + , Step(..) + + -- * Creation and elimination + , stream + , streamLn + , unstream + , reverseStream + + , length + + -- * Transformations + , reverse + + -- * Construction + -- ** Scans + , reverseScanr + + -- ** Accumulating maps + , mapAccumL + + -- ** Generation and unfolding + , unfoldrN + + -- * Indexing + , index + , findIndex + , countChar + ) where + +import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int, + Num(..), Ord(..), ($), (&&), + otherwise) +import Data.Bits (shiftL, shiftR) +import Data.Text.Internal (Text(..)) +import Data.Text.Internal.Private (runText) +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeWrite) +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Fusion.Common as S +import Data.Text.Internal.Fusion.Types +import Data.Text.Internal.Fusion.Size +import qualified Data.Text.Internal as I +import qualified Data.Text.Internal.Encoding.Utf8 as U8 +import GHC.Stack (HasCallStack) + +default(Int) + +-- | /O(n)/ Convert 'Text' into a 'Stream' 'Char'. +-- +-- __Properties__ +-- +-- @'unstream' . 'stream' = 'Data.Function.id'@ +-- +-- @'stream' . 'unstream' = 'Data.Function.id' @ +stream :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +stream = stream' False +{-# INLINE [0] stream #-} + +-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@ +-- +-- @since 2.1.2 +streamLn :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +streamLn = stream' True + +-- | Shared implementation of 'stream' and 'streamLn'. +stream' :: +#if defined(ASSERTS) + HasCallStack => +#endif + Bool -> Text -> Stream Char +stream' addNl (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) maxLen) + where + maxLen = if addNl then len + 1 else len + !end = off+len + next !i + | i < end = Yield chr (i + l) + | addNl && i == end = Yield '\n' (i + 1) + | otherwise = Done + where + n0 = A.unsafeIndex arr i + n1 = A.unsafeIndex arr (i + 1) + n2 = A.unsafeIndex arr (i + 2) + n3 = A.unsafeIndex arr (i + 3) + + l = U8.utf8LengthByLeader n0 + chr = case l of + 1 -> unsafeChr8 n0 + 2 -> U8.chr2 n0 n1 + 3 -> U8.chr3 n0 n1 n2 + _ -> U8.chr4 n0 n1 n2 n3 +{-# INLINE [0] stream' #-} + +-- | /O(n)/ Converts 'Text' into a 'Stream' 'Char', but iterates +-- backwards through the text. +-- +-- __Properties__ +-- +-- @'unstream' . 'reverseStream' = 'Data.Text.reverse' @ +reverseStream :: Text -> Stream Char +reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 2) len) + where + {-# INLINE next #-} + next !i + | i < off = Done + | n0 < 0x80 = Yield (unsafeChr8 n0) (i - 1) + | n1 >= 0xC0 = Yield (U8.chr2 n1 n0) (i - 2) + | n2 >= 0xC0 = Yield (U8.chr3 n2 n1 n0) (i - 3) + | otherwise = Yield (U8.chr4 n3 n2 n1 n0) (i - 4) + where + n0 = A.unsafeIndex arr i + n1 = A.unsafeIndex arr (i - 1) + n2 = A.unsafeIndex arr (i - 2) + n3 = A.unsafeIndex arr (i - 3) +{-# INLINE [0] reverseStream #-} + +-- | /O(n)/ Convert 'Stream' 'Char' into a 'Text'. +-- +-- __Properties__ +-- +-- @'unstream' . 'stream' = 'Data.Function.id'@ +-- +-- @'stream' . 'unstream' = 'Data.Function.id' @ +unstream :: Stream Char -> Text +unstream (Stream next0 s0 len) = runText $ \done -> do + -- Before encoding each char we perform a buffer realloc check assuming + -- worst case encoding size of four 8-bit units for the char. Just add an + -- extra space to the buffer so that we do not end up reallocating even when + -- all the chars are encoded as single unit. + let mlen = upperBound 4 len + 3 + arr0 <- A.new mlen + let outer !arr !maxi = encode + where + -- keep the common case loop as small as possible + encode !si !di = + case next0 si of + Done -> done arr di + Skip si' -> encode si' di + Yield c si' + -- simply check for the worst case + | maxi < di + 3 -> realloc si di + | otherwise -> do + n <- unsafeWrite arr di c + encode si' (di + n) + + -- keep uncommon case separate from the common case code + {-# NOINLINE realloc #-} + realloc !si !di = do + let newlen = (maxi + 1) * 2 + arr' <- A.resizeM arr newlen + outer arr' (newlen - 1) si di + + outer arr0 (mlen - 1) s0 0 +{-# INLINE [0] unstream #-} +{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-} + + +-- ---------------------------------------------------------------------------- +-- * Basic stream functions + +-- | /O(n)/ Returns the number of characters in a 'Stream'. +-- +-- __Properties__ +-- +-- @'length' . 'stream' = 'Data.Text.length' @ +length :: Stream Char -> Int +length = S.lengthI +{-# INLINE[0] length #-} + +-- | /O(n)/ Reverse the characters of a 'Stream' returning 'Text'. +-- +-- __Properties__ +-- +-- @'reverse' . 'stream' = 'Data.Text.reverse' @ +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Stream Char -> Text +reverse (Stream next s len0) + | isEmpty len0 = I.empty + | otherwise = I.text arr off' len' + where + len0' = upperBound 4 (larger len0 4) + (arr, (off', len')) = A.run2 (A.new len0' >>= loop s (len0'-1) len0') + loop !s0 !i !len marr = + case next s0 of + Done -> return (marr, (j, len-j)) + where j = i + 1 + Skip s1 -> loop s1 i len marr + Yield x s1 | i < least -> {-# SCC "reverse/resize" #-} do + let newLen = len `shiftL` 1 + marr' <- A.new newLen + A.copyM marr' (newLen-len) marr 0 len + _ <- unsafeWrite marr' (len + i - least) x + loop s1 (len + i - least - 1) newLen marr' + | otherwise -> do + _ <- unsafeWrite marr (i - least) x + loop s1 (i - least - 1) len marr + where least = U8.utf8Length x - 1 +{-# INLINE [0] reverse #-} + +-- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with +-- the input and result reversed. +-- +-- __Properties__ +-- +-- @'reverse' . 'reverseScanr' f c . 'reverseStream' = 'Data.Text.scanr' f c @ +reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char +reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low + where + {-# INLINE next #-} + next (Scan1 z s) = Yield z (Scan2 z s) + next (Scan2 z s) = case next0 s of + Yield x s' -> let !x' = f x z + in Yield x' (Scan2 x' s') + Skip s' -> Skip (Scan2 z s') + Done -> Done +{-# INLINE reverseScanr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed +-- value. However, the length of the result is limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the length of the result is known. +-- +-- __Properties__ +-- +-- @'unstream' ('unfoldrN' n f a) = 'Data.Text.unfoldrN' n f a @ +unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Stream Char +unfoldrN n = S.unfoldrNI n +{-# INLINE [0] unfoldrN #-} + +------------------------------------------------------------------------------- +-- ** Indexing streams + +-- | /O(n)/ stream index (subscript) operator, starting from 0. +-- +-- __Properties__ +-- +-- @'index' ('stream' t) n = 'Data.Text.index' t n @ +index :: HasCallStack => Stream Char -> Int -> Char +index = S.indexI +{-# INLINE [0] index #-} + +-- | The 'findIndex' function takes a predicate and a stream and +-- returns the index of the first element in the stream +-- satisfying the predicate. +-- +-- __Properties__ +-- +-- @'findIndex' p . 'stream' = 'Data.Text.findIndex' p @ +findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int +findIndex = S.findIndexI +{-# INLINE [0] findIndex #-} + +-- | /O(n)/ The 'count' function returns the number of times the query +-- element appears in the given stream. +-- +-- __Properties__ +-- +-- @'countChar' c . 'stream' = 'Data.Text.countChar' c @ +countChar :: Char -> Stream Char -> Int +countChar = S.countCharI +{-# INLINE [0] countChar #-} + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. +-- +-- __Properties__ +-- +-- @'mapAccumL' g z0 . 'stream' = 'Data.Text.mapAccumL' g z0@ +mapAccumL :: +#if defined(ASSERTS) + HasCallStack => +#endif + (a -> Char -> (a,Char)) -> a -> Stream Char -> (a, Text) +mapAccumL f z0 (Stream next0 s0 len) = (nz, I.text na 0 nl) + where + (na,(nz,nl)) = A.run2 (A.new mlen >>= \arr -> outer arr mlen z0 s0 0) + where mlen = upperBound 4 len + outer arr top = loop + where + loop !z !s !i = + case next0 s of + Done -> return (arr, (z,i)) + Skip s' -> loop z s' i + Yield x s' + | j >= top -> {-# SCC "mapAccumL/resize" #-} do + let top' = (top + 1) `shiftL` 1 + arr' <- A.resizeM arr top' + outer arr' top' z s i + | otherwise -> do d <- unsafeWrite arr i c + loop z' s' (i+d) + where (z',c) = f z x + j = i + U8.utf8Length c - 1 +{-# INLINE [0] mapAccumL #-} diff --git a/src/Data/Text/Internal/Fusion/CaseMapping.hs b/src/Data/Text/Internal/Fusion/CaseMapping.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Fusion/CaseMapping.hs @@ -0,0 +1,7984 @@ +-- AUTOMATICALLY GENERATED - DO NOT EDIT +-- Generated by scripts/CaseMapping.hs +-- CaseFolding-17.0.0.txt +-- Date: 2025-07-30, 23:54:36 GMT +-- SpecialCasing-17.0.0.txt +-- Date: 2025-07-31, 22:11:55 GMT + +{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +module Data.Text.Internal.Fusion.CaseMapping where +import GHC.Int +import GHC.Exts +import Data.Version (Version, makeVersion) +unicodeVersion :: Version +unicodeVersion = makeVersion [17, 0, 0] +unI64 :: Int64 -> _ {- unboxed Int64 -} +unI64 (I64# n) = n + +upperMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE upperMapping #-} +upperMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 174063699 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 146800710 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 153092166 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 159383622 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 321057542111302 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 334251681644614 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 176160851 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 176160851 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2856322357 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2831156548 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2795504964 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2808087876 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2831156558 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2812282180 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1931484936 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1931484937 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1931484938 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1931484939 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1931484940 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1931484941 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1931484942 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1931484943 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1931484936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1931484937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1931484938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1931484939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1931484940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1931484941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1931484942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1931484943 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1931484968 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1931484969 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1931484970 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1931484971 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1931484972 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1931484973 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1931484974 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1931484975 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1931484968 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1931484969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1931484970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1931484971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1931484972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1931484973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1931484974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1931484975 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1931485032 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1931485033 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1931485034 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1931485035 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1931485036 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1931485037 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1931485038 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1931485039 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1931485032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1931485033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1931485034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1931485035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1931485036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1931485037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1931485038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1931485039 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1931477905 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1931477905 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1931477911 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1931477911 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1931477929 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1931477929 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1931485114 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1931477894 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1931485130 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1931477897 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1931485178 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1931477903 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4050602585752465 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4050602585752471 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4050602585752489 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c5'# -> unI64 452 + '\x01c6'# -> unI64 452 + '\x01c8'# -> unI64 455 + '\x01c9'# -> unI64 455 + '\x01cb'# -> unI64 458 + '\x01cc'# -> unI64 458 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f2'# -> unI64 497 + '\x01f3'# -> unI64 497 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x10d0'# -> unI64 7312 + '\x10d1'# -> unI64 7313 + '\x10d2'# -> unI64 7314 + '\x10d3'# -> unI64 7315 + '\x10d4'# -> unI64 7316 + '\x10d5'# -> unI64 7317 + '\x10d6'# -> unI64 7318 + '\x10d7'# -> unI64 7319 + '\x10d8'# -> unI64 7320 + '\x10d9'# -> unI64 7321 + '\x10da'# -> unI64 7322 + '\x10db'# -> unI64 7323 + '\x10dc'# -> unI64 7324 + '\x10dd'# -> unI64 7325 + '\x10de'# -> unI64 7326 + '\x10df'# -> unI64 7327 + '\x10e0'# -> unI64 7328 + '\x10e1'# -> unI64 7329 + '\x10e2'# -> unI64 7330 + '\x10e3'# -> unI64 7331 + '\x10e4'# -> unI64 7332 + '\x10e5'# -> unI64 7333 + '\x10e6'# -> unI64 7334 + '\x10e7'# -> unI64 7335 + '\x10e8'# -> unI64 7336 + '\x10e9'# -> unI64 7337 + '\x10ea'# -> unI64 7338 + '\x10eb'# -> unI64 7339 + '\x10ec'# -> unI64 7340 + '\x10ed'# -> unI64 7341 + '\x10ee'# -> unI64 7342 + '\x10ef'# -> unI64 7343 + '\x10f0'# -> unI64 7344 + '\x10f1'# -> unI64 7345 + '\x10f2'# -> unI64 7346 + '\x10f3'# -> unI64 7347 + '\x10f4'# -> unI64 7348 + '\x10f5'# -> unI64 7349 + '\x10f6'# -> unI64 7350 + '\x10f7'# -> unI64 7351 + '\x10f8'# -> unI64 7352 + '\x10f9'# -> unI64 7353 + '\x10fa'# -> unI64 7354 + '\x10fd'# -> unI64 7357 + '\x10fe'# -> unI64 7358 + '\x10ff'# -> unI64 7359 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +lowerMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE lowerMapping #-} +lowerMapping = \case + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 8064 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 8065 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 8066 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 8067 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 8068 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 8069 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 8070 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 8071 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 8080 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 8081 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 8082 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 8083 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 8084 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 8085 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 8086 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 8087 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 8096 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 8097 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 8098 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 8099 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 8100 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 8101 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 8102 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 8103 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 8115 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 8131 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 8179 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c89'# -> unI64 7306 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1e9e'# -> unI64 223 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7cb'# -> unI64 612 + '\xa7cc'# -> unI64 42957 + '\xa7ce'# -> unI64 42959 + '\xa7d0'# -> unI64 42961 + '\xa7d2'# -> unI64 42963 + '\xa7d4'# -> unI64 42965 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7da'# -> unI64 42971 + '\xa7dc'# -> unI64 411 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x10d50'# -> unI64 68976 + '\x10d51'# -> unI64 68977 + '\x10d52'# -> unI64 68978 + '\x10d53'# -> unI64 68979 + '\x10d54'# -> unI64 68980 + '\x10d55'# -> unI64 68981 + '\x10d56'# -> unI64 68982 + '\x10d57'# -> unI64 68983 + '\x10d58'# -> unI64 68984 + '\x10d59'# -> unI64 68985 + '\x10d5a'# -> unI64 68986 + '\x10d5b'# -> unI64 68987 + '\x10d5c'# -> unI64 68988 + '\x10d5d'# -> unI64 68989 + '\x10d5e'# -> unI64 68990 + '\x10d5f'# -> unI64 68991 + '\x10d60'# -> unI64 68992 + '\x10d61'# -> unI64 68993 + '\x10d62'# -> unI64 68994 + '\x10d63'# -> unI64 68995 + '\x10d64'# -> unI64 68996 + '\x10d65'# -> unI64 68997 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x16ea0'# -> unI64 93883 + '\x16ea1'# -> unI64 93884 + '\x16ea2'# -> unI64 93885 + '\x16ea3'# -> unI64 93886 + '\x16ea4'# -> unI64 93887 + '\x16ea5'# -> unI64 93888 + '\x16ea6'# -> unI64 93889 + '\x16ea7'# -> unI64 93890 + '\x16ea8'# -> unI64 93891 + '\x16ea9'# -> unI64 93892 + '\x16eaa'# -> unI64 93893 + '\x16eab'# -> unI64 93894 + '\x16eac'# -> unI64 93895 + '\x16ead'# -> unI64 93896 + '\x16eae'# -> unI64 93897 + '\x16eaf'# -> unI64 93898 + '\x16eb0'# -> unI64 93899 + '\x16eb1'# -> unI64 93900 + '\x16eb2'# -> unI64 93901 + '\x16eb3'# -> unI64 93902 + '\x16eb4'# -> unI64 93903 + '\x16eb5'# -> unI64 93904 + '\x16eb6'# -> unI64 93905 + '\x16eb7'# -> unI64 93906 + '\x16eb8'# -> unI64 93907 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 +titleMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE titleMapping #-} +titleMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172563 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909574 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201030 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492486 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575494 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108806 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269715 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269715 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985653 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819844 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168260 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751172 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819854 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945476 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 8072 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 8073 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 8074 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 8075 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 8076 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 8077 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 8078 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 8079 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 8088 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 8089 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 8090 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 8091 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 8092 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 8093 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 8094 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 8095 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 8104 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 8105 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 8106 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 8107 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 8108 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 8109 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 8110 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 8111 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 8124 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 8140 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 8188 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1755324346 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1755317126 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1755324362 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1755317129 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1755324410 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1755317135 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 3681166678819729 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 3681166678819735 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 3681166678819753 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c4'# -> unI64 453 + '\x01c6'# -> unI64 453 + '\x01c7'# -> unI64 456 + '\x01c9'# -> unI64 456 + '\x01ca'# -> unI64 459 + '\x01cc'# -> unI64 459 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f1'# -> unI64 498 + '\x01f3'# -> unI64 498 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +foldMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE foldMapping #-} +foldMapping = \case + -- LATIN CAPITAL LETTER A + '\x0041'# -> unI64 97 + -- LATIN CAPITAL LETTER B + '\x0042'# -> unI64 98 + -- LATIN CAPITAL LETTER C + '\x0043'# -> unI64 99 + -- LATIN CAPITAL LETTER D + '\x0044'# -> unI64 100 + -- LATIN CAPITAL LETTER E + '\x0045'# -> unI64 101 + -- LATIN CAPITAL LETTER F + '\x0046'# -> unI64 102 + -- LATIN CAPITAL LETTER G + '\x0047'# -> unI64 103 + -- LATIN CAPITAL LETTER H + '\x0048'# -> unI64 104 + -- LATIN CAPITAL LETTER I + '\x0049'# -> unI64 105 + -- LATIN CAPITAL LETTER J + '\x004a'# -> unI64 106 + -- LATIN CAPITAL LETTER K + '\x004b'# -> unI64 107 + -- LATIN CAPITAL LETTER L + '\x004c'# -> unI64 108 + -- LATIN CAPITAL LETTER M + '\x004d'# -> unI64 109 + -- LATIN CAPITAL LETTER N + '\x004e'# -> unI64 110 + -- LATIN CAPITAL LETTER O + '\x004f'# -> unI64 111 + -- LATIN CAPITAL LETTER P + '\x0050'# -> unI64 112 + -- LATIN CAPITAL LETTER Q + '\x0051'# -> unI64 113 + -- LATIN CAPITAL LETTER R + '\x0052'# -> unI64 114 + -- LATIN CAPITAL LETTER S + '\x0053'# -> unI64 115 + -- LATIN CAPITAL LETTER T + '\x0054'# -> unI64 116 + -- LATIN CAPITAL LETTER U + '\x0055'# -> unI64 117 + -- LATIN CAPITAL LETTER V + '\x0056'# -> unI64 118 + -- LATIN CAPITAL LETTER W + '\x0057'# -> unI64 119 + -- LATIN CAPITAL LETTER X + '\x0058'# -> unI64 120 + -- LATIN CAPITAL LETTER Y + '\x0059'# -> unI64 121 + -- LATIN CAPITAL LETTER Z + '\x005a'# -> unI64 122 + -- MICRO SIGN + '\x00b5'# -> unI64 956 + -- LATIN CAPITAL LETTER A WITH GRAVE + '\x00c0'# -> unI64 224 + -- LATIN CAPITAL LETTER A WITH ACUTE + '\x00c1'# -> unI64 225 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX + '\x00c2'# -> unI64 226 + -- LATIN CAPITAL LETTER A WITH TILDE + '\x00c3'# -> unI64 227 + -- LATIN CAPITAL LETTER A WITH DIAERESIS + '\x00c4'# -> unI64 228 + -- LATIN CAPITAL LETTER A WITH RING ABOVE + '\x00c5'# -> unI64 229 + -- LATIN CAPITAL LETTER AE + '\x00c6'# -> unI64 230 + -- LATIN CAPITAL LETTER C WITH CEDILLA + '\x00c7'# -> unI64 231 + -- LATIN CAPITAL LETTER E WITH GRAVE + '\x00c8'# -> unI64 232 + -- LATIN CAPITAL LETTER E WITH ACUTE + '\x00c9'# -> unI64 233 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX + '\x00ca'# -> unI64 234 + -- LATIN CAPITAL LETTER E WITH DIAERESIS + '\x00cb'# -> unI64 235 + -- LATIN CAPITAL LETTER I WITH GRAVE + '\x00cc'# -> unI64 236 + -- LATIN CAPITAL LETTER I WITH ACUTE + '\x00cd'# -> unI64 237 + -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX + '\x00ce'# -> unI64 238 + -- LATIN CAPITAL LETTER I WITH DIAERESIS + '\x00cf'# -> unI64 239 + -- LATIN CAPITAL LETTER ETH + '\x00d0'# -> unI64 240 + -- LATIN CAPITAL LETTER N WITH TILDE + '\x00d1'# -> unI64 241 + -- LATIN CAPITAL LETTER O WITH GRAVE + '\x00d2'# -> unI64 242 + -- LATIN CAPITAL LETTER O WITH ACUTE + '\x00d3'# -> unI64 243 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX + '\x00d4'# -> unI64 244 + -- LATIN CAPITAL LETTER O WITH TILDE + '\x00d5'# -> unI64 245 + -- LATIN CAPITAL LETTER O WITH DIAERESIS + '\x00d6'# -> unI64 246 + -- LATIN CAPITAL LETTER O WITH STROKE + '\x00d8'# -> unI64 248 + -- LATIN CAPITAL LETTER U WITH GRAVE + '\x00d9'# -> unI64 249 + -- LATIN CAPITAL LETTER U WITH ACUTE + '\x00da'# -> unI64 250 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX + '\x00db'# -> unI64 251 + -- LATIN CAPITAL LETTER U WITH DIAERESIS + '\x00dc'# -> unI64 252 + -- LATIN CAPITAL LETTER Y WITH ACUTE + '\x00dd'# -> unI64 253 + -- LATIN CAPITAL LETTER THORN + '\x00de'# -> unI64 254 + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH MACRON + '\x0100'# -> unI64 257 + -- LATIN CAPITAL LETTER A WITH BREVE + '\x0102'# -> unI64 259 + -- LATIN CAPITAL LETTER A WITH OGONEK + '\x0104'# -> unI64 261 + -- LATIN CAPITAL LETTER C WITH ACUTE + '\x0106'# -> unI64 263 + -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX + '\x0108'# -> unI64 265 + -- LATIN CAPITAL LETTER C WITH DOT ABOVE + '\x010a'# -> unI64 267 + -- LATIN CAPITAL LETTER C WITH CARON + '\x010c'# -> unI64 269 + -- LATIN CAPITAL LETTER D WITH CARON + '\x010e'# -> unI64 271 + -- LATIN CAPITAL LETTER D WITH STROKE + '\x0110'# -> unI64 273 + -- LATIN CAPITAL LETTER E WITH MACRON + '\x0112'# -> unI64 275 + -- LATIN CAPITAL LETTER E WITH BREVE + '\x0114'# -> unI64 277 + -- LATIN CAPITAL LETTER E WITH DOT ABOVE + '\x0116'# -> unI64 279 + -- LATIN CAPITAL LETTER E WITH OGONEK + '\x0118'# -> unI64 281 + -- LATIN CAPITAL LETTER E WITH CARON + '\x011a'# -> unI64 283 + -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX + '\x011c'# -> unI64 285 + -- LATIN CAPITAL LETTER G WITH BREVE + '\x011e'# -> unI64 287 + -- LATIN CAPITAL LETTER G WITH DOT ABOVE + '\x0120'# -> unI64 289 + -- LATIN CAPITAL LETTER G WITH CEDILLA + '\x0122'# -> unI64 291 + -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX + '\x0124'# -> unI64 293 + -- LATIN CAPITAL LETTER H WITH STROKE + '\x0126'# -> unI64 295 + -- LATIN CAPITAL LETTER I WITH TILDE + '\x0128'# -> unI64 297 + -- LATIN CAPITAL LETTER I WITH MACRON + '\x012a'# -> unI64 299 + -- LATIN CAPITAL LETTER I WITH BREVE + '\x012c'# -> unI64 301 + -- LATIN CAPITAL LETTER I WITH OGONEK + '\x012e'# -> unI64 303 + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- LATIN CAPITAL LIGATURE IJ + '\x0132'# -> unI64 307 + -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX + '\x0134'# -> unI64 309 + -- LATIN CAPITAL LETTER K WITH CEDILLA + '\x0136'# -> unI64 311 + -- LATIN CAPITAL LETTER L WITH ACUTE + '\x0139'# -> unI64 314 + -- LATIN CAPITAL LETTER L WITH CEDILLA + '\x013b'# -> unI64 316 + -- LATIN CAPITAL LETTER L WITH CARON + '\x013d'# -> unI64 318 + -- LATIN CAPITAL LETTER L WITH MIDDLE DOT + '\x013f'# -> unI64 320 + -- LATIN CAPITAL LETTER L WITH STROKE + '\x0141'# -> unI64 322 + -- LATIN CAPITAL LETTER N WITH ACUTE + '\x0143'# -> unI64 324 + -- LATIN CAPITAL LETTER N WITH CEDILLA + '\x0145'# -> unI64 326 + -- LATIN CAPITAL LETTER N WITH CARON + '\x0147'# -> unI64 328 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 230687420 + -- LATIN CAPITAL LETTER ENG + '\x014a'# -> unI64 331 + -- LATIN CAPITAL LETTER O WITH MACRON + '\x014c'# -> unI64 333 + -- LATIN CAPITAL LETTER O WITH BREVE + '\x014e'# -> unI64 335 + -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + '\x0150'# -> unI64 337 + -- LATIN CAPITAL LIGATURE OE + '\x0152'# -> unI64 339 + -- LATIN CAPITAL LETTER R WITH ACUTE + '\x0154'# -> unI64 341 + -- LATIN CAPITAL LETTER R WITH CEDILLA + '\x0156'# -> unI64 343 + -- LATIN CAPITAL LETTER R WITH CARON + '\x0158'# -> unI64 345 + -- LATIN CAPITAL LETTER S WITH ACUTE + '\x015a'# -> unI64 347 + -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX + '\x015c'# -> unI64 349 + -- LATIN CAPITAL LETTER S WITH CEDILLA + '\x015e'# -> unI64 351 + -- LATIN CAPITAL LETTER S WITH CARON + '\x0160'# -> unI64 353 + -- LATIN CAPITAL LETTER T WITH CEDILLA + '\x0162'# -> unI64 355 + -- LATIN CAPITAL LETTER T WITH CARON + '\x0164'# -> unI64 357 + -- LATIN CAPITAL LETTER T WITH STROKE + '\x0166'# -> unI64 359 + -- LATIN CAPITAL LETTER U WITH TILDE + '\x0168'# -> unI64 361 + -- LATIN CAPITAL LETTER U WITH MACRON + '\x016a'# -> unI64 363 + -- LATIN CAPITAL LETTER U WITH BREVE + '\x016c'# -> unI64 365 + -- LATIN CAPITAL LETTER U WITH RING ABOVE + '\x016e'# -> unI64 367 + -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + '\x0170'# -> unI64 369 + -- LATIN CAPITAL LETTER U WITH OGONEK + '\x0172'# -> unI64 371 + -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX + '\x0174'# -> unI64 373 + -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + '\x0176'# -> unI64 375 + -- LATIN CAPITAL LETTER Y WITH DIAERESIS + '\x0178'# -> unI64 255 + -- LATIN CAPITAL LETTER Z WITH ACUTE + '\x0179'# -> unI64 378 + -- LATIN CAPITAL LETTER Z WITH DOT ABOVE + '\x017b'# -> unI64 380 + -- LATIN CAPITAL LETTER Z WITH CARON + '\x017d'# -> unI64 382 + -- LATIN SMALL LETTER LONG S + '\x017f'# -> unI64 115 + -- LATIN CAPITAL LETTER B WITH HOOK + '\x0181'# -> unI64 595 + -- LATIN CAPITAL LETTER B WITH TOPBAR + '\x0182'# -> unI64 387 + -- LATIN CAPITAL LETTER TONE SIX + '\x0184'# -> unI64 389 + -- LATIN CAPITAL LETTER OPEN O + '\x0186'# -> unI64 596 + -- LATIN CAPITAL LETTER C WITH HOOK + '\x0187'# -> unI64 392 + -- LATIN CAPITAL LETTER AFRICAN D + '\x0189'# -> unI64 598 + -- LATIN CAPITAL LETTER D WITH HOOK + '\x018a'# -> unI64 599 + -- LATIN CAPITAL LETTER D WITH TOPBAR + '\x018b'# -> unI64 396 + -- LATIN CAPITAL LETTER REVERSED E + '\x018e'# -> unI64 477 + -- LATIN CAPITAL LETTER SCHWA + '\x018f'# -> unI64 601 + -- LATIN CAPITAL LETTER OPEN E + '\x0190'# -> unI64 603 + -- LATIN CAPITAL LETTER F WITH HOOK + '\x0191'# -> unI64 402 + -- LATIN CAPITAL LETTER G WITH HOOK + '\x0193'# -> unI64 608 + -- LATIN CAPITAL LETTER GAMMA + '\x0194'# -> unI64 611 + -- LATIN CAPITAL LETTER IOTA + '\x0196'# -> unI64 617 + -- LATIN CAPITAL LETTER I WITH STROKE + '\x0197'# -> unI64 616 + -- LATIN CAPITAL LETTER K WITH HOOK + '\x0198'# -> unI64 409 + -- LATIN CAPITAL LETTER TURNED M + '\x019c'# -> unI64 623 + -- LATIN CAPITAL LETTER N WITH LEFT HOOK + '\x019d'# -> unI64 626 + -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE + '\x019f'# -> unI64 629 + -- LATIN CAPITAL LETTER O WITH HORN + '\x01a0'# -> unI64 417 + -- LATIN CAPITAL LETTER OI + '\x01a2'# -> unI64 419 + -- LATIN CAPITAL LETTER P WITH HOOK + '\x01a4'# -> unI64 421 + -- LATIN LETTER YR + '\x01a6'# -> unI64 640 + -- LATIN CAPITAL LETTER TONE TWO + '\x01a7'# -> unI64 424 + -- LATIN CAPITAL LETTER ESH + '\x01a9'# -> unI64 643 + -- LATIN CAPITAL LETTER T WITH HOOK + '\x01ac'# -> unI64 429 + -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + '\x01ae'# -> unI64 648 + -- LATIN CAPITAL LETTER U WITH HORN + '\x01af'# -> unI64 432 + -- LATIN CAPITAL LETTER UPSILON + '\x01b1'# -> unI64 650 + -- LATIN CAPITAL LETTER V WITH HOOK + '\x01b2'# -> unI64 651 + -- LATIN CAPITAL LETTER Y WITH HOOK + '\x01b3'# -> unI64 436 + -- LATIN CAPITAL LETTER Z WITH STROKE + '\x01b5'# -> unI64 438 + -- LATIN CAPITAL LETTER EZH + '\x01b7'# -> unI64 658 + -- LATIN CAPITAL LETTER EZH REVERSED + '\x01b8'# -> unI64 441 + -- LATIN CAPITAL LETTER TONE FIVE + '\x01bc'# -> unI64 445 + -- LATIN CAPITAL LETTER DZ WITH CARON + '\x01c4'# -> unI64 454 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + '\x01c5'# -> unI64 454 + -- LATIN CAPITAL LETTER LJ + '\x01c7'# -> unI64 457 + -- LATIN CAPITAL LETTER L WITH SMALL LETTER J + '\x01c8'# -> unI64 457 + -- LATIN CAPITAL LETTER NJ + '\x01ca'# -> unI64 460 + -- LATIN CAPITAL LETTER N WITH SMALL LETTER J + '\x01cb'# -> unI64 460 + -- LATIN CAPITAL LETTER A WITH CARON + '\x01cd'# -> unI64 462 + -- LATIN CAPITAL LETTER I WITH CARON + '\x01cf'# -> unI64 464 + -- LATIN CAPITAL LETTER O WITH CARON + '\x01d1'# -> unI64 466 + -- LATIN CAPITAL LETTER U WITH CARON + '\x01d3'# -> unI64 468 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + '\x01d5'# -> unI64 470 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + '\x01d7'# -> unI64 472 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + '\x01d9'# -> unI64 474 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + '\x01db'# -> unI64 476 + -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + '\x01de'# -> unI64 479 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + '\x01e0'# -> unI64 481 + -- LATIN CAPITAL LETTER AE WITH MACRON + '\x01e2'# -> unI64 483 + -- LATIN CAPITAL LETTER G WITH STROKE + '\x01e4'# -> unI64 485 + -- LATIN CAPITAL LETTER G WITH CARON + '\x01e6'# -> unI64 487 + -- LATIN CAPITAL LETTER K WITH CARON + '\x01e8'# -> unI64 489 + -- LATIN CAPITAL LETTER O WITH OGONEK + '\x01ea'# -> unI64 491 + -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + '\x01ec'# -> unI64 493 + -- LATIN CAPITAL LETTER EZH WITH CARON + '\x01ee'# -> unI64 495 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778666 + -- LATIN CAPITAL LETTER DZ + '\x01f1'# -> unI64 499 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z + '\x01f2'# -> unI64 499 + -- LATIN CAPITAL LETTER G WITH ACUTE + '\x01f4'# -> unI64 501 + -- LATIN CAPITAL LETTER HWAIR + '\x01f6'# -> unI64 405 + -- LATIN CAPITAL LETTER WYNN + '\x01f7'# -> unI64 447 + -- LATIN CAPITAL LETTER N WITH GRAVE + '\x01f8'# -> unI64 505 + -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + '\x01fa'# -> unI64 507 + -- LATIN CAPITAL LETTER AE WITH ACUTE + '\x01fc'# -> unI64 509 + -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + '\x01fe'# -> unI64 511 + -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + '\x0200'# -> unI64 513 + -- LATIN CAPITAL LETTER A WITH INVERTED BREVE + '\x0202'# -> unI64 515 + -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + '\x0204'# -> unI64 517 + -- LATIN CAPITAL LETTER E WITH INVERTED BREVE + '\x0206'# -> unI64 519 + -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + '\x0208'# -> unI64 521 + -- LATIN CAPITAL LETTER I WITH INVERTED BREVE + '\x020a'# -> unI64 523 + -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + '\x020c'# -> unI64 525 + -- LATIN CAPITAL LETTER O WITH INVERTED BREVE + '\x020e'# -> unI64 527 + -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + '\x0210'# -> unI64 529 + -- LATIN CAPITAL LETTER R WITH INVERTED BREVE + '\x0212'# -> unI64 531 + -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + '\x0214'# -> unI64 533 + -- LATIN CAPITAL LETTER U WITH INVERTED BREVE + '\x0216'# -> unI64 535 + -- LATIN CAPITAL LETTER S WITH COMMA BELOW + '\x0218'# -> unI64 537 + -- LATIN CAPITAL LETTER T WITH COMMA BELOW + '\x021a'# -> unI64 539 + -- LATIN CAPITAL LETTER YOGH + '\x021c'# -> unI64 541 + -- LATIN CAPITAL LETTER H WITH CARON + '\x021e'# -> unI64 543 + -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + '\x0220'# -> unI64 414 + -- LATIN CAPITAL LETTER OU + '\x0222'# -> unI64 547 + -- LATIN CAPITAL LETTER Z WITH HOOK + '\x0224'# -> unI64 549 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE + '\x0226'# -> unI64 551 + -- LATIN CAPITAL LETTER E WITH CEDILLA + '\x0228'# -> unI64 553 + -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + '\x022a'# -> unI64 555 + -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON + '\x022c'# -> unI64 557 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE + '\x022e'# -> unI64 559 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + '\x0230'# -> unI64 561 + -- LATIN CAPITAL LETTER Y WITH MACRON + '\x0232'# -> unI64 563 + -- LATIN CAPITAL LETTER A WITH STROKE + '\x023a'# -> unI64 11365 + -- LATIN CAPITAL LETTER C WITH STROKE + '\x023b'# -> unI64 572 + -- LATIN CAPITAL LETTER L WITH BAR + '\x023d'# -> unI64 410 + -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + '\x023e'# -> unI64 11366 + -- LATIN CAPITAL LETTER GLOTTAL STOP + '\x0241'# -> unI64 578 + -- LATIN CAPITAL LETTER B WITH STROKE + '\x0243'# -> unI64 384 + -- LATIN CAPITAL LETTER U BAR + '\x0244'# -> unI64 649 + -- LATIN CAPITAL LETTER TURNED V + '\x0245'# -> unI64 652 + -- LATIN CAPITAL LETTER E WITH STROKE + '\x0246'# -> unI64 583 + -- LATIN CAPITAL LETTER J WITH STROKE + '\x0248'# -> unI64 585 + -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL + '\x024a'# -> unI64 587 + -- LATIN CAPITAL LETTER R WITH STROKE + '\x024c'# -> unI64 589 + -- LATIN CAPITAL LETTER Y WITH STROKE + '\x024e'# -> unI64 591 + -- COMBINING GREEK YPOGEGRAMMENI + '\x0345'# -> unI64 953 + -- GREEK CAPITAL LETTER HETA + '\x0370'# -> unI64 881 + -- GREEK CAPITAL LETTER ARCHAIC SAMPI + '\x0372'# -> unI64 883 + -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA + '\x0376'# -> unI64 887 + -- GREEK CAPITAL LETTER YOT + '\x037f'# -> unI64 1011 + -- GREEK CAPITAL LETTER ALPHA WITH TONOS + '\x0386'# -> unI64 940 + -- GREEK CAPITAL LETTER EPSILON WITH TONOS + '\x0388'# -> unI64 941 + -- GREEK CAPITAL LETTER ETA WITH TONOS + '\x0389'# -> unI64 942 + -- GREEK CAPITAL LETTER IOTA WITH TONOS + '\x038a'# -> unI64 943 + -- GREEK CAPITAL LETTER OMICRON WITH TONOS + '\x038c'# -> unI64 972 + -- GREEK CAPITAL LETTER UPSILON WITH TONOS + '\x038e'# -> unI64 973 + -- GREEK CAPITAL LETTER OMEGA WITH TONOS + '\x038f'# -> unI64 974 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429881 + -- GREEK CAPITAL LETTER ALPHA + '\x0391'# -> unI64 945 + -- GREEK CAPITAL LETTER BETA + '\x0392'# -> unI64 946 + -- GREEK CAPITAL LETTER GAMMA + '\x0393'# -> unI64 947 + -- GREEK CAPITAL LETTER DELTA + '\x0394'# -> unI64 948 + -- GREEK CAPITAL LETTER EPSILON + '\x0395'# -> unI64 949 + -- GREEK CAPITAL LETTER ZETA + '\x0396'# -> unI64 950 + -- GREEK CAPITAL LETTER ETA + '\x0397'# -> unI64 951 + -- GREEK CAPITAL LETTER THETA + '\x0398'# -> unI64 952 + -- GREEK CAPITAL LETTER IOTA + '\x0399'# -> unI64 953 + -- GREEK CAPITAL LETTER KAPPA + '\x039a'# -> unI64 954 + -- GREEK CAPITAL LETTER LAMDA + '\x039b'# -> unI64 955 + -- GREEK CAPITAL LETTER MU + '\x039c'# -> unI64 956 + -- GREEK CAPITAL LETTER NU + '\x039d'# -> unI64 957 + -- GREEK CAPITAL LETTER XI + '\x039e'# -> unI64 958 + -- GREEK CAPITAL LETTER OMICRON + '\x039f'# -> unI64 959 + -- GREEK CAPITAL LETTER PI + '\x03a0'# -> unI64 960 + -- GREEK CAPITAL LETTER RHO + '\x03a1'# -> unI64 961 + -- GREEK CAPITAL LETTER SIGMA + '\x03a3'# -> unI64 963 + -- GREEK CAPITAL LETTER TAU + '\x03a4'# -> unI64 964 + -- GREEK CAPITAL LETTER UPSILON + '\x03a5'# -> unI64 965 + -- GREEK CAPITAL LETTER PHI + '\x03a6'# -> unI64 966 + -- GREEK CAPITAL LETTER CHI + '\x03a7'# -> unI64 967 + -- GREEK CAPITAL LETTER PSI + '\x03a8'# -> unI64 968 + -- GREEK CAPITAL LETTER OMEGA + '\x03a9'# -> unI64 969 + -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + '\x03aa'# -> unI64 970 + -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + '\x03ab'# -> unI64 971 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER FINAL SIGMA + '\x03c2'# -> unI64 963 + -- GREEK CAPITAL KAI SYMBOL + '\x03cf'# -> unI64 983 + -- GREEK BETA SYMBOL + '\x03d0'# -> unI64 946 + -- GREEK THETA SYMBOL + '\x03d1'# -> unI64 952 + -- GREEK PHI SYMBOL + '\x03d5'# -> unI64 966 + -- GREEK PI SYMBOL + '\x03d6'# -> unI64 960 + -- GREEK LETTER ARCHAIC KOPPA + '\x03d8'# -> unI64 985 + -- GREEK LETTER STIGMA + '\x03da'# -> unI64 987 + -- GREEK LETTER DIGAMMA + '\x03dc'# -> unI64 989 + -- GREEK LETTER KOPPA + '\x03de'# -> unI64 991 + -- GREEK LETTER SAMPI + '\x03e0'# -> unI64 993 + -- COPTIC CAPITAL LETTER SHEI + '\x03e2'# -> unI64 995 + -- COPTIC CAPITAL LETTER FEI + '\x03e4'# -> unI64 997 + -- COPTIC CAPITAL LETTER KHEI + '\x03e6'# -> unI64 999 + -- COPTIC CAPITAL LETTER HORI + '\x03e8'# -> unI64 1001 + -- COPTIC CAPITAL LETTER GANGIA + '\x03ea'# -> unI64 1003 + -- COPTIC CAPITAL LETTER SHIMA + '\x03ec'# -> unI64 1005 + -- COPTIC CAPITAL LETTER DEI + '\x03ee'# -> unI64 1007 + -- GREEK KAPPA SYMBOL + '\x03f0'# -> unI64 954 + -- GREEK RHO SYMBOL + '\x03f1'# -> unI64 961 + -- GREEK CAPITAL THETA SYMBOL + '\x03f4'# -> unI64 952 + -- GREEK LUNATE EPSILON SYMBOL + '\x03f5'# -> unI64 949 + -- GREEK CAPITAL LETTER SHO + '\x03f7'# -> unI64 1016 + -- GREEK CAPITAL LUNATE SIGMA SYMBOL + '\x03f9'# -> unI64 1010 + -- GREEK CAPITAL LETTER SAN + '\x03fa'# -> unI64 1019 + -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL + '\x03fd'# -> unI64 891 + -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL + '\x03fe'# -> unI64 892 + -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + '\x03ff'# -> unI64 893 + -- CYRILLIC CAPITAL LETTER IE WITH GRAVE + '\x0400'# -> unI64 1104 + -- CYRILLIC CAPITAL LETTER IO + '\x0401'# -> unI64 1105 + -- CYRILLIC CAPITAL LETTER DJE + '\x0402'# -> unI64 1106 + -- CYRILLIC CAPITAL LETTER GJE + '\x0403'# -> unI64 1107 + -- CYRILLIC CAPITAL LETTER UKRAINIAN IE + '\x0404'# -> unI64 1108 + -- CYRILLIC CAPITAL LETTER DZE + '\x0405'# -> unI64 1109 + -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + '\x0406'# -> unI64 1110 + -- CYRILLIC CAPITAL LETTER YI + '\x0407'# -> unI64 1111 + -- CYRILLIC CAPITAL LETTER JE + '\x0408'# -> unI64 1112 + -- CYRILLIC CAPITAL LETTER LJE + '\x0409'# -> unI64 1113 + -- CYRILLIC CAPITAL LETTER NJE + '\x040a'# -> unI64 1114 + -- CYRILLIC CAPITAL LETTER TSHE + '\x040b'# -> unI64 1115 + -- CYRILLIC CAPITAL LETTER KJE + '\x040c'# -> unI64 1116 + -- CYRILLIC CAPITAL LETTER I WITH GRAVE + '\x040d'# -> unI64 1117 + -- CYRILLIC CAPITAL LETTER SHORT U + '\x040e'# -> unI64 1118 + -- CYRILLIC CAPITAL LETTER DZHE + '\x040f'# -> unI64 1119 + -- CYRILLIC CAPITAL LETTER A + '\x0410'# -> unI64 1072 + -- CYRILLIC CAPITAL LETTER BE + '\x0411'# -> unI64 1073 + -- CYRILLIC CAPITAL LETTER VE + '\x0412'# -> unI64 1074 + -- CYRILLIC CAPITAL LETTER GHE + '\x0413'# -> unI64 1075 + -- CYRILLIC CAPITAL LETTER DE + '\x0414'# -> unI64 1076 + -- CYRILLIC CAPITAL LETTER IE + '\x0415'# -> unI64 1077 + -- CYRILLIC CAPITAL LETTER ZHE + '\x0416'# -> unI64 1078 + -- CYRILLIC CAPITAL LETTER ZE + '\x0417'# -> unI64 1079 + -- CYRILLIC CAPITAL LETTER I + '\x0418'# -> unI64 1080 + -- CYRILLIC CAPITAL LETTER SHORT I + '\x0419'# -> unI64 1081 + -- CYRILLIC CAPITAL LETTER KA + '\x041a'# -> unI64 1082 + -- CYRILLIC CAPITAL LETTER EL + '\x041b'# -> unI64 1083 + -- CYRILLIC CAPITAL LETTER EM + '\x041c'# -> unI64 1084 + -- CYRILLIC CAPITAL LETTER EN + '\x041d'# -> unI64 1085 + -- CYRILLIC CAPITAL LETTER O + '\x041e'# -> unI64 1086 + -- CYRILLIC CAPITAL LETTER PE + '\x041f'# -> unI64 1087 + -- CYRILLIC CAPITAL LETTER ER + '\x0420'# -> unI64 1088 + -- CYRILLIC CAPITAL LETTER ES + '\x0421'# -> unI64 1089 + -- CYRILLIC CAPITAL LETTER TE + '\x0422'# -> unI64 1090 + -- CYRILLIC CAPITAL LETTER U + '\x0423'# -> unI64 1091 + -- CYRILLIC CAPITAL LETTER EF + '\x0424'# -> unI64 1092 + -- CYRILLIC CAPITAL LETTER HA + '\x0425'# -> unI64 1093 + -- CYRILLIC CAPITAL LETTER TSE + '\x0426'# -> unI64 1094 + -- CYRILLIC CAPITAL LETTER CHE + '\x0427'# -> unI64 1095 + -- CYRILLIC CAPITAL LETTER SHA + '\x0428'# -> unI64 1096 + -- CYRILLIC CAPITAL LETTER SHCHA + '\x0429'# -> unI64 1097 + -- CYRILLIC CAPITAL LETTER HARD SIGN + '\x042a'# -> unI64 1098 + -- CYRILLIC CAPITAL LETTER YERU + '\x042b'# -> unI64 1099 + -- CYRILLIC CAPITAL LETTER SOFT SIGN + '\x042c'# -> unI64 1100 + -- CYRILLIC CAPITAL LETTER E + '\x042d'# -> unI64 1101 + -- CYRILLIC CAPITAL LETTER YU + '\x042e'# -> unI64 1102 + -- CYRILLIC CAPITAL LETTER YA + '\x042f'# -> unI64 1103 + -- CYRILLIC CAPITAL LETTER OMEGA + '\x0460'# -> unI64 1121 + -- CYRILLIC CAPITAL LETTER YAT + '\x0462'# -> unI64 1123 + -- CYRILLIC CAPITAL LETTER IOTIFIED E + '\x0464'# -> unI64 1125 + -- CYRILLIC CAPITAL LETTER LITTLE YUS + '\x0466'# -> unI64 1127 + -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS + '\x0468'# -> unI64 1129 + -- CYRILLIC CAPITAL LETTER BIG YUS + '\x046a'# -> unI64 1131 + -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS + '\x046c'# -> unI64 1133 + -- CYRILLIC CAPITAL LETTER KSI + '\x046e'# -> unI64 1135 + -- CYRILLIC CAPITAL LETTER PSI + '\x0470'# -> unI64 1137 + -- CYRILLIC CAPITAL LETTER FITA + '\x0472'# -> unI64 1139 + -- CYRILLIC CAPITAL LETTER IZHITSA + '\x0474'# -> unI64 1141 + -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + '\x0476'# -> unI64 1143 + -- CYRILLIC CAPITAL LETTER UK + '\x0478'# -> unI64 1145 + -- CYRILLIC CAPITAL LETTER ROUND OMEGA + '\x047a'# -> unI64 1147 + -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO + '\x047c'# -> unI64 1149 + -- CYRILLIC CAPITAL LETTER OT + '\x047e'# -> unI64 1151 + -- CYRILLIC CAPITAL LETTER KOPPA + '\x0480'# -> unI64 1153 + -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL + '\x048a'# -> unI64 1163 + -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN + '\x048c'# -> unI64 1165 + -- CYRILLIC CAPITAL LETTER ER WITH TICK + '\x048e'# -> unI64 1167 + -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN + '\x0490'# -> unI64 1169 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE + '\x0492'# -> unI64 1171 + -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK + '\x0494'# -> unI64 1173 + -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + '\x0496'# -> unI64 1175 + -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + '\x0498'# -> unI64 1177 + -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER + '\x049a'# -> unI64 1179 + -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + '\x049c'# -> unI64 1181 + -- CYRILLIC CAPITAL LETTER KA WITH STROKE + '\x049e'# -> unI64 1183 + -- CYRILLIC CAPITAL LETTER BASHKIR KA + '\x04a0'# -> unI64 1185 + -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER + '\x04a2'# -> unI64 1187 + -- CYRILLIC CAPITAL LIGATURE EN GHE + '\x04a4'# -> unI64 1189 + -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK + '\x04a6'# -> unI64 1191 + -- CYRILLIC CAPITAL LETTER ABKHASIAN HA + '\x04a8'# -> unI64 1193 + -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER + '\x04aa'# -> unI64 1195 + -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER + '\x04ac'# -> unI64 1197 + -- CYRILLIC CAPITAL LETTER STRAIGHT U + '\x04ae'# -> unI64 1199 + -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + '\x04b0'# -> unI64 1201 + -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER + '\x04b2'# -> unI64 1203 + -- CYRILLIC CAPITAL LIGATURE TE TSE + '\x04b4'# -> unI64 1205 + -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + '\x04b6'# -> unI64 1207 + -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + '\x04b8'# -> unI64 1209 + -- CYRILLIC CAPITAL LETTER SHHA + '\x04ba'# -> unI64 1211 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE + '\x04bc'# -> unI64 1213 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER + '\x04be'# -> unI64 1215 + -- CYRILLIC LETTER PALOCHKA + '\x04c0'# -> unI64 1231 + -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE + '\x04c1'# -> unI64 1218 + -- CYRILLIC CAPITAL LETTER KA WITH HOOK + '\x04c3'# -> unI64 1220 + -- CYRILLIC CAPITAL LETTER EL WITH TAIL + '\x04c5'# -> unI64 1222 + -- CYRILLIC CAPITAL LETTER EN WITH HOOK + '\x04c7'# -> unI64 1224 + -- CYRILLIC CAPITAL LETTER EN WITH TAIL + '\x04c9'# -> unI64 1226 + -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE + '\x04cb'# -> unI64 1228 + -- CYRILLIC CAPITAL LETTER EM WITH TAIL + '\x04cd'# -> unI64 1230 + -- CYRILLIC CAPITAL LETTER A WITH BREVE + '\x04d0'# -> unI64 1233 + -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS + '\x04d2'# -> unI64 1235 + -- CYRILLIC CAPITAL LIGATURE A IE + '\x04d4'# -> unI64 1237 + -- CYRILLIC CAPITAL LETTER IE WITH BREVE + '\x04d6'# -> unI64 1239 + -- CYRILLIC CAPITAL LETTER SCHWA + '\x04d8'# -> unI64 1241 + -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + '\x04da'# -> unI64 1243 + -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + '\x04dc'# -> unI64 1245 + -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + '\x04de'# -> unI64 1247 + -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE + '\x04e0'# -> unI64 1249 + -- CYRILLIC CAPITAL LETTER I WITH MACRON + '\x04e2'# -> unI64 1251 + -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS + '\x04e4'# -> unI64 1253 + -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS + '\x04e6'# -> unI64 1255 + -- CYRILLIC CAPITAL LETTER BARRED O + '\x04e8'# -> unI64 1257 + -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + '\x04ea'# -> unI64 1259 + -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS + '\x04ec'# -> unI64 1261 + -- CYRILLIC CAPITAL LETTER U WITH MACRON + '\x04ee'# -> unI64 1263 + -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS + '\x04f0'# -> unI64 1265 + -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + '\x04f2'# -> unI64 1267 + -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + '\x04f4'# -> unI64 1269 + -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER + '\x04f6'# -> unI64 1271 + -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + '\x04f8'# -> unI64 1273 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK + '\x04fa'# -> unI64 1275 + -- CYRILLIC CAPITAL LETTER HA WITH HOOK + '\x04fc'# -> unI64 1277 + -- CYRILLIC CAPITAL LETTER HA WITH STROKE + '\x04fe'# -> unI64 1279 + -- CYRILLIC CAPITAL LETTER KOMI DE + '\x0500'# -> unI64 1281 + -- CYRILLIC CAPITAL LETTER KOMI DJE + '\x0502'# -> unI64 1283 + -- CYRILLIC CAPITAL LETTER KOMI ZJE + '\x0504'# -> unI64 1285 + -- CYRILLIC CAPITAL LETTER KOMI DZJE + '\x0506'# -> unI64 1287 + -- CYRILLIC CAPITAL LETTER KOMI LJE + '\x0508'# -> unI64 1289 + -- CYRILLIC CAPITAL LETTER KOMI NJE + '\x050a'# -> unI64 1291 + -- CYRILLIC CAPITAL LETTER KOMI SJE + '\x050c'# -> unI64 1293 + -- CYRILLIC CAPITAL LETTER KOMI TJE + '\x050e'# -> unI64 1295 + -- CYRILLIC CAPITAL LETTER REVERSED ZE + '\x0510'# -> unI64 1297 + -- CYRILLIC CAPITAL LETTER EL WITH HOOK + '\x0512'# -> unI64 1299 + -- CYRILLIC CAPITAL LETTER LHA + '\x0514'# -> unI64 1301 + -- CYRILLIC CAPITAL LETTER RHA + '\x0516'# -> unI64 1303 + -- CYRILLIC CAPITAL LETTER YAE + '\x0518'# -> unI64 1305 + -- CYRILLIC CAPITAL LETTER QA + '\x051a'# -> unI64 1307 + -- CYRILLIC CAPITAL LETTER WE + '\x051c'# -> unI64 1309 + -- CYRILLIC CAPITAL LETTER ALEUT KA + '\x051e'# -> unI64 1311 + -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK + '\x0520'# -> unI64 1313 + -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK + '\x0522'# -> unI64 1315 + -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER + '\x0524'# -> unI64 1317 + -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER + '\x0526'# -> unI64 1319 + -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK + '\x0528'# -> unI64 1321 + -- CYRILLIC CAPITAL LETTER DZZHE + '\x052a'# -> unI64 1323 + -- CYRILLIC CAPITAL LETTER DCHE + '\x052c'# -> unI64 1325 + -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER + '\x052e'# -> unI64 1327 + -- ARMENIAN CAPITAL LETTER AYB + '\x0531'# -> unI64 1377 + -- ARMENIAN CAPITAL LETTER BEN + '\x0532'# -> unI64 1378 + -- ARMENIAN CAPITAL LETTER GIM + '\x0533'# -> unI64 1379 + -- ARMENIAN CAPITAL LETTER DA + '\x0534'# -> unI64 1380 + -- ARMENIAN CAPITAL LETTER ECH + '\x0535'# -> unI64 1381 + -- ARMENIAN CAPITAL LETTER ZA + '\x0536'# -> unI64 1382 + -- ARMENIAN CAPITAL LETTER EH + '\x0537'# -> unI64 1383 + -- ARMENIAN CAPITAL LETTER ET + '\x0538'# -> unI64 1384 + -- ARMENIAN CAPITAL LETTER TO + '\x0539'# -> unI64 1385 + -- ARMENIAN CAPITAL LETTER ZHE + '\x053a'# -> unI64 1386 + -- ARMENIAN CAPITAL LETTER INI + '\x053b'# -> unI64 1387 + -- ARMENIAN CAPITAL LETTER LIWN + '\x053c'# -> unI64 1388 + -- ARMENIAN CAPITAL LETTER XEH + '\x053d'# -> unI64 1389 + -- ARMENIAN CAPITAL LETTER CA + '\x053e'# -> unI64 1390 + -- ARMENIAN CAPITAL LETTER KEN + '\x053f'# -> unI64 1391 + -- ARMENIAN CAPITAL LETTER HO + '\x0540'# -> unI64 1392 + -- ARMENIAN CAPITAL LETTER JA + '\x0541'# -> unI64 1393 + -- ARMENIAN CAPITAL LETTER GHAD + '\x0542'# -> unI64 1394 + -- ARMENIAN CAPITAL LETTER CHEH + '\x0543'# -> unI64 1395 + -- ARMENIAN CAPITAL LETTER MEN + '\x0544'# -> unI64 1396 + -- ARMENIAN CAPITAL LETTER YI + '\x0545'# -> unI64 1397 + -- ARMENIAN CAPITAL LETTER NOW + '\x0546'# -> unI64 1398 + -- ARMENIAN CAPITAL LETTER SHA + '\x0547'# -> unI64 1399 + -- ARMENIAN CAPITAL LETTER VO + '\x0548'# -> unI64 1400 + -- ARMENIAN CAPITAL LETTER CHA + '\x0549'# -> unI64 1401 + -- ARMENIAN CAPITAL LETTER PEH + '\x054a'# -> unI64 1402 + -- ARMENIAN CAPITAL LETTER JHEH + '\x054b'# -> unI64 1403 + -- ARMENIAN CAPITAL LETTER RA + '\x054c'# -> unI64 1404 + -- ARMENIAN CAPITAL LETTER SEH + '\x054d'# -> unI64 1405 + -- ARMENIAN CAPITAL LETTER VEW + '\x054e'# -> unI64 1406 + -- ARMENIAN CAPITAL LETTER TIWN + '\x054f'# -> unI64 1407 + -- ARMENIAN CAPITAL LETTER REH + '\x0550'# -> unI64 1408 + -- ARMENIAN CAPITAL LETTER CO + '\x0551'# -> unI64 1409 + -- ARMENIAN CAPITAL LETTER YIWN + '\x0552'# -> unI64 1410 + -- ARMENIAN CAPITAL LETTER PIWR + '\x0553'# -> unI64 1411 + -- ARMENIAN CAPITAL LETTER KEH + '\x0554'# -> unI64 1412 + -- ARMENIAN CAPITAL LETTER OH + '\x0555'# -> unI64 1413 + -- ARMENIAN CAPITAL LETTER FEH + '\x0556'# -> unI64 1414 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985701 + -- GEORGIAN CAPITAL LETTER AN + '\x10a0'# -> unI64 11520 + -- GEORGIAN CAPITAL LETTER BAN + '\x10a1'# -> unI64 11521 + -- GEORGIAN CAPITAL LETTER GAN + '\x10a2'# -> unI64 11522 + -- GEORGIAN CAPITAL LETTER DON + '\x10a3'# -> unI64 11523 + -- GEORGIAN CAPITAL LETTER EN + '\x10a4'# -> unI64 11524 + -- GEORGIAN CAPITAL LETTER VIN + '\x10a5'# -> unI64 11525 + -- GEORGIAN CAPITAL LETTER ZEN + '\x10a6'# -> unI64 11526 + -- GEORGIAN CAPITAL LETTER TAN + '\x10a7'# -> unI64 11527 + -- GEORGIAN CAPITAL LETTER IN + '\x10a8'# -> unI64 11528 + -- GEORGIAN CAPITAL LETTER KAN + '\x10a9'# -> unI64 11529 + -- GEORGIAN CAPITAL LETTER LAS + '\x10aa'# -> unI64 11530 + -- GEORGIAN CAPITAL LETTER MAN + '\x10ab'# -> unI64 11531 + -- GEORGIAN CAPITAL LETTER NAR + '\x10ac'# -> unI64 11532 + -- GEORGIAN CAPITAL LETTER ON + '\x10ad'# -> unI64 11533 + -- GEORGIAN CAPITAL LETTER PAR + '\x10ae'# -> unI64 11534 + -- GEORGIAN CAPITAL LETTER ZHAR + '\x10af'# -> unI64 11535 + -- GEORGIAN CAPITAL LETTER RAE + '\x10b0'# -> unI64 11536 + -- GEORGIAN CAPITAL LETTER SAN + '\x10b1'# -> unI64 11537 + -- GEORGIAN CAPITAL LETTER TAR + '\x10b2'# -> unI64 11538 + -- GEORGIAN CAPITAL LETTER UN + '\x10b3'# -> unI64 11539 + -- GEORGIAN CAPITAL LETTER PHAR + '\x10b4'# -> unI64 11540 + -- GEORGIAN CAPITAL LETTER KHAR + '\x10b5'# -> unI64 11541 + -- GEORGIAN CAPITAL LETTER GHAN + '\x10b6'# -> unI64 11542 + -- GEORGIAN CAPITAL LETTER QAR + '\x10b7'# -> unI64 11543 + -- GEORGIAN CAPITAL LETTER SHIN + '\x10b8'# -> unI64 11544 + -- GEORGIAN CAPITAL LETTER CHIN + '\x10b9'# -> unI64 11545 + -- GEORGIAN CAPITAL LETTER CAN + '\x10ba'# -> unI64 11546 + -- GEORGIAN CAPITAL LETTER JIL + '\x10bb'# -> unI64 11547 + -- GEORGIAN CAPITAL LETTER CIL + '\x10bc'# -> unI64 11548 + -- GEORGIAN CAPITAL LETTER CHAR + '\x10bd'# -> unI64 11549 + -- GEORGIAN CAPITAL LETTER XAN + '\x10be'# -> unI64 11550 + -- GEORGIAN CAPITAL LETTER JHAN + '\x10bf'# -> unI64 11551 + -- GEORGIAN CAPITAL LETTER HAE + '\x10c0'# -> unI64 11552 + -- GEORGIAN CAPITAL LETTER HE + '\x10c1'# -> unI64 11553 + -- GEORGIAN CAPITAL LETTER HIE + '\x10c2'# -> unI64 11554 + -- GEORGIAN CAPITAL LETTER WE + '\x10c3'# -> unI64 11555 + -- GEORGIAN CAPITAL LETTER HAR + '\x10c4'# -> unI64 11556 + -- GEORGIAN CAPITAL LETTER HOE + '\x10c5'# -> unI64 11557 + -- GEORGIAN CAPITAL LETTER YN + '\x10c7'# -> unI64 11559 + -- GEORGIAN CAPITAL LETTER AEN + '\x10cd'# -> unI64 11565 + -- CHEROKEE SMALL LETTER YE + '\x13f8'# -> unI64 5104 + -- CHEROKEE SMALL LETTER YI + '\x13f9'# -> unI64 5105 + -- CHEROKEE SMALL LETTER YO + '\x13fa'# -> unI64 5106 + -- CHEROKEE SMALL LETTER YU + '\x13fb'# -> unI64 5107 + -- CHEROKEE SMALL LETTER YV + '\x13fc'# -> unI64 5108 + -- CHEROKEE SMALL LETTER MV + '\x13fd'# -> unI64 5109 + -- CYRILLIC SMALL LETTER ROUNDED VE + '\x1c80'# -> unI64 1074 + -- CYRILLIC SMALL LETTER LONG-LEGGED DE + '\x1c81'# -> unI64 1076 + -- CYRILLIC SMALL LETTER NARROW O + '\x1c82'# -> unI64 1086 + -- CYRILLIC SMALL LETTER WIDE ES + '\x1c83'# -> unI64 1089 + -- CYRILLIC SMALL LETTER TALL TE + '\x1c84'# -> unI64 1090 + -- CYRILLIC SMALL LETTER THREE-LEGGED TE + '\x1c85'# -> unI64 1090 + -- CYRILLIC SMALL LETTER TALL HARD SIGN + '\x1c86'# -> unI64 1098 + -- CYRILLIC SMALL LETTER TALL YAT + '\x1c87'# -> unI64 1123 + -- CYRILLIC SMALL LETTER UNBLENDED UK + '\x1c88'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER TJE + '\x1c89'# -> unI64 7306 + -- GEORGIAN MTAVRULI CAPITAL LETTER AN + '\x1c90'# -> unI64 4304 + -- GEORGIAN MTAVRULI CAPITAL LETTER BAN + '\x1c91'# -> unI64 4305 + -- GEORGIAN MTAVRULI CAPITAL LETTER GAN + '\x1c92'# -> unI64 4306 + -- GEORGIAN MTAVRULI CAPITAL LETTER DON + '\x1c93'# -> unI64 4307 + -- GEORGIAN MTAVRULI CAPITAL LETTER EN + '\x1c94'# -> unI64 4308 + -- GEORGIAN MTAVRULI CAPITAL LETTER VIN + '\x1c95'# -> unI64 4309 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN + '\x1c96'# -> unI64 4310 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAN + '\x1c97'# -> unI64 4311 + -- GEORGIAN MTAVRULI CAPITAL LETTER IN + '\x1c98'# -> unI64 4312 + -- GEORGIAN MTAVRULI CAPITAL LETTER KAN + '\x1c99'# -> unI64 4313 + -- GEORGIAN MTAVRULI CAPITAL LETTER LAS + '\x1c9a'# -> unI64 4314 + -- GEORGIAN MTAVRULI CAPITAL LETTER MAN + '\x1c9b'# -> unI64 4315 + -- GEORGIAN MTAVRULI CAPITAL LETTER NAR + '\x1c9c'# -> unI64 4316 + -- GEORGIAN MTAVRULI CAPITAL LETTER ON + '\x1c9d'# -> unI64 4317 + -- GEORGIAN MTAVRULI CAPITAL LETTER PAR + '\x1c9e'# -> unI64 4318 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR + '\x1c9f'# -> unI64 4319 + -- GEORGIAN MTAVRULI CAPITAL LETTER RAE + '\x1ca0'# -> unI64 4320 + -- GEORGIAN MTAVRULI CAPITAL LETTER SAN + '\x1ca1'# -> unI64 4321 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAR + '\x1ca2'# -> unI64 4322 + -- GEORGIAN MTAVRULI CAPITAL LETTER UN + '\x1ca3'# -> unI64 4323 + -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR + '\x1ca4'# -> unI64 4324 + -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR + '\x1ca5'# -> unI64 4325 + -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN + '\x1ca6'# -> unI64 4326 + -- GEORGIAN MTAVRULI CAPITAL LETTER QAR + '\x1ca7'# -> unI64 4327 + -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN + '\x1ca8'# -> unI64 4328 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN + '\x1ca9'# -> unI64 4329 + -- GEORGIAN MTAVRULI CAPITAL LETTER CAN + '\x1caa'# -> unI64 4330 + -- GEORGIAN MTAVRULI CAPITAL LETTER JIL + '\x1cab'# -> unI64 4331 + -- GEORGIAN MTAVRULI CAPITAL LETTER CIL + '\x1cac'# -> unI64 4332 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR + '\x1cad'# -> unI64 4333 + -- GEORGIAN MTAVRULI CAPITAL LETTER XAN + '\x1cae'# -> unI64 4334 + -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN + '\x1caf'# -> unI64 4335 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAE + '\x1cb0'# -> unI64 4336 + -- GEORGIAN MTAVRULI CAPITAL LETTER HE + '\x1cb1'# -> unI64 4337 + -- GEORGIAN MTAVRULI CAPITAL LETTER HIE + '\x1cb2'# -> unI64 4338 + -- GEORGIAN MTAVRULI CAPITAL LETTER WE + '\x1cb3'# -> unI64 4339 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAR + '\x1cb4'# -> unI64 4340 + -- GEORGIAN MTAVRULI CAPITAL LETTER HOE + '\x1cb5'# -> unI64 4341 + -- GEORGIAN MTAVRULI CAPITAL LETTER FI + '\x1cb6'# -> unI64 4342 + -- GEORGIAN MTAVRULI CAPITAL LETTER YN + '\x1cb7'# -> unI64 4343 + -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI + '\x1cb8'# -> unI64 4344 + -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN + '\x1cb9'# -> unI64 4345 + -- GEORGIAN MTAVRULI CAPITAL LETTER AIN + '\x1cba'# -> unI64 4346 + -- GEORGIAN MTAVRULI CAPITAL LETTER AEN + '\x1cbd'# -> unI64 4349 + -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN + '\x1cbe'# -> unI64 4350 + -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + '\x1cbf'# -> unI64 4351 + -- LATIN CAPITAL LETTER A WITH RING BELOW + '\x1e00'# -> unI64 7681 + -- LATIN CAPITAL LETTER B WITH DOT ABOVE + '\x1e02'# -> unI64 7683 + -- LATIN CAPITAL LETTER B WITH DOT BELOW + '\x1e04'# -> unI64 7685 + -- LATIN CAPITAL LETTER B WITH LINE BELOW + '\x1e06'# -> unI64 7687 + -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + '\x1e08'# -> unI64 7689 + -- LATIN CAPITAL LETTER D WITH DOT ABOVE + '\x1e0a'# -> unI64 7691 + -- LATIN CAPITAL LETTER D WITH DOT BELOW + '\x1e0c'# -> unI64 7693 + -- LATIN CAPITAL LETTER D WITH LINE BELOW + '\x1e0e'# -> unI64 7695 + -- LATIN CAPITAL LETTER D WITH CEDILLA + '\x1e10'# -> unI64 7697 + -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + '\x1e12'# -> unI64 7699 + -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + '\x1e14'# -> unI64 7701 + -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + '\x1e16'# -> unI64 7703 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + '\x1e18'# -> unI64 7705 + -- LATIN CAPITAL LETTER E WITH TILDE BELOW + '\x1e1a'# -> unI64 7707 + -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + '\x1e1c'# -> unI64 7709 + -- LATIN CAPITAL LETTER F WITH DOT ABOVE + '\x1e1e'# -> unI64 7711 + -- LATIN CAPITAL LETTER G WITH MACRON + '\x1e20'# -> unI64 7713 + -- LATIN CAPITAL LETTER H WITH DOT ABOVE + '\x1e22'# -> unI64 7715 + -- LATIN CAPITAL LETTER H WITH DOT BELOW + '\x1e24'# -> unI64 7717 + -- LATIN CAPITAL LETTER H WITH DIAERESIS + '\x1e26'# -> unI64 7719 + -- LATIN CAPITAL LETTER H WITH CEDILLA + '\x1e28'# -> unI64 7721 + -- LATIN CAPITAL LETTER H WITH BREVE BELOW + '\x1e2a'# -> unI64 7723 + -- LATIN CAPITAL LETTER I WITH TILDE BELOW + '\x1e2c'# -> unI64 7725 + -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + '\x1e2e'# -> unI64 7727 + -- LATIN CAPITAL LETTER K WITH ACUTE + '\x1e30'# -> unI64 7729 + -- LATIN CAPITAL LETTER K WITH DOT BELOW + '\x1e32'# -> unI64 7731 + -- LATIN CAPITAL LETTER K WITH LINE BELOW + '\x1e34'# -> unI64 7733 + -- LATIN CAPITAL LETTER L WITH DOT BELOW + '\x1e36'# -> unI64 7735 + -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + '\x1e38'# -> unI64 7737 + -- LATIN CAPITAL LETTER L WITH LINE BELOW + '\x1e3a'# -> unI64 7739 + -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + '\x1e3c'# -> unI64 7741 + -- LATIN CAPITAL LETTER M WITH ACUTE + '\x1e3e'# -> unI64 7743 + -- LATIN CAPITAL LETTER M WITH DOT ABOVE + '\x1e40'# -> unI64 7745 + -- LATIN CAPITAL LETTER M WITH DOT BELOW + '\x1e42'# -> unI64 7747 + -- LATIN CAPITAL LETTER N WITH DOT ABOVE + '\x1e44'# -> unI64 7749 + -- LATIN CAPITAL LETTER N WITH DOT BELOW + '\x1e46'# -> unI64 7751 + -- LATIN CAPITAL LETTER N WITH LINE BELOW + '\x1e48'# -> unI64 7753 + -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + '\x1e4a'# -> unI64 7755 + -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + '\x1e4c'# -> unI64 7757 + -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + '\x1e4e'# -> unI64 7759 + -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + '\x1e50'# -> unI64 7761 + -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + '\x1e52'# -> unI64 7763 + -- LATIN CAPITAL LETTER P WITH ACUTE + '\x1e54'# -> unI64 7765 + -- LATIN CAPITAL LETTER P WITH DOT ABOVE + '\x1e56'# -> unI64 7767 + -- LATIN CAPITAL LETTER R WITH DOT ABOVE + '\x1e58'# -> unI64 7769 + -- LATIN CAPITAL LETTER R WITH DOT BELOW + '\x1e5a'# -> unI64 7771 + -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + '\x1e5c'# -> unI64 7773 + -- LATIN CAPITAL LETTER R WITH LINE BELOW + '\x1e5e'# -> unI64 7775 + -- LATIN CAPITAL LETTER S WITH DOT ABOVE + '\x1e60'# -> unI64 7777 + -- LATIN CAPITAL LETTER S WITH DOT BELOW + '\x1e62'# -> unI64 7779 + -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + '\x1e64'# -> unI64 7781 + -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + '\x1e66'# -> unI64 7783 + -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + '\x1e68'# -> unI64 7785 + -- LATIN CAPITAL LETTER T WITH DOT ABOVE + '\x1e6a'# -> unI64 7787 + -- LATIN CAPITAL LETTER T WITH DOT BELOW + '\x1e6c'# -> unI64 7789 + -- LATIN CAPITAL LETTER T WITH LINE BELOW + '\x1e6e'# -> unI64 7791 + -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + '\x1e70'# -> unI64 7793 + -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + '\x1e72'# -> unI64 7795 + -- LATIN CAPITAL LETTER U WITH TILDE BELOW + '\x1e74'# -> unI64 7797 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + '\x1e76'# -> unI64 7799 + -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + '\x1e78'# -> unI64 7801 + -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + '\x1e7a'# -> unI64 7803 + -- LATIN CAPITAL LETTER V WITH TILDE + '\x1e7c'# -> unI64 7805 + -- LATIN CAPITAL LETTER V WITH DOT BELOW + '\x1e7e'# -> unI64 7807 + -- LATIN CAPITAL LETTER W WITH GRAVE + '\x1e80'# -> unI64 7809 + -- LATIN CAPITAL LETTER W WITH ACUTE + '\x1e82'# -> unI64 7811 + -- LATIN CAPITAL LETTER W WITH DIAERESIS + '\x1e84'# -> unI64 7813 + -- LATIN CAPITAL LETTER W WITH DOT ABOVE + '\x1e86'# -> unI64 7815 + -- LATIN CAPITAL LETTER W WITH DOT BELOW + '\x1e88'# -> unI64 7817 + -- LATIN CAPITAL LETTER X WITH DOT ABOVE + '\x1e8a'# -> unI64 7819 + -- LATIN CAPITAL LETTER X WITH DIAERESIS + '\x1e8c'# -> unI64 7821 + -- LATIN CAPITAL LETTER Y WITH DOT ABOVE + '\x1e8e'# -> unI64 7823 + -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + '\x1e90'# -> unI64 7825 + -- LATIN CAPITAL LETTER Z WITH DOT BELOW + '\x1e92'# -> unI64 7827 + -- LATIN CAPITAL LETTER Z WITH LINE BELOW + '\x1e94'# -> unI64 7829 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373288 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390068 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584375 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584377 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200801 + -- LATIN SMALL LETTER LONG S WITH DOT ABOVE + '\x1e9b'# -> unI64 7777 + -- LATIN CAPITAL LETTER SHARP S + '\x1e9e'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH DOT BELOW + '\x1ea0'# -> unI64 7841 + -- LATIN CAPITAL LETTER A WITH HOOK ABOVE + '\x1ea2'# -> unI64 7843 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + '\x1ea4'# -> unI64 7845 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + '\x1ea6'# -> unI64 7847 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ea8'# -> unI64 7849 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + '\x1eaa'# -> unI64 7851 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + '\x1eac'# -> unI64 7853 + -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + '\x1eae'# -> unI64 7855 + -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + '\x1eb0'# -> unI64 7857 + -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + '\x1eb2'# -> unI64 7859 + -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE + '\x1eb4'# -> unI64 7861 + -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + '\x1eb6'# -> unI64 7863 + -- LATIN CAPITAL LETTER E WITH DOT BELOW + '\x1eb8'# -> unI64 7865 + -- LATIN CAPITAL LETTER E WITH HOOK ABOVE + '\x1eba'# -> unI64 7867 + -- LATIN CAPITAL LETTER E WITH TILDE + '\x1ebc'# -> unI64 7869 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + '\x1ebe'# -> unI64 7871 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + '\x1ec0'# -> unI64 7873 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ec2'# -> unI64 7875 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + '\x1ec4'# -> unI64 7877 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + '\x1ec6'# -> unI64 7879 + -- LATIN CAPITAL LETTER I WITH HOOK ABOVE + '\x1ec8'# -> unI64 7881 + -- LATIN CAPITAL LETTER I WITH DOT BELOW + '\x1eca'# -> unI64 7883 + -- LATIN CAPITAL LETTER O WITH DOT BELOW + '\x1ecc'# -> unI64 7885 + -- LATIN CAPITAL LETTER O WITH HOOK ABOVE + '\x1ece'# -> unI64 7887 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + '\x1ed0'# -> unI64 7889 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + '\x1ed2'# -> unI64 7891 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ed4'# -> unI64 7893 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + '\x1ed6'# -> unI64 7895 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + '\x1ed8'# -> unI64 7897 + -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE + '\x1eda'# -> unI64 7899 + -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE + '\x1edc'# -> unI64 7901 + -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + '\x1ede'# -> unI64 7903 + -- LATIN CAPITAL LETTER O WITH HORN AND TILDE + '\x1ee0'# -> unI64 7905 + -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + '\x1ee2'# -> unI64 7907 + -- LATIN CAPITAL LETTER U WITH DOT BELOW + '\x1ee4'# -> unI64 7909 + -- LATIN CAPITAL LETTER U WITH HOOK ABOVE + '\x1ee6'# -> unI64 7911 + -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE + '\x1ee8'# -> unI64 7913 + -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE + '\x1eea'# -> unI64 7915 + -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + '\x1eec'# -> unI64 7917 + -- LATIN CAPITAL LETTER U WITH HORN AND TILDE + '\x1eee'# -> unI64 7919 + -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + '\x1ef0'# -> unI64 7921 + -- LATIN CAPITAL LETTER Y WITH GRAVE + '\x1ef2'# -> unI64 7923 + -- LATIN CAPITAL LETTER Y WITH DOT BELOW + '\x1ef4'# -> unI64 7925 + -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE + '\x1ef6'# -> unI64 7927 + -- LATIN CAPITAL LETTER Y WITH TILDE + '\x1ef8'# -> unI64 7929 + -- LATIN CAPITAL LETTER MIDDLE-WELSH LL + '\x1efa'# -> unI64 7931 + -- LATIN CAPITAL LETTER MIDDLE-WELSH V + '\x1efc'# -> unI64 7933 + -- LATIN CAPITAL LETTER Y WITH LOOP + '\x1efe'# -> unI64 7935 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI + '\x1f08'# -> unI64 7936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA + '\x1f09'# -> unI64 7937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA + '\x1f0a'# -> unI64 7938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA + '\x1f0b'# -> unI64 7939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA + '\x1f0c'# -> unI64 7940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA + '\x1f0d'# -> unI64 7941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI + '\x1f0e'# -> unI64 7942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + '\x1f0f'# -> unI64 7943 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI + '\x1f18'# -> unI64 7952 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA + '\x1f19'# -> unI64 7953 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA + '\x1f1a'# -> unI64 7954 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA + '\x1f1b'# -> unI64 7955 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA + '\x1f1c'# -> unI64 7956 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + '\x1f1d'# -> unI64 7957 + -- GREEK CAPITAL LETTER ETA WITH PSILI + '\x1f28'# -> unI64 7968 + -- GREEK CAPITAL LETTER ETA WITH DASIA + '\x1f29'# -> unI64 7969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA + '\x1f2a'# -> unI64 7970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA + '\x1f2b'# -> unI64 7971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA + '\x1f2c'# -> unI64 7972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA + '\x1f2d'# -> unI64 7973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI + '\x1f2e'# -> unI64 7974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + '\x1f2f'# -> unI64 7975 + -- GREEK CAPITAL LETTER IOTA WITH PSILI + '\x1f38'# -> unI64 7984 + -- GREEK CAPITAL LETTER IOTA WITH DASIA + '\x1f39'# -> unI64 7985 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA + '\x1f3a'# -> unI64 7986 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA + '\x1f3b'# -> unI64 7987 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA + '\x1f3c'# -> unI64 7988 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA + '\x1f3d'# -> unI64 7989 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI + '\x1f3e'# -> unI64 7990 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + '\x1f3f'# -> unI64 7991 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI + '\x1f48'# -> unI64 8000 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA + '\x1f49'# -> unI64 8001 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA + '\x1f4a'# -> unI64 8002 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA + '\x1f4b'# -> unI64 8003 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA + '\x1f4c'# -> unI64 8004 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + '\x1f4d'# -> unI64 8005 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459589 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987461 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498565 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720325 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA + '\x1f59'# -> unI64 8017 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + '\x1f5b'# -> unI64 8019 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + '\x1f5d'# -> unI64 8021 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + '\x1f5f'# -> unI64 8023 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI + '\x1f68'# -> unI64 8032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA + '\x1f69'# -> unI64 8033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA + '\x1f6a'# -> unI64 8034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA + '\x1f6b'# -> unI64 8035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA + '\x1f6c'# -> unI64 8036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA + '\x1f6d'# -> unI64 8037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI + '\x1f6e'# -> unI64 8038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + '\x1f6f'# -> unI64 8039 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1998593792 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1998593793 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1998593794 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1998593795 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1998593796 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1998593797 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1998593798 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1998593799 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1998593792 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1998593793 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1998593794 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1998593795 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1998593796 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1998593797 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1998593798 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1998593799 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1998593824 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1998593825 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1998593826 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1998593827 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1998593828 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1998593829 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1998593830 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1998593831 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1998593824 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1998593825 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1998593826 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1998593827 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1998593828 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1998593829 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1998593830 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1998593831 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1998593888 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1998593889 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1998593890 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1998593891 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1998593892 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1998593893 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1998593894 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1998593895 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1998593888 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1998593889 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1998593890 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1998593891 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1998593892 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1998593893 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1998593894 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1998593895 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1998593904 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1998586801 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1998586796 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025713 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4191340074107825 + -- GREEK CAPITAL LETTER ALPHA WITH VRACHY + '\x1fb8'# -> unI64 8112 + -- GREEK CAPITAL LETTER ALPHA WITH MACRON + '\x1fb9'# -> unI64 8113 + -- GREEK CAPITAL LETTER ALPHA WITH VARIA + '\x1fba'# -> unI64 8048 + -- GREEK CAPITAL LETTER ALPHA WITH OXIA + '\x1fbb'# -> unI64 8049 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1998586801 + -- GREEK PROSGEGRAMMENI + '\x1fbe'# -> unI64 953 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1998593908 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1998586807 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1998586798 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025719 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4191340074107831 + -- GREEK CAPITAL LETTER EPSILON WITH VARIA + '\x1fc8'# -> unI64 8050 + -- GREEK CAPITAL LETTER EPSILON WITH OXIA + '\x1fc9'# -> unI64 8051 + -- GREEK CAPITAL LETTER ETA WITH VARIA + '\x1fca'# -> unI64 8052 + -- GREEK CAPITAL LETTER ETA WITH OXIA + '\x1fcb'# -> unI64 8053 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1998586807 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918777 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025721 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651641 + -- GREEK CAPITAL LETTER IOTA WITH VRACHY + '\x1fd8'# -> unI64 8144 + -- GREEK CAPITAL LETTER IOTA WITH MACRON + '\x1fd9'# -> unI64 8145 + -- GREEK CAPITAL LETTER IOTA WITH VARIA + '\x1fda'# -> unI64 8054 + -- GREEK CAPITAL LETTER IOTA WITH OXIA + '\x1fdb'# -> unI64 8055 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918789 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459585 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025733 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651653 + -- GREEK CAPITAL LETTER UPSILON WITH VRACHY + '\x1fe8'# -> unI64 8160 + -- GREEK CAPITAL LETTER UPSILON WITH MACRON + '\x1fe9'# -> unI64 8161 + -- GREEK CAPITAL LETTER UPSILON WITH VARIA + '\x1fea'# -> unI64 8058 + -- GREEK CAPITAL LETTER UPSILON WITH OXIA + '\x1feb'# -> unI64 8059 + -- GREEK CAPITAL LETTER RHO WITH DASIA + '\x1fec'# -> unI64 8165 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1998593916 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1998586825 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1998586830 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025737 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4191340074107849 + -- GREEK CAPITAL LETTER OMICRON WITH VARIA + '\x1ff8'# -> unI64 8056 + -- GREEK CAPITAL LETTER OMICRON WITH OXIA + '\x1ff9'# -> unI64 8057 + -- GREEK CAPITAL LETTER OMEGA WITH VARIA + '\x1ffa'# -> unI64 8060 + -- GREEK CAPITAL LETTER OMEGA WITH OXIA + '\x1ffb'# -> unI64 8061 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1998586825 + -- OHM SIGN + '\x2126'# -> unI64 969 + -- KELVIN SIGN + '\x212a'# -> unI64 107 + -- ANGSTROM SIGN + '\x212b'# -> unI64 229 + -- TURNED CAPITAL F + '\x2132'# -> unI64 8526 + -- ROMAN NUMERAL ONE + '\x2160'# -> unI64 8560 + -- ROMAN NUMERAL TWO + '\x2161'# -> unI64 8561 + -- ROMAN NUMERAL THREE + '\x2162'# -> unI64 8562 + -- ROMAN NUMERAL FOUR + '\x2163'# -> unI64 8563 + -- ROMAN NUMERAL FIVE + '\x2164'# -> unI64 8564 + -- ROMAN NUMERAL SIX + '\x2165'# -> unI64 8565 + -- ROMAN NUMERAL SEVEN + '\x2166'# -> unI64 8566 + -- ROMAN NUMERAL EIGHT + '\x2167'# -> unI64 8567 + -- ROMAN NUMERAL NINE + '\x2168'# -> unI64 8568 + -- ROMAN NUMERAL TEN + '\x2169'# -> unI64 8569 + -- ROMAN NUMERAL ELEVEN + '\x216a'# -> unI64 8570 + -- ROMAN NUMERAL TWELVE + '\x216b'# -> unI64 8571 + -- ROMAN NUMERAL FIFTY + '\x216c'# -> unI64 8572 + -- ROMAN NUMERAL ONE HUNDRED + '\x216d'# -> unI64 8573 + -- ROMAN NUMERAL FIVE HUNDRED + '\x216e'# -> unI64 8574 + -- ROMAN NUMERAL ONE THOUSAND + '\x216f'# -> unI64 8575 + -- ROMAN NUMERAL REVERSED ONE HUNDRED + '\x2183'# -> unI64 8580 + -- CIRCLED LATIN CAPITAL LETTER A + '\x24b6'# -> unI64 9424 + -- CIRCLED LATIN CAPITAL LETTER B + '\x24b7'# -> unI64 9425 + -- CIRCLED LATIN CAPITAL LETTER C + '\x24b8'# -> unI64 9426 + -- CIRCLED LATIN CAPITAL LETTER D + '\x24b9'# -> unI64 9427 + -- CIRCLED LATIN CAPITAL LETTER E + '\x24ba'# -> unI64 9428 + -- CIRCLED LATIN CAPITAL LETTER F + '\x24bb'# -> unI64 9429 + -- CIRCLED LATIN CAPITAL LETTER G + '\x24bc'# -> unI64 9430 + -- CIRCLED LATIN CAPITAL LETTER H + '\x24bd'# -> unI64 9431 + -- CIRCLED LATIN CAPITAL LETTER I + '\x24be'# -> unI64 9432 + -- CIRCLED LATIN CAPITAL LETTER J + '\x24bf'# -> unI64 9433 + -- CIRCLED LATIN CAPITAL LETTER K + '\x24c0'# -> unI64 9434 + -- CIRCLED LATIN CAPITAL LETTER L + '\x24c1'# -> unI64 9435 + -- CIRCLED LATIN CAPITAL LETTER M + '\x24c2'# -> unI64 9436 + -- CIRCLED LATIN CAPITAL LETTER N + '\x24c3'# -> unI64 9437 + -- CIRCLED LATIN CAPITAL LETTER O + '\x24c4'# -> unI64 9438 + -- CIRCLED LATIN CAPITAL LETTER P + '\x24c5'# -> unI64 9439 + -- CIRCLED LATIN CAPITAL LETTER Q + '\x24c6'# -> unI64 9440 + -- CIRCLED LATIN CAPITAL LETTER R + '\x24c7'# -> unI64 9441 + -- CIRCLED LATIN CAPITAL LETTER S + '\x24c8'# -> unI64 9442 + -- CIRCLED LATIN CAPITAL LETTER T + '\x24c9'# -> unI64 9443 + -- CIRCLED LATIN CAPITAL LETTER U + '\x24ca'# -> unI64 9444 + -- CIRCLED LATIN CAPITAL LETTER V + '\x24cb'# -> unI64 9445 + -- CIRCLED LATIN CAPITAL LETTER W + '\x24cc'# -> unI64 9446 + -- CIRCLED LATIN CAPITAL LETTER X + '\x24cd'# -> unI64 9447 + -- CIRCLED LATIN CAPITAL LETTER Y + '\x24ce'# -> unI64 9448 + -- CIRCLED LATIN CAPITAL LETTER Z + '\x24cf'# -> unI64 9449 + -- GLAGOLITIC CAPITAL LETTER AZU + '\x2c00'# -> unI64 11312 + -- GLAGOLITIC CAPITAL LETTER BUKY + '\x2c01'# -> unI64 11313 + -- GLAGOLITIC CAPITAL LETTER VEDE + '\x2c02'# -> unI64 11314 + -- GLAGOLITIC CAPITAL LETTER GLAGOLI + '\x2c03'# -> unI64 11315 + -- GLAGOLITIC CAPITAL LETTER DOBRO + '\x2c04'# -> unI64 11316 + -- GLAGOLITIC CAPITAL LETTER YESTU + '\x2c05'# -> unI64 11317 + -- GLAGOLITIC CAPITAL LETTER ZHIVETE + '\x2c06'# -> unI64 11318 + -- GLAGOLITIC CAPITAL LETTER DZELO + '\x2c07'# -> unI64 11319 + -- GLAGOLITIC CAPITAL LETTER ZEMLJA + '\x2c08'# -> unI64 11320 + -- GLAGOLITIC CAPITAL LETTER IZHE + '\x2c09'# -> unI64 11321 + -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE + '\x2c0a'# -> unI64 11322 + -- GLAGOLITIC CAPITAL LETTER I + '\x2c0b'# -> unI64 11323 + -- GLAGOLITIC CAPITAL LETTER DJERVI + '\x2c0c'# -> unI64 11324 + -- GLAGOLITIC CAPITAL LETTER KAKO + '\x2c0d'# -> unI64 11325 + -- GLAGOLITIC CAPITAL LETTER LJUDIJE + '\x2c0e'# -> unI64 11326 + -- GLAGOLITIC CAPITAL LETTER MYSLITE + '\x2c0f'# -> unI64 11327 + -- GLAGOLITIC CAPITAL LETTER NASHI + '\x2c10'# -> unI64 11328 + -- GLAGOLITIC CAPITAL LETTER ONU + '\x2c11'# -> unI64 11329 + -- GLAGOLITIC CAPITAL LETTER POKOJI + '\x2c12'# -> unI64 11330 + -- GLAGOLITIC CAPITAL LETTER RITSI + '\x2c13'# -> unI64 11331 + -- GLAGOLITIC CAPITAL LETTER SLOVO + '\x2c14'# -> unI64 11332 + -- GLAGOLITIC CAPITAL LETTER TVRIDO + '\x2c15'# -> unI64 11333 + -- GLAGOLITIC CAPITAL LETTER UKU + '\x2c16'# -> unI64 11334 + -- GLAGOLITIC CAPITAL LETTER FRITU + '\x2c17'# -> unI64 11335 + -- GLAGOLITIC CAPITAL LETTER HERU + '\x2c18'# -> unI64 11336 + -- GLAGOLITIC CAPITAL LETTER OTU + '\x2c19'# -> unI64 11337 + -- GLAGOLITIC CAPITAL LETTER PE + '\x2c1a'# -> unI64 11338 + -- GLAGOLITIC CAPITAL LETTER SHTA + '\x2c1b'# -> unI64 11339 + -- GLAGOLITIC CAPITAL LETTER TSI + '\x2c1c'# -> unI64 11340 + -- GLAGOLITIC CAPITAL LETTER CHRIVI + '\x2c1d'# -> unI64 11341 + -- GLAGOLITIC CAPITAL LETTER SHA + '\x2c1e'# -> unI64 11342 + -- GLAGOLITIC CAPITAL LETTER YERU + '\x2c1f'# -> unI64 11343 + -- GLAGOLITIC CAPITAL LETTER YERI + '\x2c20'# -> unI64 11344 + -- GLAGOLITIC CAPITAL LETTER YATI + '\x2c21'# -> unI64 11345 + -- GLAGOLITIC CAPITAL LETTER SPIDERY HA + '\x2c22'# -> unI64 11346 + -- GLAGOLITIC CAPITAL LETTER YU + '\x2c23'# -> unI64 11347 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS + '\x2c24'# -> unI64 11348 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL + '\x2c25'# -> unI64 11349 + -- GLAGOLITIC CAPITAL LETTER YO + '\x2c26'# -> unI64 11350 + -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS + '\x2c27'# -> unI64 11351 + -- GLAGOLITIC CAPITAL LETTER BIG YUS + '\x2c28'# -> unI64 11352 + -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS + '\x2c29'# -> unI64 11353 + -- GLAGOLITIC CAPITAL LETTER FITA + '\x2c2a'# -> unI64 11354 + -- GLAGOLITIC CAPITAL LETTER IZHITSA + '\x2c2b'# -> unI64 11355 + -- GLAGOLITIC CAPITAL LETTER SHTAPIC + '\x2c2c'# -> unI64 11356 + -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A + '\x2c2d'# -> unI64 11357 + -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE + '\x2c2e'# -> unI64 11358 + -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI + '\x2c2f'# -> unI64 11359 + -- LATIN CAPITAL LETTER L WITH DOUBLE BAR + '\x2c60'# -> unI64 11361 + -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE + '\x2c62'# -> unI64 619 + -- LATIN CAPITAL LETTER P WITH STROKE + '\x2c63'# -> unI64 7549 + -- LATIN CAPITAL LETTER R WITH TAIL + '\x2c64'# -> unI64 637 + -- LATIN CAPITAL LETTER H WITH DESCENDER + '\x2c67'# -> unI64 11368 + -- LATIN CAPITAL LETTER K WITH DESCENDER + '\x2c69'# -> unI64 11370 + -- LATIN CAPITAL LETTER Z WITH DESCENDER + '\x2c6b'# -> unI64 11372 + -- LATIN CAPITAL LETTER ALPHA + '\x2c6d'# -> unI64 593 + -- LATIN CAPITAL LETTER M WITH HOOK + '\x2c6e'# -> unI64 625 + -- LATIN CAPITAL LETTER TURNED A + '\x2c6f'# -> unI64 592 + -- LATIN CAPITAL LETTER TURNED ALPHA + '\x2c70'# -> unI64 594 + -- LATIN CAPITAL LETTER W WITH HOOK + '\x2c72'# -> unI64 11379 + -- LATIN CAPITAL LETTER HALF H + '\x2c75'# -> unI64 11382 + -- LATIN CAPITAL LETTER S WITH SWASH TAIL + '\x2c7e'# -> unI64 575 + -- LATIN CAPITAL LETTER Z WITH SWASH TAIL + '\x2c7f'# -> unI64 576 + -- COPTIC CAPITAL LETTER ALFA + '\x2c80'# -> unI64 11393 + -- COPTIC CAPITAL LETTER VIDA + '\x2c82'# -> unI64 11395 + -- COPTIC CAPITAL LETTER GAMMA + '\x2c84'# -> unI64 11397 + -- COPTIC CAPITAL LETTER DALDA + '\x2c86'# -> unI64 11399 + -- COPTIC CAPITAL LETTER EIE + '\x2c88'# -> unI64 11401 + -- COPTIC CAPITAL LETTER SOU + '\x2c8a'# -> unI64 11403 + -- COPTIC CAPITAL LETTER ZATA + '\x2c8c'# -> unI64 11405 + -- COPTIC CAPITAL LETTER HATE + '\x2c8e'# -> unI64 11407 + -- COPTIC CAPITAL LETTER THETHE + '\x2c90'# -> unI64 11409 + -- COPTIC CAPITAL LETTER IAUDA + '\x2c92'# -> unI64 11411 + -- COPTIC CAPITAL LETTER KAPA + '\x2c94'# -> unI64 11413 + -- COPTIC CAPITAL LETTER LAULA + '\x2c96'# -> unI64 11415 + -- COPTIC CAPITAL LETTER MI + '\x2c98'# -> unI64 11417 + -- COPTIC CAPITAL LETTER NI + '\x2c9a'# -> unI64 11419 + -- COPTIC CAPITAL LETTER KSI + '\x2c9c'# -> unI64 11421 + -- COPTIC CAPITAL LETTER O + '\x2c9e'# -> unI64 11423 + -- COPTIC CAPITAL LETTER PI + '\x2ca0'# -> unI64 11425 + -- COPTIC CAPITAL LETTER RO + '\x2ca2'# -> unI64 11427 + -- COPTIC CAPITAL LETTER SIMA + '\x2ca4'# -> unI64 11429 + -- COPTIC CAPITAL LETTER TAU + '\x2ca6'# -> unI64 11431 + -- COPTIC CAPITAL LETTER UA + '\x2ca8'# -> unI64 11433 + -- COPTIC CAPITAL LETTER FI + '\x2caa'# -> unI64 11435 + -- COPTIC CAPITAL LETTER KHI + '\x2cac'# -> unI64 11437 + -- COPTIC CAPITAL LETTER PSI + '\x2cae'# -> unI64 11439 + -- COPTIC CAPITAL LETTER OOU + '\x2cb0'# -> unI64 11441 + -- COPTIC CAPITAL LETTER DIALECT-P ALEF + '\x2cb2'# -> unI64 11443 + -- COPTIC CAPITAL LETTER OLD COPTIC AIN + '\x2cb4'# -> unI64 11445 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE + '\x2cb6'# -> unI64 11447 + -- COPTIC CAPITAL LETTER DIALECT-P KAPA + '\x2cb8'# -> unI64 11449 + -- COPTIC CAPITAL LETTER DIALECT-P NI + '\x2cba'# -> unI64 11451 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI + '\x2cbc'# -> unI64 11453 + -- COPTIC CAPITAL LETTER OLD COPTIC OOU + '\x2cbe'# -> unI64 11455 + -- COPTIC CAPITAL LETTER SAMPI + '\x2cc0'# -> unI64 11457 + -- COPTIC CAPITAL LETTER CROSSED SHEI + '\x2cc2'# -> unI64 11459 + -- COPTIC CAPITAL LETTER OLD COPTIC SHEI + '\x2cc4'# -> unI64 11461 + -- COPTIC CAPITAL LETTER OLD COPTIC ESH + '\x2cc6'# -> unI64 11463 + -- COPTIC CAPITAL LETTER AKHMIMIC KHEI + '\x2cc8'# -> unI64 11465 + -- COPTIC CAPITAL LETTER DIALECT-P HORI + '\x2cca'# -> unI64 11467 + -- COPTIC CAPITAL LETTER OLD COPTIC HORI + '\x2ccc'# -> unI64 11469 + -- COPTIC CAPITAL LETTER OLD COPTIC HA + '\x2cce'# -> unI64 11471 + -- COPTIC CAPITAL LETTER L-SHAPED HA + '\x2cd0'# -> unI64 11473 + -- COPTIC CAPITAL LETTER OLD COPTIC HEI + '\x2cd2'# -> unI64 11475 + -- COPTIC CAPITAL LETTER OLD COPTIC HAT + '\x2cd4'# -> unI64 11477 + -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA + '\x2cd6'# -> unI64 11479 + -- COPTIC CAPITAL LETTER OLD COPTIC DJA + '\x2cd8'# -> unI64 11481 + -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA + '\x2cda'# -> unI64 11483 + -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA + '\x2cdc'# -> unI64 11485 + -- COPTIC CAPITAL LETTER OLD NUBIAN NGI + '\x2cde'# -> unI64 11487 + -- COPTIC CAPITAL LETTER OLD NUBIAN NYI + '\x2ce0'# -> unI64 11489 + -- COPTIC CAPITAL LETTER OLD NUBIAN WAU + '\x2ce2'# -> unI64 11491 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI + '\x2ceb'# -> unI64 11500 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA + '\x2ced'# -> unI64 11502 + -- COPTIC CAPITAL LETTER BOHAIRIC KHEI + '\x2cf2'# -> unI64 11507 + -- CYRILLIC CAPITAL LETTER ZEMLYA + '\xa640'# -> unI64 42561 + -- CYRILLIC CAPITAL LETTER DZELO + '\xa642'# -> unI64 42563 + -- CYRILLIC CAPITAL LETTER REVERSED DZE + '\xa644'# -> unI64 42565 + -- CYRILLIC CAPITAL LETTER IOTA + '\xa646'# -> unI64 42567 + -- CYRILLIC CAPITAL LETTER DJERV + '\xa648'# -> unI64 42569 + -- CYRILLIC CAPITAL LETTER MONOGRAPH UK + '\xa64a'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER BROAD OMEGA + '\xa64c'# -> unI64 42573 + -- CYRILLIC CAPITAL LETTER NEUTRAL YER + '\xa64e'# -> unI64 42575 + -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER + '\xa650'# -> unI64 42577 + -- CYRILLIC CAPITAL LETTER IOTIFIED YAT + '\xa652'# -> unI64 42579 + -- CYRILLIC CAPITAL LETTER REVERSED YU + '\xa654'# -> unI64 42581 + -- CYRILLIC CAPITAL LETTER IOTIFIED A + '\xa656'# -> unI64 42583 + -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS + '\xa658'# -> unI64 42585 + -- CYRILLIC CAPITAL LETTER BLENDED YUS + '\xa65a'# -> unI64 42587 + -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS + '\xa65c'# -> unI64 42589 + -- CYRILLIC CAPITAL LETTER YN + '\xa65e'# -> unI64 42591 + -- CYRILLIC CAPITAL LETTER REVERSED TSE + '\xa660'# -> unI64 42593 + -- CYRILLIC CAPITAL LETTER SOFT DE + '\xa662'# -> unI64 42595 + -- CYRILLIC CAPITAL LETTER SOFT EL + '\xa664'# -> unI64 42597 + -- CYRILLIC CAPITAL LETTER SOFT EM + '\xa666'# -> unI64 42599 + -- CYRILLIC CAPITAL LETTER MONOCULAR O + '\xa668'# -> unI64 42601 + -- CYRILLIC CAPITAL LETTER BINOCULAR O + '\xa66a'# -> unI64 42603 + -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O + '\xa66c'# -> unI64 42605 + -- CYRILLIC CAPITAL LETTER DWE + '\xa680'# -> unI64 42625 + -- CYRILLIC CAPITAL LETTER DZWE + '\xa682'# -> unI64 42627 + -- CYRILLIC CAPITAL LETTER ZHWE + '\xa684'# -> unI64 42629 + -- CYRILLIC CAPITAL LETTER CCHE + '\xa686'# -> unI64 42631 + -- CYRILLIC CAPITAL LETTER DZZE + '\xa688'# -> unI64 42633 + -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK + '\xa68a'# -> unI64 42635 + -- CYRILLIC CAPITAL LETTER TWE + '\xa68c'# -> unI64 42637 + -- CYRILLIC CAPITAL LETTER TSWE + '\xa68e'# -> unI64 42639 + -- CYRILLIC CAPITAL LETTER TSSE + '\xa690'# -> unI64 42641 + -- CYRILLIC CAPITAL LETTER TCHE + '\xa692'# -> unI64 42643 + -- CYRILLIC CAPITAL LETTER HWE + '\xa694'# -> unI64 42645 + -- CYRILLIC CAPITAL LETTER SHWE + '\xa696'# -> unI64 42647 + -- CYRILLIC CAPITAL LETTER DOUBLE O + '\xa698'# -> unI64 42649 + -- CYRILLIC CAPITAL LETTER CROSSED O + '\xa69a'# -> unI64 42651 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF + '\xa722'# -> unI64 42787 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN + '\xa724'# -> unI64 42789 + -- LATIN CAPITAL LETTER HENG + '\xa726'# -> unI64 42791 + -- LATIN CAPITAL LETTER TZ + '\xa728'# -> unI64 42793 + -- LATIN CAPITAL LETTER TRESILLO + '\xa72a'# -> unI64 42795 + -- LATIN CAPITAL LETTER CUATRILLO + '\xa72c'# -> unI64 42797 + -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA + '\xa72e'# -> unI64 42799 + -- LATIN CAPITAL LETTER AA + '\xa732'# -> unI64 42803 + -- LATIN CAPITAL LETTER AO + '\xa734'# -> unI64 42805 + -- LATIN CAPITAL LETTER AU + '\xa736'# -> unI64 42807 + -- LATIN CAPITAL LETTER AV + '\xa738'# -> unI64 42809 + -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR + '\xa73a'# -> unI64 42811 + -- LATIN CAPITAL LETTER AY + '\xa73c'# -> unI64 42813 + -- LATIN CAPITAL LETTER REVERSED C WITH DOT + '\xa73e'# -> unI64 42815 + -- LATIN CAPITAL LETTER K WITH STROKE + '\xa740'# -> unI64 42817 + -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + '\xa742'# -> unI64 42819 + -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + '\xa744'# -> unI64 42821 + -- LATIN CAPITAL LETTER BROKEN L + '\xa746'# -> unI64 42823 + -- LATIN CAPITAL LETTER L WITH HIGH STROKE + '\xa748'# -> unI64 42825 + -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + '\xa74a'# -> unI64 42827 + -- LATIN CAPITAL LETTER O WITH LOOP + '\xa74c'# -> unI64 42829 + -- LATIN CAPITAL LETTER OO + '\xa74e'# -> unI64 42831 + -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + '\xa750'# -> unI64 42833 + -- LATIN CAPITAL LETTER P WITH FLOURISH + '\xa752'# -> unI64 42835 + -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + '\xa754'# -> unI64 42837 + -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + '\xa756'# -> unI64 42839 + -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + '\xa758'# -> unI64 42841 + -- LATIN CAPITAL LETTER R ROTUNDA + '\xa75a'# -> unI64 42843 + -- LATIN CAPITAL LETTER RUM ROTUNDA + '\xa75c'# -> unI64 42845 + -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + '\xa75e'# -> unI64 42847 + -- LATIN CAPITAL LETTER VY + '\xa760'# -> unI64 42849 + -- LATIN CAPITAL LETTER VISIGOTHIC Z + '\xa762'# -> unI64 42851 + -- LATIN CAPITAL LETTER THORN WITH STROKE + '\xa764'# -> unI64 42853 + -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER + '\xa766'# -> unI64 42855 + -- LATIN CAPITAL LETTER VEND + '\xa768'# -> unI64 42857 + -- LATIN CAPITAL LETTER ET + '\xa76a'# -> unI64 42859 + -- LATIN CAPITAL LETTER IS + '\xa76c'# -> unI64 42861 + -- LATIN CAPITAL LETTER CON + '\xa76e'# -> unI64 42863 + -- LATIN CAPITAL LETTER INSULAR D + '\xa779'# -> unI64 42874 + -- LATIN CAPITAL LETTER INSULAR F + '\xa77b'# -> unI64 42876 + -- LATIN CAPITAL LETTER INSULAR G + '\xa77d'# -> unI64 7545 + -- LATIN CAPITAL LETTER TURNED INSULAR G + '\xa77e'# -> unI64 42879 + -- LATIN CAPITAL LETTER TURNED L + '\xa780'# -> unI64 42881 + -- LATIN CAPITAL LETTER INSULAR R + '\xa782'# -> unI64 42883 + -- LATIN CAPITAL LETTER INSULAR S + '\xa784'# -> unI64 42885 + -- LATIN CAPITAL LETTER INSULAR T + '\xa786'# -> unI64 42887 + -- LATIN CAPITAL LETTER SALTILLO + '\xa78b'# -> unI64 42892 + -- LATIN CAPITAL LETTER TURNED H + '\xa78d'# -> unI64 613 + -- LATIN CAPITAL LETTER N WITH DESCENDER + '\xa790'# -> unI64 42897 + -- LATIN CAPITAL LETTER C WITH BAR + '\xa792'# -> unI64 42899 + -- LATIN CAPITAL LETTER B WITH FLOURISH + '\xa796'# -> unI64 42903 + -- LATIN CAPITAL LETTER F WITH STROKE + '\xa798'# -> unI64 42905 + -- LATIN CAPITAL LETTER VOLAPUK AE + '\xa79a'# -> unI64 42907 + -- LATIN CAPITAL LETTER VOLAPUK OE + '\xa79c'# -> unI64 42909 + -- LATIN CAPITAL LETTER VOLAPUK UE + '\xa79e'# -> unI64 42911 + -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + '\xa7a0'# -> unI64 42913 + -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + '\xa7a2'# -> unI64 42915 + -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + '\xa7a4'# -> unI64 42917 + -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + '\xa7a6'# -> unI64 42919 + -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + '\xa7a8'# -> unI64 42921 + -- LATIN CAPITAL LETTER H WITH HOOK + '\xa7aa'# -> unI64 614 + -- LATIN CAPITAL LETTER REVERSED OPEN E + '\xa7ab'# -> unI64 604 + -- LATIN CAPITAL LETTER SCRIPT G + '\xa7ac'# -> unI64 609 + -- LATIN CAPITAL LETTER L WITH BELT + '\xa7ad'# -> unI64 620 + -- LATIN CAPITAL LETTER SMALL CAPITAL I + '\xa7ae'# -> unI64 618 + -- LATIN CAPITAL LETTER TURNED K + '\xa7b0'# -> unI64 670 + -- LATIN CAPITAL LETTER TURNED T + '\xa7b1'# -> unI64 647 + -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL + '\xa7b2'# -> unI64 669 + -- LATIN CAPITAL LETTER CHI + '\xa7b3'# -> unI64 43859 + -- LATIN CAPITAL LETTER BETA + '\xa7b4'# -> unI64 42933 + -- LATIN CAPITAL LETTER OMEGA + '\xa7b6'# -> unI64 42935 + -- LATIN CAPITAL LETTER U WITH STROKE + '\xa7b8'# -> unI64 42937 + -- LATIN CAPITAL LETTER GLOTTAL A + '\xa7ba'# -> unI64 42939 + -- LATIN CAPITAL LETTER GLOTTAL I + '\xa7bc'# -> unI64 42941 + -- LATIN CAPITAL LETTER GLOTTAL U + '\xa7be'# -> unI64 42943 + -- LATIN CAPITAL LETTER OLD POLISH O + '\xa7c0'# -> unI64 42945 + -- LATIN CAPITAL LETTER ANGLICANA W + '\xa7c2'# -> unI64 42947 + -- LATIN CAPITAL LETTER C WITH PALATAL HOOK + '\xa7c4'# -> unI64 42900 + -- LATIN CAPITAL LETTER S WITH HOOK + '\xa7c5'# -> unI64 642 + -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK + '\xa7c6'# -> unI64 7566 + -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + '\xa7c7'# -> unI64 42952 + -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + '\xa7c9'# -> unI64 42954 + -- LATIN CAPITAL LETTER RAMS HORN + '\xa7cb'# -> unI64 612 + -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE + '\xa7cc'# -> unI64 42957 + -- LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE + '\xa7ce'# -> unI64 42959 + -- LATIN CAPITAL LETTER CLOSED INSULAR G + '\xa7d0'# -> unI64 42961 + -- LATIN CAPITAL LETTER DOUBLE THORN + '\xa7d2'# -> unI64 42963 + -- LATIN CAPITAL LETTER DOUBLE WYNN + '\xa7d4'# -> unI64 42965 + -- LATIN CAPITAL LETTER MIDDLE SCOTS S + '\xa7d6'# -> unI64 42967 + -- LATIN CAPITAL LETTER SIGMOID S + '\xa7d8'# -> unI64 42969 + -- LATIN CAPITAL LETTER LAMBDA + '\xa7da'# -> unI64 42971 + -- LATIN CAPITAL LETTER LAMBDA WITH STROKE + '\xa7dc'# -> unI64 411 + -- LATIN CAPITAL LETTER REVERSED HALF H + '\xa7f5'# -> unI64 42998 + -- CHEROKEE SMALL LETTER A + '\xab70'# -> unI64 5024 + -- CHEROKEE SMALL LETTER E + '\xab71'# -> unI64 5025 + -- CHEROKEE SMALL LETTER I + '\xab72'# -> unI64 5026 + -- CHEROKEE SMALL LETTER O + '\xab73'# -> unI64 5027 + -- CHEROKEE SMALL LETTER U + '\xab74'# -> unI64 5028 + -- CHEROKEE SMALL LETTER V + '\xab75'# -> unI64 5029 + -- CHEROKEE SMALL LETTER GA + '\xab76'# -> unI64 5030 + -- CHEROKEE SMALL LETTER KA + '\xab77'# -> unI64 5031 + -- CHEROKEE SMALL LETTER GE + '\xab78'# -> unI64 5032 + -- CHEROKEE SMALL LETTER GI + '\xab79'# -> unI64 5033 + -- CHEROKEE SMALL LETTER GO + '\xab7a'# -> unI64 5034 + -- CHEROKEE SMALL LETTER GU + '\xab7b'# -> unI64 5035 + -- CHEROKEE SMALL LETTER GV + '\xab7c'# -> unI64 5036 + -- CHEROKEE SMALL LETTER HA + '\xab7d'# -> unI64 5037 + -- CHEROKEE SMALL LETTER HE + '\xab7e'# -> unI64 5038 + -- CHEROKEE SMALL LETTER HI + '\xab7f'# -> unI64 5039 + -- CHEROKEE SMALL LETTER HO + '\xab80'# -> unI64 5040 + -- CHEROKEE SMALL LETTER HU + '\xab81'# -> unI64 5041 + -- CHEROKEE SMALL LETTER HV + '\xab82'# -> unI64 5042 + -- CHEROKEE SMALL LETTER LA + '\xab83'# -> unI64 5043 + -- CHEROKEE SMALL LETTER LE + '\xab84'# -> unI64 5044 + -- CHEROKEE SMALL LETTER LI + '\xab85'# -> unI64 5045 + -- CHEROKEE SMALL LETTER LO + '\xab86'# -> unI64 5046 + -- CHEROKEE SMALL LETTER LU + '\xab87'# -> unI64 5047 + -- CHEROKEE SMALL LETTER LV + '\xab88'# -> unI64 5048 + -- CHEROKEE SMALL LETTER MA + '\xab89'# -> unI64 5049 + -- CHEROKEE SMALL LETTER ME + '\xab8a'# -> unI64 5050 + -- CHEROKEE SMALL LETTER MI + '\xab8b'# -> unI64 5051 + -- CHEROKEE SMALL LETTER MO + '\xab8c'# -> unI64 5052 + -- CHEROKEE SMALL LETTER MU + '\xab8d'# -> unI64 5053 + -- CHEROKEE SMALL LETTER NA + '\xab8e'# -> unI64 5054 + -- CHEROKEE SMALL LETTER HNA + '\xab8f'# -> unI64 5055 + -- CHEROKEE SMALL LETTER NAH + '\xab90'# -> unI64 5056 + -- CHEROKEE SMALL LETTER NE + '\xab91'# -> unI64 5057 + -- CHEROKEE SMALL LETTER NI + '\xab92'# -> unI64 5058 + -- CHEROKEE SMALL LETTER NO + '\xab93'# -> unI64 5059 + -- CHEROKEE SMALL LETTER NU + '\xab94'# -> unI64 5060 + -- CHEROKEE SMALL LETTER NV + '\xab95'# -> unI64 5061 + -- CHEROKEE SMALL LETTER QUA + '\xab96'# -> unI64 5062 + -- CHEROKEE SMALL LETTER QUE + '\xab97'# -> unI64 5063 + -- CHEROKEE SMALL LETTER QUI + '\xab98'# -> unI64 5064 + -- CHEROKEE SMALL LETTER QUO + '\xab99'# -> unI64 5065 + -- CHEROKEE SMALL LETTER QUU + '\xab9a'# -> unI64 5066 + -- CHEROKEE SMALL LETTER QUV + '\xab9b'# -> unI64 5067 + -- CHEROKEE SMALL LETTER SA + '\xab9c'# -> unI64 5068 + -- CHEROKEE SMALL LETTER S + '\xab9d'# -> unI64 5069 + -- CHEROKEE SMALL LETTER SE + '\xab9e'# -> unI64 5070 + -- CHEROKEE SMALL LETTER SI + '\xab9f'# -> unI64 5071 + -- CHEROKEE SMALL LETTER SO + '\xaba0'# -> unI64 5072 + -- CHEROKEE SMALL LETTER SU + '\xaba1'# -> unI64 5073 + -- CHEROKEE SMALL LETTER SV + '\xaba2'# -> unI64 5074 + -- CHEROKEE SMALL LETTER DA + '\xaba3'# -> unI64 5075 + -- CHEROKEE SMALL LETTER TA + '\xaba4'# -> unI64 5076 + -- CHEROKEE SMALL LETTER DE + '\xaba5'# -> unI64 5077 + -- CHEROKEE SMALL LETTER TE + '\xaba6'# -> unI64 5078 + -- CHEROKEE SMALL LETTER DI + '\xaba7'# -> unI64 5079 + -- CHEROKEE SMALL LETTER TI + '\xaba8'# -> unI64 5080 + -- CHEROKEE SMALL LETTER DO + '\xaba9'# -> unI64 5081 + -- CHEROKEE SMALL LETTER DU + '\xabaa'# -> unI64 5082 + -- CHEROKEE SMALL LETTER DV + '\xabab'# -> unI64 5083 + -- CHEROKEE SMALL LETTER DLA + '\xabac'# -> unI64 5084 + -- CHEROKEE SMALL LETTER TLA + '\xabad'# -> unI64 5085 + -- CHEROKEE SMALL LETTER TLE + '\xabae'# -> unI64 5086 + -- CHEROKEE SMALL LETTER TLI + '\xabaf'# -> unI64 5087 + -- CHEROKEE SMALL LETTER TLO + '\xabb0'# -> unI64 5088 + -- CHEROKEE SMALL LETTER TLU + '\xabb1'# -> unI64 5089 + -- CHEROKEE SMALL LETTER TLV + '\xabb2'# -> unI64 5090 + -- CHEROKEE SMALL LETTER TSA + '\xabb3'# -> unI64 5091 + -- CHEROKEE SMALL LETTER TSE + '\xabb4'# -> unI64 5092 + -- CHEROKEE SMALL LETTER TSI + '\xabb5'# -> unI64 5093 + -- CHEROKEE SMALL LETTER TSO + '\xabb6'# -> unI64 5094 + -- CHEROKEE SMALL LETTER TSU + '\xabb7'# -> unI64 5095 + -- CHEROKEE SMALL LETTER TSV + '\xabb8'# -> unI64 5096 + -- CHEROKEE SMALL LETTER WA + '\xabb9'# -> unI64 5097 + -- CHEROKEE SMALL LETTER WE + '\xabba'# -> unI64 5098 + -- CHEROKEE SMALL LETTER WI + '\xabbb'# -> unI64 5099 + -- CHEROKEE SMALL LETTER WO + '\xabbc'# -> unI64 5100 + -- CHEROKEE SMALL LETTER WU + '\xabbd'# -> unI64 5101 + -- CHEROKEE SMALL LETTER WV + '\xabbe'# -> unI64 5102 + -- CHEROKEE SMALL LETTER YA + '\xabbf'# -> unI64 5103 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909606 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201062 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492518 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575526 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108838 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269747 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269747 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819892 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168308 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751220 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819902 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945524 + -- FULLWIDTH LATIN CAPITAL LETTER A + '\xff21'# -> unI64 65345 + -- FULLWIDTH LATIN CAPITAL LETTER B + '\xff22'# -> unI64 65346 + -- FULLWIDTH LATIN CAPITAL LETTER C + '\xff23'# -> unI64 65347 + -- FULLWIDTH LATIN CAPITAL LETTER D + '\xff24'# -> unI64 65348 + -- FULLWIDTH LATIN CAPITAL LETTER E + '\xff25'# -> unI64 65349 + -- FULLWIDTH LATIN CAPITAL LETTER F + '\xff26'# -> unI64 65350 + -- FULLWIDTH LATIN CAPITAL LETTER G + '\xff27'# -> unI64 65351 + -- FULLWIDTH LATIN CAPITAL LETTER H + '\xff28'# -> unI64 65352 + -- FULLWIDTH LATIN CAPITAL LETTER I + '\xff29'# -> unI64 65353 + -- FULLWIDTH LATIN CAPITAL LETTER J + '\xff2a'# -> unI64 65354 + -- FULLWIDTH LATIN CAPITAL LETTER K + '\xff2b'# -> unI64 65355 + -- FULLWIDTH LATIN CAPITAL LETTER L + '\xff2c'# -> unI64 65356 + -- FULLWIDTH LATIN CAPITAL LETTER M + '\xff2d'# -> unI64 65357 + -- FULLWIDTH LATIN CAPITAL LETTER N + '\xff2e'# -> unI64 65358 + -- FULLWIDTH LATIN CAPITAL LETTER O + '\xff2f'# -> unI64 65359 + -- FULLWIDTH LATIN CAPITAL LETTER P + '\xff30'# -> unI64 65360 + -- FULLWIDTH LATIN CAPITAL LETTER Q + '\xff31'# -> unI64 65361 + -- FULLWIDTH LATIN CAPITAL LETTER R + '\xff32'# -> unI64 65362 + -- FULLWIDTH LATIN CAPITAL LETTER S + '\xff33'# -> unI64 65363 + -- FULLWIDTH LATIN CAPITAL LETTER T + '\xff34'# -> unI64 65364 + -- FULLWIDTH LATIN CAPITAL LETTER U + '\xff35'# -> unI64 65365 + -- FULLWIDTH LATIN CAPITAL LETTER V + '\xff36'# -> unI64 65366 + -- FULLWIDTH LATIN CAPITAL LETTER W + '\xff37'# -> unI64 65367 + -- FULLWIDTH LATIN CAPITAL LETTER X + '\xff38'# -> unI64 65368 + -- FULLWIDTH LATIN CAPITAL LETTER Y + '\xff39'# -> unI64 65369 + -- FULLWIDTH LATIN CAPITAL LETTER Z + '\xff3a'# -> unI64 65370 + -- DESERET CAPITAL LETTER LONG I + '\x10400'# -> unI64 66600 + -- DESERET CAPITAL LETTER LONG E + '\x10401'# -> unI64 66601 + -- DESERET CAPITAL LETTER LONG A + '\x10402'# -> unI64 66602 + -- DESERET CAPITAL LETTER LONG AH + '\x10403'# -> unI64 66603 + -- DESERET CAPITAL LETTER LONG O + '\x10404'# -> unI64 66604 + -- DESERET CAPITAL LETTER LONG OO + '\x10405'# -> unI64 66605 + -- DESERET CAPITAL LETTER SHORT I + '\x10406'# -> unI64 66606 + -- DESERET CAPITAL LETTER SHORT E + '\x10407'# -> unI64 66607 + -- DESERET CAPITAL LETTER SHORT A + '\x10408'# -> unI64 66608 + -- DESERET CAPITAL LETTER SHORT AH + '\x10409'# -> unI64 66609 + -- DESERET CAPITAL LETTER SHORT O + '\x1040a'# -> unI64 66610 + -- DESERET CAPITAL LETTER SHORT OO + '\x1040b'# -> unI64 66611 + -- DESERET CAPITAL LETTER AY + '\x1040c'# -> unI64 66612 + -- DESERET CAPITAL LETTER OW + '\x1040d'# -> unI64 66613 + -- DESERET CAPITAL LETTER WU + '\x1040e'# -> unI64 66614 + -- DESERET CAPITAL LETTER YEE + '\x1040f'# -> unI64 66615 + -- DESERET CAPITAL LETTER H + '\x10410'# -> unI64 66616 + -- DESERET CAPITAL LETTER PEE + '\x10411'# -> unI64 66617 + -- DESERET CAPITAL LETTER BEE + '\x10412'# -> unI64 66618 + -- DESERET CAPITAL LETTER TEE + '\x10413'# -> unI64 66619 + -- DESERET CAPITAL LETTER DEE + '\x10414'# -> unI64 66620 + -- DESERET CAPITAL LETTER CHEE + '\x10415'# -> unI64 66621 + -- DESERET CAPITAL LETTER JEE + '\x10416'# -> unI64 66622 + -- DESERET CAPITAL LETTER KAY + '\x10417'# -> unI64 66623 + -- DESERET CAPITAL LETTER GAY + '\x10418'# -> unI64 66624 + -- DESERET CAPITAL LETTER EF + '\x10419'# -> unI64 66625 + -- DESERET CAPITAL LETTER VEE + '\x1041a'# -> unI64 66626 + -- DESERET CAPITAL LETTER ETH + '\x1041b'# -> unI64 66627 + -- DESERET CAPITAL LETTER THEE + '\x1041c'# -> unI64 66628 + -- DESERET CAPITAL LETTER ES + '\x1041d'# -> unI64 66629 + -- DESERET CAPITAL LETTER ZEE + '\x1041e'# -> unI64 66630 + -- DESERET CAPITAL LETTER ESH + '\x1041f'# -> unI64 66631 + -- DESERET CAPITAL LETTER ZHEE + '\x10420'# -> unI64 66632 + -- DESERET CAPITAL LETTER ER + '\x10421'# -> unI64 66633 + -- DESERET CAPITAL LETTER EL + '\x10422'# -> unI64 66634 + -- DESERET CAPITAL LETTER EM + '\x10423'# -> unI64 66635 + -- DESERET CAPITAL LETTER EN + '\x10424'# -> unI64 66636 + -- DESERET CAPITAL LETTER ENG + '\x10425'# -> unI64 66637 + -- DESERET CAPITAL LETTER OI + '\x10426'# -> unI64 66638 + -- DESERET CAPITAL LETTER EW + '\x10427'# -> unI64 66639 + -- OSAGE CAPITAL LETTER A + '\x104b0'# -> unI64 66776 + -- OSAGE CAPITAL LETTER AI + '\x104b1'# -> unI64 66777 + -- OSAGE CAPITAL LETTER AIN + '\x104b2'# -> unI64 66778 + -- OSAGE CAPITAL LETTER AH + '\x104b3'# -> unI64 66779 + -- OSAGE CAPITAL LETTER BRA + '\x104b4'# -> unI64 66780 + -- OSAGE CAPITAL LETTER CHA + '\x104b5'# -> unI64 66781 + -- OSAGE CAPITAL LETTER EHCHA + '\x104b6'# -> unI64 66782 + -- OSAGE CAPITAL LETTER E + '\x104b7'# -> unI64 66783 + -- OSAGE CAPITAL LETTER EIN + '\x104b8'# -> unI64 66784 + -- OSAGE CAPITAL LETTER HA + '\x104b9'# -> unI64 66785 + -- OSAGE CAPITAL LETTER HYA + '\x104ba'# -> unI64 66786 + -- OSAGE CAPITAL LETTER I + '\x104bb'# -> unI64 66787 + -- OSAGE CAPITAL LETTER KA + '\x104bc'# -> unI64 66788 + -- OSAGE CAPITAL LETTER EHKA + '\x104bd'# -> unI64 66789 + -- OSAGE CAPITAL LETTER KYA + '\x104be'# -> unI64 66790 + -- OSAGE CAPITAL LETTER LA + '\x104bf'# -> unI64 66791 + -- OSAGE CAPITAL LETTER MA + '\x104c0'# -> unI64 66792 + -- OSAGE CAPITAL LETTER NA + '\x104c1'# -> unI64 66793 + -- OSAGE CAPITAL LETTER O + '\x104c2'# -> unI64 66794 + -- OSAGE CAPITAL LETTER OIN + '\x104c3'# -> unI64 66795 + -- OSAGE CAPITAL LETTER PA + '\x104c4'# -> unI64 66796 + -- OSAGE CAPITAL LETTER EHPA + '\x104c5'# -> unI64 66797 + -- OSAGE CAPITAL LETTER SA + '\x104c6'# -> unI64 66798 + -- OSAGE CAPITAL LETTER SHA + '\x104c7'# -> unI64 66799 + -- OSAGE CAPITAL LETTER TA + '\x104c8'# -> unI64 66800 + -- OSAGE CAPITAL LETTER EHTA + '\x104c9'# -> unI64 66801 + -- OSAGE CAPITAL LETTER TSA + '\x104ca'# -> unI64 66802 + -- OSAGE CAPITAL LETTER EHTSA + '\x104cb'# -> unI64 66803 + -- OSAGE CAPITAL LETTER TSHA + '\x104cc'# -> unI64 66804 + -- OSAGE CAPITAL LETTER DHA + '\x104cd'# -> unI64 66805 + -- OSAGE CAPITAL LETTER U + '\x104ce'# -> unI64 66806 + -- OSAGE CAPITAL LETTER WA + '\x104cf'# -> unI64 66807 + -- OSAGE CAPITAL LETTER KHA + '\x104d0'# -> unI64 66808 + -- OSAGE CAPITAL LETTER GHA + '\x104d1'# -> unI64 66809 + -- OSAGE CAPITAL LETTER ZA + '\x104d2'# -> unI64 66810 + -- OSAGE CAPITAL LETTER ZHA + '\x104d3'# -> unI64 66811 + -- VITHKUQI CAPITAL LETTER A + '\x10570'# -> unI64 66967 + -- VITHKUQI CAPITAL LETTER BBE + '\x10571'# -> unI64 66968 + -- VITHKUQI CAPITAL LETTER BE + '\x10572'# -> unI64 66969 + -- VITHKUQI CAPITAL LETTER CE + '\x10573'# -> unI64 66970 + -- VITHKUQI CAPITAL LETTER CHE + '\x10574'# -> unI64 66971 + -- VITHKUQI CAPITAL LETTER DE + '\x10575'# -> unI64 66972 + -- VITHKUQI CAPITAL LETTER DHE + '\x10576'# -> unI64 66973 + -- VITHKUQI CAPITAL LETTER EI + '\x10577'# -> unI64 66974 + -- VITHKUQI CAPITAL LETTER E + '\x10578'# -> unI64 66975 + -- VITHKUQI CAPITAL LETTER FE + '\x10579'# -> unI64 66976 + -- VITHKUQI CAPITAL LETTER GA + '\x1057a'# -> unI64 66977 + -- VITHKUQI CAPITAL LETTER HA + '\x1057c'# -> unI64 66979 + -- VITHKUQI CAPITAL LETTER HHA + '\x1057d'# -> unI64 66980 + -- VITHKUQI CAPITAL LETTER I + '\x1057e'# -> unI64 66981 + -- VITHKUQI CAPITAL LETTER IJE + '\x1057f'# -> unI64 66982 + -- VITHKUQI CAPITAL LETTER JE + '\x10580'# -> unI64 66983 + -- VITHKUQI CAPITAL LETTER KA + '\x10581'# -> unI64 66984 + -- VITHKUQI CAPITAL LETTER LA + '\x10582'# -> unI64 66985 + -- VITHKUQI CAPITAL LETTER LLA + '\x10583'# -> unI64 66986 + -- VITHKUQI CAPITAL LETTER ME + '\x10584'# -> unI64 66987 + -- VITHKUQI CAPITAL LETTER NE + '\x10585'# -> unI64 66988 + -- VITHKUQI CAPITAL LETTER NJE + '\x10586'# -> unI64 66989 + -- VITHKUQI CAPITAL LETTER O + '\x10587'# -> unI64 66990 + -- VITHKUQI CAPITAL LETTER PE + '\x10588'# -> unI64 66991 + -- VITHKUQI CAPITAL LETTER QA + '\x10589'# -> unI64 66992 + -- VITHKUQI CAPITAL LETTER RE + '\x1058a'# -> unI64 66993 + -- VITHKUQI CAPITAL LETTER SE + '\x1058c'# -> unI64 66995 + -- VITHKUQI CAPITAL LETTER SHE + '\x1058d'# -> unI64 66996 + -- VITHKUQI CAPITAL LETTER TE + '\x1058e'# -> unI64 66997 + -- VITHKUQI CAPITAL LETTER THE + '\x1058f'# -> unI64 66998 + -- VITHKUQI CAPITAL LETTER U + '\x10590'# -> unI64 66999 + -- VITHKUQI CAPITAL LETTER VE + '\x10591'# -> unI64 67000 + -- VITHKUQI CAPITAL LETTER XE + '\x10592'# -> unI64 67001 + -- VITHKUQI CAPITAL LETTER Y + '\x10594'# -> unI64 67003 + -- VITHKUQI CAPITAL LETTER ZE + '\x10595'# -> unI64 67004 + -- OLD HUNGARIAN CAPITAL LETTER A + '\x10c80'# -> unI64 68800 + -- OLD HUNGARIAN CAPITAL LETTER AA + '\x10c81'# -> unI64 68801 + -- OLD HUNGARIAN CAPITAL LETTER EB + '\x10c82'# -> unI64 68802 + -- OLD HUNGARIAN CAPITAL LETTER AMB + '\x10c83'# -> unI64 68803 + -- OLD HUNGARIAN CAPITAL LETTER EC + '\x10c84'# -> unI64 68804 + -- OLD HUNGARIAN CAPITAL LETTER ENC + '\x10c85'# -> unI64 68805 + -- OLD HUNGARIAN CAPITAL LETTER ECS + '\x10c86'# -> unI64 68806 + -- OLD HUNGARIAN CAPITAL LETTER ED + '\x10c87'# -> unI64 68807 + -- OLD HUNGARIAN CAPITAL LETTER AND + '\x10c88'# -> unI64 68808 + -- OLD HUNGARIAN CAPITAL LETTER E + '\x10c89'# -> unI64 68809 + -- OLD HUNGARIAN CAPITAL LETTER CLOSE E + '\x10c8a'# -> unI64 68810 + -- OLD HUNGARIAN CAPITAL LETTER EE + '\x10c8b'# -> unI64 68811 + -- OLD HUNGARIAN CAPITAL LETTER EF + '\x10c8c'# -> unI64 68812 + -- OLD HUNGARIAN CAPITAL LETTER EG + '\x10c8d'# -> unI64 68813 + -- OLD HUNGARIAN CAPITAL LETTER EGY + '\x10c8e'# -> unI64 68814 + -- OLD HUNGARIAN CAPITAL LETTER EH + '\x10c8f'# -> unI64 68815 + -- OLD HUNGARIAN CAPITAL LETTER I + '\x10c90'# -> unI64 68816 + -- OLD HUNGARIAN CAPITAL LETTER II + '\x10c91'# -> unI64 68817 + -- OLD HUNGARIAN CAPITAL LETTER EJ + '\x10c92'# -> unI64 68818 + -- OLD HUNGARIAN CAPITAL LETTER EK + '\x10c93'# -> unI64 68819 + -- OLD HUNGARIAN CAPITAL LETTER AK + '\x10c94'# -> unI64 68820 + -- OLD HUNGARIAN CAPITAL LETTER UNK + '\x10c95'# -> unI64 68821 + -- OLD HUNGARIAN CAPITAL LETTER EL + '\x10c96'# -> unI64 68822 + -- OLD HUNGARIAN CAPITAL LETTER ELY + '\x10c97'# -> unI64 68823 + -- OLD HUNGARIAN CAPITAL LETTER EM + '\x10c98'# -> unI64 68824 + -- OLD HUNGARIAN CAPITAL LETTER EN + '\x10c99'# -> unI64 68825 + -- OLD HUNGARIAN CAPITAL LETTER ENY + '\x10c9a'# -> unI64 68826 + -- OLD HUNGARIAN CAPITAL LETTER O + '\x10c9b'# -> unI64 68827 + -- OLD HUNGARIAN CAPITAL LETTER OO + '\x10c9c'# -> unI64 68828 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE + '\x10c9d'# -> unI64 68829 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE + '\x10c9e'# -> unI64 68830 + -- OLD HUNGARIAN CAPITAL LETTER OEE + '\x10c9f'# -> unI64 68831 + -- OLD HUNGARIAN CAPITAL LETTER EP + '\x10ca0'# -> unI64 68832 + -- OLD HUNGARIAN CAPITAL LETTER EMP + '\x10ca1'# -> unI64 68833 + -- OLD HUNGARIAN CAPITAL LETTER ER + '\x10ca2'# -> unI64 68834 + -- OLD HUNGARIAN CAPITAL LETTER SHORT ER + '\x10ca3'# -> unI64 68835 + -- OLD HUNGARIAN CAPITAL LETTER ES + '\x10ca4'# -> unI64 68836 + -- OLD HUNGARIAN CAPITAL LETTER ESZ + '\x10ca5'# -> unI64 68837 + -- OLD HUNGARIAN CAPITAL LETTER ET + '\x10ca6'# -> unI64 68838 + -- OLD HUNGARIAN CAPITAL LETTER ENT + '\x10ca7'# -> unI64 68839 + -- OLD HUNGARIAN CAPITAL LETTER ETY + '\x10ca8'# -> unI64 68840 + -- OLD HUNGARIAN CAPITAL LETTER ECH + '\x10ca9'# -> unI64 68841 + -- OLD HUNGARIAN CAPITAL LETTER U + '\x10caa'# -> unI64 68842 + -- OLD HUNGARIAN CAPITAL LETTER UU + '\x10cab'# -> unI64 68843 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE + '\x10cac'# -> unI64 68844 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE + '\x10cad'# -> unI64 68845 + -- OLD HUNGARIAN CAPITAL LETTER EV + '\x10cae'# -> unI64 68846 + -- OLD HUNGARIAN CAPITAL LETTER EZ + '\x10caf'# -> unI64 68847 + -- OLD HUNGARIAN CAPITAL LETTER EZS + '\x10cb0'# -> unI64 68848 + -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN + '\x10cb1'# -> unI64 68849 + -- OLD HUNGARIAN CAPITAL LETTER US + '\x10cb2'# -> unI64 68850 + -- GARAY CAPITAL LETTER A + '\x10d50'# -> unI64 68976 + -- GARAY CAPITAL LETTER CA + '\x10d51'# -> unI64 68977 + -- GARAY CAPITAL LETTER MA + '\x10d52'# -> unI64 68978 + -- GARAY CAPITAL LETTER KA + '\x10d53'# -> unI64 68979 + -- GARAY CAPITAL LETTER BA + '\x10d54'# -> unI64 68980 + -- GARAY CAPITAL LETTER JA + '\x10d55'# -> unI64 68981 + -- GARAY CAPITAL LETTER SA + '\x10d56'# -> unI64 68982 + -- GARAY CAPITAL LETTER WA + '\x10d57'# -> unI64 68983 + -- GARAY CAPITAL LETTER LA + '\x10d58'# -> unI64 68984 + -- GARAY CAPITAL LETTER GA + '\x10d59'# -> unI64 68985 + -- GARAY CAPITAL LETTER DA + '\x10d5a'# -> unI64 68986 + -- GARAY CAPITAL LETTER XA + '\x10d5b'# -> unI64 68987 + -- GARAY CAPITAL LETTER YA + '\x10d5c'# -> unI64 68988 + -- GARAY CAPITAL LETTER TA + '\x10d5d'# -> unI64 68989 + -- GARAY CAPITAL LETTER RA + '\x10d5e'# -> unI64 68990 + -- GARAY CAPITAL LETTER NYA + '\x10d5f'# -> unI64 68991 + -- GARAY CAPITAL LETTER FA + '\x10d60'# -> unI64 68992 + -- GARAY CAPITAL LETTER NA + '\x10d61'# -> unI64 68993 + -- GARAY CAPITAL LETTER PA + '\x10d62'# -> unI64 68994 + -- GARAY CAPITAL LETTER HA + '\x10d63'# -> unI64 68995 + -- GARAY CAPITAL LETTER OLD KA + '\x10d64'# -> unI64 68996 + -- GARAY CAPITAL LETTER OLD NA + '\x10d65'# -> unI64 68997 + -- WARANG CITI CAPITAL LETTER NGAA + '\x118a0'# -> unI64 71872 + -- WARANG CITI CAPITAL LETTER A + '\x118a1'# -> unI64 71873 + -- WARANG CITI CAPITAL LETTER WI + '\x118a2'# -> unI64 71874 + -- WARANG CITI CAPITAL LETTER YU + '\x118a3'# -> unI64 71875 + -- WARANG CITI CAPITAL LETTER YA + '\x118a4'# -> unI64 71876 + -- WARANG CITI CAPITAL LETTER YO + '\x118a5'# -> unI64 71877 + -- WARANG CITI CAPITAL LETTER II + '\x118a6'# -> unI64 71878 + -- WARANG CITI CAPITAL LETTER UU + '\x118a7'# -> unI64 71879 + -- WARANG CITI CAPITAL LETTER E + '\x118a8'# -> unI64 71880 + -- WARANG CITI CAPITAL LETTER O + '\x118a9'# -> unI64 71881 + -- WARANG CITI CAPITAL LETTER ANG + '\x118aa'# -> unI64 71882 + -- WARANG CITI CAPITAL LETTER GA + '\x118ab'# -> unI64 71883 + -- WARANG CITI CAPITAL LETTER KO + '\x118ac'# -> unI64 71884 + -- WARANG CITI CAPITAL LETTER ENY + '\x118ad'# -> unI64 71885 + -- WARANG CITI CAPITAL LETTER YUJ + '\x118ae'# -> unI64 71886 + -- WARANG CITI CAPITAL LETTER UC + '\x118af'# -> unI64 71887 + -- WARANG CITI CAPITAL LETTER ENN + '\x118b0'# -> unI64 71888 + -- WARANG CITI CAPITAL LETTER ODD + '\x118b1'# -> unI64 71889 + -- WARANG CITI CAPITAL LETTER TTE + '\x118b2'# -> unI64 71890 + -- WARANG CITI CAPITAL LETTER NUNG + '\x118b3'# -> unI64 71891 + -- WARANG CITI CAPITAL LETTER DA + '\x118b4'# -> unI64 71892 + -- WARANG CITI CAPITAL LETTER AT + '\x118b5'# -> unI64 71893 + -- WARANG CITI CAPITAL LETTER AM + '\x118b6'# -> unI64 71894 + -- WARANG CITI CAPITAL LETTER BU + '\x118b7'# -> unI64 71895 + -- WARANG CITI CAPITAL LETTER PU + '\x118b8'# -> unI64 71896 + -- WARANG CITI CAPITAL LETTER HIYO + '\x118b9'# -> unI64 71897 + -- WARANG CITI CAPITAL LETTER HOLO + '\x118ba'# -> unI64 71898 + -- WARANG CITI CAPITAL LETTER HORR + '\x118bb'# -> unI64 71899 + -- WARANG CITI CAPITAL LETTER HAR + '\x118bc'# -> unI64 71900 + -- WARANG CITI CAPITAL LETTER SSUU + '\x118bd'# -> unI64 71901 + -- WARANG CITI CAPITAL LETTER SII + '\x118be'# -> unI64 71902 + -- WARANG CITI CAPITAL LETTER VIYO + '\x118bf'# -> unI64 71903 + -- MEDEFAIDRIN CAPITAL LETTER M + '\x16e40'# -> unI64 93792 + -- MEDEFAIDRIN CAPITAL LETTER S + '\x16e41'# -> unI64 93793 + -- MEDEFAIDRIN CAPITAL LETTER V + '\x16e42'# -> unI64 93794 + -- MEDEFAIDRIN CAPITAL LETTER W + '\x16e43'# -> unI64 93795 + -- MEDEFAIDRIN CAPITAL LETTER ATIU + '\x16e44'# -> unI64 93796 + -- MEDEFAIDRIN CAPITAL LETTER Z + '\x16e45'# -> unI64 93797 + -- MEDEFAIDRIN CAPITAL LETTER KP + '\x16e46'# -> unI64 93798 + -- MEDEFAIDRIN CAPITAL LETTER P + '\x16e47'# -> unI64 93799 + -- MEDEFAIDRIN CAPITAL LETTER T + '\x16e48'# -> unI64 93800 + -- MEDEFAIDRIN CAPITAL LETTER G + '\x16e49'# -> unI64 93801 + -- MEDEFAIDRIN CAPITAL LETTER F + '\x16e4a'# -> unI64 93802 + -- MEDEFAIDRIN CAPITAL LETTER I + '\x16e4b'# -> unI64 93803 + -- MEDEFAIDRIN CAPITAL LETTER K + '\x16e4c'# -> unI64 93804 + -- MEDEFAIDRIN CAPITAL LETTER A + '\x16e4d'# -> unI64 93805 + -- MEDEFAIDRIN CAPITAL LETTER J + '\x16e4e'# -> unI64 93806 + -- MEDEFAIDRIN CAPITAL LETTER E + '\x16e4f'# -> unI64 93807 + -- MEDEFAIDRIN CAPITAL LETTER B + '\x16e50'# -> unI64 93808 + -- MEDEFAIDRIN CAPITAL LETTER C + '\x16e51'# -> unI64 93809 + -- MEDEFAIDRIN CAPITAL LETTER U + '\x16e52'# -> unI64 93810 + -- MEDEFAIDRIN CAPITAL LETTER YU + '\x16e53'# -> unI64 93811 + -- MEDEFAIDRIN CAPITAL LETTER L + '\x16e54'# -> unI64 93812 + -- MEDEFAIDRIN CAPITAL LETTER Q + '\x16e55'# -> unI64 93813 + -- MEDEFAIDRIN CAPITAL LETTER HP + '\x16e56'# -> unI64 93814 + -- MEDEFAIDRIN CAPITAL LETTER NY + '\x16e57'# -> unI64 93815 + -- MEDEFAIDRIN CAPITAL LETTER X + '\x16e58'# -> unI64 93816 + -- MEDEFAIDRIN CAPITAL LETTER D + '\x16e59'# -> unI64 93817 + -- MEDEFAIDRIN CAPITAL LETTER OE + '\x16e5a'# -> unI64 93818 + -- MEDEFAIDRIN CAPITAL LETTER N + '\x16e5b'# -> unI64 93819 + -- MEDEFAIDRIN CAPITAL LETTER R + '\x16e5c'# -> unI64 93820 + -- MEDEFAIDRIN CAPITAL LETTER O + '\x16e5d'# -> unI64 93821 + -- MEDEFAIDRIN CAPITAL LETTER AI + '\x16e5e'# -> unI64 93822 + -- MEDEFAIDRIN CAPITAL LETTER Y + '\x16e5f'# -> unI64 93823 + -- BERIA ERFE CAPITAL LETTER ARKAB + '\x16ea0'# -> unI64 93883 + -- BERIA ERFE CAPITAL LETTER BASIGNA + '\x16ea1'# -> unI64 93884 + -- BERIA ERFE CAPITAL LETTER DARBAI + '\x16ea2'# -> unI64 93885 + -- BERIA ERFE CAPITAL LETTER EH + '\x16ea3'# -> unI64 93886 + -- BERIA ERFE CAPITAL LETTER FITKO + '\x16ea4'# -> unI64 93887 + -- BERIA ERFE CAPITAL LETTER GOWAY + '\x16ea5'# -> unI64 93888 + -- BERIA ERFE CAPITAL LETTER HIRDEABO + '\x16ea6'# -> unI64 93889 + -- BERIA ERFE CAPITAL LETTER I + '\x16ea7'# -> unI64 93890 + -- BERIA ERFE CAPITAL LETTER DJAI + '\x16ea8'# -> unI64 93891 + -- BERIA ERFE CAPITAL LETTER KOBO + '\x16ea9'# -> unI64 93892 + -- BERIA ERFE CAPITAL LETTER LAKKO + '\x16eaa'# -> unI64 93893 + -- BERIA ERFE CAPITAL LETTER MERI + '\x16eab'# -> unI64 93894 + -- BERIA ERFE CAPITAL LETTER NINI + '\x16eac'# -> unI64 93895 + -- BERIA ERFE CAPITAL LETTER GNA + '\x16ead'# -> unI64 93896 + -- BERIA ERFE CAPITAL LETTER NGAY + '\x16eae'# -> unI64 93897 + -- BERIA ERFE CAPITAL LETTER OI + '\x16eaf'# -> unI64 93898 + -- BERIA ERFE CAPITAL LETTER PI + '\x16eb0'# -> unI64 93899 + -- BERIA ERFE CAPITAL LETTER ERIGO + '\x16eb1'# -> unI64 93900 + -- BERIA ERFE CAPITAL LETTER ERIGO TAMURA + '\x16eb2'# -> unI64 93901 + -- BERIA ERFE CAPITAL LETTER SERI + '\x16eb3'# -> unI64 93902 + -- BERIA ERFE CAPITAL LETTER SHEP + '\x16eb4'# -> unI64 93903 + -- BERIA ERFE CAPITAL LETTER TATASOUE + '\x16eb5'# -> unI64 93904 + -- BERIA ERFE CAPITAL LETTER UI + '\x16eb6'# -> unI64 93905 + -- BERIA ERFE CAPITAL LETTER WASSE + '\x16eb7'# -> unI64 93906 + -- BERIA ERFE CAPITAL LETTER AY + '\x16eb8'# -> unI64 93907 + -- ADLAM CAPITAL LETTER ALIF + '\x1e900'# -> unI64 125218 + -- ADLAM CAPITAL LETTER DAALI + '\x1e901'# -> unI64 125219 + -- ADLAM CAPITAL LETTER LAAM + '\x1e902'# -> unI64 125220 + -- ADLAM CAPITAL LETTER MIIM + '\x1e903'# -> unI64 125221 + -- ADLAM CAPITAL LETTER BA + '\x1e904'# -> unI64 125222 + -- ADLAM CAPITAL LETTER SINNYIIYHE + '\x1e905'# -> unI64 125223 + -- ADLAM CAPITAL LETTER PE + '\x1e906'# -> unI64 125224 + -- ADLAM CAPITAL LETTER BHE + '\x1e907'# -> unI64 125225 + -- ADLAM CAPITAL LETTER RA + '\x1e908'# -> unI64 125226 + -- ADLAM CAPITAL LETTER E + '\x1e909'# -> unI64 125227 + -- ADLAM CAPITAL LETTER FA + '\x1e90a'# -> unI64 125228 + -- ADLAM CAPITAL LETTER I + '\x1e90b'# -> unI64 125229 + -- ADLAM CAPITAL LETTER O + '\x1e90c'# -> unI64 125230 + -- ADLAM CAPITAL LETTER DHA + '\x1e90d'# -> unI64 125231 + -- ADLAM CAPITAL LETTER YHE + '\x1e90e'# -> unI64 125232 + -- ADLAM CAPITAL LETTER WAW + '\x1e90f'# -> unI64 125233 + -- ADLAM CAPITAL LETTER NUN + '\x1e910'# -> unI64 125234 + -- ADLAM CAPITAL LETTER KAF + '\x1e911'# -> unI64 125235 + -- ADLAM CAPITAL LETTER YA + '\x1e912'# -> unI64 125236 + -- ADLAM CAPITAL LETTER U + '\x1e913'# -> unI64 125237 + -- ADLAM CAPITAL LETTER JIIM + '\x1e914'# -> unI64 125238 + -- ADLAM CAPITAL LETTER CHI + '\x1e915'# -> unI64 125239 + -- ADLAM CAPITAL LETTER HA + '\x1e916'# -> unI64 125240 + -- ADLAM CAPITAL LETTER QAAF + '\x1e917'# -> unI64 125241 + -- ADLAM CAPITAL LETTER GA + '\x1e918'# -> unI64 125242 + -- ADLAM CAPITAL LETTER NYA + '\x1e919'# -> unI64 125243 + -- ADLAM CAPITAL LETTER TU + '\x1e91a'# -> unI64 125244 + -- ADLAM CAPITAL LETTER NHA + '\x1e91b'# -> unI64 125245 + -- ADLAM CAPITAL LETTER VA + '\x1e91c'# -> unI64 125246 + -- ADLAM CAPITAL LETTER KHA + '\x1e91d'# -> unI64 125247 + -- ADLAM CAPITAL LETTER GBE + '\x1e91e'# -> unI64 125248 + -- ADLAM CAPITAL LETTER ZAL + '\x1e91f'# -> unI64 125249 + -- ADLAM CAPITAL LETTER KPO + '\x1e920'# -> unI64 125250 + -- ADLAM CAPITAL LETTER SHA + '\x1e921'# -> unI64 125251 + _ -> unI64 0 diff --git a/src/Data/Text/Internal/Fusion/Common.hs b/src/Data/Text/Internal/Fusion/Common.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Fusion/Common.hs @@ -0,0 +1,1213 @@ +{-# LANGUAGE BangPatterns, MagicHash, RankNTypes, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +-- | +-- Module : Data.Text.Internal.Fusion.Common +-- Copyright : (c) Bryan O'Sullivan 2009, 2012 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- This module provides a common stream fusion interface for text. +-- The stream interface allows us to write text pipelines which +-- do not allocate intermediate text values. For example, we could +-- guarantee no intermediate text is allocated by writing the following: +-- +-- @ +-- getNucleotides :: 'Data.Text.Internal.Text' -> 'Data.Text.Internal.Text' +-- getNucleotides = +-- 'Data.Text.Internal.Fusion.unstream' +-- . 'filter' isNucleotide +-- . 'toLower' +-- . 'Data.Text.Internal.Fusion.stream' +-- where +-- isNucleotide chr = +-- chr == \'a\' || +-- chr == \'c\' || +-- chr == \'t\' || +-- chr == \'g\' +-- @ + +module Data.Text.Internal.Fusion.Common + ( + -- * Creation and elimination + singleton + , streamList + , unstreamList + , streamCString# + + -- * Basic interface + , cons + , snoc + , append + , head + , uncons + , last + , tail + , init + , null + , lengthI + , compareLengthI + , isSingleton + + -- * Transformations + , map + , intercalate + , intersperse + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toTitle + , toUpper + + -- ** Justification + , justifyLeftI + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + + -- * Construction + -- ** Scans + , scanl + + -- ** Generation and unfolding + , replicateCharI + , replicateI + , unfoldr + , unfoldrNI + + -- * Substrings + -- ** Breaking strings + , take + , drop + , takeWhile + , dropWhile + + -- * Predicates + , isPrefixOf + + -- * Searching + , elem + , filter + + -- * Indexing + , findBy + , indexI + , findIndexI + , countCharI + + -- * Zipping and unzipping + , zipWith + ) where + +import Prelude (Bool(..), Char, Eq, (==), Int, Integral, Maybe(..), + Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++), + (&&), fromIntegral, otherwise) +import qualified Data.List as L hiding (head, tail) +import qualified Prelude as P +import Data.Bits (shiftL, shiftR, (.&.)) +import Data.Char (isLetter, isSpace) +import GHC.Int (Int64(..)) +import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader) +import Data.Text.Internal.Fusion.Types +import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping, + upperMapping) +import Data.Text.Internal.Fusion.Size +import GHC.Exts (Char(..), Char#, chr#) +import GHC.Prim (Addr#, indexWord8OffAddr#) +import GHC.Stack (HasCallStack) +import GHC.Types (Int(..)) +import Data.Text.Internal.Unsafe.Char (unsafeChr8) +import GHC.Word + +-- | /O(1)/ Convert a character into a 'Stream' +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'singleton' = 'Data.Text.singleton'@ +singleton :: Char -> Stream Char +singleton c = Stream next False (codePointsSize 1) + where next False = Yield c True + next True = Done +{-# INLINE [0] singleton #-} + +-- | /O(n)/ Convert a list into a 'Stream'. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'streamList' = 'Data.Text.pack'@ +streamList :: [a] -> Stream a +{-# INLINE [0] streamList #-} +streamList s = Stream next s unknownSize + where next [] = Done + next (x:xs) = Yield x xs + +-- | /O(n)/ Convert a 'Stream' into a list. +-- +-- __Properties__ +-- +-- @'unstreamList' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.unpack'@ +unstreamList :: Stream a -> [a] +unstreamList (Stream next s0 _len) = unfold s0 + where unfold !s = case next s of + Done -> [] + Skip s' -> unfold s' + Yield x s' -> x : unfold s' +{-# INLINE [0] unstreamList #-} + +{-# RULES "STREAM streamList/unstreamList fusion" forall s. streamList (unstreamList s) = s #-} + +-- | Stream the UTF-8-like packed encoding used by GHC to represent +-- constant strings in generated code. +-- +-- This encoding uses the byte sequence "\xc0\x80" to represent NUL, +-- and the string is NUL-terminated. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'streamCString#' addr# = 'Data.Text.Show.unpackCString#' addr#@ +streamCString# :: Addr# -> Stream Char +streamCString# addr = Stream step 0 unknownSize + where + step !i + | b == 0 = Done + | otherwise = Yield chr (i + l) + where b = at# i + l = utf8LengthByLeader b + next n = at# (i+n) + chr = case l of + 1 -> unsafeChr8 b + 2 -> chr2 b (next 1) + 3 -> chr3 b (next 1) (next 2) + _ -> chr4 b (next 1) (next 2) (next 3) + at# (I# i#) = W8# (indexWord8OffAddr# addr i#) +{-# INLINE [0] streamCString# #-} + +-- ---------------------------------------------------------------------------- +-- * Basic stream functions + +data C s = C0 !s + | C1 !s + +-- | /O(n)/ Adds a character to the front of a Stream Char. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'cons' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.cons' c @ +cons :: Char -> Stream Char -> Stream Char +cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1) + where + next (C1 s) = Yield w (C0 s) + next (C0 s) = case next0 s of + Done -> Done + Skip s' -> Skip (C0 s') + Yield x s' -> Yield x (C0 s') +{-# INLINE [0] cons #-} + +data Snoc a = N + | J !a + +-- | /O(n)/ Adds a character to the end of a stream. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'snoc' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.snoc' c @ +snoc :: Stream Char -> Char -> Stream Char +snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1) + where + next (J xs) = case next0 xs of + Done -> Yield w N + Skip xs' -> Skip (J xs') + Yield x xs' -> Yield x (J xs') + next N = Done +{-# INLINE [0] snoc #-} + +data E l r = L !l + | R !r + +-- | /O(n)/ Appends one Stream to the other. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' ('append' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.append' t1 t2@ +append :: Stream Char -> Stream Char -> Stream Char +append (Stream next0 s01 len1) (Stream next1 s02 len2) = + Stream next (L s01) (len1 + len2) + where + next (L s1) = case next0 s1 of + Done -> Skip (R s02) + Skip s1' -> Skip (L s1') + Yield x s1' -> Yield x (L s1') + next (R s2) = case next1 s2 of + Done -> Done + Skip s2' -> Skip (R s2') + Yield x s2' -> Yield x (R s2') +{-# INLINE [0] append #-} + +-- | /O(1)/ Returns the first character of a 'Stream' 'Char', which must be non-empty. +-- This is a partial function, consider using 'uncons'. +-- +-- __Properties__ +-- +-- @ 'head' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.head' @ +head :: HasCallStack => Stream Char -> Char +head (Stream next s0 _len) = loop_head s0 + where + loop_head !s = case next s of + Yield x _ -> x + Skip s' -> loop_head s' + Done -> head_empty +{-# INLINE [0] head #-} + +head_empty :: HasCallStack => a +head_empty = streamError "head" "Empty stream" +{-# NOINLINE head_empty #-} + +-- | /O(1)/ Returns the first character and remainder of a 'Stream' +-- 'Char', or 'Nothing' if empty. +-- +-- __Properties__ +-- +-- @ 'Data.Functor.fmap' 'Data.Tuple.fst' . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.fst' . 'Data.Text.uncons' @ +-- +-- @ 'Data.Functor.fmap' ('Data.Text.Internal.Fusion.unstream' . 'Data.Tuple.snd') . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.snd' . 'Data.Text.uncons' @ +uncons :: Stream Char -> Maybe (Char, Stream Char) +uncons (Stream next s0 len) = loop_uncons s0 + where + loop_uncons !s = case next s of + Yield x s1 -> Just (x, Stream next s1 (len - codePointsSize 1)) + Skip s' -> loop_uncons s' + Done -> Nothing +{-# INLINE [0] uncons #-} + +-- | /O(n)/ Returns the last character of a 'Stream' 'Char', which must +-- be non-empty. +-- +-- __Properties__ +-- +-- @ 'last' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.last' @ +last :: HasCallStack => Stream Char -> Char +last (Stream next s0 _len) = loop0_last s0 + where + loop0_last !s = case next s of + Done -> emptyError "last" + Skip s' -> loop0_last s' + Yield x s' -> loop_last x s' + loop_last !x !s = case next s of + Done -> x + Skip s' -> loop_last x s' + Yield x' s' -> loop_last x' s' +{-# INLINE[0] last #-} + +-- | /O(1)/ Returns all characters after the head of a 'Stream' 'Char', which must +-- be non-empty. This is a partial function, consider using 'uncons'. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'tail' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.tail' @ +tail :: HasCallStack => Stream Char -> Stream Char +tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1) + where + next (C0 s) = case next0 s of + Done -> emptyError "tail" + Skip s' -> Skip (C0 s') + Yield _ s' -> Skip (C1 s') + next (C1 s) = case next0 s of + Done -> Done + Skip s' -> Skip (C1 s') + Yield x s' -> Yield x (C1 s') +{-# INLINE [0] tail #-} + +data Init s = Init0 !s + | Init1 {-# UNPACK #-} !Char !s + +-- | /O(1)/ Returns all but the last character of a 'Stream' 'Char', which +-- must be non-empty. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'init' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.init' @ +init :: HasCallStack => Stream Char -> Stream Char +init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1) + where + next (Init0 s) = case next0 s of + Done -> emptyError "init" + Skip s' -> Skip (Init0 s') + Yield x s' -> Skip (Init1 x s') + next (Init1 x s) = case next0 s of + Done -> Done + Skip s' -> Skip (Init1 x s') + Yield x' s' -> Yield x (Init1 x' s') +{-# INLINE [0] init #-} + +-- | /O(1)/ Tests whether a 'Stream' 'Char' is empty or not. +-- +-- __Properties__ +-- +-- @ 'null' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.null' @ +null :: Stream Char -> Bool +null (Stream next s0 _len) = loop_null s0 + where + loop_null !s = case next s of + Done -> True + Yield _ _ -> False + Skip s' -> loop_null s' +{-# INLINE[0] null #-} + +-- | /O(n)/ Returns the number of characters in a string. +lengthI :: Integral a => Stream Char -> a +lengthI (Stream next s0 _len) = loop_length 0 s0 + where + loop_length !z s = case next s of + Done -> z + Skip s' -> loop_length z s' + Yield _ s' -> loop_length (z + 1) s' +{-# INLINE[0] lengthI #-} + +-- | /O(n)/ Compares the count of characters in a string to a number. +-- +-- This function gives the same answer as comparing against the result +-- of 'lengthI', but can short circuit if the count of characters is +-- greater than the number or if the stream can't possibly be as long +-- as the number supplied, and hence be more efficient. +compareLengthI :: Integral a => Stream Char -> a -> Ordering +compareLengthI (Stream next s0 len) n + -- Note that @len@ tracks code units whereas we want to compare the length + -- in code points. Specifically, a stream with hint @len@ may consist of + -- anywhere from @len/2@ to @len@ code points. + | n < 0 = GT + | Just r <- compareSize len n' = r + | otherwise = loop_cmp 0 s0 + where + n' = codePointsSize $ fromIntegral n + loop_cmp !z s = case next s of + Done -> compare z n + Skip s' -> loop_cmp z s' + Yield _ s' | z > n -> GT + | otherwise -> loop_cmp (z + 1) s' +{-# INLINE[0] compareLengthI #-} + +-- | /O(n)/ Indicate whether a string contains exactly one element. +-- +-- __Properties__ +-- +-- @ 'isSingleton' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.isSingleton' @ +isSingleton :: Stream Char -> Bool +isSingleton (Stream next s0 _len) = loop 0 s0 + where + loop !z s = case next s of + Done -> z == (1::Int) + Skip s' -> loop z s' + Yield _ s' + | z >= 1 -> False + | otherwise -> loop (z+1) s' +{-# INLINE[0] isSingleton #-} + +-- ---------------------------------------------------------------------------- +-- * Stream transformations + +-- | /O(n)/ 'map' @f @xs is the 'Stream' 'Char' obtained by applying @f@ +-- to each element of @xs@. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'map' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.map' f @ +map :: (Char -> Char) -> Stream Char -> Stream Char +map f (Stream next0 s0 len) = Stream next s0 len + where + next !s = case next0 s of + Done -> Done + Skip s' -> Skip s' + Yield x s' -> Yield (f x) s' +{-# INLINE [0] map #-} + +{-# + RULES "STREAM map/map fusion" forall f g s. + map f (map g s) = map (\x -> f (g x)) s + #-} + +data I s = I1 !s + | I2 !s {-# UNPACK #-} !Char + | I3 !s + +-- | /O(n)/ Take a character and place it between each of the +-- characters of a 'Stream Char'. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'intersperse' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.intersperse' c @ +intersperse :: Char -> Stream Char -> Stream Char +intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize) + where + next (I1 s) = case next0 s of + Done -> Done + Skip s' -> Skip (I1 s') + Yield x s' -> Skip (I2 s' x) + next (I2 s x) = Yield x (I3 s) + next (I3 s) = case next0 s of + Done -> Done + Skip s' -> Skip (I3 s') + Yield x s' -> Yield c (I2 s' x) +{-# INLINE [0] intersperse #-} + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- With Unicode text, it is incorrect to use combinators like @map +-- toUpper@ to case convert each character of a string individually. +-- Instead, use the whole-string case conversion functions from this +-- module. For correctness in different writing systems, these +-- functions may map one input character to two or three output +-- characters. + +-- | Map a 'Stream' through the given case-mapping function. +caseConvert :: (Char# -> _ {- unboxed Int64 -}) + -> Stream Char -> Stream Char +caseConvert remap (Stream next0 s0 len) = + Stream next (CC s0 0) (len `unionSize` (3*len)) + where + next (CC s 0) = + case next0 s of + Done -> Done + Skip s' -> Skip (CC s' 0) + Yield c@(C# c#) s' -> case I64# (remap c#) of + 0 -> Yield c (CC s' 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC s' b) + next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b) + +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (I# n) = C# (chr# n) + mask = (1 `shiftL` 21) - 1 + a = fromIntegral $ ab .&. mask + +-- | /O(n)/ Convert a string to folded case. This function is mainly +-- useful for performing caseless (or case insensitive) string +-- comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @'toCaseFold' x == 'toCaseFold' y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature men now (U+FB13) is case folded to the +-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is +-- case folded to the Greek small letter letter mu (U+03BC) instead of +-- itself. +toCaseFold :: Stream Char -> Stream Char +toCaseFold = caseConvert foldMapping +{-# INLINE [0] toCaseFold #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. The result string may be longer than the input string. +-- For instance, the German eszett (U+00DF) maps to the two-letter +-- sequence SS. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toUpper' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toUpper' @ +toUpper :: Stream Char -> Stream Char +toUpper = caseConvert upperMapping +{-# INLINE [0] toUpper #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. The result string may be longer than the input string. +-- For instance, the Latin capital letter I with dot above (U+0130) +-- maps to the sequence Latin small letter i (U+0069) followed by +-- combining dot above (U+0307). +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toLower' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toLower' @ +toLower :: Stream Char -> Stream Char +toLower = caseConvert lowerMapping +{-# INLINE [0] toLower #-} + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toTitle' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toTitle' @ +toTitle :: Stream Char -> Stream Char +toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) 0) (len + unknownSize) + where + next (CC (letter :*: s) 0) = + case next0 s of + Done -> Done + Skip s' -> Skip (CC (letter :*: s') 0) + Yield c@(C# c#) s' + | nonSpace, letter -> case I64# (lowerMapping c#) of + 0 -> Yield c (CC (nonSpace :*: s') 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC (nonSpace :*: s') b) + | nonSpace -> case I64# (titleMapping c#) of + 0 -> Yield c (CC (letter' :*: s') 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC (letter' :*: s') b) + | otherwise -> Yield c (CC (letter' :*: s') 0) + where nonSpace = P.not (isSpace c) + letter' = isLetter c + next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b) +{-# INLINE [0] toTitle #-} + +data Justify i s = Just1 !i !s + | Just2 !i !s + +justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char +justifyLeftI k c (Stream next0 s0 len) = + Stream next (Just1 0 s0) (larger (fromIntegral k * charSize c + len) len) + where + next (Just1 n s) = + case next0 s of + Done -> next (Just2 n s) + Skip s' -> Skip (Just1 n s') + Yield x s' -> Yield x (Just1 (n+1) s') + next (Just2 n s) + | n < k = Yield c (Just2 (n+1) s) + | otherwise = Done + {-# INLINE next #-} +{-# INLINE [0] justifyLeftI #-} + +-- ---------------------------------------------------------------------------- +-- * Reducing Streams (folds) + +-- | foldl, applied to a binary operator, a starting value (typically the +-- left-identity of the operator), and a 'Stream', reduces the 'Stream' using the +-- binary operator, from left to right. +-- +-- __Properties__ +-- +-- @ 'foldl' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl' f z0 @ +foldl :: (b -> Char -> b) -> b -> Stream Char -> b +foldl f z0 (Stream next s0 _len) = loop_foldl z0 s0 + where + loop_foldl z !s = case next s of + Done -> z + Skip s' -> loop_foldl z s' + Yield x s' -> loop_foldl (f z x) s' +{-# INLINE [0] foldl #-} + +-- | A strict version of foldl. +-- +-- __Properties__ +-- +-- @ 'foldl'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl'' f z0 @ +foldl' :: (b -> Char -> b) -> b -> Stream Char -> b +foldl' f z0 (Stream next s0 _len) = loop_foldl' z0 s0 + where + loop_foldl' !z !s = case next s of + Done -> z + Skip s' -> loop_foldl' z s' + Yield x s' -> loop_foldl' (f z x) s' +{-# INLINE [0] foldl' #-} + +-- | foldl1 is a variant of foldl that has no starting value argument, +-- and thus must be applied to non-empty Streams. +-- +-- __Properties__ +-- +-- @ 'foldl1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1' f @ +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char +foldl1 f (Stream next s0 _len) = loop0_foldl1 s0 + where + loop0_foldl1 !s = case next s of + Skip s' -> loop0_foldl1 s' + Yield x s' -> loop_foldl1 x s' + Done -> emptyError "foldl1" + loop_foldl1 z !s = case next s of + Done -> z + Skip s' -> loop_foldl1 z s' + Yield x s' -> loop_foldl1 (f z x) s' +{-# INLINE [0] foldl1 #-} + +-- | A strict version of foldl1. +-- +-- __Properties__ +-- +-- @ 'foldl1'' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1'' f @ +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char +foldl1' f (Stream next s0 _len) = loop0_foldl1' s0 + where + loop0_foldl1' !s = case next s of + Skip s' -> loop0_foldl1' s' + Yield x s' -> loop_foldl1' x s' + Done -> emptyError "foldl1" + loop_foldl1' !z !s = case next s of + Done -> z + Skip s' -> loop_foldl1' z s' + Yield x s' -> loop_foldl1' (f z x) s' +{-# INLINE [0] foldl1' #-} + +-- | A monadic version of 'foldl'. +-- +-- __Properties__ +-- +-- @ 'foldlM'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldlM'' f z0 @ +foldlM' :: P.Monad m => (b -> Char -> m b) -> b -> Stream Char -> m b +foldlM' f z0 (Stream next s0 _len) = loop_foldlM' z0 s0 + where + loop_foldlM' !z !s = case next s of + Done -> P.pure z + Skip s' -> loop_foldlM' z s' + Yield x s' -> f z x P.>>= \z' -> loop_foldlM' z' s' +{-# INLINE [0] foldlM' #-} + +-- | 'foldr', applied to a binary operator, a starting value (typically the +-- right-identity of the operator), and a stream, reduces the stream using the +-- binary operator, from right to left. +-- +-- __Properties__ +-- +-- @ 'foldr' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr' f z0 @ +foldr :: (Char -> b -> b) -> b -> Stream Char -> b +foldr f z (Stream next s0 _len) = loop_foldr s0 + where + loop_foldr !s = case next s of + Done -> z + Skip s' -> loop_foldr s' + Yield x s' -> f x (loop_foldr s') +{-# INLINE [0] foldr #-} + +-- | foldr1 is a variant of 'foldr' that has no starting value argument, +-- and thus must be applied to non-empty streams. +-- +-- __Properties__ +-- +-- @ 'foldr1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr1' f @ +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char +foldr1 f (Stream next s0 _len) = loop0_foldr1 s0 + where + loop0_foldr1 !s = case next s of + Done -> emptyError "foldr1" + Skip s' -> loop0_foldr1 s' + Yield x s' -> loop_foldr1 x s' + + loop_foldr1 x !s = case next s of + Done -> x + Skip s' -> loop_foldr1 x s' + Yield x' s' -> f x (loop_foldr1 x' s') +{-# INLINE [0] foldr1 #-} + +-- | intercalate str strs inserts the stream str in between the streams strs and +-- concatenates the result. +-- +-- __Properties__ +-- +-- @ 'intercalate' s = 'concat' . 'L.intersperse' s @ +intercalate :: Stream Char -> [Stream Char] -> Stream Char +intercalate s = concat . (L.intersperse s) +{-# INLINE [0] intercalate #-} + +-- ---------------------------------------------------------------------------- +-- ** Special folds + +-- | /O(n)/ Concatenate a list of streams. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'concat' . 'Data.Functor.fmap' 'Data.Text.Internal.Fusion.stream' = 'Data.Text.concat'@ +concat :: [Stream Char] -> Stream Char +concat = L.foldr append empty +{-# INLINE [0] concat #-} + +-- | Map a function over a stream that results in a stream and concatenate the +-- results. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'concatMap' ('Data.Text.Fusion.stream' . f) . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.concatMap' f@ +concatMap :: (Char -> Stream Char) -> Stream Char -> Stream Char +concatMap f = foldr (append . f) empty +{-# INLINE [0] concatMap #-} + +-- | /O(n)/ any @p @xs determines if any character in the stream +-- @xs@ satisfies the predicate @p@. +-- +-- __Properties__ +-- +-- @'any' f . 'Data.Text.Fusion.stream' = 'Data.Text.any' f@ +any :: (Char -> Bool) -> Stream Char -> Bool +any p (Stream next0 s0 _len) = loop_any s0 + where + loop_any !s = case next0 s of + Done -> False + Skip s' -> loop_any s' + Yield x s' | p x -> True + | otherwise -> loop_any s' +{-# INLINE [0] any #-} + +-- | /O(n)/ all @p @xs determines if all characters in the 'Text' +-- @xs@ satisfy the predicate @p@. +-- +-- __Properties__ +-- +-- @'all' f . 'Data.Text.Fusion.stream' = 'Data.Text.all' f@ +all :: (Char -> Bool) -> Stream Char -> Bool +all p (Stream next0 s0 _len) = loop_all s0 + where + loop_all !s = case next0 s of + Done -> True + Skip s' -> loop_all s' + Yield x s' | p x -> loop_all s' + | otherwise -> False +{-# INLINE [0] all #-} + +-- | /O(n)/ maximum returns the maximum value from a stream, which must be +-- non-empty. +-- +-- __Properties__ +-- +-- @'maximum' . 'Data.Text.Fusion.stream' = 'Data.Text.maximum'@ +maximum :: HasCallStack => Stream Char -> Char +maximum (Stream next0 s0 _len) = loop0_maximum s0 + where + loop0_maximum !s = case next0 s of + Done -> emptyError "maximum" + Skip s' -> loop0_maximum s' + Yield x s' -> loop_maximum x s' + loop_maximum !z !s = case next0 s of + Done -> z + Skip s' -> loop_maximum z s' + Yield x s' + | x > z -> loop_maximum x s' + | otherwise -> loop_maximum z s' +{-# INLINE [0] maximum #-} + +-- | /O(n)/ minimum returns the minimum value from a 'Text', which must be +-- non-empty. +-- +-- __Properties__ +-- +-- @'minimum' . 'Data.Text.Fusion.stream' = 'Data.Text.minimum'@ +minimum :: HasCallStack => Stream Char -> Char +minimum (Stream next0 s0 _len) = loop0_minimum s0 + where + loop0_minimum !s = case next0 s of + Done -> emptyError "minimum" + Skip s' -> loop0_minimum s' + Yield x s' -> loop_minimum x s' + loop_minimum !z !s = case next0 s of + Done -> z + Skip s' -> loop_minimum z s' + Yield x s' + | x < z -> loop_minimum x s' + | otherwise -> loop_minimum z s' +{-# INLINE [0] minimum #-} + +-- ----------------------------------------------------------------------------- +-- * Building streams +-- +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a stream of +-- successive reduced values from the left. Conceptually, if we +-- write the input stream as a list then we have: +-- +-- > scanl f z [x1, x2, ...] == [z, z 'f' x1, (z 'f' x1) 'f' x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ +scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char +scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low + where + {-# INLINE next #-} + next (Scan1 z s) = Yield z (Scan2 z s) + next (Scan2 z s) = case next0 s of + Yield x s' -> let !x' = f z x + in Yield x' (Scan2 x' s') + Skip s' -> Skip (Scan2 z s') + Done -> Done +{-# INLINE [0] scanl #-} + +-- ----------------------------------------------------------------------------- +-- ** Generating and unfolding streams + +-- | /O(n)/ 'replicateCharI' @n@ @c@ is a 'Stream' 'Char' of length @n@ with @c@ the +-- value of every element. +replicateCharI :: Integral a => a -> Char -> Stream Char +replicateCharI !n !c + | n < 0 = empty + | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low + where + next !i | i >= n = Done + | otherwise = Yield c (i + 1) +{-# INLINE [0] replicateCharI #-} + +data RI s = RI !s {-# UNPACK #-} !Int64 + + +-- | /O(n*m)/ 'replicateI' @n@ @t@ is a 'Stream' 'Char' consisting of the input +-- @t@ repeated @n@ times. +replicateI :: Int64 -> Stream Char -> Stream Char +replicateI n (Stream next0 s0 len) = + Stream next (RI s0 0) (int64ToSize (max 0 n) * len) + where + next (RI s k) + | k >= n = Done + | otherwise = case next0 s of + Done -> Skip (RI s0 (k+1)) + Skip s' -> Skip (RI s' k) + Yield x s' -> Yield x (RI s' k) +{-# INLINE [0] replicateI #-} + +-- | /O(n)/, where @n@ is the length of the result. The unfoldr function +-- is analogous to the List 'unfoldr'. unfoldr builds a stream +-- from a seed value. The function takes the element and returns +-- Nothing if it is done producing the stream or returns Just +-- (a,b), in which case, a is the next Char in the string, and b is +-- the seed value for further production. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'unfoldr' f z = 'Data.Text.unfoldr' f z@ +unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char +unfoldr f s0 = Stream next s0 unknownSize + where + {-# INLINE next #-} + next !s = case f s of + Nothing -> Done + Just (w, s') -> Yield w s' +{-# INLINE [0] unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrNI' builds a stream from a seed +-- value. However, the length of the result is limited by the +-- first argument to 'unfoldrNI'. This function is more efficient than +-- 'unfoldr' when the length of the result is known. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' ('unfoldrNI' n f z) = 'Data.Text.unfoldrN' n f z@ +unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char +unfoldrNI n f s0 | n < 0 = empty + | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2)) + where + {-# INLINE next #-} + next (z :*: s) = case f s of + Nothing -> Done + Just (w, s') | z >= n -> Done + | otherwise -> Yield w ((z + 1) :*: s') +{-# INLINE unfoldrNI #-} + +------------------------------------------------------------------------------- +-- * Substreams + +-- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the +-- stream of length @n@, or the stream itself if @n@ is greater than the +-- length of the stream. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'take' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.take' n@ +take :: Integral a => a -> Stream Char -> Stream Char +take n0 (Stream next0 s0 len) = + Stream next (n0' :*: s0) (smaller len (codePointsSize $ fromIntegral n0')) + where + n0' = max n0 0 + + {-# INLINE next #-} + next (n :*: s) | n <= 0 = Done + | otherwise = case next0 s of + Done -> Done + Skip s' -> Skip (n :*: s') + Yield x s' -> Yield x ((n-1) :*: s') +{-# INLINE [0] take #-} + +data Drop a s = NS !s + | JS !a !s + +-- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the +-- stream after the first @n@ characters, or the empty stream if @n@ +-- is greater than the length of the stream. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'drop' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.drop' n@ +drop :: Integral a => a -> Stream Char -> Stream Char +drop n0 (Stream next0 s0 len) = + Stream next (JS n0' s0) (len - codePointsSize (fromIntegral n0')) + where + n0' = max n0 0 + + {-# INLINE next #-} + next (JS n s) + | n <= 0 = Skip (NS s) + | otherwise = case next0 s of + Done -> Done + Skip s' -> Skip (JS n s') + Yield _ s' -> Skip (JS (n-1) s') + next (NS s) = case next0 s of + Done -> Done + Skip s' -> Skip (NS s') + Yield x s' -> Yield x (NS s') +{-# INLINE [0] drop #-} + +-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the +-- longest prefix (possibly empty) of elements that satisfy @p@. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'takeWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.takeWhile' p@ +takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char +takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize) + where + {-# INLINE next #-} + next !s = case next0 s of + Done -> Done + Skip s' -> Skip s' + Yield x s' | p x -> Yield x s' + | otherwise -> Done +{-# INLINE [0] takeWhile #-} + +-- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'dropWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.dropWhile' p@ +dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char +dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize) + where + {-# INLINE next #-} + next (L s) = case next0 s of + Done -> Done + Skip s' -> Skip (L s') + Yield x s' | p x -> Skip (L s') + | otherwise -> Yield x (R s') + next (R s) = case next0 s of + Done -> Done + Skip s' -> Skip (R s') + Yield x s' -> Yield x (R s') +{-# INLINE [0] dropWhile #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns +-- 'True' if and only if the first is a prefix of the second. +-- +-- __Properties__ +-- +-- @ 'isPrefixOf' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2) = 'Data.Text.isPrefixOf' t1 t2@ +isPrefixOf :: (Eq a) => Stream a -> Stream a -> Bool +isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) + where + loop Done _ = True + loop _ Done = False + loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') + loop (Skip s1') x2 = loop (next1 s1') x2 + loop x1 (Skip s2') = loop x1 (next2 s2') + loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 && + loop (next1 s1') (next2 s2') +{-# INLINE [0] isPrefixOf #-} + +-- ---------------------------------------------------------------------------- +-- * Searching + +------------------------------------------------------------------------------- +-- ** Searching by equality + +-- | /O(n)/ 'elem' is the stream membership predicate. +-- +-- __Properties__ +-- +-- @ 'elem' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.elem' c@ +elem :: Char -> Stream Char -> Bool +elem w (Stream next s0 _len) = loop_elem s0 + where + loop_elem !s = case next s of + Done -> False + Skip s' -> loop_elem s' + Yield x s' | x == w -> True + | otherwise -> loop_elem s' +{-# INLINE [0] elem #-} + +------------------------------------------------------------------------------- +-- ** Searching with a predicate + +-- | /O(n)/ The 'findBy' function takes a predicate and a stream, +-- and returns the first element in matching the predicate, or 'Nothing' +-- if there is no such element. +-- +-- __Properties__ +-- +-- @ 'findBy' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.find' p@ +findBy :: (Char -> Bool) -> Stream Char -> Maybe Char +findBy p (Stream next s0 _len) = loop_find s0 + where + loop_find !s = case next s of + Done -> Nothing + Skip s' -> loop_find s' + Yield x s' | p x -> Just x + | otherwise -> loop_find s' +{-# INLINE [0] findBy #-} + +-- | /O(n)/ Stream index (subscript) operator, starting from 0. +-- +-- __Properties__ +-- +-- @ 'indexI' ('Data.Text.Internal.Fusion.stream' t) n = 'Data.Text.index' t n@ +indexI :: (HasCallStack, Integral a) => Stream Char -> a -> Char +indexI (Stream next s0 _len) n0 + | n0 < 0 = streamError "index" "Negative index" + | otherwise = loop_index n0 s0 + where + loop_index !n !s = case next s of + Done -> streamError "index" "Index too large" + Skip s' -> loop_index n s' + Yield x s' | n == 0 -> x + | otherwise -> loop_index (n-1) s' +{-# INLINE [0] indexI #-} + +-- | /O(n)/ 'filter', applied to a predicate and a stream, +-- returns a stream containing those characters that satisfy the +-- predicate. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'filter' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.filter' p @ +filter :: (Char -> Bool) -> Stream Char -> Stream Char +filter p (Stream next0 s0 len) = + Stream next s0 (len - unknownSize) -- HINT maybe too high + where + next !s = case next0 s of + Done -> Done + Skip s' -> Skip s' + Yield x s' | p x -> Yield x s' + | otherwise -> Skip s' +{-# INLINE [0] filter #-} + +{-# RULES + "STREAM filter/filter fusion" forall p q s. + filter p (filter q s) = filter (\x -> q x && p x) s + #-} + +-- | The 'findIndexI' function takes a predicate and a stream and +-- returns the index of the first element in the stream satisfying the +-- predicate. +-- +-- __Properties__ +-- +-- @'findIndexI' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.findIndex' p @ +findIndexI :: Integral a => (Char -> Bool) -> Stream Char -> Maybe a +findIndexI p s = case findIndicesI p s of + (i:_) -> Just i + _ -> Nothing +{-# INLINE [0] findIndexI #-} + +-- | The 'findIndicesI' function takes a predicate and a stream and +-- returns all indices of the elements in the stream satisfying the +-- predicate. +findIndicesI :: Integral a => (Char -> Bool) -> Stream Char -> [a] +findIndicesI p (Stream next s0 _len) = loop_findIndex 0 s0 + where + loop_findIndex !i !s = case next s of + Done -> [] + Skip s' -> loop_findIndex i s' -- hmm. not caught by QC + Yield x s' | p x -> i : loop_findIndex (i+1) s' + | otherwise -> loop_findIndex (i+1) s' +{-# INLINE [0] findIndicesI #-} + +------------------------------------------------------------------------------- +-- * Zipping + +-- | Strict triple. +data Zip a b m = Z1 !a !b + | Z2 !a !b !m + +-- | zipWith generalises 'zip' by zipping with the function given as +-- the first argument, instead of a tupling function. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' ('zipWith' f ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.zipWith' f t1 t2@ +zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b +zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) = + Stream next (Z1 sa0 sb0) (smaller len1 len2) + where + next (Z1 sa sb) = case next0 sa of + Done -> Done + Skip sa' -> Skip (Z1 sa' sb) + Yield a sa' -> Skip (Z2 sa' sb a) + + next (Z2 sa' sb a) = case next1 sb of + Done -> Done + Skip sb' -> Skip (Z2 sa' sb' a) + Yield b sb' -> Yield (f a b) (Z1 sa' sb') +{-# INLINE [0] zipWith #-} + +-- | /O(n)/ The 'countCharI' function returns the number of times the +-- query element appears in the given stream. +-- +-- __Properties__ +-- +-- @'countCharI' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.countChar' c @ +countCharI :: Integral a => Char -> Stream Char -> a +countCharI a (Stream next s0 _len) = loop 0 s0 + where + loop !i !s = case next s of + Done -> i + Skip s' -> loop i s' + Yield x s' | a == x -> loop (i+1) s' + | otherwise -> loop i s' +{-# INLINE [0] countCharI #-} + +streamError :: HasCallStack => String -> String -> a +streamError func msg = P.error $ "Data.Text.Internal.Fusion.Common." ++ func ++ ": " ++ msg + +emptyError :: HasCallStack => String -> a +emptyError func = internalError func "Empty input" + +internalError :: HasCallStack => String -> a +internalError func = streamError func "Internal error" + +int64ToSize :: Int64 -> Size +int64ToSize = fromIntegral diff --git a/src/Data/Text/Internal/Fusion/Size.hs b/src/Data/Text/Internal/Fusion/Size.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Fusion/Size.hs @@ -0,0 +1,185 @@ +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-missing-methods #-} +-- | +-- Module : Data.Text.Internal.Fusion.Internal +-- Copyright : (c) Roman Leshchinskiy 2008, +-- (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Size hints. + +module Data.Text.Internal.Fusion.Size + ( + Size + -- * Sizes + , exactSize + , maxSize + , betweenSize + , unknownSize + , unionSize + , charSize + , codePointsSize + -- * Querying sizes + , exactly + , smaller + , larger + , upperBound + , lowerBound + , compareSize + , isEmpty + ) where + +import Data.Text.Internal.Encoding.Utf8 (utf8Length) +import Data.Text.Internal (mul) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif + +-- | A size in UTF-8 code units (which is bytes). +data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size. + | Unknown -- ^ Unknown size. + deriving (Eq, Show) + +exactly :: Size -> Maybe Int +exactly (Between na nb) | na == nb = Just na +exactly _ = Nothing +{-# INLINE exactly #-} + +-- | The 'Size' of the given code point. +charSize :: Char -> Size +charSize = exactSize . utf8Length + +-- | The 'Size' of @n@ code points. +codePointsSize :: Int -> Size +codePointsSize n = +#if defined(ASSERTS) + assert (n >= 0) +#endif + Between n (4*n) +{-# INLINE codePointsSize #-} + +exactSize :: Int -> Size +exactSize n = +#if defined(ASSERTS) + assert (n >= 0) +#endif + Between n n +{-# INLINE exactSize #-} + +maxSize :: Int -> Size +maxSize n = +#if defined(ASSERTS) + assert (n >= 0) +#endif + Between 0 n +{-# INLINE maxSize #-} + +betweenSize :: Int -> Int -> Size +betweenSize m n = +#if defined(ASSERTS) + assert (m >= 0) + assert (n >= m) +#endif + Between m n +{-# INLINE betweenSize #-} + +unionSize :: Size -> Size -> Size +unionSize (Between a b) (Between c d) = Between (min a c) (max b d) +unionSize _ _ = Unknown + +unknownSize :: Size +unknownSize = Unknown +{-# INLINE unknownSize #-} + +instance Num Size where + (+) = addSize + (-) = subtractSize + (*) = mulSize + + fromInteger = f where f = exactSize . fromInteger + {-# INLINE f #-} + +add :: Int -> Int -> Int +add m n | mn >= 0 = mn + | otherwise = overflowError + where mn = m + n +{-# INLINE add #-} + +addSize :: Size -> Size -> Size +addSize (Between ma mb) (Between na nb) = Between (add ma na) (add mb nb) +addSize _ _ = Unknown +{-# INLINE addSize #-} + +subtractSize :: Size -> Size -> Size +subtractSize (Between ma mb) (Between na nb) = Between (max (ma-nb) 0) (max (mb-na) 0) +subtractSize a@(Between 0 _) Unknown = a +subtractSize (Between _ mb) Unknown = Between 0 mb +subtractSize _ _ = Unknown +{-# INLINE subtractSize #-} + +mulSize :: Size -> Size -> Size +mulSize (Between ma mb) (Between na nb) = Between (mul ma na) (mul mb nb) +mulSize _ _ = Unknown +{-# INLINE mulSize #-} + +-- | Minimum of two size hints. +smaller :: Size -> Size -> Size +smaller a@(Between ma mb) b@(Between na nb) + | mb <= na = a + | nb <= ma = b + | otherwise = Between (ma `min` na) (mb `min` nb) +smaller a@(Between 0 _) Unknown = a +smaller (Between _ mb) Unknown = Between 0 mb +smaller Unknown b@(Between 0 _) = b +smaller Unknown (Between _ nb) = Between 0 nb +smaller Unknown Unknown = Unknown +{-# INLINE smaller #-} + +-- | Maximum of two size hints. +larger :: Size -> Size -> Size +larger a@(Between ma mb) b@(Between na nb) + | ma >= nb = a + | na >= mb = b + | otherwise = Between (ma `max` na) (mb `max` nb) +larger _ _ = Unknown +{-# INLINE larger #-} + +-- | Compute the maximum size from a size hint, if possible. +upperBound :: Int -> Size -> Int +upperBound _ (Between _ n) = n +upperBound k _ = k +{-# INLINE upperBound #-} + +-- | Compute the minimum size from a size hint, if possible. +lowerBound :: Int -> Size -> Int +lowerBound _ (Between n _) = n +lowerBound k _ = k +{-# INLINE lowerBound #-} + +-- | Determine the ordering relationship between two 'Size's, or 'Nothing' in +-- the indeterminate case. +compareSize :: Size -> Size -> Maybe Ordering +compareSize (Between ma mb) (Between na nb) + | mb < na = Just LT + | ma > nb = Just GT + | ma == mb + , ma == na + , ma == nb = Just EQ +compareSize _ _ = Nothing + + +isEmpty :: Size -> Bool +isEmpty (Between _ n) = n <= 0 +isEmpty _ = False +{-# INLINE isEmpty #-} + +overflowError :: Int +overflowError = error "Data.Text.Internal.Fusion.Size: size overflow" diff --git a/src/Data/Text/Internal/Fusion/Types.hs b/src/Data/Text/Internal/Fusion/Types.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Fusion/Types.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE BangPatterns, ExistentialQuantification #-} +-- | +-- Module : Data.Text.Internal.Fusion.Types +-- Copyright : (c) Tom Harper 2008-2009, +-- (c) Bryan O'Sullivan 2009, +-- (c) Duncan Coutts 2009, +-- (c) Jasper Van der Jeugt 2011 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Core stream fusion functionality for text. + +module Data.Text.Internal.Fusion.Types + ( + CC(..) + , PairS(..) + , Scan(..) + , RS(..) + , Step(..) + , Stream(..) + , empty + ) where + +import Data.Text.Internal.Fusion.Size +import Data.Int (Int64) +import Data.Word (Word8) + +-- | Specialised tuple for case conversion. +data CC s = CC !s {-# UNPACK #-} !Int64 + +-- | Restreaming state. +data RS s + = RS0 !s + | RS1 !s {-# UNPACK #-} !Word8 + | RS2 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 + | RS3 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 + +-- | Strict pair. +data PairS a b = !a :*: !b + -- deriving (Eq, Ord, Show) +infixl 2 :*: + +-- | An intermediate result in a scan. +data Scan s = Scan1 {-# UNPACK #-} !Char !s + | Scan2 {-# UNPACK #-} !Char !s + +-- | Intermediate result in a processing pipeline. +data Step s a = Done + | Skip !s + | Yield !a !s + +{- +instance (Show a) => Show (Step s a) + where show Done = "Done" + show (Skip _) = "Skip" + show (Yield x _) = "Yield " ++ show x +-} + +instance (Eq a) => Eq (Stream a) where + (==) = eq + +instance (Ord a) => Ord (Stream a) where + compare = cmp + +-- The length hint in a Stream has two roles. If its value is zero, +-- we trust it, and treat the stream as empty. Otherwise, we treat it +-- as a hint: it should usually be accurate, so we use it when +-- unstreaming to decide what size array to allocate. However, the +-- unstreaming functions must be able to cope with the hint being too +-- small or too large. +-- +-- The size hint tries to track the UTF-8 code units in a stream, +-- but often counts the number of code points instead. It can easily +-- undercount if, for instance, a transformed stream contains astral +-- plane code points (those above 0x10000). + +-- | A co-recursive type yielding a single element at a time depending +-- on the internal state it carries. +data Stream a = + forall s. Stream + (s -> Step s a) -- stepper function + !s -- current state + !Size -- size hint in code units + +-- | /O(n)/ Determines if two streams are equal. +eq :: (Eq a) => Stream a -> Stream a -> Bool +eq (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) + where + loop Done Done = True + loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') + loop (Skip s1') x2 = loop (next1 s1') x2 + loop x1 (Skip s2') = loop x1 (next2 s2') + loop Done _ = False + loop _ Done = False + loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 && + loop (next1 s1') (next2 s2') +{-# INLINE [0] eq #-} + +cmp :: (Ord a) => Stream a -> Stream a -> Ordering +cmp (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) + where + loop Done Done = EQ + loop (Skip s1') (Skip s2') = loop (next1 s1') (next2 s2') + loop (Skip s1') x2 = loop (next1 s1') x2 + loop x1 (Skip s2') = loop x1 (next2 s2') + loop Done _ = LT + loop _ Done = GT + loop (Yield x1 s1') (Yield x2 s2') = + case compare x1 x2 of + EQ -> loop (next1 s1') (next2 s2') + other -> other +{-# INLINE [0] cmp #-} + +-- | The empty stream. +empty :: Stream a +empty = Stream next () 0 + where next _ = Done +{-# INLINE [0] empty #-} diff --git a/src/Data/Text/Internal/IO.hs b/src/Data/Text/Internal/IO.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/IO.hs @@ -0,0 +1,308 @@ +{-# LANGUAGE BangPatterns, RecordWildCards #-} +{-# LANGUAGE MagicHash #-} +-- | +-- Module : Data.Text.Internal.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Low-level support for text I\/O. + +module Data.Text.Internal.IO + ( + hGetLineWith + , readChunk + , hPutStream + , hPutStr + , hPutStrLn + ) where + +import qualified Control.Exception as E +import qualified Data.ByteString as B +import Data.ByteString.Builder (hPutBuilder, charUtf8) +import Data.IORef (readIORef, writeIORef) +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, streamLn, unstream) +import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size (exactSize, maxSize) +import Data.Text.Unsafe (inlinePerformIO) +import Foreign.Storable (peekElemOff) +import GHC.Exts (reallyUnsafePtrEquality#, isTrue#) +import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer, + bufferAdjustL, bufferElems, charSize, emptyBuffer, + isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer, + writeCharBuf) +import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_, + wantWritableHandle) +import GHC.IO.Handle.Text (commitBuffer') +import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..)) +import System.IO (Handle, hPutChar, utf8) +import System.IO.Error (isEOFError) +import qualified Data.Text as T + +-- | Read a single line of input from a handle, constructing a list of +-- decoded chunks as we go. When we're done, transform them into the +-- destination type. +hGetLineWith :: ([Text] -> t) -> Handle -> IO t +hGetLineWith f h = wantReadableHandle_ "hGetLine" h go + where + go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] + +hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] +hGetLineLoop hh@Handle__{..} = go where + go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do + let findEOL raw r | r == w = return (False, w) + | otherwise = do + (c,r') <- readCharBuf raw r + if c == '\n' + then return (True, r) + else findEOL raw r' + (eol, off) <- findEOL raw0 r0 + (t,r') <- if haInputNL == CRLF + then unpack_nl raw0 r0 off + else do t <- unpack raw0 r0 off + return (t,off) + if eol + then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) + return $ reverse (t:ts) + else do + let buf1 = bufferAdjustL r' buf + maybe_buf <- maybeFillReadBuffer hh buf1 + case maybe_buf of + -- Nothing indicates we caught an EOF, and we may have a + -- partial line to return. + Nothing -> do + -- we reached EOF. There might be a lone \r left + -- in the buffer, so check for that and + -- append it to the line if necessary. + let pre | isEmptyBuffer buf1 = T.empty + | otherwise = T.singleton '\r' + writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } + let str = reverse . filter (not . T.null) $ pre:t:ts + if null str + then ioe_EOF + else return str + Just new_buf -> go (t:ts) new_buf + +-- This function is lifted almost verbatim from GHC.IO.Handle.Text. +maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) +maybeFillReadBuffer handle_ buf + = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> + if isEOFError e + then return Nothing + else ioError e + +unpack :: RawCharBuffer -> Int -> Int -> IO Text +unpack !buf !r !w + | charSize /= 4 = sizeError "unpack" + | r >= w = return T.empty + | otherwise = withRawBuffer buf go + where + go pbuf = return $! unstream (Stream next r (exactSize (w-r))) + where + next !i | i >= w = Done + | otherwise = Yield (ix i) (i+1) + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- Variant of 'unpack' with CRLF decoding. If there is a trailing '\r', leave it in the buffer. +unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) +unpack_nl !buf !r !w + | charSize /= 4 = sizeError "unpack_nl" + | r >= w = return (T.empty, 0) + | otherwise = withRawBuffer buf $ go + where + go pbuf = do + let !t = unstream (Stream next r (maxSize (w-r))) + w' = w - 1 + return $ if ix w' == '\r' + then (t,w') + else (t,w) + where + next !i | i >= w = Done + | c == '\r' = let i' = i + 1 + in if i' < w + then if ix i' == '\n' + then Yield '\n' (i+2) + else Yield '\r' i' + else Done + | otherwise = Yield c (i+1) + where c = ix i + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer +getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = + case bufferElems buf of + -- buffer empty: read some more + 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf + + -- if the buffer has a single '\r' in it and we're doing newline + -- translation: read some more + 1 | haInputNL == CRLF -> do + (c,_) <- readCharBuf bufRaw bufL + if c == '\r' + then do -- shuffle the '\r' to the beginning. This is only safe + -- if we're about to call readTextDevice, otherwise it + -- would mess up flushCharBuffer. + -- See [note Buffer Flushing], GHC.IO.Handle.Types + _ <- writeCharBuf bufRaw 0 '\r' + let buf' = buf{ bufL=0, bufR=1 } + readTextDevice handle_ buf' + else do + return buf + + -- buffer has some chars in it already: just return it + _otherwise -> {-# SCC "otherwise" #-} return buf + +-- | Read a single chunk of strict text from a buffer. Used by both +-- the strict and lazy implementations of hGetContents. +readChunk :: Handle__ -> CharBuffer -> IO Text +readChunk hh@Handle__{..} buf = do + buf'@Buffer{..} <- getSomeCharacters hh buf + (t,r) <- if haInputNL == CRLF + then unpack_nl bufRaw bufL bufR + else do t <- unpack bufRaw bufL bufR + return (t,bufR) + writeIORef haCharBuffer (bufferAdjustL r buf') + return t + +-- | Print a @Stream Char@. +hPutStream :: Handle -> Stream Char -> IO () +hPutStream h str = hPutStreamOrUtf8 h str Nothing + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8) + where + putUtf8 = B.hPutStr h (encodeUtf8 t) + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8) + where + -- Not using B.hPutStrLn because it's not necessarily atomic: + -- https://github.com/haskell/bytestring/issues/200 + putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n') + +-- | 'hPutStream' with an optional special case when the output encoding is +-- UTF-8 and without newline conversion. +hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO () +-- This function is modified from GHC.IO.Handle.Text. +hPutStreamOrUtf8 h str mPutUtf8 = do + (buffer_mode, nl, isUtf8) <- + wantWritableHandle "hPutStr" h $ \h_ -> do + bmode <- getSpareBuffer h_ + return (bmode, haOutputNL h_, eqUTF8 h_) + case buffer_mode of + _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8 + (NoBuffering, _) -> hPutChars h str + (LineBuffering, buf) -> writeLines h nl buf str + (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str + + where + -- If the encoding is UTF-8, it's most likely pointer-equal to + -- 'System.IO.utf8', letting us avoid a String comparison. + -- If it is somehow UTF-8 but not pointer-equal to 'utf8', + -- we will just take a slower branch, but the result is still correct. + eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec +{-# INLINE hPutStreamOrUtf8 #-} + +hPutChars :: Handle -> Stream Char -> IO () +hPutChars h (Stream next0 s0 _len) = loop s0 + where + loop !s = case next0 s of + Done -> return () + Skip s' -> loop s' + Yield x s' -> hPutChar h x >> loop s' + +-- The following functions are largely lifted from GHC.IO.Handle.Text, +-- but adapted to a coinductive stream of data instead of an inductive +-- list. +-- +-- We have several variations of more or less the same code for +-- performance reasons. Splitting the original buffered write +-- function into line- and block-oriented versions gave us a 2.1x +-- performance improvement. Lifting out the raw/cooked newline +-- handling gave a few more percent on top. + +writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO () +writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' -> do + n' <- if nl == CRLF + then do n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' + else writeCharBuf' raw len n x + commit n' True{-needs flush-} False >>= outer s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO () +writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + -- Leave room for two characters for CRLF decoding + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' && isCRLF -> do + n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' >>= inner s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +-- | Only modifies the raw buffer and not the buffer attributes +writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int +writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $ + writeCharBuf bufRaw n c + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) +getSpareBuffer Handle__{haCharBuffer=ref, + haBuffers=spare_ref, + haBufferMode=mode} + = do + case mode of + NoBuffering -> return (mode, error "no buffer!") + _ -> do + bufs <- readIORef spare_ref + buf <- readIORef ref + case bufs of + BufferListCons b rest -> do + writeIORef spare_ref rest + return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) + BufferListNil -> do + new_buf <- newCharBuffer (bufSize buf) WriteBuffer + return (mode, new_buf) + + +-- This function is modified from GHC.Internal.IO.Handle.Text. +commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool + -> IO CharBuffer +commitBuffer hdl !raw !sz !count flush release = + wantWritableHandle "commitAndReleaseBuffer" hdl $ + commitBuffer' raw sz count flush release +{-# INLINE commitBuffer #-} + +sizeError :: String -> a +sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size" diff --git a/src/Data/Text/Internal/IsAscii.hs b/src/Data/Text/Internal/IsAscii.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/IsAscii.hs @@ -0,0 +1,94 @@ +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE UnliftedFFITypes #-} +#if defined(PURE_HASKELL) +{-# LANGUAGE BangPatterns #-} +#endif + +{-# OPTIONS_HADDOCK not-home #-} + +-- | Implements 'isAscii', using efficient C routines by default. +-- +-- Similarly implements asciiPrefixLength, used internally in Data.Text.Encoding. +module Data.Text.Internal.IsAscii where + +#if defined(PURE_HASKELL) +import Prelude hiding (all) +import qualified Data.Char as Char +import qualified Data.ByteString as BS +import Data.Text.Unsafe (iter, Iter(..)) +#else +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Word (Word8) +import Foreign.C.Types +import Foreign.Ptr (Ptr, plusPtr) +import GHC.Base (ByteArray#) +import Prelude (Bool(..), Int, (==), ($), IO, (<$>)) +import qualified Data.Text.Array as A +#endif +import Data.ByteString (ByteString) +import Data.Text.Internal (Text(..)) +import qualified Prelude as P + +-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only +-- U+0000 through U+007F). +-- +-- This is a more efficient version of @'all' 'Data.Char.isAscii'@. +-- +-- >>> isAscii "" +-- True +-- +-- >>> isAscii "abc\NUL" +-- True +-- +-- >>> isAscii "abcd€" +-- False +-- +-- prop> isAscii t == all (< '\x80') t +-- +-- @since 2.0.2 +isAscii :: Text -> Bool +#if defined(PURE_HASKELL) +isAscii = all Char.isAscii + +-- | (Re)implemented to avoid circular dependency on Data.Text. +all :: (Char -> Bool) -> Text -> Bool +all p t@(Text _ _ len) = go 0 + where + go i | i >= len = True + | otherwise = + let !(Iter c j) = iter t i + in p c && go (i+j) +#else +cSizeToInt :: CSize -> Int +cSizeToInt = P.fromIntegral +{-# INLINE cSizeToInt #-} + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +isAscii (Text (A.ByteArray arr) off len) = + cSizeToInt (c_is_ascii_offset arr (intToCSize off) (intToCSize len)) == len +#endif +{-# INLINE isAscii #-} + +-- | Length of the longest ASCII prefix. +asciiPrefixLength :: ByteString -> Int +#if defined(PURE_HASKELL) +asciiPrefixLength = BS.length P.. BS.takeWhile (P.< 0x80) +#else +asciiPrefixLength bs = unsafeDupablePerformIO $ withBS bs $ \ fp len -> + unsafeWithForeignPtr fp $ \src -> do + P.fromIntegral <$> c_is_ascii src (src `plusPtr` len) +#endif +{-# INLINE asciiPrefixLength #-} + +#if !defined(PURE_HASKELL) +foreign import ccall unsafe "_hs_text_is_ascii_offset" c_is_ascii_offset + :: ByteArray# -> CSize -> CSize -> CSize + +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize +#endif diff --git a/src/Data/Text/Internal/Lazy.hs b/src/Data/Text/Internal/Lazy.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Lazy.hs @@ -0,0 +1,151 @@ +{-# LANGUAGE BangPatterns #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal.Lazy +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. + +module Data.Text.Internal.Lazy + ( + Text(..) + , LazyText + , chunk + , empty + , foldrChunks + , foldlChunks + -- * Data type invariant and abstraction functions + + -- $invariant + , strictInvariant + , lazyInvariant + , showStructure + + -- * Chunk allocation sizes + , defaultChunkSize + , smallChunkSize + , chunkOverhead + + , equal + ) where + +import Data.Bits (shiftL) +import Data.Text () +import Foreign.Storable (sizeOf) +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text as T + +data Text = Empty + -- ^ Empty text. + -- + -- @since 2.1.2 + | Chunk {-# UNPACK #-} !T.Text Text + -- ^ Chunks must be non-empty, this invariant is not checked. + +-- | Type synonym for the lazy flavour of 'Text'. +-- +-- @since 2.1.1 +type LazyText = Text + +-- $invariant +-- +-- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or +-- consists of non-null 'T.Text's. All functions must preserve this, +-- and the QC properties must check this. + +-- | Check the invariant strictly. +strictInvariant :: Text -> Bool +strictInvariant Empty = True +strictInvariant x@(Chunk (T.Text _ _ len) cs) + | len > 0 = strictInvariant cs + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Check the invariant lazily. +lazyInvariant :: Text -> Text +lazyInvariant Empty = Empty +lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) + | len > 0 = Chunk c (lazyInvariant cs) + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Display the internal structure of a lazy 'Text'. +showStructure :: Text -> String +showStructure Empty = "Empty" +showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" +showStructure (Chunk t ts) = + "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" + +-- | Smart constructor for 'Chunk'. Guarantees the data type invariant. +chunk :: T.Text -> Text -> Text +{-# INLINE [0] chunk #-} +chunk t ts | T.null t = ts + | otherwise = Chunk t ts + +{-# RULES +"TEXT chunk/text" forall arr off len. + chunk (T.text arr off len) = chunk (T.Text arr off len) +"TEXT chunk/empty" forall ts. + chunk T.empty ts = ts +#-} + +-- | Smart constructor for 'Empty'. +empty :: Text +{-# INLINE [0] empty #-} +empty = Empty + +-- | Consume the chunks of a lazy 'Text' with a natural right fold. +foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a +foldrChunks f z = go + where go Empty = z + go (Chunk c cs) = f c (go cs) +{-# INLINE foldrChunks #-} + +-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, +-- accumulating left fold. +foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a +foldlChunks f z = go z + where go !a Empty = a + go !a (Chunk c cs) = go (f a c) cs +{-# INLINE foldlChunks #-} + +-- | Currently set to 16 KiB, less the memory management overhead. +defaultChunkSize :: Int +defaultChunkSize = 16384 - chunkOverhead +{-# INLINE defaultChunkSize #-} + +-- | Currently set to 128 bytes, less the memory management overhead. +smallChunkSize :: Int +smallChunkSize = 128 - chunkOverhead +{-# INLINE smallChunkSize #-} + +-- | The memory management overhead. Currently this is tuned for GHC only. +chunkOverhead :: Int +chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 +{-# INLINE chunkOverhead #-} + +equal :: Text -> Text -> Bool +equal Empty Empty = True +equal Empty _ = False +equal _ Empty = False +equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + case compare lenA lenB of + LT -> A.equal arrA offA arrB offB lenA && + as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs + EQ -> A.equal arrA offA arrB offB lenA && + as `equal` bs + GT -> A.equal arrA offA arrB offB lenB && + Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs diff --git a/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs b/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs @@ -0,0 +1,332 @@ +{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} + +-- | +-- Module : Data.Text.Lazy.Encoding.Fusion +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Fusible 'Stream'-oriented functions for converting between lazy +-- 'Text' and several common encodings. + +module Data.Text.Internal.Lazy.Encoding.Fusion + ( + -- * Streaming + -- streamASCII + streamUtf8 + , streamUtf16LE + , streamUtf16BE + , streamUtf32LE + , streamUtf32BE + + -- * Unstreaming + , unstream + + , module Data.Text.Internal.Encoding.Fusion.Common + ) where + +import Data.Bits (shiftL) +import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize) +import qualified Data.ByteString as B +import qualified Data.ByteString.Unsafe as B +import Data.Text.Internal.ByteStringCompat +import Data.Text.Internal.Encoding.Fusion.Common +import Data.Text.Encoding.Error +import Data.Text.Internal.Fusion (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Word (Word8, Word16, Word32) +import qualified Data.Text.Internal.Encoding.Utf8 as U8 +import qualified Data.Text.Internal.Encoding.Utf16 as U16 +import qualified Data.Text.Internal.Encoding.Utf32 as U32 +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Foreign.ForeignPtr (ForeignPtr) +import Foreign.Marshal.Utils (copyBytes) +import Foreign.Storable (pokeByteOff) +import Data.ByteString.Internal (mallocByteString) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif + +data S = S0 + | S1 {-# UNPACK #-} !Word8 + | S2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 + | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 + | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 + +data T = T !ByteString !S {-# UNPACK #-} !Int + +-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using +-- UTF-8 encoding. +streamUtf8 :: OnDecodeError -> ByteString -> Stream Char +streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize + where + next (T bs@(Chunk ps _) S0 i) + | i < len && U8.validate1 a = + Yield (unsafeChr8 a) (T bs S0 (i+1)) + | i + 1 < len && U8.validate2 a b = + Yield (U8.chr2 a b) (T bs S0 (i+2)) + | i + 2 < len && U8.validate3 a b c = + Yield (U8.chr3 a b c) (T bs S0 (i+3)) + | i + 3 < len && U8.validate4 a b c d = + Yield (U8.chr4 a b c d) (T bs S0 (i+4)) + where len = B.length ps + a = B.unsafeIndex ps i + b = B.unsafeIndex ps (i+1) + c = B.unsafeIndex ps (i+2) + d = B.unsafeIndex ps (i+3) + next st@(T bs s i) = + case s of + S1 a | U8.validate1 a -> Yield (unsafeChr8 a) es + S2 a b | U8.validate2 a b -> Yield (U8.chr2 a b) es + S3 a b c | U8.validate3 a b c -> Yield (U8.chr3 a b c) es + S4 a b c d | U8.validate4 a b c d -> Yield (U8.chr4 a b c d) es + _ -> consume st + where es = T bs S0 i + consume (T bs@(Chunk ps rest) s i) + | i >= B.length ps = consume (T rest s 0) + | otherwise = + case s of + S0 -> next (T bs (S1 x) (i+1)) + S1 a -> next (T bs (S2 a x) (i+1)) + S2 a b -> next (T bs (S3 a b x) (i+1)) + S3 a b c -> next (T bs (S4 a b c x) (i+1)) + S4 a b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a) + (T bs (S4 b c d x) (i+1)) + where x = B.unsafeIndex ps i + consume (T Empty S0 _) = Done + consume (T Empty _ i) = decodeError "streamUtf8" "UTF-8" onErr Nothing (T Empty S0 i) +{-# INLINE [0] streamUtf8 #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little +-- endian UTF-16 encoding. +streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char +streamUtf16LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize + where + next (T bs@(Chunk ps _) S0 i) + | i + 1 < len && U16.validate1 x1 = + Yield (unsafeChr16 x1) (T bs S0 (i+2)) + | i + 3 < len && U16.validate2 x1 x2 = + Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) + where len = B.length ps + x1 = c (idx i) (idx (i + 1)) + x2 = c (idx (i + 2)) (idx (i + 3)) + c w1 w2 = w1 + (w2 `shiftL` 8) + idx = word8ToWord16 . B.unsafeIndex ps :: Int -> Word16 + next st@(T bs s i) = + case s of + S2 w1 w2 | U16.validate1 (c w1 w2) -> + Yield (unsafeChr16 (c w1 w2)) es + S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> + Yield (U16.chr2 (c w1 w2) (c w3 w4)) es + _ -> consume st + where es = T bs S0 i + c :: Word8 -> Word8 -> Word16 + c w1 w2 = word8ToWord16 w1 + (word8ToWord16 w2 `shiftL` 8) + consume (T bs@(Chunk ps rest) s i) + | i >= B.length ps = consume (T rest s 0) + | otherwise = + case s of + S0 -> next (T bs (S1 x) (i+1)) + S1 w1 -> next (T bs (S2 w1 x) (i+1)) + S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) + S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) + S4 w1 w2 w3 w4 -> decodeError "streamUtf16LE" "UTF-16LE" onErr (Just w1) + (T bs (S4 w2 w3 w4 x) (i+1)) + where x = B.unsafeIndex ps i + consume (T Empty S0 _) = Done + consume (T Empty _ i) = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (T Empty S0 i) +{-# INLINE [0] streamUtf16LE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big +-- endian UTF-16 encoding. +streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char +streamUtf16BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize + where + next (T bs@(Chunk ps _) S0 i) + | i + 1 < len && U16.validate1 x1 = + Yield (unsafeChr16 x1) (T bs S0 (i+2)) + | i + 3 < len && U16.validate2 x1 x2 = + Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) + where len = B.length ps + x1 = c (idx i) (idx (i + 1)) + x2 = c (idx (i + 2)) (idx (i + 3)) + c w1 w2 = (w1 `shiftL` 8) + w2 + idx = word8ToWord16 . B.unsafeIndex ps :: Int -> Word16 + next st@(T bs s i) = + case s of + S2 w1 w2 | U16.validate1 (c w1 w2) -> + Yield (unsafeChr16 (c w1 w2)) es + S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> + Yield (U16.chr2 (c w1 w2) (c w3 w4)) es + _ -> consume st + where es = T bs S0 i + c :: Word8 -> Word8 -> Word16 + c w1 w2 = (word8ToWord16 w1 `shiftL` 8) + word8ToWord16 w2 + consume (T bs@(Chunk ps rest) s i) + | i >= B.length ps = consume (T rest s 0) + | otherwise = + case s of + S0 -> next (T bs (S1 x) (i+1)) + S1 w1 -> next (T bs (S2 w1 x) (i+1)) + S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) + S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) + S4 w1 w2 w3 w4 -> decodeError "streamUtf16BE" "UTF-16BE" onErr (Just w1) + (T bs (S4 w2 w3 w4 x) (i+1)) + where x = B.unsafeIndex ps i + consume (T Empty S0 _) = Done + consume (T Empty _ i) = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (T Empty S0 i) +{-# INLINE [0] streamUtf16BE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big +-- endian UTF-32 encoding. +streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char +streamUtf32BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize + where + next (T bs@(Chunk ps _) S0 i) + | i + 3 < len && U32.validate x = + Yield (unsafeChr32 x) (T bs S0 (i+4)) + where len = B.length ps + x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 + x1 = idx i + x2 = idx (i+1) + x3 = idx (i+2) + x4 = idx (i+3) + idx = word8ToWord32 . B.unsafeIndex ps :: Int -> Word32 + next st@(T bs s i) = + case s of + S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) -> + Yield (unsafeChr32 (c w1 w2 w3 w4)) es + _ -> consume st + where es = T bs S0 i + c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32 + c w1 w2 w3 w4 = shifted + where + shifted = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4 + x1 = word8ToWord32 w1 + x2 = word8ToWord32 w2 + x3 = word8ToWord32 w3 + x4 = word8ToWord32 w4 + consume (T bs@(Chunk ps rest) s i) + | i >= B.length ps = consume (T rest s 0) + | otherwise = + case s of + S0 -> next (T bs (S1 x) (i+1)) + S1 w1 -> next (T bs (S2 w1 x) (i+1)) + S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) + S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) + S4 w1 w2 w3 w4 -> decodeError "streamUtf32BE" "UTF-32BE" onErr (Just w1) + (T bs (S4 w2 w3 w4 x) (i+1)) + where x = B.unsafeIndex ps i + consume (T Empty S0 _) = Done + consume (T Empty _ i) = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (T Empty S0 i) +{-# INLINE [0] streamUtf32BE #-} + +-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little +-- endian UTF-32 encoding. +streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char +streamUtf32LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize + where + next (T bs@(Chunk ps _) S0 i) + | i + 3 < len && U32.validate x = + Yield (unsafeChr32 x) (T bs S0 (i+4)) + where len = B.length ps + x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 + x1 = idx i + x2 = idx (i+1) + x3 = idx (i+2) + x4 = idx (i+3) + idx = word8ToWord32 . B.unsafeIndex ps :: Int -> Word32 + next st@(T bs s i) = + case s of + S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) -> + Yield (unsafeChr32 (c w1 w2 w3 w4)) es + _ -> consume st + where es = T bs S0 i + c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32 + c w1 w2 w3 w4 = shifted + where + shifted = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1 + x1 = word8ToWord32 w1 + x2 = word8ToWord32 w2 + x3 = word8ToWord32 w3 + x4 = word8ToWord32 w4 + consume (T bs@(Chunk ps rest) s i) + | i >= B.length ps = consume (T rest s 0) + | otherwise = + case s of + S0 -> next (T bs (S1 x) (i+1)) + S1 w1 -> next (T bs (S2 w1 x) (i+1)) + S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) + S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) + S4 w1 w2 w3 w4 -> decodeError "streamUtf32LE" "UTF-32LE" onErr (Just w1) + (T bs (S4 w2 w3 w4 x) (i+1)) + where x = B.unsafeIndex ps i + consume (T Empty S0 _) = Done + consume (T Empty _ i) = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (T Empty S0 i) +{-# INLINE [0] streamUtf32LE #-} + +-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'. +unstreamChunks :: Int -> Stream Word8 -> ByteString +unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 (upperBound 4 len0) + where chunk s1 len1 = unsafeDupablePerformIO $ do + let len = max 4 (min len1 chunkSize) + mallocByteString len >>= loop len 0 s1 + where + loop !n !off !s fp = case next s of + Done | off == 0 -> return Empty + | otherwise -> return $! Chunk (trimUp fp off) Empty + Skip s' -> loop n off s' fp + Yield x s' + | off == chunkSize -> do + let !newLen = n - off + return $! Chunk (trimUp fp off) (chunk s newLen) + | off == n -> realloc fp n off s' x + | otherwise -> do + unsafeWithForeignPtr fp $ \p -> pokeByteOff p off x + loop n (off+1) s' fp + {-# NOINLINE realloc #-} + realloc fp n off s x = do + let n' = min (n+n) chunkSize + fp' <- copy0 fp n n' + unsafeWithForeignPtr fp' $ \p -> pokeByteOff p off x + loop n' (off+1) s fp' + trimUp fp off = mkBS fp off + copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8) + copy0 !src !srcLen !destLen = +#if defined(ASSERTS) + assert (srcLen <= destLen) $ +#endif + do + dest <- mallocByteString destLen + unsafeWithForeignPtr src $ \src' -> + unsafeWithForeignPtr dest $ \dest' -> + copyBytes dest' src' srcLen + return dest + +-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'. +unstream :: Stream Word8 -> ByteString +unstream = unstreamChunks defaultChunkSize + +decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8 + -> s -> Step s Char +decodeError func kind onErr mb i = + case onErr desc mb of + Nothing -> Skip i + Just c -> Yield c i + where desc = "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Invalid " ++ + kind ++ " stream" + +word8ToWord16 :: Word8 -> Word16 +word8ToWord16 = fromIntegral + +word8ToWord32 :: Word8 -> Word32 +word8ToWord32 = fromIntegral diff --git a/src/Data/Text/Internal/Lazy/Fusion.hs b/src/Data/Text/Internal/Lazy/Fusion.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Lazy/Fusion.hs @@ -0,0 +1,156 @@ +{-# LANGUAGE BangPatterns, CPP #-} +-- | +-- Module : Data.Text.Lazy.Fusion +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Core stream fusion functionality for text. + +module Data.Text.Internal.Lazy.Fusion + ( + stream + , streamLn + , unstream + , unstreamChunks + , length + , unfoldrN + , index + , countChar + ) where + +import Prelude hiding (length) +import Data.Bits (shiftL) +import qualified Data.Text.Internal.Fusion.Common as S +import Control.Monad.ST (runST) +import Data.Text.Internal.Fusion.Types +import Data.Text.Internal.Fusion.Size (isEmpty, unknownSize) +import Data.Text.Internal.Lazy +import qualified Data.Text.Internal as I +import qualified Data.Text.Array as A +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Unsafe (Iter(..), iter) +import Data.Int (Int64) +import GHC.Stack (HasCallStack) + +default(Int64) + +-- | /O(n)/ Convert a 'Text' into a 'Stream Char'. +stream :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +stream = stream' False +{-# INLINE [0] stream #-} + +-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@ +-- +-- @since 2.1.2 +streamLn :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +streamLn = stream' True + +-- | Shared implementation of 'stream' and 'streamLn'. +stream' :: +#if defined(ASSERTS) + HasCallStack => +#endif + Bool -> Text -> Stream Char +stream' addNl text = Stream next (text :*: 0) unknownSize + where + next (Empty :*: i) + | addNl && i <= 0 = Yield '\n' (Empty :*: 1) + | otherwise = Done + next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i) + | i >= len = next (ts :*: 0) + | otherwise = Yield c (txt :*: i+d) + where Iter c d = iter t i +{-# INLINE [0] stream' #-} + +-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given +-- chunk size. +unstreamChunks :: +#if defined(ASSERTS) + HasCallStack => +#endif + Int -> Stream Char -> Text +unstreamChunks !chunkSize (Stream next s0 len0) + | isEmpty len0 = Empty + | otherwise = outer s0 + where + outer so = {-# SCC "unstreamChunks/outer" #-} + case next so of + Done -> Empty + Skip s' -> outer s' + Yield x s' -> runST $ do + a <- A.new unknownLength + unsafeWrite a 0 x >>= inner a unknownLength s' + where unknownLength = 4 + where + inner marr !len s !i + | i + 3 >= chunkSize = finish marr i s + | i + 3 >= len = {-# SCC "unstreamChunks/resize" #-} do + let newLen = min (len `shiftL` 1) chunkSize + marr' <- A.new newLen + A.copyM marr' 0 marr 0 len + inner marr' newLen s i + | otherwise = + {-# SCC "unstreamChunks/inner" #-} + case next s of + Done -> finish marr i s + Skip s' -> inner marr len s' i + Yield x s' -> do d <- unsafeWrite marr i x + inner marr len s' (i+d) + finish marr len s' = do + A.shrinkM marr len + arr <- A.unsafeFreeze marr + return (I.Text arr 0 len `Chunk` outer s') +{-# INLINE [0] unstreamChunks #-} + +-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using +-- 'defaultChunkSize'. +unstream :: +#if defined(ASSERTS) + HasCallStack => +#endif + Stream Char -> Text +unstream = unstreamChunks defaultChunkSize +{-# INLINE [0] unstream #-} + +-- | /O(n)/ Returns the number of characters in a text. +length :: Stream Char -> Int64 +length = S.lengthI +{-# INLINE[0] length #-} + +{-# RULES "LAZY STREAM stream/unstream fusion" forall s. + stream (unstream s) = s #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed +-- value. However, the length of the result is limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the length of the result is known. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Stream Char +unfoldrN n = S.unfoldrNI n +{-# INLINE [0] unfoldrN #-} + +-- | /O(n)/ stream index (subscript) operator, starting from 0. +index :: HasCallStack => Stream Char -> Int64 -> Char +index = S.indexI +{-# INLINE [0] index #-} + +-- | /O(n)/ The 'count' function returns the number of times the query +-- element appears in the given stream. +countChar :: Char -> Stream Char -> Int64 +countChar = S.countCharI +{-# INLINE [0] countChar #-} diff --git a/src/Data/Text/Internal/Lazy/Search.hs b/src/Data/Text/Internal/Lazy/Search.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Lazy/Search.hs @@ -0,0 +1,132 @@ +{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} + +-- | +-- Module : Data.Text.Lazy.Search +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Fast substring search for lazy 'Text', based on work by Boyer, +-- Moore, Horspool, Sunday, and Lundh. Adapted from the strict +-- implementation. + +module Data.Text.Internal.Lazy.Search + ( + indices + ) where + +import Data.Bits (unsafeShiftL, (.|.), (.&.)) +import qualified Data.Text.Array as A +import Data.Int (Int64) +import Data.Word (Word8, Word64) +import qualified Data.Text.Internal as T +import qualified Data.Text as T (concat, isPrefixOf) +import Data.Text.Internal.ArrayUtils (memchr) +import Data.Text.Internal.Fusion.Types (PairS(..)) +import Data.Text.Internal.Lazy (Text(..), foldrChunks) + +-- | /O(n+m)/ Find the offsets of all non-overlapping indices of +-- @needle@ within @haystack@. +-- +-- This function is strict in @needle@, and lazy (as far as possible) +-- in the chunks of @haystack@. +-- +-- In (unlikely) bad cases, this algorithm's complexity degrades +-- towards /O(n*m)/. +indices :: Text -- ^ Substring to search for (@needle@) + -> Text -- ^ Text to search in (@haystack@) + -> [Int64] +indices needle + | nlen <= 0 = const [] + | nlen == 1 = indicesOne (A.unsafeIndex narr noff) 0 + | otherwise = advance 0 0 + where + T.Text narr noff nlen = T.concat (foldrChunks (:) [] needle) + + advance !_ !_ Empty = [] + advance !(g :: Int64) !(i :: Int) xxs@(Chunk x@(T.Text xarr@(A.ByteArray xarr#) xoff l) xs) + | i >= l = advance g (i - l) xs + | lackingHay (i + nlen) x xs = [] + | c == z && candidateMatch = g : advance (g + intToInt64 nlen) (i + nlen) xxs + | otherwise = advance (g + intToInt64 delta) (i + delta) xxs + where + c = index xxs (i + nlast) + delta | nextInPattern = nlen + 1 + | c == z = skip + 1 + | l >= i + nlen = case + memchr xarr# (xoff + i + nlen) (l - i - nlen) z of + -1 -> max 1 (l - i - nlen) + s -> s + 1 + | otherwise = 1 + nextInPattern = mask .&. swizzle (index xxs (i + nlen)) == 0 + + candidateMatch + | i + nlen <= l = A.equal narr noff xarr (xoff + i) nlen + | otherwise = A.equal narr noff xarr (xoff + i) (l - i) && + T.Text narr (noff + l - i) (nlen - l + i) `isPrefixOf` xs + + nlast = nlen - 1 + z = A.unsafeIndex narr (noff + nlen - 1) + (mask :: Word64) :*: skip = buildTable 0 0 0 (nlen-2) + + swizzle :: Word8 -> Word64 + swizzle w = 1 `unsafeShiftL` (word8ToInt w .&. 0x3f) + + buildTable !g !i !msk !skp + | i >= nlast = (msk .|. swizzle z) :*: skp + | otherwise = buildTable (g+1) (i+1) msk' skp' + where c = A.unsafeIndex narr (noff+i) + msk' = msk .|. swizzle c + skp' | c == z = nlen - g - 2 + | otherwise = skp + + -- | Check whether an attempt to index into the haystack at the + -- given offset would fail. + lackingHay :: Int -> T.Text -> Text -> Bool + lackingHay q (T.Text _ _ l) ps = l < q && case ps of + Empty -> True + Chunk r rs -> lackingHay (q - l) r rs + +-- | Fast index into a partly unpacked 'Text'. We take into account +-- the possibility that the caller might try to access one element +-- past the end. +index :: Text -> Int -> Word8 +index Empty !_ = 0 +index (Chunk (T.Text arr off len) xs) !i + | i < len = A.unsafeIndex arr (off + i) + | otherwise = index xs (i - len) + +-- | A variant of 'indices' that scans linearly for a single 'Word8'. +indicesOne :: Word8 -> Int64 -> Text -> [Int64] +indicesOne c = chunk + where + chunk :: Int64 -> Text -> [Int64] + chunk !_ Empty = [] + chunk !i (Chunk (T.Text oarr ooff olen) os) = go 0 + where + go h | h >= olen = chunk (i+intToInt64 olen) os + | on == c = i + intToInt64 h : go (h+1) + | otherwise = go (h+1) + where on = A.unsafeIndex oarr (ooff+h) + +-- | First argument is a strict Text, and second is a lazy one. +isPrefixOf :: T.Text -> Text -> Bool +isPrefixOf (T.Text _ _ xlen) Empty = xlen == 0 +isPrefixOf x@(T.Text xarr xoff xlen) (Chunk y@(T.Text _ _ ylen) ys) + | xlen <= ylen = x `T.isPrefixOf` y + | otherwise = y `T.isPrefixOf` x && T.Text xarr (xoff + ylen) (xlen - ylen) `isPrefixOf` ys + +intToInt64 :: Int -> Int64 +intToInt64 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral diff --git a/src/Data/Text/Internal/Measure.hs b/src/Data/Text/Internal/Measure.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Measure.hs @@ -0,0 +1,53 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} + +#if defined(PURE_HASKELL) +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MultiWayIf #-} +#endif + +#if !defined(PURE_HASKELL) +{-# LANGUAGE UnliftedFFITypes #-} +#endif + +{-# OPTIONS_HADDOCK not-home #-} + +-- | Implements 'measure_off', using efficient C routines by default. +module Data.Text.Internal.Measure + ( measure_off + ) +where + +import GHC.Exts + +#if defined(PURE_HASKELL) +import GHC.Word +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader) +#endif + +import Foreign.C.Types (CSize(..)) +import System.Posix.Types (CSsize(..)) + +#if defined(PURE_HASKELL) + +measure_off :: ByteArray# -> CSize -> CSize -> CSize -> CSsize +measure_off ba off len cnt = go 0 0 + where + go !cc !i + -- return the number of bytes for the first cnt codepoints, + | cc == cnt = fromIntegral i + -- return negated number of codepoints if there are fewer than cnt + | i >= len = negate (fromIntegral cc) + | otherwise = + let !(I# o) = fromIntegral (off+i) + !b = indexWord8Array# ba o + in go (cc+1) (i + fromIntegral (utf8LengthByLeader (W8# b))) + +#else + +-- | The input buffer (arr :: ByteArray#, off :: CSize, len :: CSize) +-- must specify a valid UTF-8 sequence, this condition is not checked. +foreign import ccall unsafe "_hs_text_measure_off" measure_off + :: ByteArray# -> CSize -> CSize -> CSize -> CSsize + +#endif diff --git a/src/Data/Text/Internal/PrimCompat.hs b/src/Data/Text/Internal/PrimCompat.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/PrimCompat.hs @@ -0,0 +1,35 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} + +module Data.Text.Internal.PrimCompat + ( word8ToWord# + , wordToWord8# + + , word16ToWord# + , wordToWord16# + + , wordToWord32# + , word32ToWord# + ) where + +#if MIN_VERSION_base(4,16,0) +import GHC.Exts (wordToWord8#,word8ToWord#,wordToWord16#,word16ToWord#,wordToWord32#,word32ToWord#) +#else +import GHC.Prim (Word#) +#endif + +#if !(MIN_VERSION_base(4,16,0)) +wordToWord8#, word8ToWord# :: Word# -> Word# +wordToWord16#, word16ToWord# :: Word# -> Word# +wordToWord32#, word32ToWord# :: Word# -> Word# +word8ToWord# w = w +word16ToWord# w = w +word32ToWord# w = w +wordToWord8# w = w +wordToWord16# w = w +wordToWord32# w = w +{-# INLINE wordToWord16# #-} +{-# INLINE word16ToWord# #-} +{-# INLINE wordToWord32# #-} +{-# INLINE word32ToWord# #-} +#endif diff --git a/src/Data/Text/Internal/Private.hs b/src/Data/Text/Internal/Private.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Private.hs @@ -0,0 +1,61 @@ +{-# LANGUAGE BangPatterns, CPP, RankNTypes, UnboxedTuples #-} + +-- | +-- Module : Data.Text.Internal.Private +-- Copyright : (c) 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC + +module Data.Text.Internal.Private + ( + runText + , span_ + , spanAscii_ + ) where + +import Control.Monad.ST (ST, runST) +import Data.Text.Internal (Text(..), text) +import Data.Text.Unsafe (Iter(..), iter) +import qualified Data.Text.Array as A +import Data.Word (Word8) + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +span_ :: (Char -> Bool) -> Text -> (# Text, Text #) +span_ p t@(Text arr off len) = (# hd,tl #) + where hd = text arr off k + tl = text arr (off+k) (len-k) + !k = loop 0 + loop !i | i < len && p c = loop (i+d) + | otherwise = i + where Iter c d = iter t i +{-# INLINE span_ #-} + +-- | For the sake of performance this function does not check +-- that a char is in ASCII range; it is a responsibility of @p@. +-- +-- @since 2.0 +spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #) +spanAscii_ p (Text arr off len) = (# hd, tl #) + where hd = text arr off k + tl = text arr (off + k) (len - k) + !k = loop 0 + loop !i | i < len && p (A.unsafeIndex arr (off + i)) = loop (i + 1) + | otherwise = i +{-# INLINE spanAscii_ #-} + +runText :: +#if defined(ASSERTS) + HasCallStack => +#endif + (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text +runText act = runST (act $ \ !marr !len -> do + A.shrinkM marr len + arr <- A.unsafeFreeze marr + return $! text arr 0 len) +{-# INLINE runText #-} diff --git a/src/Data/Text/Internal/Read.hs b/src/Data/Text/Internal/Read.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Read.hs @@ -0,0 +1,80 @@ +-- | +-- Module : Data.Text.Internal.Read +-- Copyright : (c) 2014 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- Common internal functions for reading textual data. +module Data.Text.Internal.Read + ( + IReader + , IParser(..) + , T(..) + , digitToInt + , hexDigitToInt + , perhaps + ) where + +import Control.Applicative as App (Applicative(..)) +import Control.Arrow (first) +import Control.Monad (ap) +import Data.Char (ord) + +type IReader t a = t -> Either String (a,t) + +newtype IParser t a = P { + runP :: IReader t a + } + +instance Functor (IParser t) where + fmap f m = P $ fmap (first f) . runP m + +instance Applicative (IParser t) where + pure a = P $ \t -> Right (a,t) + {-# INLINE pure #-} + (<*>) = ap + +instance Monad (IParser t) where + return = App.pure + m >>= k = P $ \t -> case runP m t of + Left err -> Left err + Right (a,t') -> runP (k a) t' + {-# INLINE (>>=) #-} + +-- If we ever need a `MonadFail` instance the definition below can be used +-- +-- > instance MonadFail (IParser t) where +-- > fail msg = P $ \_ -> Left msg +-- +-- But given the code compiles fine with a post-MFP GHC 8.6+ we don't need +-- one just yet. + +data T = T !Integer !Int + +perhaps :: a -> IParser t a -> IParser t a +perhaps def m = P $ \t -> case runP m t of + Left _ -> Right (def,t) + r@(Right _) -> r + +hexDigitToInt :: Char -> Int +hexDigitToInt c + | to0 < 10 = wordToInt to0 + | toa < 6 = wordToInt toa + 10 + | otherwise = wordToInt toA + 10 + where + ordW = intToWord (ord c) + to0 = ordW - intToWord (ord '0') + toa = ordW - intToWord (ord 'a') + toA = ordW - intToWord (ord 'A') + +digitToInt :: Char -> Int +digitToInt c = ord c - ord '0' + +intToWord :: Int -> Word +intToWord = fromIntegral + +wordToInt :: Word -> Int +wordToInt = fromIntegral diff --git a/src/Data/Text/Internal/Reverse.hs b/src/Data/Text/Internal/Reverse.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Reverse.hs @@ -0,0 +1,108 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE NoImplicitPrelude #-} +#if defined(PURE_HASKELL) +{-# LANGUAGE BangPatterns #-} +#endif + +{-# OPTIONS_HADDOCK not-home #-} + +-- | Implements 'reverse', using efficient C routines by default. +module Data.Text.Internal.Reverse (reverse, reverseNonEmpty) where + +#if !defined(PURE_HASKELL) +import GHC.Exts as Exts +import Control.Monad.ST.Unsafe (unsafeIOToST) +import Foreign.C.Types (CSize(..)) +#else +import Control.Monad.ST (ST) +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader) +#endif +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif +import Prelude hiding (reverse) +import Data.Text.Internal (Text(..), empty) +import Control.Monad.ST (runST) +import qualified Data.Text.Array as A + +-- | /O(n)/ Reverse the characters of a string. +-- +-- Example: +-- +-- $setup +-- >>> T.reverse "desrever" +-- "reversed" +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text +reverse (Text _ _ 0) = empty +reverse t = reverseNonEmpty t +{-# INLINE reverse #-} + +-- | /O(n)/ Reverse the characters of a string. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +reverseNonEmpty :: + Text -> Text +#if defined(PURE_HASKELL) +reverseNonEmpty (Text src off len) = runST $ do + dest <- A.new len + _ <- reversePoints src off dest len + result <- A.unsafeFreeze dest + pure $ Text result 0 len + +-- Step 0: +-- +-- Input: R E D R U M +-- ^ +-- x +-- Output: _ _ _ _ _ _ +-- ^ +-- y +-- +-- Step 1: +-- +-- Input: R E D R U M +-- ^ +-- x +-- +-- Output: _ _ _ _ _ R +-- ^ +-- y +reversePoints + :: A.Array -- ^ Input array + -> Int -- ^ Input index + -> A.MArray s -- ^ Output array + -> Int -- ^ Output index + -> ST s () +reversePoints src xx dest yy = go xx yy where + go !_ y | y <= 0 = pure () + go x y = + let pLen = utf8LengthByLeader (A.unsafeIndex src x) + -- The next y is also the start of the current point in the output + yNext = y - pLen + in do + A.copyI pLen dest yNext src x + go (x + pLen) yNext +#else +reverseNonEmpty (Text (A.ByteArray ba) off len) = runST $ do + marr@(A.MutableByteArray mba) <- A.new len + unsafeIOToST $ c_reverse mba ba (fromIntegral off) (fromIntegral len) + brr <- A.unsafeFreeze marr + return $ Text brr 0 len +#endif +{-# INLINE reverseNonEmpty #-} + +#if !defined(PURE_HASKELL) +-- | The input buffer (src :: ByteArray#, off :: CSize, len :: CSize) +-- must specify a valid UTF-8 sequence, this condition is not checked. +foreign import ccall unsafe "_hs_text_reverse" c_reverse + :: Exts.MutableByteArray# s -> ByteArray# -> CSize -> CSize -> IO () +#endif + +-- $setup +-- >>> :set -XOverloadedStrings +-- >>> import qualified Data.Text.Internal.Reverse as T diff --git a/src/Data/Text/Internal/Search.hs b/src/Data/Text/Internal/Search.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Search.hs @@ -0,0 +1,103 @@ +{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} + +-- | +-- Module : Data.Text.Internal.Search +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- Fast substring search for 'Text', based on work by Boyer, Moore, +-- Horspool, Sunday, and Lundh. +-- +-- References: +-- +-- * R. S. Boyer, J. S. Moore: A Fast String Searching Algorithm. +-- Communications of the ACM, 20, 10, 762-772 (1977) +-- +-- * R. N. Horspool: Practical Fast Searching in Strings. Software - +-- Practice and Experience 10, 501-506 (1980) +-- +-- * D. M. Sunday: A Very Fast Substring Search Algorithm. +-- Communications of the ACM, 33, 8, 132-142 (1990) +-- +-- * F. Lundh: +-- . (2006) + +module Data.Text.Internal.Search + ( + indices + ) where + +import qualified Data.Text.Array as A +import Data.Word (Word64, Word8) +import Data.Text.Internal (Text(..)) +import Data.Bits ((.|.), (.&.), unsafeShiftL) +import Data.Text.Internal.ArrayUtils (memchr) + +data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int + +-- | /O(n+m)/ Find the offsets of all non-overlapping indices of +-- @needle@ within @haystack@. +-- +-- In (unlikely) bad cases, this algorithm's complexity degrades +-- towards /O(n*m)/. +indices :: Text -- ^ Substring to search for (@needle@) + -> Text -- ^ Text to search in (@haystack@) + -> [Int] +indices needle@(Text narr noff nlen) + | nlen == 1 = scanOne (A.unsafeIndex narr noff) + | nlen <= 0 = const [] + | otherwise = indices' needle +{-# INLINE indices #-} + +-- | nlen must be >= 2, otherwise nindex causes access violation +indices' :: Text -> Text -> [Int] +indices' (Text narr noff nlen) (Text harr@(A.ByteArray harr#) hoff hlen) = loop (hoff + nlen) + where + nlast = nlen - 1 + !z = nindex nlast + nindex k = A.unsafeIndex narr (noff+k) + buildTable !i !msk !skp + | i >= nlast = (msk .|. swizzle z) :* skp + | otherwise = buildTable (i+1) (msk .|. swizzle c) skp' + where !c = nindex i + skp' | c == z = nlen - i - 2 + | otherwise = skp + !(mask :* skip) = buildTable 0 0 (nlen-2) + + swizzle :: Word8 -> Word64 + swizzle !k = 1 `unsafeShiftL` (word8ToInt k .&. 0x3f) + + loop !i + | i > hlen + hoff + = [] + | A.unsafeIndex harr (i - 1) == z + = if A.equal narr noff harr (i - nlen) nlen + then i - nlen - hoff : loop (i + nlen) + else loop (i + skip + 1) + | i == hlen + hoff + = [] + | mask .&. swizzle (A.unsafeIndex harr i) == 0 + = loop (i + nlen + 1) + | otherwise + = case memchr harr# i (hlen + hoff - i) z of + -1 -> [] + x -> loop (i + x + 1) +{-# INLINE indices' #-} + +scanOne :: Word8 -> Text -> [Int] +scanOne c (Text harr hoff hlen) = loop 0 + where + loop !i + | i >= hlen = [] + | A.unsafeIndex harr (hoff+i) == c = i : loop (i+1) + | otherwise = loop (i+1) +{-# INLINE scanOne #-} + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral diff --git a/src/Data/Text/Internal/StrictBuilder.hs b/src/Data/Text/Internal/StrictBuilder.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/StrictBuilder.hs @@ -0,0 +1,129 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeSynonymInstances #-} + +-- | +-- Module : Data.Text.Internal.Builder +-- License : BSD-style (see LICENSE) +-- Stability : experimental +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- @since 2.0.2 + +module Data.Text.Internal.StrictBuilder + ( StrictTextBuilder(..) + , StrictBuilder + , toText + , fromChar + , fromText + + -- * Unsafe + -- $unsafe + , unsafeFromByteString + , unsafeFromWord8 + ) where + +import Control.Monad.ST (ST, runST) +import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) +import Data.Functor (void) +import Data.Word (Word8) +import Data.ByteString (ByteString) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Text.Internal (Text(..), empty, safe) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Encoding.Utf8 (utf8Length) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import qualified Data.ByteString as B +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Unsafe.Char as Char + +-- | A delayed representation of strict 'Text'. +-- +-- @since 2.1.2 +data StrictTextBuilder = StrictTextBuilder + { sbLength :: {-# UNPACK #-} !Int + , sbWrite :: forall s. A.MArray s -> Int -> ST s () + } + +-- | A delayed representation of strict 'Text'. +-- +-- @since 2.0.2 +{-# DEPRECATED StrictBuilder "Use StrictTextBuilder instead" #-} +type StrictBuilder = StrictTextBuilder + +-- | Use 'StrictBuilder' to build 'Text'. +-- +-- @since 2.0.2 +toText :: StrictTextBuilder -> Text +toText (StrictTextBuilder 0 _) = empty +toText (StrictTextBuilder n write) = runST (do + dst <- A.new n + write dst 0 + arr <- A.unsafeFreeze dst + pure (Text arr 0 n)) + +-- | Concatenation of 'StrictBuilder' is right-biased: +-- the right builder will be run first. This allows a builder to +-- run tail-recursively when it was accumulated left-to-right. +instance Semigroup StrictTextBuilder where + (<>) = appendRStrictBuilder + +instance Monoid StrictTextBuilder where + mempty = emptyStrictBuilder + mappend = (<>) + +emptyStrictBuilder :: StrictTextBuilder +emptyStrictBuilder = StrictTextBuilder 0 (\_ _ -> pure ()) + +appendRStrictBuilder :: StrictTextBuilder -> StrictTextBuilder -> StrictTextBuilder +appendRStrictBuilder (StrictTextBuilder 0 _) b2 = b2 +appendRStrictBuilder b1 (StrictTextBuilder 0 _) = b1 +appendRStrictBuilder (StrictTextBuilder n1 write1) (StrictTextBuilder n2 write2) = + StrictTextBuilder (n1 + n2) (\dst ofs -> do + write2 dst (ofs + n1) + write1 dst ofs) + +copyFromByteString :: A.MArray s -> Int -> ByteString -> ST s () +copyFromByteString dst ofs src = withBS src $ \ srcFPtr len -> + unsafeIOToST $ unsafeWithForeignPtr srcFPtr $ \ srcPtr -> do + unsafeSTToIO $ A.copyFromPointer dst ofs srcPtr len + +-- | Copy a 'ByteString'. +-- +-- Unsafe: This may not be valid UTF-8 text. +-- +-- @since 2.0.2 +unsafeFromByteString :: ByteString -> StrictTextBuilder +unsafeFromByteString bs = + StrictTextBuilder (B.length bs) (\dst ofs -> copyFromByteString dst ofs bs) + +-- | +-- @since 2.0.2 +{-# INLINE fromChar #-} +fromChar :: Char -> StrictTextBuilder +fromChar c = + StrictTextBuilder (utf8Length c) (\dst ofs -> void (Char.unsafeWrite dst ofs (safe c))) + +-- $unsafe +-- For internal purposes, we abuse 'StrictBuilder' as a delayed 'Array' rather +-- than 'Text': it may not actually be valid 'Text'. + +-- | Unsafe: This may not be valid UTF-8 text. +-- +-- @since 2.0.2 +unsafeFromWord8 :: Word8 -> StrictTextBuilder +unsafeFromWord8 !w = + StrictTextBuilder 1 (\dst ofs -> A.unsafeWrite dst ofs w) + +-- | Copy 'Text' in a 'StrictBuilder' +-- +-- @since 2.0.2 +fromText :: Text -> StrictTextBuilder +fromText (Text src srcOfs n) = StrictTextBuilder n (\dst dstOfs -> + A.copyI n dst dstOfs src srcOfs) diff --git a/src/Data/Text/Internal/Transformation.hs b/src/Data/Text/Internal/Transformation.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Transformation.hs @@ -0,0 +1,340 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text.Internal.Transformation +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- This module holds functions shared between the strict and lazy implementations of @Text@ transformations. + +module Data.Text.Internal.Transformation + ( mapNonEmpty + , toCaseFoldNonEmpty + , toLowerNonEmpty + , toUpperNonEmpty + , toTitleNonEmpty + , filter_ + ) where + +import Prelude (Char, Bool(..), Int, + Ord(..), + Monad(..), pure, + (+), (-), ($), (&&), (||), (==), + not, return, otherwise) +import Data.Bits ((.&.), shiftR, shiftL) +import Data.Char (isLetter, isSpace) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4) +import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping) +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iterArray) +import Data.Word (Word8) +import qualified GHC.Exts as Exts +import GHC.Int (Int64(..)) + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +mapNonEmpty :: (Char -> Char) -> Text -> Text +mapNonEmpty f = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + | otherwise = do + let !(Iter c d) = iterArray src srcOff + d' <- unsafeWrite dst dstOff (safe (f c)) + inner (srcOff + d) (dstOff + d') +{-# INLINE mapNonEmpty #-} + +caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text +caseConvert ascii remap (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff (ascii m0) + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' + +{-# INLINABLE caseConvert #-} + +writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int +writeMapping !_ 0 !dstOff = pure dstOff +writeMapping dst i dstOff = do + let (ch, j) = chopOffChar i + d <- unsafeWrite dst dstOff ch + writeMapping dst j (dstOff + d) + +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (Exts.I# n) = Exts.C# (Exts.chr# n) + mask = (1 `shiftL` 21) - 1 + a = P.fromIntegral $ ab .&. mask + +-- | /O(n)/ Convert a string to folded case. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toCaseFoldNonEmpty :: Text -> Text +toCaseFoldNonEmpty = \xs -> caseConvert asciiToLower foldMapping xs +{-# INLINE toCaseFoldNonEmpty #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toLowerNonEmpty :: Text -> Text +toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs +{-# INLINE toLowerNonEmpty #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toUpperNonEmpty :: Text -> Text +toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs +{-# INLINE toUpperNonEmpty #-} + +asciiToLower :: Word8 -> Word8 +asciiToLower w = if w - 65 <= 25 then w + 32 else w + +asciiToUpper :: Word8 -> Word8 +asciiToUpper w = if w - 97 <= 25 then w - 32 else w + +isAsciiLetter :: Word8 -> Bool +isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25 + +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) + +-- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toTitleNonEmpty :: Text -> Text +toTitleNonEmpty (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 False + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !mode + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff mode + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + + case d of + 1 -> do + let (mode', m0') = asciiAdvance mode m0 + A.unsafeWrite dst dstOff m0' + inner (srcOff + 1) (dstOff + 1) mode' + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' mode' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + isSpace3 ch + = m0 == 0xE1 && m1 == 0x9A && m2 == 0x80 + || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F) + || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80 + !(# mode', c' #) = advance isSpace3 mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' mode' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + !(# mode', c' #) = advance (\_ -> False) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' mode' + + asciiAdvance :: Bool -> Word8 -> (Bool, Word8) + asciiAdvance False w = (isAsciiLetter w, asciiToUpper w) + asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w) + + advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #) + advance _ False c = (# isLetter (Exts.C# c), titleMapping c #) + advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #) + {-# INLINE advance #-} + +-- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@, +-- calls the continuation with the @Text@ containing only the characters satisfying the predicate. +filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a +filter_ mkText p = go + where + go (Text src o l) = runST $ do + -- It's tempting to allocate l elements at once and avoid resizing. + -- However, this can be unacceptable in scenarios where a huge array + -- is filtered with a rare predicate, resulting in a much shorter buffer. + let !dstLen = min l 64 + dst <- A.new dstLen + outer dst dstLen o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return $ mkText arr 0 dstOff + | dstOff + 4 > dstLen = do + -- Double size of the buffer, unless it becomes longer than + -- source string. Ensure to extend it by least 4 bytes. + let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- In case of success, filter writes exactly the same character + -- it just read (this is not a case for map, for example). + -- We leverage this fact below: no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + let !c = unsafeChr8 m0 + if not (p c) then inner (srcOff + 1) dstOff else do + A.unsafeWrite dst dstOff m0 + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !c = chr2 m0 m1 + if not (p c) then inner (srcOff + 2) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + inner (srcOff + 2) (dstOff + 2) + 3 -> do + let !c = chr3 m0 m1 m2 + if not (p c) then inner (srcOff + 3) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + inner (srcOff + 3) (dstOff + 3) + _ -> do + let !c = chr4 m0 m1 m2 m3 + if not (p c) then inner (srcOff + 4) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + inner (srcOff + 4) (dstOff + 4) +{-# INLINE filter_ #-} diff --git a/src/Data/Text/Internal/Unsafe.hs b/src/Data/Text/Internal/Unsafe.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Unsafe.hs @@ -0,0 +1,65 @@ +{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal.Unsafe +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : portable +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- A module containing /unsafe/ operations, for /very very careful/ use +-- in /heavily tested/ code. +module Data.Text.Internal.Unsafe + ( + inlineInterleaveST + , inlinePerformIO + , unsafeWithForeignPtr + ) where + +import Foreign.Ptr (Ptr) +import Foreign.ForeignPtr (ForeignPtr) +#if MIN_VERSION_base(4,15,0) +import qualified GHC.ForeignPtr (unsafeWithForeignPtr) +#else +import qualified Foreign.ForeignPtr (withForeignPtr) +#endif + +import GHC.ST (ST(..)) +import GHC.IO (IO(IO)) +import GHC.Base (realWorld#) + +-- | Just like unsafePerformIO, but we inline it. Big performance gains as +-- it exposes lots of things to further inlining. /Very unsafe/. In +-- particular, you should do no memory allocation inside an +-- 'inlinePerformIO' block. +-- +{-# INLINE inlinePerformIO #-} +inlinePerformIO :: IO a -> a +inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r + +-- | Allow an 'ST' computation to be deferred lazily. When passed an +-- action of type 'ST' @s@ @a@, the action will only be performed when +-- the value of @a@ is demanded. +-- +-- This function is identical to the normal unsafeInterleaveST, but is +-- inlined and hence faster. +-- +-- /Note/: This operation is highly unsafe, as it can introduce +-- externally visible non-determinism into an 'ST' action. +inlineInterleaveST :: ST s a -> ST s a +inlineInterleaveST (ST m) = ST $ \ s -> + let r = case m s of (# _, res #) -> res in (# s, r #) +{-# INLINE inlineInterleaveST #-} + +unsafeWithForeignPtr :: ForeignPtr a -> (Ptr a -> IO b) -> IO b +#if MIN_VERSION_base(4,15,0) +unsafeWithForeignPtr = GHC.ForeignPtr.unsafeWithForeignPtr +#else +unsafeWithForeignPtr = Foreign.ForeignPtr.withForeignPtr +#endif diff --git a/src/Data/Text/Internal/Unsafe/Char.hs b/src/Data/Text/Internal/Unsafe/Char.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Unsafe/Char.hs @@ -0,0 +1,88 @@ +{-# LANGUAGE CPP, MagicHash #-} + +-- | +-- Module : Data.Text.Internal.Unsafe.Char +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Fast character manipulation functions. +module Data.Text.Internal.Unsafe.Char + ( + ord + , unsafeChr16 + , unsafeChr8 + , unsafeChr32 + , unsafeWrite + ) where + +import Control.Monad.ST (ST) +import Data.Text.Internal.Encoding.Utf8 +import GHC.Exts (Char(..), Int(..), chr#, ord#, word2Int#) +import GHC.Word (Word8(..), Word16(..), Word32(..)) +import qualified Data.Text.Array as A +import Data.Text.Internal.PrimCompat ( word8ToWord#, word16ToWord#, word32ToWord# ) +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +ord :: Char -> Int +ord (C# c#) = I# (ord# c#) +{-# INLINE ord #-} + +-- | @since 2.0 +unsafeChr16 :: Word16 -> Char +unsafeChr16 (W16# w#) = C# (chr# (word2Int# (word16ToWord# w#))) +{-# INLINE unsafeChr16 #-} + +unsafeChr8 :: Word8 -> Char +unsafeChr8 (W8# w#) = C# (chr# (word2Int# (word8ToWord# w#))) +{-# INLINE unsafeChr8 #-} + +unsafeChr32 :: Word32 -> Char +unsafeChr32 (W32# w#) = C# (chr# (word2Int# (word32ToWord# w#))) +{-# INLINE unsafeChr32 #-} + +-- | Write a character into the array at the given offset. Returns +-- the number of 'Word8's written. +unsafeWrite :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.MArray s -> Int -> Char -> ST s Int +unsafeWrite marr i c = case utf8Length c of + 1 -> do + let n0 = intToWord8 (ord c) + A.unsafeWrite marr i n0 + return 1 + 2 -> do + let (n0, n1) = ord2 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 + return 2 + 3 -> do + let (n0, n1, n2) = ord3 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 + A.unsafeWrite marr (i+2) n2 + return 3 + _ -> do + let (n0, n1, n2, n3) = ord4 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 + A.unsafeWrite marr (i+2) n2 + A.unsafeWrite marr (i+3) n3 + return 4 +{-# INLINE unsafeWrite #-} + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral diff --git a/src/Data/Text/Internal/Validate.hs b/src/Data/Text/Internal/Validate.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Validate.hs @@ -0,0 +1,107 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} + +-- | Test whether or not a sequence of bytes is a valid UTF-8 byte sequence. +-- In the GHC Haskell ecosystem, there are several representations of byte +-- sequences. The only one that the stable @text@ API concerns itself with is +-- 'ByteString'. Part of bytestring-to-text decoding is 'isValidUtf8ByteString', +-- a high-performance UTF-8 validation routine written in C++ with fallbacks +-- for various platforms. The C++ code backing this routine is nontrivial, +-- so in the interest of reuse, this module additionally exports functions +-- for working with the GC-managed @ByteArray@ type. These @ByteArray@ +-- functions are not used anywhere else in @text@. They are for the benefit +-- of library and application authors who do not use 'ByteString' but still +-- need to interoperate with @text@. +module Data.Text.Internal.Validate + ( + -- * ByteString + isValidUtf8ByteString + -- * ByteArray + -- + -- | Is the slice of a byte array a valid UTF-8 byte sequence? These + -- functions all accept an offset and a length. + , isValidUtf8ByteArray + , isValidUtf8ByteArrayUnpinned + , isValidUtf8ByteArrayPinned + ) where + +import Data.Array.Byte (ByteArray(ByteArray)) +import Data.ByteString (ByteString) +import GHC.Exts (isTrue#,isByteArrayPinned#) + +#ifdef SIMDUTF +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Internal.Validate.Simd (c_is_valid_utf8_bytearray_safe,c_is_valid_utf8_bytearray_unsafe,c_is_valid_utf8_ptr_unsafe) +#else +import qualified Data.ByteString as B +import qualified Data.Text.Internal.Validate.Native as N +#endif + +-- | Is the ByteString a valid UTF-8 byte sequence? +isValidUtf8ByteString :: ByteString -> Bool +#ifdef SIMDUTF +isValidUtf8ByteString bs = withBS bs $ \fp len -> unsafeDupablePerformIO $ + unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> c_is_valid_utf8_ptr_unsafe ptr (fromIntegral len) +#else +-- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0. +-- MIN_VERSION_bytestring does not allow us to differentiate +-- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+. +#if MIN_VERSION_bytestring(0,12,1) +isValidUtf8ByteString = B.isValidUtf8 +#else +isValidUtf8ByteString = N.isValidUtf8ByteStringHaskell +#endif +#endif + +-- | For pinned byte arrays larger than 128KiB, this switches to the safe FFI +-- so that it does not prevent GC. This threshold (128KiB) was chosen +-- somewhat arbitrarily and may change in the future. +isValidUtf8ByteArray :: + ByteArray -- ^ Bytes + -> Int -- ^ Offset + -> Int -- ^ Length + -> Bool +isValidUtf8ByteArray b@(ByteArray b#) !off !len + | len >= 131072 -- 128KiB + , isTrue# (isByteArrayPinned# b#) + = isValidUtf8ByteArrayPinned b off len + | otherwise = isValidUtf8ByteArrayUnpinned b off len + +-- | This uses the @unsafe@ FFI. GC waits for all @unsafe@ FFI calls +-- to complete before starting. Consequently, an @unsafe@ FFI call does not +-- run concurrently with GC and is not interrupted by GC. Since relocation +-- cannot happen concurrently with an @unsafe@ FFI call, it is safe +-- to call this function with an unpinned byte array argument. +-- It is also safe to call this with a pinned @ByteArray@ argument. +isValidUtf8ByteArrayUnpinned :: + ByteArray -- ^ Bytes + -> Int -- ^ Offset + -> Int -- ^ Length + -> Bool +#ifdef SIMDUTF +isValidUtf8ByteArrayUnpinned (ByteArray bs) !off !len = + unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_unsafe bs (fromIntegral off) (fromIntegral len) +#else +isValidUtf8ByteArrayUnpinned = N.isValidUtf8ByteArrayHaskell +#endif + +-- | This uses the @safe@ FFI. GC may run concurrently with @safe@ +-- FFI calls. Consequently, unpinned objects may be relocated while a +-- @safe@ FFI call is executing. The byte array argument /must/ be pinned, +-- and the calling context is responsible for enforcing this. If the +-- byte array is not pinned, this function's behavior is undefined. +isValidUtf8ByteArrayPinned :: + ByteArray -- ^ Bytes + -> Int -- ^ Offset + -> Int -- ^ Length + -> Bool +#ifdef SIMDUTF +isValidUtf8ByteArrayPinned (ByteArray bs) !off !len = + unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_safe bs (fromIntegral off) (fromIntegral len) +#else +isValidUtf8ByteArrayPinned = N.isValidUtf8ByteArrayHaskell +#endif diff --git a/src/Data/Text/Internal/Validate/Native.hs b/src/Data/Text/Internal/Validate/Native.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Validate/Native.hs @@ -0,0 +1,60 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} + +-- | Native implementation of 'Data.Text.Internal.Validate'. +module Data.Text.Internal.Validate.Native + ( isValidUtf8ByteStringHaskell + , isValidUtf8ByteArrayHaskell + ) where + +import Data.Array.Byte (ByteArray(ByteArray)) +import Data.ByteString (ByteString) +import GHC.Exts (ByteArray#,Int(I#),indexWord8Array#) +import GHC.Word (Word8(W8#)) +import Data.Text.Internal.Encoding.Utf8 (CodePoint(..),DecoderResult(..),utf8DecodeStart,utf8DecodeContinue) +import qualified Data.ByteString as B +import qualified Data.ByteString.Unsafe as B + +-- | Native implementation of 'Data.Text.Internal.Validate.isValidUtf8ByteString'. +isValidUtf8ByteStringHaskell :: ByteString -> Bool +isValidUtf8ByteStringHaskell bs = start 0 + where + start ix + | ix >= B.length bs = True + | otherwise = case utf8DecodeStart (B.unsafeIndex bs ix) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st _ -> step (ix + 1) st + step ix st + | ix >= B.length bs = False + -- We do not use decoded code point, so passing a dummy value to save an argument. + | otherwise = case utf8DecodeContinue (B.unsafeIndex bs ix) st (CodePoint 0) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st' _ -> step (ix + 1) st' + +-- | Native implementation of +-- 'Data.Text.Internal.Validate.isValidUtf8ByteArrayUnpinned' +-- and 'Data.Text.Internal.Validate.isValidUtf8ByteArrayPinned'. +isValidUtf8ByteArrayHaskell :: + ByteArray -- ^ Bytes + -> Int -- ^ Offset + -> Int -- ^ Length + -> Bool +isValidUtf8ByteArrayHaskell (ByteArray b) !off !len = start off + where + indexWord8 :: ByteArray# -> Int -> Word8 + indexWord8 !x (I# i) = W8# (indexWord8Array# x i) + start ix + | ix >= off + len = True + | otherwise = case utf8DecodeStart (indexWord8 b ix) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st _ -> step (ix + 1) st + step ix st + | ix >= off + len = False + -- We do not use decoded code point, so passing a dummy value to save an argument. + | otherwise = case utf8DecodeContinue (indexWord8 b ix) st (CodePoint 0) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st' _ -> step (ix + 1) st' diff --git a/src/Data/Text/Internal/Validate/Simd.hs b/src/Data/Text/Internal/Validate/Simd.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Internal/Validate/Simd.hs @@ -0,0 +1,42 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} + +-- | Validate that a byte sequence is UTF-8-encoded text. All of these +-- functions return zero when the byte sequence is not UTF-8-encoded text, +-- and they return an unspecified non-zero value when the byte sequence +-- is UTF-8-encoded text. +-- +-- Variants are provided for both @ByteArray#@ and @Ptr@. Additionally, +-- variants are provided that use both the @safe@ and @unsafe@ FFI. +-- +-- If compiling with SIMDUTF turned off, this module exports nothing. +module Data.Text.Internal.Validate.Simd + ( c_is_valid_utf8_ptr_unsafe + , c_is_valid_utf8_ptr_safe + , c_is_valid_utf8_bytearray_unsafe + , c_is_valid_utf8_bytearray_safe + ) where + +import Data.Word (Word8) +import Foreign.C.Types (CSize(..),CInt(..)) +import GHC.Exts (Ptr,ByteArray#) + +foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_unsafe + :: Ptr Word8 -- ^ Bytes + -> CSize -- ^ Length + -> IO CInt +foreign import ccall safe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_safe + :: Ptr Word8 -- ^ Bytes + -> CSize -- ^ Length + -> IO CInt +foreign import ccall unsafe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_unsafe + :: ByteArray# -- ^ Bytes + -> CSize -- ^ Offset into bytes + -> CSize -- ^ Length + -> IO CInt +foreign import ccall safe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_safe + :: ByteArray# -- ^ Bytes + -> CSize -- ^ Offset into bytes + -> CSize -- ^ Length + -> IO CInt diff --git a/src/Data/Text/Lazy.hs b/src/Data/Text/Lazy.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy.hs @@ -0,0 +1,1911 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy +-- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text using +-- lists of packed arrays. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- The representation used by this module is suitable for high +-- performance use and for streaming large quantities of data. It +-- provides a means to manipulate a large body of text without +-- requiring that the entire content be resident in memory. +-- +-- Some operations, such as 'concat', 'append', 'reverse' and 'cons', +-- have better time complexity than their "Data.Text" equivalents, due +-- to the underlying representation being a list of chunks. For other +-- operations, lazy 'Text's are usually within a few percent of strict +-- ones, but often with better heap usage if used in a streaming +-- fashion. For data larger than available memory, or if you have +-- tight memory constraints, this module will be the only option. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions. eg. +-- +-- > import qualified Data.Text.Lazy as L + +module Data.Text.Lazy + ( + -- * Fusion + -- $fusion + + -- * Acceptable data + -- $replacement + + -- * Types + Text + , LazyText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + , fromChunks + , toChunks + , toStrict + , fromStrict + , foldrChunks + , foldlChunks + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , repeat + , replicate + , cycle + , iterate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , span + , spanM + , spanEndM + , breakOn + , breakOnEnd + , break + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + -- , breakSubstring + + -- ** Breaking into lines and words + , lines + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , find + , elem + , breakOnAll + , partition + + -- , findSubstring + + -- * Indexing + , index + , count + + -- * Zipping and unzipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + ) where + +import Prelude (Char, Bool(..), Maybe(..), String, + Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec), + Monad(..), pure, (<$>), + (&&), (+), (-), (.), ($), (++), + error, flip, fmap, fromIntegral, not, otherwise, quot) +import qualified Prelude as P +import Control.Arrow (first) +import Control.DeepSeq (NFData(..)) +import Data.Bits (finiteBitSize, toIntegralSized) +import Data.Int (Int64) +import qualified Data.List as L hiding (head, tail) +import Data.Char (isSpace) +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Unsafe as T +import qualified Data.Text.Internal.Lazy.Fusion as S +import Data.Text.Internal.Fusion.Types (PairS(..)) +import Data.Text.Internal.Lazy.Fusion (stream, unstream) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, + foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText) +import Data.Text.Internal (firstf, safe, text) +import Data.Text.Internal.Reverse (reverseNonEmpty) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) +import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Lazy.Search (indices) +import qualified GHC.CString as GHC +import qualified GHC.Exts as Exts +import GHC.Prim (Addr#) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Syntax as TH +import qualified Language.Haskell.TH.Lib as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- . +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- () +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- .) + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +instance Eq Text where + (==) = equal + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +compareText :: Text -> Text -> Ordering +compareText Empty Empty = EQ +compareText Empty _ = LT +compareText _ Empty = GT +compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of + LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) + EQ -> compareText as bs + GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +instance Show Text where + showsPrec p ps r = showsPrec p (unpack ps) r + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +instance Semigroup Text where + (<>) = append + stimes n _ | n < 0 = P.error "Data.Text.Lazy.stimes: given number is negative!" + stimes n a = + let nInt64 = fromIntegral n :: Int64 + len = if n == fromIntegral nInt64 && nInt64 >= 0 then nInt64 else P.maxBound + -- We clamp the length to maxBound :: Int64. + -- To tell the difference, the caller would have to skip through 2^63 chunks. + in replicate len a + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Data.Text.Lazy.Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Data.Text.Lazy.Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Text".@'Data.Text.Text' +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> error "Data.Text.Lazy.Text.gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 900 + lift x = [| fromStrict $(TH.lift . toStrict $ x) |] +#else + lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- +-- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555") +-- "\65533" +pack :: +#if defined(ASSERTS) + HasCallStack => +#endif + String -> Text +pack = unstream . S.streamList . L.map safe +{-# INLINE [1] pack #-} + +-- | /O(n)/ Convert a 'Text' into a 'String'. +unpack :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> String +unpack t = S.unstreamList (stream t) +{-# NOINLINE unpack #-} + +foldrFB :: (Char -> b -> b) -> b -> Text -> b +foldrFB = foldr +{-# INLINE [0] foldrFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us `foldrFB (:) []` +-- and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrFB` inlines and `foldr` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.Lazy.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrFB lcons lnil t) +"Text.Lazy.unpackBack" [1] foldrFB (:) [] = unpack + #-} + +-- | /O(n)/ Convert a literal string into a Text. +unpackCString# :: Addr# -> Text +unpackCString# addr# = unstream (S.streamCString# addr#) +{-# NOINLINE unpackCString# #-} + +{-# RULES "TEXT literal" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCString# a))) + = unpackCString# a #-} + +{-# RULES "TEXT literal UTF8" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) + = unpackCString# a #-} + +{-# RULES "LAZY TEXT empty literal" + unstream (S.streamList (L.map safe [])) + = Empty #-} + +{-# RULES "LAZY TEXT empty literal" forall a. + unstream (S.streamList (L.map safe [a])) + = Chunk (T.singleton a) Empty #-} + +-- | /O(1)/ Convert a character into a Text. +-- Performs replacement on invalid scalar values. +singleton :: Char -> Text +singleton c = Chunk (T.singleton c) Empty +{-# INLINE [1] singleton #-} + +-- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. +fromChunks :: [T.Text] -> Text +fromChunks cs = L.foldr chunk Empty cs + +-- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. +toChunks :: Text -> [T.Text] +toChunks cs = foldrChunks (:) [] cs + +-- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. +toStrict :: LazyText -> T.StrictText +toStrict t = T.concat (toChunks t) +{-# INLINE [1] toStrict #-} + +-- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. +fromStrict :: T.StrictText -> LazyText +fromStrict t = chunk t Empty +{-# INLINE [1] fromStrict #-} + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(1)/ Adds a character to the front of a 'Text'. +cons :: Char -> Text -> Text +cons c t = Chunk (T.singleton c) t +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +snoc :: Text -> Char -> Text +snoc t c = foldrChunks Chunk (singleton c) t +{-# INLINE [1] snoc #-} + +-- | /O(n\/c)/ Appends one 'Text' to another. +append :: Text -> Text -> Text +append xs ys = foldrChunks Chunk ys xs +{-# INLINE [1] append #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons Empty = Nothing +uncons (Chunk t ts) = Just (T.unsafeHead t, ts') + where ts' | T.compareLength t 1 == EQ = ts + | otherwise = Chunk (T.unsafeTail t) ts +{-# INLINE uncons #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t = S.head (stream t) +{-# INLINE head #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail (Chunk t ts) = chunk (T.tail t) ts +tail Empty = emptyError "tail" +{-# INLINE [1] tail #-} + +-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init (Chunk t0 ts0) = go t0 ts0 + where go t (Chunk t' ts) = Chunk t (go t' ts) + go t Empty = chunk (T.init t) Empty +init Empty = emptyError "init" +{-# INLINE [1] init #-} + +-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if +-- empty. +-- +-- * It is no faster than using 'init' and 'last'. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc Empty = Nothing +unsnoc ts@(Chunk _ _) = Just (init ts, last ts) +{-# INLINE unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null Empty = True +null _ = False +{-# INLINE [1] null #-} + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + +-- | /O(n\/c)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last Empty = emptyError "last" +last (Chunk t ts) = go t ts + where go _ (Chunk t' ts') = go t' ts' + go t' Empty = T.last t' +{-# INLINE [1] last #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: Text -> Int64 +length = foldlChunks go 0 + where + go :: Int64 -> T.Text -> Int64 + go l t = l + intToInt64 (T.length t) +{-# INLINE [1] length #-} + +{-# RULES +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = max 0 n P.* length t +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (2 P.* length t - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int64 -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +-- We don't apply those otherwise appealing length-to-compareLength +-- rewrite rules here, because they can change the strictness +-- properties of code. + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. Performs replacement on +-- invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = foldrChunks (Chunk . mapNonEmpty f) Empty +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. Performs +-- replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t = unstream (S.intersperse (safe c) (stream t)) +{-# INLINE [1] intersperse #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. Performs +-- replacement on invalid scalar values. +-- +-- Examples: +-- +-- > justifyLeft 7 'x' "foo" == "fooxxxx" +-- > justifyLeft 3 'x' "foobar" == "foobar" +justifyLeft :: Int64 -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChunk (k-len) (T.singleton c) + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- > justifyRight 7 'x' "bar" == "xxxxbar" +-- > justifyRight 3 'x' "foobar" == "foobar" +justifyRight :: Int64 -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChunk (k-len) (T.singleton c) `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- > center 8 'x' "HS" = "xxxHSxxx" +center :: Int64 -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +transpose :: [Text] -> [Text] +transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) + (L.transpose (L.map unpack ts)) +-- TODO: make this fast + +-- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text +reverse = rev Empty + where rev a Empty = a + rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- > replace "oo" "foo" "oo" == "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- > replace "ofo" "bar" "ofofo" == "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace s d = intercalate d . splitOn s +{-# INLINE replace #-} + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- With Unicode text, it is incorrect to use combinators like @map +-- toUpper@ to case convert each character of a string individually. +-- Instead, use the whole-string case conversion functions from this +-- module. For correctness in different writing systems, these +-- functions may map one input character to two or three output +-- characters. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (or case +-- insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature men now (U+FB13) is case folded to the +-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is +-- case folded to the Greek small letter letter mu (U+03BC) instead of +-- itself. +toCaseFold :: Text -> Text +toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the Latin capital letter I with dot above (U+0130) maps +-- to the sequence Latin small letter i (U+0069) followed by combining +-- dot above (U+0307). +toLower :: Text -> Text +toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German eszett (U+00DF) maps to the two-letter +-- sequence SS. +toUpper :: Text -> Text +toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty +{-# INLINE toUpper #-} + + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty +{-# INLINE toTitle #-} + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z t = S.foldl f z (stream t) +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +-- +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z t = S.foldl' f z (stream t) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = S.foldl1 f (stream t) +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = S.foldl1' f (stream t) +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z t = S.foldlM' f z (stream t) +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z t = S.foldr f z (stream t) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = S.foldr1 f (stream t) +{-# INLINE foldr1 #-} + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat [] = Empty +concat (Empty : css) = concat css +concat (Chunk c Empty : css) = Chunk c (concat css) +concat (Chunk c cs : css) = Chunk c (concat (cs : css)) +{-# INLINE concat #-} + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p t = S.any p (stream t) +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p t = S.all p (stream t) +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum t = S.maximum (stream t) +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum t = S.minimum (stream t) +{-# INLINE minimum #-} + +-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only +-- U+0000 through U+007F). +-- +-- This is a more efficient version of @'all' 'Data.Char.isAscii'@. +-- +-- >>> isAscii "" +-- True +-- +-- >>> isAscii "abc\NUL" +-- True +-- +-- >>> isAscii "abcd€" +-- False +-- +-- prop> isAscii t == all (< '\x80') t +-- +-- @since 2.0.2 +isAscii :: Text -> Bool +isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True + +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- Note that +-- +-- > last (scanl f z xs) == foldl f z xs. +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f z t = cons z $ P.snd $ + mapAccumL (\acc c -> let c' = f acc c in (c', c')) (safe z) t +-- This is a bit suboptimal: we could have used +-- Data.Text.scanl for the first chunk and mapAccumL +-- for subsequent ones, but but I doubt anyone cares +-- about the performance of 'scanl' much. +{-# INLINE scanl #-} + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t0 = case uncons t0 of + Nothing -> empty + Just (t,ts) -> scanl f t ts +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f z t = (`snoc` z) $ P.snd $ + mapAccumR (\acc c -> let c' = f c acc in (c', c')) (safe z) t +-- See the comment for 'scanl' above. +{-# INLINE scanr #-} + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumL f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z', c') = T.mapAccumL f z c + (z'', cs') = go z' cs + go z Empty = (z, Empty) +{-# INLINE mapAccumL #-} + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. Performs replacement on invalid scalar values. +mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumR f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z'', c') = T.mapAccumR f z' c + (z', cs') = go z cs + go z Empty = (z, Empty) +{-# INLINE mapAccumR #-} + +-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every +-- element. +-- +-- @since 1.2.0.5 +repeat :: Char -> Text +repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t + in t + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int64 -> Text -> Text +replicate n + | n <= 0 = P.const Empty + | otherwise = \case + Empty -> Empty + Chunk t Empty -> replicateChunk n t + t -> concat (rep n) + where + rep 0 = [] + rep i = t : rep (i - 1) +{-# INLINE [1] replicate #-} + +replicateChunk :: Int64 -> T.Text -> Text +replicateChunk !n !t@(T.Text _ _ len) + | n <= 0 = Empty + | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) + where + perChunk = defaultChunkSize `quot` len + normalChunk = T.replicate perChunk t + (q, r) = n `P.quotRem` intToInt64 perChunk + headChunk = T.replicate (int64ToInt r) t +{-# INLINE replicateChunk #-} + +-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or +-- equivalently, the infinite repetition of the original 'Text'. +-- +-- @since 1.2.0.5 +cycle :: HasCallStack => Text -> Text +cycle Empty = emptyError "cycle" +cycle t = let t' = foldrChunks Chunk t' t + in t' + +-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications +-- of @f@ to @x@: +-- +-- > iterate f x == [x, f x, f (f x), ...] +-- +-- @since 1.2.0.5 +iterate :: (Char -> Char) -> Char -> Text +iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) + in t c + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int64 -> Text -> Text +take i _ | i <= 0 = Empty +take i t0 = take' i t0 + where + take' :: Int64 -> Text -> Text + take' 0 _ = Empty + take' _ Empty = Empty + take' n (Chunk t@(T.Text arr off _) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then fromStrict (T.Text arr off m) + else Chunk t (take' (n + intToInt64 m) ts) + + | n < l = Chunk (T.take (int64ToInt n) t) Empty + | otherwise = Chunk t (take' (n - l) ts) + where l = intToInt64 (T.length t) +{-# INLINE [1] take #-} + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > takeEnd 3 "foobar" == "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int64 -> Text -> Text +takeEnd n t0 + | n <= 0 = empty + | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 + where + takeChunk :: Int64 -> Text -> [T.Text] -> Text + takeChunk _ acc [] = acc + takeChunk i acc (t:ts) + | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc + | otherwise = takeChunk (i-l) (Chunk t acc) ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int64 -> Text -> Text +drop i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk t@(T.Text arr off len) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then chunk (T.Text arr (off + m) (len - m)) ts + else drop' (n + intToInt64 m) ts + + | n < l = Chunk (T.drop (int64ToInt n) t) ts + | otherwise = drop' (n - l) ts + where l = intToInt64 (T.length t) +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > dropEnd 3 "foobar" == "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int64 -> Text -> Text +dropEnd n t0 + | n <= 0 = t0 + | otherwise = dropChunk n . L.reverse . toChunks $ t0 + where + dropChunk :: Int64 -> [T.Text] -> Text + dropChunk _ [] = empty + dropChunk m (t:ts) + | m >= l = dropChunk (m-l) ts + | otherwise = fromChunks . L.reverse $ + T.dropEnd (int64ToInt m) t : ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' +-- values dropped, or the empty 'Text' if @n@ is greater than the +-- number of 'Word8' values present. +dropWords :: Int64 -> Text -> Text +dropWords i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk (T.Text arr off len) ts) + | n < len' = chunk (text arr (off+n') (len-n')) ts + | otherwise = drop' (n - len') ts + where len' = intToInt64 len + n' = int64ToInt n + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t0 = takeWhile' t0 + where takeWhile' Empty = Empty + takeWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n | n > 0 -> Chunk (T.take n t) Empty + | otherwise -> Empty + Nothing -> Chunk t (takeWhile' ts) +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- > takeWhileEnd (=='o') "foo" == "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p = takeChunk empty . L.reverse . toChunks + where takeChunk acc [] = acc + takeChunk acc (t:ts) + | T.lengthWord8 t' < T.lengthWord8 t + = chunk t' acc + | otherwise = takeChunk (Chunk t' acc) ts + where t' = T.takeWhileEnd p t +{-# INLINE takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t0 = dropWhile' t0 + where dropWhile' Empty = Empty + dropWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n -> Chunk (T.drop n t) ts + Nothing -> dropWhile' ts +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- > dropWhileEnd (=='.') "foo..." == "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p = go + where go Empty = Empty + go (Chunk t Empty) = if T.null t' + then Empty + else Chunk t' Empty + where t' = T.dropWhileEnd p t + go (Chunk t ts) = case go ts of + Empty -> go (Chunk t Empty) + ts' -> Chunk t ts' +{-# INLINE dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int64 -> Text -> (Text, Text) +splitAt = loop + where + loop :: Int64 -> Text -> (Text, Text) + loop !_ Empty = (empty, empty) + loop n t | n <= 0 = (empty, t) + loop n (Chunk t@(T.Text arr off len) ts) + | n > mx = let (ts', ts'') = loop (n - intToInt64 (T.length t)) ts + in (Chunk t ts', ts'') + | m > 0, m >= len = (Chunk t Empty, ts) + | m > 0 = let t' = T.Text arr off m + t'' = T.Text arr (off+m) (len-m) + in (Chunk t' Empty, Chunk t'' ts) + | otherwise = let (ts', ts'') = loop (n + intToInt64 m) ts + in (Chunk t ts', ts'') + where + mx = intToInt64 P.maxBound + m = T.measureOff (int64ToInt n) t + + +-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first +-- element is a prefix of @t@ whose chunks contain @n@ 'Word8' +-- values, and whose second is the remainder of the string. +splitAtWord :: Int64 -> Text -> PairS Text Text +splitAtWord !_ Empty = empty :*: empty +splitAtWord x (Chunk c@(T.Text arr off len) cs) + | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs + in Chunk c h :*: t + | otherwise = chunk (text arr off y) empty :*: + chunk (text arr (off+y) (len-y)) cs + where y = int64ToInt x + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> let h :*: t = splitAtWord x src + in (h, t) + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) + in (reverse b, reverse a) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- > breakOnAll "::" "" +-- > ==> [] +-- > breakOnAll "/" "a/b/c/" +-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src + | null pat = emptyError "breakOnAll" + | otherwise = go 0 empty src (indices pat src) + where + go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s + h' = append p h + in (h',t) : go x h' t xs + go _ _ _ _ = [] + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is over +-- elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p t0 = break' t0 + where break' Empty = (empty, empty) + break' c@(Chunk t ts) = + case T.findIndex p t of + Nothing -> let (ts', ts'') = break' ts + in (Chunk t ts', ts'') + Just n | n == 0 -> (Empty, c) + | otherwise -> let (a,b) = T.splitAt n t + in (Chunk a Empty, Chunk b ts) + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p = break (not . p) +{-# INLINE span #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t1, t2) <- T.spanM p t + if T.null t2 then first (chunk t) <$> go ts + else pure (chunk t1 empty, Chunk t2 ts) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t3, t4) <- go ts + if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t + else pure (Chunk t t3, t4) +{-# INLINE spanEndM #-} + +-- | The 'group' function takes a 'Text' and returns a list of 'Text's +-- such that the concatenation of the result is equal to the argument. +-- Moreover, each sublist in the result contains only equal elements. +-- For example, +-- +-- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] +-- +-- It is a special case of 'groupBy', which allows the programmer to +-- supply their own equality test. +group :: Text -> [Text] +group = groupBy (==) +{-# INLINE group #-} + +-- | The 'groupBy' function is the non-overloaded version of 'group'. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy _ Empty = [] +groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs + where (ys,zs) = span (eq x) xs + x = T.unsafeHead t + xs = chunk (T.unsafeTail t) ts + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +inits :: Text -> [Text] +inits = (NE.toList P.$!) . initsNE + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmpty Text +initsNE ts0 = Empty NE.:| inits' 0 ts0 + where + inits' :: Int64 -- Number of previous chunks i + -> Text -- The remainder after dropping i chunks from ts0 + -> [Text] -- Prefixes longer than the first i chunks of ts0. + inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t)) + ++ inits' (i + 1) ts + inits' _ Empty = [] + +takeChunks :: Int64 -> Text -> T.Text -> Text +takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk) +takeChunks _ _ lastChunk = Chunk lastChunk Empty + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NE.toList P.$!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmpty Text +tailsNE Empty = Empty :| [] +tailsNE ts@(Chunk t ts') + | T.length t == 1 = ts :| tails ts' + | otherwise = ts :| tails (Chunk (T.unsafeTail t) ts') + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be an empty string), consuming the +-- delimiter. An empty delimiter is invalid, and will cause an error +-- to be raised. +-- +-- Examples: +-- +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat src + | null pat = emptyError "splitOn" + | isSingleton pat = split (== head pat) src + | otherwise = go 0 (indices pat src) src + where + go _ [] cs = [cs] + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in h : go (x+l) xs (dropWords l t) + l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat +{-# INLINE [1] splitOn #-} + +{-# RULES +"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') [] == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ Empty = [Empty] +split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 + where comb acc (s:[]) Empty = revChunks (s:acc) : [] + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts + comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts + comb _ [] _ = impossibleError "split" +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] +-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] +chunksOf :: Int64 -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines Empty = [] +lines t = NE.toList $ go t + where + go :: Text -> NonEmpty Text + go Empty = Empty :| [] + go (Chunk x xs) + -- x is non-empty, so T.lines x is non-empty as well + | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs + | otherwise = case unsnocList (T.lines x) of + Nothing -> impossibleError "lines" + Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls + +prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text +prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! + +unsnocList :: [a] -> Maybe ([a], a) +unsnocList [] = Nothing +unsnocList (x : xs) = Just $ go x xs + where + go y [] = ([], y) + go y (z : zs) = first (y :) (go z zs) + +hasNlEnd :: T.Text -> Bool +hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words = L.filter (not . null) . split isSpace +{-# INLINE words #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf Empty _ = True +isPrefixOf _ Empty = False +isPrefixOf (Chunk x xs) (Chunk y ys) + | lx == ly = x == y && isPrefixOf xs ys + | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) + | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys + where (xh,xt) = T.splitAt ly x + (yh,yt) = T.splitAt lx y + lx = T.length x + ly = T.length y + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf x y = reverse x `isPrefixOf` reverse y +{-# INLINE isSuffixOf #-} +-- TODO: a better implementation + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (head needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripPrefix "foo" "foobar" == Just "bar" +-- > stripPrefix "" "baz" == Just "baz" +-- > stripPrefix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p t + | null p = Just t + | otherwise = case commonPrefixes p t of + Just (_,c,r) | null c -> Just r + _ -> Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") +-- > commonPrefixes "veeble" "fetzer" == Nothing +-- > commonPrefixes "" "baz" == Nothing +commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) +commonPrefixes Empty _ = Nothing +commonPrefixes _ Empty = Nothing +commonPrefixes a0 b0 = Just (go a0 b0 []) + where + go t0@(Chunk x xs) t1@(Chunk y ys) ps + = case T.commonPrefixes x y of + Just (p,a,b) + | T.null a -> go xs (chunk b ys) (p:ps) + | T.null b -> go (chunk a xs) ys (p:ps) + | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) + Nothing -> (fromChunks (L.reverse ps),t0,t1) + go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripSuffix "bar" "foobar" == Just "foo" +-- > stripSuffix "" "baz" == Just "baz" +-- > stripSuffix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = foldrChunks (chunk . filter_ T.Text p) Empty +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' +-- if there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem c t = S.any (== c) (stream t) +{-# INLINE elem #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int64 -> Char +index lazyText ix + | ix < 0 = P.error $ "Data.Text.Lazy.index: negative index " ++ P.show ix + | otherwise = go lazyText ix + where + go :: Text -> Int64 -> Char + go Empty _ = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + go (Chunk t@(T.Text _ _ lenInBytes) ts) n = case toIntegralSized n of + Nothing -> + go ts (n - fromIntegral (T.length t)) + Just n' + | off < 0 -> go ts (n + fromIntegral off) + | off == lenInBytes -> go ts 0 + | otherwise -> ch + where + off = T.measureOff n' t + T.Iter ch _ = T.iter t off +{-# INLINE index #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int64 +count pat + | null pat = emptyError "count" + | otherwise = go 0 . indices pat + where go !n [] = n + go !n (_:xs) = go (n+1) xs +{-# INLINE [1] count #-} + +{-# RULES +"LAZY TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int64 +countChar c t = S.countChar c (stream t) + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE [0] zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [0] zipWith #-} + +-- | Convert a value to lazy 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +revChunks :: [T.Text] -> Text +revChunks = L.foldl' (flip chunk) Empty + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") + +impossibleError :: HasCallStack => String -> a +impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") + +intToInt64 :: Exts.Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Exts.Int +int64ToInt = fromIntegral diff --git a/src/Data/Text/Lazy/Builder.hs b/src/Data/Text/Lazy/Builder.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Builder.hs @@ -0,0 +1,56 @@ +{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} +{-# LANGUAGE Trustworthy #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Data.Text.Lazy.Builder +-- Copyright : (c) 2013 Bryan O'Sullivan +-- (c) 2010 Johan Tibell +-- License : BSD-style (see LICENSE) +-- +-- Maintainer : Johan Tibell +-- Portability : portable to Hugs and GHC +-- +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. +-- +-- To get maximum performance when building lazy @Text@ values using a +-- builder, associate @mappend@ calls to the right. For example, +-- prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. Or, equivalently, +-- prefer +-- +-- > singleton 'a' <> singleton 'b' <> singleton 'c' +-- +-- since the '<>' from recent versions of 'Data.Monoid' associates +-- to the right. + +----------------------------------------------------------------------------- + +module Data.Text.Lazy.Builder + ( -- * The Builder type + Builder + , LazyTextBuilder + , toLazyText + , toLazyTextWith + + -- * Constructing Builders + , singleton + , fromText + , fromLazyText + , fromString + + -- * Flushing the buffer state + , flush + ) where + +import Data.Text.Internal.Builder diff --git a/src/Data/Text/Lazy/Builder/Int.hs b/src/Data/Text/Lazy/Builder/Int.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Builder/Int.hs @@ -0,0 +1,244 @@ +{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, ScopedTypeVariables, + UnboxedTuples #-} +{-# LANGUAGE Trustworthy #-} + +-- Module: Data.Text.Lazy.Builder.Int +-- Copyright: (c) 2013 Bryan O'Sullivan +-- (c) 2011 MailRank, Inc. +-- License: BSD-style +-- Maintainer: Bryan O'Sullivan +-- Portability: portable +-- +-- Efficiently write an integral value to a 'Builder'. + +module Data.Text.Lazy.Builder.Int + ( + decimal + , hexadecimal + ) where + +import Data.Int (Int8, Int16, Int32, Int64) +import Data.Monoid (mempty) +import qualified Data.ByteString.Unsafe as B +import Data.Text.Internal.Builder.Functions ((<>), i2d) +import Data.Text.Internal.Builder +import Data.Text.Internal.Builder.Int.Digits (digits) +import Data.Text.Array +import Data.Word (Word, Word8, Word16, Word32, Word64) +import GHC.Base (quotInt, remInt) +import Control.Monad.ST +#if MIN_VERSION_base(4,11,0) +import Prelude hiding ((<>)) +#endif + +decimal :: Integral a => a -> Builder +{-# RULES "decimal/Int8" decimal = boundedDecimal :: Int8 -> Builder #-} +{-# RULES "decimal/Int" decimal = boundedDecimal :: Int -> Builder #-} +{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-} +{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-} +{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-} +{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-} +{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-} +{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-} +{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-} +{-# RULES "decimal/Word64" decimal = positive :: Word64 -> Builder #-} +{-# RULES "decimal/Integer" decimal = integer 10 :: Integer -> Builder #-} +decimal i = decimal' (<= -128) i +{-# NOINLINE decimal #-} + +boundedDecimal :: (Integral a, Bounded a) => a -> Builder +{-# SPECIALIZE boundedDecimal :: Int -> Builder #-} +{-# SPECIALIZE boundedDecimal :: Int8 -> Builder #-} +{-# SPECIALIZE boundedDecimal :: Int16 -> Builder #-} +{-# SPECIALIZE boundedDecimal :: Int32 -> Builder #-} +{-# SPECIALIZE boundedDecimal :: Int64 -> Builder #-} +boundedDecimal i = decimal' (== minBound) i + +decimal' :: (Integral a) => (a -> Bool) -> a -> Builder +{-# INLINE decimal' #-} +decimal' p i + | i < 0 = if p i + then let (q, r) = i `quotRem` 10 + qq = -q + !n = countDigits qq + in writeN (n + 2) $ \marr off -> do + unsafeWrite marr off minus + posDecimal marr (off+1) n qq + unsafeWrite marr (off+n+1) (i2w (-r)) + else let j = -i + !n = countDigits j + in writeN (n + 1) $ \marr off -> + unsafeWrite marr off minus >> posDecimal marr (off+1) n j + | otherwise = positive i + +positive :: (Integral a) => a -> Builder +{-# SPECIALIZE positive :: Int -> Builder #-} +{-# SPECIALIZE positive :: Int8 -> Builder #-} +{-# SPECIALIZE positive :: Int16 -> Builder #-} +{-# SPECIALIZE positive :: Int32 -> Builder #-} +{-# SPECIALIZE positive :: Int64 -> Builder #-} +{-# SPECIALIZE positive :: Word -> Builder #-} +{-# SPECIALIZE positive :: Word8 -> Builder #-} +{-# SPECIALIZE positive :: Word16 -> Builder #-} +{-# SPECIALIZE positive :: Word32 -> Builder #-} +{-# SPECIALIZE positive :: Word64 -> Builder #-} +positive i + | i < 10 = writeN 1 $ \marr off -> unsafeWrite marr off (i2w i) + | otherwise = let !n = countDigits i + in writeN n $ \marr off -> posDecimal marr off n i + +posDecimal :: (Integral a) => + forall s. MArray s -> Int -> Int -> a -> ST s () +{-# INLINE posDecimal #-} +posDecimal marr off0 ds v0 = go (off0 + ds - 1) v0 + where go off v + | v >= 100 = do + let (q, r) = v `quotRem` 100 + write2 off r + go (off - 2) q + | v < 10 = unsafeWrite marr off (i2w v) + | otherwise = write2 off v + write2 off i0 = do + let i = fromIntegral i0; j = i + i + unsafeWrite marr off $ get (j + 1) + unsafeWrite marr (off - 1) $ get j + get = B.unsafeIndex digits + +minus, zero :: Word8 +{-# INLINE minus #-} +{-# INLINE zero #-} +minus = 45 +zero = 48 + +i2w :: (Integral a) => a -> Word8 +{-# INLINE i2w #-} +i2w v = zero + fromIntegral v + +countDigits :: (Integral a) => a -> Int +{-# INLINE countDigits #-} +countDigits v0 + | fromIntegral v64 == v0 = go 1 v64 + | otherwise = goBig 1 (toInteger v0) + where v64 = fromIntegral v0 + goBig !k (v :: Integer) + | v > big = goBig (k + 19) (v `quot` big) + | otherwise = go k (fromInteger v) + big = 10000000000000000000 + go !k (v :: Word64) + | v < 10 = k + | v < 100 = k + 1 + | v < 1000 = k + 2 + | v < 1000000000000 = + k + if v < 100000000 + then if v < 1000000 + then if v < 10000 + then 3 + else 4 + fin v 100000 + else 6 + fin v 10000000 + else if v < 10000000000 + then 8 + fin v 1000000000 + else 10 + fin v 100000000000 + | otherwise = go (k + 12) (v `quot` 1000000000000) + fin v n = if v >= n then 1 else 0 + +hexadecimal :: Integral a => a -> Builder +{-# SPECIALIZE hexadecimal :: Int -> Builder #-} +{-# SPECIALIZE hexadecimal :: Int8 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Int16 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Int32 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Int64 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Word -> Builder #-} +{-# SPECIALIZE hexadecimal :: Word8 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Word16 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Word32 -> Builder #-} +{-# SPECIALIZE hexadecimal :: Word64 -> Builder #-} +{-# RULES "hexadecimal/Integer" + hexadecimal = hexInteger :: Integer -> Builder #-} +hexadecimal i + | i < 0 = error hexErrMsg + | otherwise = go i + where + go n | n < 16 = hexDigit n + | otherwise = go (n `quot` 16) <> hexDigit (n `rem` 16) +{-# NOINLINE[0] hexadecimal #-} + +hexInteger :: Integer -> Builder +hexInteger i + | i < 0 = error hexErrMsg + | otherwise = integer 16 i + +hexErrMsg :: String +hexErrMsg = "Data.Text.Lazy.Builder.Int.hexadecimal: applied to negative number" + +hexDigit :: Integral a => a -> Builder +hexDigit n + | n <= 9 = singleton $! i2d (fromIntegral n) + | otherwise = singleton $! toEnum (fromIntegral n + 87) +{-# INLINE hexDigit #-} + +data T = T !Integer !Int + +integer :: Int -> Integer -> Builder +integer 10 i + | i' <- fromInteger i, toInteger i' == i = decimal (i' :: Int) +integer 16 i + | i' <- fromInteger i, toInteger i' == i = hexadecimal (i' :: Int) +integer base i + | i < 0 = singleton '-' <> go (-i) + | otherwise = go i + where + go n | n < maxInt = int (fromInteger n) + | otherwise = putH (splitf (maxInt * maxInt) n) + + splitf p n + | p > n = [n] + | otherwise = splith p (splitf (p*p) n) + + splith p (n:ns) = case n `quotRem` p of + (q, r) | q > 0 -> q : r : splitb p ns + | otherwise -> r : splitb p ns + splith _ _ = error "splith: the impossible happened." + + splitb p (n:ns) = case n `quotRem` p of + (q, r) -> q : r : splitb p ns + splitb _ _ = [] + + T maxInt10 maxDigits10 = + until ((>mi) . (*10) . fstT) (\(T n d) -> T (n*10) (d+1)) (T 10 1) + where mi = toInteger (maxBound :: Int) + T maxInt16 maxDigits16 = + until ((>mi) . (*16) . fstT) (\(T n d) -> T (n*16) (d+1)) (T 16 1) + where mi = toInteger (maxBound :: Int) + + fstT (T a _) = a + + maxInt | base == 10 = maxInt10 + | otherwise = maxInt16 + maxDigits | base == 10 = maxDigits10 + | otherwise = maxDigits16 + + putH (n:ns) = case n `quotRem` maxInt of + (x, y) + | q > 0 -> int q <> pblock r <> putB ns + | otherwise -> int r <> putB ns + where q = fromInteger x + r = fromInteger y + putH _ = error "putH: the impossible happened" + + putB (n:ns) = case n `quotRem` maxInt of + (x, y) -> pblock q <> pblock r <> putB ns + where q = fromInteger x + r = fromInteger y + putB _ = Data.Monoid.mempty + + int :: Int -> Builder + int x | base == 10 = decimal x + | otherwise = hexadecimal x + + pblock = loop maxDigits + where + loop !d !n + | d == 1 = hexDigit n + | otherwise = loop (d-1) q <> hexDigit r + where q = n `quotInt` base + r = n `remInt` base diff --git a/src/Data/Text/Lazy/Builder/RealFloat.hs b/src/Data/Text/Lazy/Builder/RealFloat.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Builder/RealFloat.hs @@ -0,0 +1,257 @@ +{-# LANGUAGE CPP, OverloadedStrings #-} +{-# LANGUAGE Trustworthy #-} + +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +-- | +-- Module: Data.Text.Lazy.Builder.RealFloat +-- Copyright: (c) The University of Glasgow 1994-2002 +-- License: see libraries/base/LICENSE +-- +-- Write a floating point value to a 'Builder'. + +module Data.Text.Lazy.Builder.RealFloat + ( + FPFormat(..) + , realFloat + , formatRealFloat + ) where + +import Data.Array.Base (unsafeAt) +import Data.Array.IArray +import Data.Text.Internal.Builder.Functions ((<>), i2d) +import Data.Text.Lazy.Builder.Int (decimal) +import Data.Text.Internal.Builder.RealFloat.Functions (roundTo) +import Data.Text.Lazy.Builder +import qualified Data.Text as T +#if MIN_VERSION_base(4,11,0) +import Prelude hiding ((<>)) +#endif + +-- | Control the rendering of floating point numbers. +data FPFormat = Exponent + -- ^ Scientific notation (e.g. @2.3e123@). + | Fixed + -- ^ Standard decimal notation. + | Generic + -- ^ Use decimal notation for values between @0.1@ and + -- @9,999,999@, and scientific notation otherwise. + deriving (Enum, Read, Show, Bounded) + +-- | Show a signed 'RealFloat' value to full precision, +-- using standard decimal notation for arguments whose absolute value lies +-- between @0.1@ and @9,999,999@, and scientific notation otherwise. +realFloat :: (RealFloat a) => a -> Builder +{-# SPECIALIZE realFloat :: Float -> Builder #-} +{-# SPECIALIZE realFloat :: Double -> Builder #-} +realFloat x = formatRealFloat Generic Nothing x + +-- | Encode a signed 'RealFloat' according to 'FPFormat' and optionally requested precision. +-- +-- This corresponds to the @show{E,F,G}Float@ operations provided by @base@'s "Numeric" module. +-- +-- __NOTE__: The functions in @base-4.12@ changed the serialisation in +-- case of a @Just 0@ precision; this version of @text@ still provides +-- the serialisation as implemented in @base-4.11@. The next major +-- version of @text@ will switch to the more correct @base-4.12@ serialisation. +formatRealFloat :: (RealFloat a) => + FPFormat + -> Maybe Int -- ^ Number of decimal places to render. + -> a + -> Builder +{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Float -> Builder #-} +{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Double -> Builder #-} +formatRealFloat fmt decs x + | isNaN x = "NaN" + | isInfinite x = if x < 0 then "-Infinity" else "Infinity" + | x < 0 || isNegativeZero x = singleton '-' <> doFmt fmt (floatToDigits (-x)) + | otherwise = doFmt fmt (floatToDigits x) + where + doFmt format (is, e) = + let ds = map i2d is in + case format of + Generic -> + doFmt (if e < 0 || e > 7 then Exponent else Fixed) + (is,e) + Exponent -> + case decs of + Nothing -> + let show_e' = decimal (e-1) in + case ds of + "0" -> "0.0e0" + [d] -> singleton d <> ".0e" <> show_e' + (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> show_e' + [] -> error "formatRealFloat/doFmt/Exponent/Nothing: []" + Just dec -> + let dec' = max dec 1 in + case is of + [0] -> "0." <> fromText (T.replicate dec' "0") <> "e0" + _ -> + let (ei,is') = roundTo (dec'+1) is + is'' = map i2d (if ei > 0 then init is' else is') + in case is'' of + [] -> error "formatRealFloat/doFmt/Exponent/Just: []" + (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> decimal (e-1+ei) + Fixed -> + let + mk0 ls = case ls of { "" -> "0" ; _ -> fromString ls} + in + case decs of + Nothing + | e <= 0 -> "0." <> fromText (T.replicate (-e) "0") <> fromString ds + | otherwise -> + let + f 0 s rs = mk0 (reverse s) <> singleton '.' <> mk0 rs + f n s "" = f (n-1) ('0':s) "" + f n s (r:rs) = f (n-1) (r:s) rs + in + f e "" ds + Just dec -> + let dec' = max dec 0 in + if e >= 0 then + let + (ei,is') = roundTo (dec' + e) is + (ls,rs) = splitAt (e+ei) (map i2d is') + in + mk0 ls <> (if null rs then "" else singleton '.' <> fromString rs) + else + let (ei,is') = roundTo dec' (replicate (-e) 0 ++ is) + is'' = map i2d (if ei > 0 then is' else 0:is') + in case is'' of + [] -> error "formatRealFloat/doFmt/Fixed: []" + (d:ds') -> singleton d <> (if null ds' then "" else singleton '.' <> fromString ds') + + +-- Based on "Printing Floating-Point Numbers Quickly and Accurately" +-- by R.G. Burger and R.K. Dybvig in PLDI 96. +-- This version uses a much slower logarithm estimator. It should be improved. + +-- | 'floatToDigits' takes a base and a non-negative 'RealFloat' number, +-- and returns a list of digits and an exponent. +-- In particular, if @x>=0@, and +-- +-- > floatToDigits base x = ([d1,d2,...,dn], e) +-- +-- then +-- +-- (1) @n >= 1@ +-- +-- (2) @x = 0.d1d2...dn * (base**e)@ +-- +-- (3) @0 <= di <= base-1@ + +floatToDigits :: (RealFloat a) => a -> ([Int], Int) +{-# SPECIALIZE floatToDigits :: Float -> ([Int], Int) #-} +{-# SPECIALIZE floatToDigits :: Double -> ([Int], Int) #-} +floatToDigits 0 = ([0], 0) +floatToDigits x = + let + (f0, e0) = decodeFloat x + (minExp0, _) = floatRange x + p = floatDigits x + b = floatRadix x + minExp = minExp0 - p -- the real minimum exponent + -- Haskell requires that f be adjusted so denormalized numbers + -- will have an impossibly low exponent. Adjust for this. + (f, e) = + let n = minExp - e0 in + if n > 0 then (f0 `quot` (expt b n), e0+n) else (f0, e0) + (r, s, mUp, mDn) = + if e >= 0 then + let be = expt b e in + if f == expt b (p-1) then + (f*be*b*2, 2*b, be*b, be) -- according to Burger and Dybvig + else + (f*be*2, 2, be, be) + else + if e > minExp && f == expt b (p-1) then + (f*b*2, expt b (-e+1)*2, b, 1) + else + (f*2, expt b (-e)*2, 1, 1) + k :: Int + k = + let + k0 :: Int + k0 = + if b == 2 then + -- logBase 10 2 is very slightly larger than 8651/28738 + -- (about 5.3558e-10), so if log x >= 0, the approximation + -- k1 is too small, hence we add one and need one fixup step less. + -- If log x < 0, the approximation errs rather on the high side. + -- That is usually more than compensated for by ignoring the + -- fractional part of logBase 2 x, but when x is a power of 1/2 + -- or slightly larger and the exponent is a multiple of the + -- denominator of the rational approximation to logBase 10 2, + -- k1 is larger than logBase 10 x. If k1 > 1 + logBase 10 x, + -- we get a leading zero-digit we don't want. + -- With the approximation 3/10, this happened for + -- 0.5^1030, 0.5^1040, ..., 0.5^1070 and values close above. + -- The approximation 8651/28738 guarantees k1 < 1 + logBase 10 x + -- for IEEE-ish floating point types with exponent fields + -- <= 17 bits and mantissae of several thousand bits, earlier + -- convergents to logBase 10 2 would fail for long double. + -- Using quot instead of div is a little faster and requires + -- fewer fixup steps for negative lx. + let lx = p - 1 + e0 + k1 = (lx * 8651) `quot` 28738 + in if lx >= 0 then k1 + 1 else k1 + else + -- f :: Integer, log :: Float -> Float, + -- ceiling :: Float -> Int + ceiling ((log (fromInteger (f+1) :: Float) + + intToFloat e * log (fromInteger b)) / + log 10) +--WAS: fromInt e * log (fromInteger b)) + + fixup n = + if n >= 0 then + if r + mUp <= expt 10 n * s then n else fixup (n+1) + else + if expt 10 (-n) * (r + mUp) <= s then n else fixup (n+1) + in + fixup k0 + + gen ds rn sN mUpN mDnN = + let + (dn, rn') = (rn * 10) `quotRem` sN + mUpN' = mUpN * 10 + mDnN' = mDnN * 10 + in + case (rn' < mDnN', rn' + mUpN' > sN) of + (True, False) -> dn : ds + (False, True) -> dn+1 : ds + (True, True) -> if rn' * 2 < sN then dn : ds else dn+1 : ds + (False, False) -> gen (dn:ds) rn' sN mUpN' mDnN' + + rds = + if k >= 0 then + gen [] r (s * expt 10 k) mUp mDn + else + let bk = expt 10 (-k) in + gen [] (r * bk) s (mUp * bk) (mDn * bk) + in + (map fromInteger (reverse rds), k) + +-- Exponentiation with a cache for the most common numbers. +minExpt, maxExpt :: Int +minExpt = 0 +maxExpt = 1100 + +expt :: Integer -> Int -> Integer +expt base n + | base == 2 && n >= minExpt && n <= maxExpt = expts `unsafeAt` n + | base == 10 && n <= maxExpt10 = expts10 `unsafeAt` n + | otherwise = base^n + +expts :: Array Int Integer +expts = array (minExpt,maxExpt) [(n,2^n) | n <- [minExpt .. maxExpt]] + +maxExpt10 :: Int +maxExpt10 = 324 + +expts10 :: Array Int Integer +expts10 = array (minExpt,maxExpt10) [(n,10^n) | n <- [minExpt .. maxExpt10]] + +intToFloat :: Int -> Float +intToFloat = fromIntegral diff --git a/src/Data/Text/Lazy/Encoding.hs b/src/Data/Text/Lazy/Encoding.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Encoding.hs @@ -0,0 +1,254 @@ +{-# LANGUAGE BangPatterns,CPP #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy.Encoding +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting lazy 'Text' values to and from lazy +-- 'ByteString', using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- . + +module Data.Text.Lazy.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeUtf8Lenient + + -- *** Catchable failure + , decodeUtf8' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + ) where + +import Control.Exception (evaluate, try) +import Data.Monoid (Monoid(..)) +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) +import Data.Word (Word8) +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Lazy as B +import qualified Data.ByteString.Lazy.Internal as B +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Internal.Encoding as TE +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as F +import qualified Data.Text.Internal.StrictBuilder as SB +import Data.Text.Unsafe (unsafeDupablePerformIO) + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing 7-bit ASCII +-- encoded text. +decodeASCII :: B.ByteString -> Text +decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +decodeLatin1 :: B.ByteString -> Text +decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +decodeUtf8With :: OnDecodeError -> B.ByteString -> Text +decodeUtf8With onErr = loop TE.startUtf8State + where + chunkb builder t | SB.sbLength builder == 0 = t + | otherwise = Chunk (TE.strictBuilderToText builder) t + loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of + (builder, _, s') -> chunkb builder (loop s' bs) + loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty + msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +decodeUtf8 :: B.ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text.. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +-- +-- /Note/: this function is /not/ lazy, as it must decode its entire +-- input before it can return a result. If you need lazy (streaming) +-- decoding, use 'decodeUtf8With' in lenient mode. +decodeUtf8' :: B.ByteString -> Either UnicodeException Text +decodeUtf8' bs = unsafeDupablePerformIO $ do + let t = decodeUtf8 bs + try (evaluate (rnf t `seq` t)) + where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts +{-# INLINE decodeUtf8' #-} + +-- | Decode a lazy 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.1.4 +decodeUtf8Lenient :: B.ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> B.ByteString +encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped prim = + foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: B.ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: B.ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> B.ByteString +encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> B.ByteString +encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: B.ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: B.ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> B.ByteString +encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> B.ByteString +encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) +{-# INLINE encodeUtf32BE #-} diff --git a/src/Data/Text/Lazy/IO.hs b/src/Data/Text/Lazy/IO.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/IO.hs @@ -0,0 +1,159 @@ +{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-} +{-# LANGUAGE Trustworthy #-} +-- | +-- Module : Data.Text.Lazy.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Efficient locale-sensitive support for lazy text I\/O. +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding (e.g. UTF-8), and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. + +module Data.Text.Lazy.IO + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Data.Text.Lazy (Text) +import Prelude hiding (appendFile, getContents, getLine, interact, + putStr, putStrLn, readFile, writeFile) +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, + withFile) +import qualified Data.Text.Lazy as L +import qualified Control.Exception as E +import Control.Monad (when) +import Data.IORef (readIORef) +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStream) +import Data.Text.Internal.Lazy (chunk, empty) +import Data.Text.Internal.Lazy.Fusion (stream, streamLn) +import GHC.IO.Buffer (isEmptyBuffer) +import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException) +import GHC.IO.Handle.Internals (augmentIOError, hClose_help, + wantReadableHandle, withHandle) +import GHC.IO.Handle.Types (Handle__(..), HandleType(..)) +import System.IO (BufferMode(..), hGetBuffering, hSetBuffering) +import System.IO.Error (isEOFError) +import System.IO.Unsafe (unsafeInterleaveIO) + +-- | Read a file and return its contents as a string. The file is +-- read lazily, as with 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Lazy.Encoding.decodeUtf8' '.' 'Data.ByteString.Lazy.readFile' +-- is a much faster and safer alternative. +readFile :: FilePath -> IO Text +readFile name = openFile name ReadMode >>= hGetContents + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile p = withFile p WriteMode . flip hPutStr + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile p = withFile p AppendMode . flip hPutStr + +-- | Lazily read the remaining contents of a 'Handle'. The 'Handle' +-- will be closed after the read completes, or on error. +hGetContents :: Handle -> IO Text +hGetContents h = do + chooseGoodBuffering h + wantReadableHandle "hGetContents" h $ \hh -> do + ts <- lazyRead h + return (hh{haType=SemiClosedHandle}, ts) + +-- | Use a more efficient buffer size if we're reading in +-- block-buffered mode with the default buffer size. +chooseGoodBuffering :: Handle -> IO () +chooseGoodBuffering h = do + bufMode <- hGetBuffering h + when (bufMode == BlockBuffering Nothing) $ + hSetBuffering h (BlockBuffering (Just 16384)) + +lazyRead :: Handle -> IO Text +lazyRead h = unsafeInterleaveIO $ + withHandle "hGetContents" h $ \hh -> do + case haType hh of + ClosedHandle -> return (hh, L.empty) + SemiClosedHandle -> lazyReadBuffered h hh + _ -> ioException + (IOError (Just h) IllegalOperation "hGetContents" + "illegal handle type" Nothing Nothing) + +lazyReadBuffered :: Handle -> Handle__ -> IO (Handle__, Text) +lazyReadBuffered h hh@Handle__{..} = do + buf <- readIORef haCharBuffer + (do t <- readChunk hh buf + ts <- lazyRead h + return (hh, chunk t ts)) `E.catch` \e -> do + (hh', _) <- hClose_help hh + if isEOFError e + then return $ if isEmptyBuffer buf + then (hh', empty) + else (hh', L.singleton '\r') + else E.throwIO (augmentIOError e "hGetContents" h) + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = hGetLineWith L.fromChunks + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h = hPutStream h . stream + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h = hPutStream h . streamLn + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed (lazily) to this function as its argument, and the resulting +-- string is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Lazily read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = hGetContents stdin + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = hGetLine stdin + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = hPutStr stdout + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = hPutStrLn stdout diff --git a/src/Data/Text/Lazy/Internal.hs b/src/Data/Text/Lazy/Internal.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Internal.hs @@ -0,0 +1,19 @@ +-- | +-- Module : Data.Text.Lazy.Internal +-- Copyright : (c) 2013 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- This module has been renamed to 'Data.Text.Internal.Lazy'. This +-- name for the module will be removed in the next major release. + +module Data.Text.Lazy.Internal + {-# DEPRECATED "Use Data.Text.Internal.Lazy instead" #-} + ( + module Data.Text.Internal.Lazy + ) where + +import Data.Text.Internal.Lazy diff --git a/src/Data/Text/Lazy/Read.hs b/src/Data/Text/Lazy/Read.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Lazy/Read.hs @@ -0,0 +1,221 @@ +{-# LANGUAGE OverloadedStrings, CPP #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE UnboxedTuples #-} + +-- | +-- Module : Data.Text.Lazy.Read +-- Copyright : (c) 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Functions used frequently when reading textual data. +module Data.Text.Lazy.Read + ( + Reader + , decimal + , hexadecimal + , signed + , rational + , double + ) where + +import Control.Monad (liftM) +import Data.Char (ord) +import Data.Int (Int8, Int16, Int32, Int64) +import Data.Ratio ((%)) +import Data.Text.Internal.Read +import Data.Text.Array as A +import Data.Text.Lazy as T +import Data.Text.Internal.Lazy as T (Text(..)) +import qualified Data.Text.Internal as T (Text(..)) +import qualified Data.Text.Internal.Private as T (spanAscii_) +import Data.Word (Word, Word8, Word16, Word32, Word64) + +-- | Read some text. If the read succeeds, return its value and the +-- remaining text, otherwise an error message. +type Reader a = Text -> Either String (a, Text) +type Parser = IParser Text + +-- | Read a decimal integer. The input must begin with at least one +-- decimal digit, and is consumed until a non-digit or end of string +-- is reached. +-- +-- This function does not handle leading sign characters. If you need +-- to handle signed input, use @'signed' 'decimal'@. +-- +-- /Note/: For fixed-width integer types, this function does not +-- attempt to detect overflow, so a sufficiently long input may give +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. +decimal :: Integral a => Reader a +{-# SPECIALIZE decimal :: Reader Int #-} +{-# SPECIALIZE decimal :: Reader Int8 #-} +{-# SPECIALIZE decimal :: Reader Int16 #-} +{-# SPECIALIZE decimal :: Reader Int32 #-} +{-# SPECIALIZE decimal :: Reader Int64 #-} +{-# SPECIALIZE decimal :: Reader Integer #-} +{-# SPECIALIZE decimal :: Reader Data.Word.Word #-} +{-# SPECIALIZE decimal :: Reader Word8 #-} +{-# SPECIALIZE decimal :: Reader Word16 #-} +{-# SPECIALIZE decimal :: Reader Word32 #-} +{-# SPECIALIZE decimal :: Reader Word64 #-} +decimal txt + | T.null h = Left "input does not start with a digit" + | otherwise = Right (T.foldl' go 0 h, t) + where (# h, t #) = spanAscii_ (\w -> w - ord8 '0' < 10) txt + go n d = (n * 10 + fromIntegral (digitToInt d)) + +-- | Read a hexadecimal integer, consisting of an optional leading +-- @\"0x\"@ followed by at least one hexadecimal digit. Input is +-- consumed until a non-hex-digit or end of string is reached. +-- This function is case insensitive. +-- +-- This function does not handle leading sign characters. If you need +-- to handle signed input, use @'signed' 'hexadecimal'@. +-- +-- /Note/: For fixed-width integer types, this function does not +-- attempt to detect overflow, so a sufficiently long input may give +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. +hexadecimal :: Integral a => Reader a +{-# SPECIALIZE hexadecimal :: Reader Int #-} +{-# SPECIALIZE hexadecimal :: Reader Integer #-} +hexadecimal txt + | h == "0x" || h == "0X" = hex t + | otherwise = hex txt + where (h,t) = T.splitAt 2 txt + +hex :: Integral a => Reader a +{-# SPECIALIZE hexadecimal :: Reader Int #-} +{-# SPECIALIZE hexadecimal :: Reader Int8 #-} +{-# SPECIALIZE hexadecimal :: Reader Int16 #-} +{-# SPECIALIZE hexadecimal :: Reader Int32 #-} +{-# SPECIALIZE hexadecimal :: Reader Int64 #-} +{-# SPECIALIZE hexadecimal :: Reader Integer #-} +{-# SPECIALIZE hexadecimal :: Reader Word #-} +{-# SPECIALIZE hexadecimal :: Reader Word8 #-} +{-# SPECIALIZE hexadecimal :: Reader Word16 #-} +{-# SPECIALIZE hexadecimal :: Reader Word32 #-} +{-# SPECIALIZE hexadecimal :: Reader Word64 #-} +hex txt + | T.null h = Left "input does not start with a hexadecimal digit" + | otherwise = Right (T.foldl' go 0 h, t) + where (# h, t #) = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt + go n d = (n * 16 + fromIntegral (hexDigitToInt d)) + +-- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and +-- apply it to the result of applying the given reader. +signed :: Num a => Reader a -> Reader a +{-# INLINE signed #-} +signed f = runP (signa (P f)) + +-- | Read a rational number. +-- +-- This function accepts an optional leading sign character, followed +-- by at least one decimal digit. The syntax similar to that accepted +-- by the 'read' function, with the exception that a trailing @\'.\'@ +-- or @\'e\'@ /not/ followed by a number is not consumed. +-- +-- Examples: +-- +-- >rational "3" == Right (3.0, "") +-- >rational "3.1" == Right (3.1, "") +-- >rational "3e4" == Right (30000.0, "") +-- >rational "3.1e4" == Right (31000.0, "") +-- >rational ".3" == Left "input does not start with a digit" +-- >rational "e3" == Left "input does not start with a digit" +-- +-- Examples of differences from 'read': +-- +-- >rational "3.foo" == Right (3.0, ".foo") +-- >rational "3e" == Right (3.0, "e") +rational :: Fractional a => Reader a +{-# SPECIALIZE rational :: Reader Double #-} +rational = floaty $ \real frac fracDenom power -> + -- We must be careful to prevent DDoS attacks: if the return type is 'Double', + -- a client rightfully expects 'rational' to operate within bounded memory. + -- Thus if power is small, we can compute fraction with full precision and divide. + -- Otherwise divide first, apply fromRational and scale last: + -- the small loss of precision for Double does not matter much because the result is + -- likely infinity or zero anyway. + if abs power < 1000 + then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power)) + else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) + +-- | Read a rational number. +-- +-- The syntax accepted by this function is the same as for 'rational'. +-- +-- /Note/: This function is almost ten times faster than 'rational', +-- but is slightly less accurate. +-- +-- The 'Double' type supports about 16 decimal places of accuracy. +-- For 94.2% of numbers, this function and 'rational' give identical +-- results, but for the remaining 5.8%, this function loses precision +-- around the 15th decimal place. For 0.001% of numbers, this +-- function will lose precision at the 13th or 14th decimal place. +double :: Reader Double +double = floaty $ \real frac fracDenom power -> + (fromInteger real + + fromInteger frac / fromInteger fracDenom) * (10 ^^ power) + +signa :: Num a => Parser a -> Parser a +{-# SPECIALIZE signa :: Parser Int -> Parser Int #-} +{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-} +{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-} +{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-} +{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} +{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} +signa p = do + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + if sign == ord8 '+' then p else negate `liftM` p + +charAscii :: (Word8 -> Bool) -> Parser Word8 +charAscii p = P $ \case + Empty -> Left "character does not match" + -- len is > 0, unless the internal invariant of Text is violated + Chunk (T.Text arr off len) ts -> let c = A.unsafeIndex arr off in + if p c + then Right (c, if len <= 1 then ts else Chunk (T.Text arr (off + 1) (len - 1)) ts) + else Left "character does not match" + +floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a +{-# INLINE floaty #-} +floaty f = runP $ do + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + real <- P decimal + T fraction fracDigits <- perhaps (T 0 0) $ do + _ <- charAscii (== ord8 '.') + digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in int64ToInt (T.length hd), t) + n <- P decimal + return $ T n digits + let e c = c == ord8 'e' || c == ord8 'E' + power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int) + let n = if fracDigits == 0 + then if power == 0 + then fromInteger real + else fromInteger real * (10 ^^ power) + else f real fraction (10 ^ fracDigits) power + return $! if sign == ord8 '+' + then n + else -n + +int64ToInt :: Int64 -> Int +int64ToInt = fromIntegral + +ord8 :: Char -> Word8 +ord8 = fromIntegral . ord + +-- | For the sake of performance this function does not check +-- that a char is in ASCII range; it is a responsibility of @p@. +spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #) +spanAscii_ p = loop + where + loop Empty = (# Empty, Empty #) + loop (Chunk t ts) = let (# t', t''@(T.Text _ _ len) #) = T.spanAscii_ p t in + if len == 0 + then let (# ts', ts'' #) = loop ts in (# Chunk t ts', ts'' #) + else (# Chunk t' Empty, Chunk t'' ts #) diff --git a/src/Data/Text/Read.hs b/src/Data/Text/Read.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Read.hs @@ -0,0 +1,210 @@ +{-# LANGUAGE OverloadedStrings, UnboxedTuples, CPP #-} +{-# LANGUAGE Trustworthy #-} + +-- | +-- Module : Data.Text.Read +-- Copyright : (c) 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Functions used frequently when reading textual data. +module Data.Text.Read + ( + Reader + , decimal + , hexadecimal + , signed + , rational + , double + ) where + +import Control.Monad (liftM) +import Data.Char (ord) +import Data.Int (Int8, Int16, Int32, Int64) +import Data.Ratio ((%)) +import Data.Text as T +import Data.Text.Internal as T (Text(..)) +import Data.Text.Array as A +import Data.Text.Internal.Private (spanAscii_) +import Data.Text.Internal.Read +import Data.Word (Word, Word8, Word16, Word32, Word64) + +-- | Read some text. If the read succeeds, return its value and the +-- remaining text, otherwise an error message. +type Reader a = Text -> Either String (a, Text) +type Parser a = IParser Text a + +-- | Read a decimal integer. The input must begin with at least one +-- decimal digit, and is consumed until a non-digit or end of string +-- is reached. +-- +-- This function does not handle leading sign characters. If you need +-- to handle signed input, use @'signed' 'decimal'@. +-- +-- /Note/: For fixed-width integer types, this function does not +-- attempt to detect overflow, so a sufficiently long input may give +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. +decimal :: Integral a => Reader a +{-# SPECIALIZE decimal :: Reader Int #-} +{-# SPECIALIZE decimal :: Reader Int8 #-} +{-# SPECIALIZE decimal :: Reader Int16 #-} +{-# SPECIALIZE decimal :: Reader Int32 #-} +{-# SPECIALIZE decimal :: Reader Int64 #-} +{-# SPECIALIZE decimal :: Reader Integer #-} +{-# SPECIALIZE decimal :: Reader Data.Word.Word #-} +{-# SPECIALIZE decimal :: Reader Word8 #-} +{-# SPECIALIZE decimal :: Reader Word16 #-} +{-# SPECIALIZE decimal :: Reader Word32 #-} +{-# SPECIALIZE decimal :: Reader Word64 #-} +decimal txt + | T.null h = Left "input does not start with a digit" + | otherwise = Right (T.foldl' go 0 h, t) + where (# h,t #) = spanAscii_ (\w -> w - ord8 '0' < 10) txt + go n d = (n * 10 + fromIntegral (digitToInt d)) + +-- | Read a hexadecimal integer, consisting of an optional leading +-- @\"0x\"@ followed by at least one hexadecimal digit. Input is +-- consumed until a non-hex-digit or end of string is reached. +-- This function is case insensitive. +-- +-- This function does not handle leading sign characters. If you need +-- to handle signed input, use @'signed' 'hexadecimal'@. +-- +-- /Note/: For fixed-width integer types, this function does not +-- attempt to detect overflow, so a sufficiently long input may give +-- incorrect results. If you are worried about overflow, use +-- 'Integer' for your result type. +hexadecimal :: Integral a => Reader a +{-# SPECIALIZE hexadecimal :: Reader Int #-} +{-# SPECIALIZE hexadecimal :: Reader Int8 #-} +{-# SPECIALIZE hexadecimal :: Reader Int16 #-} +{-# SPECIALIZE hexadecimal :: Reader Int32 #-} +{-# SPECIALIZE hexadecimal :: Reader Int64 #-} +{-# SPECIALIZE hexadecimal :: Reader Integer #-} +{-# SPECIALIZE hexadecimal :: Reader Word #-} +{-# SPECIALIZE hexadecimal :: Reader Word8 #-} +{-# SPECIALIZE hexadecimal :: Reader Word16 #-} +{-# SPECIALIZE hexadecimal :: Reader Word32 #-} +{-# SPECIALIZE hexadecimal :: Reader Word64 #-} +hexadecimal txt + | h == "0x" || h == "0X" = hex t + | otherwise = hex txt + where (h,t) = T.splitAt 2 txt + +hex :: Integral a => Reader a +{-# SPECIALIZE hex :: Reader Int #-} +{-# SPECIALIZE hex :: Reader Int8 #-} +{-# SPECIALIZE hex :: Reader Int16 #-} +{-# SPECIALIZE hex :: Reader Int32 #-} +{-# SPECIALIZE hex :: Reader Int64 #-} +{-# SPECIALIZE hex :: Reader Integer #-} +{-# SPECIALIZE hex :: Reader Word #-} +{-# SPECIALIZE hex :: Reader Word8 #-} +{-# SPECIALIZE hex :: Reader Word16 #-} +{-# SPECIALIZE hex :: Reader Word32 #-} +{-# SPECIALIZE hex :: Reader Word64 #-} +hex txt + | T.null h = Left "input does not start with a hexadecimal digit" + | otherwise = Right (T.foldl' go 0 h, t) + where (# h,t #) = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt + go n d = (n * 16 + fromIntegral (hexDigitToInt d)) + +-- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and +-- apply it to the result of applying the given reader. +signed :: Num a => Reader a -> Reader a +{-# INLINE signed #-} +signed f = runP (signa (P f)) + +-- | Read a rational number. +-- +-- This function accepts an optional leading sign character, followed +-- by at least one decimal digit. The syntax similar to that accepted +-- by the 'read' function, with the exception that a trailing @\'.\'@ +-- or @\'e\'@ /not/ followed by a number is not consumed. +-- +-- Examples (with behaviour identical to 'read'): +-- +-- >rational "3" == Right (3.0, "") +-- >rational "3.1" == Right (3.1, "") +-- >rational "3e4" == Right (30000.0, "") +-- >rational "3.1e4" == Right (31000.0, "") +-- >rational ".3" == Left "input does not start with a digit" +-- >rational "e3" == Left "input does not start with a digit" +-- +-- Examples of differences from 'read': +-- +-- >rational "3.foo" == Right (3.0, ".foo") +-- >rational "3e" == Right (3.0, "e") +rational :: Fractional a => Reader a +{-# SPECIALIZE rational :: Reader Double #-} +rational = floaty $ \real frac fracDenom power -> + -- We must be careful to prevent DDoS attacks: if the return type is 'Double', + -- a client rightfully expects 'rational' to operate within bounded memory. + -- Thus if power is small, we can compute fraction with full precision and divide. + -- Otherwise divide first, apply fromRational and scale last: + -- the small loss of precision for Double does not matter much because the result is + -- likely infinity or zero anyway. + if abs power < 1000 + then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power)) + else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) + +-- | Read a rational number. +-- +-- The syntax accepted by this function is the same as for 'rational'. +-- +-- /Note/: This function is almost ten times faster than 'rational', +-- but is slightly less accurate. +-- +-- The 'Double' type supports about 16 decimal places of accuracy. +-- For 94.2% of numbers, this function and 'rational' give identical +-- results, but for the remaining 5.8%, this function loses precision +-- around the 15th decimal place. For 0.001% of numbers, this +-- function will lose precision at the 13th or 14th decimal place. +double :: Reader Double +double = floaty $ \real frac fracDenom power -> + (fromInteger real + + fromInteger frac / fromInteger fracDenom) * (10 ^^ power) + +signa :: Num a => Parser a -> Parser a +{-# SPECIALIZE signa :: Parser Int -> Parser Int #-} +{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-} +{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-} +{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-} +{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} +{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} +signa p = do + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + if sign == ord8 '+' then p else negate `liftM` p + +charAscii :: (Word8 -> Bool) -> Parser Word8 +charAscii p = P $ \(Text arr off len) -> let c = A.unsafeIndex arr off in + if len > 0 && p c + then Right (c, Text arr (off + 1) (len - 1)) + else Left "character does not match" + +floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a +{-# INLINE floaty #-} +floaty f = runP $ do + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + real <- P decimal + T fraction fracDigits <- perhaps (T 0 0) $ do + _ <- charAscii (== ord8 '.') + digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in T.length hd, t) + n <- P decimal + return $ T n digits + let e c = c == ord8 'e' || c == ord8 'E' + power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int) + let n = if fracDigits == 0 + then if power == 0 + then fromInteger real + else fromInteger real * (10 ^^ power) + else f real fraction (10 ^ fracDigits) power + return $! if sign == ord8 '+' + then n + else -n + +ord8 :: Char -> Word8 +ord8 = fromIntegral . ord diff --git a/src/Data/Text/Show.hs b/src/Data/Text/Show.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Show.hs @@ -0,0 +1,171 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CApiFFI #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +-- | +-- Module : Data.Text.Show +-- Copyright : (c) 2009-2015 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC + +module Data.Text.Show + ( + addrLen + , singleton + , unpack + , unpackCString# + , unpackCStringAscii# + ) where + +import Control.Monad.ST (ST, runST) +import Data.Text.Internal (Text(..), empty, safe, pack) +import Data.Text.Internal.Encoding.Utf8 (utf8Length) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Unsafe (Iter(..), iterArray) +import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#) +import qualified GHC.Exts as Exts +import GHC.Word (Word8(..)) +import qualified Data.Text.Array as A +#if !MIN_VERSION_ghc_prim(0,7,0) +import Foreign.C.String (CString) +import Foreign.C.Types (CSize(..)) +#endif + +import qualified GHC.CString as GHC + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +instance Show Text where + showsPrec p ps r = showsPrec p (unpack ps) r + +-- | /O(n)/ Convert a 'Text' into a 'String'. +unpack :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> String +unpack t = foldrText (:) [] t +{-# NOINLINE unpack #-} + +foldrText :: (Char -> b -> b) -> b -> Text -> b +foldrText f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldrText #-} + +foldrTextFB :: (Char -> b -> b) -> b -> Text -> b +foldrTextFB = foldrText +{-# INLINE [0] foldrTextFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us +-- `foldrTextFB (:) []` and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrTextFB` inlines and `foldrText` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrTextFB lcons lnil t) +"Text.unpackBack" [1] foldrTextFB (:) [] = unpack + #-} + +-- | /O(n)/ Convert a null-terminated +-- +-- (but with a standard UTF-8 representation of characters from supplementary planes) +-- string to a 'Text'. Counterpart to 'GHC.unpackCStringUtf8#'. +-- No validation is performed, malformed input can lead to memory access violation. +-- +-- @since 1.2.1.1 +unpackCString# :: Addr# -> Text +unpackCString# addr# = runST $ do + let l = addrLen addr# + at (I# i#) = W8# (indexWord8OffAddr# addr# i#) + marr <- A.new l + let go srcOff@(at -> w8) dstOff + | srcOff >= l + = return dstOff + -- Surrogate halves take 3 bytes and are replaced by \xfffd (also 3 bytes long). + -- Cf. Data.Text.Internal.safe + | w8 == 0xed, at (srcOff + 1) >= 0xa0 = do + A.unsafeWrite marr dstOff 0xef + A.unsafeWrite marr (dstOff + 1) 0xbf + A.unsafeWrite marr (dstOff + 2) 0xbd + go (srcOff + 3) (dstOff + 3) + -- Byte sequence "\xc0\x80" is used to represent NUL + | w8 == 0xc0, at (srcOff + 1) == 0x80 + = A.unsafeWrite marr dstOff 0 >> go (srcOff + 2) (dstOff + 1) + | otherwise + = A.unsafeWrite marr dstOff w8 >> go (srcOff + 1) (dstOff + 1) + actualLen <- go 0 0 + A.shrinkM marr actualLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 actualLen + +-- When a module contains many literal strings, 'unpackCString#' can easily +-- bloat generated code to insane size. There is also very little to gain +-- from inlining. Thus explicit NOINLINE is desired. +{-# NOINLINE unpackCString# #-} + +-- | /O(n)/ Convert a null-terminated ASCII string to a 'Text'. +-- Counterpart to 'GHC.unpackCString#'. +-- No validation is performed, malformed input can lead to memory access violation. +-- +-- @since 2.0 +unpackCStringAscii# :: Addr# -> Text +unpackCStringAscii# addr# = Text ba 0 l + where + l = addrLen addr# + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Ptr addr#) l + A.unsafeFreeze marr +{-# NOINLINE unpackCStringAscii# #-} + +addrLen :: Addr# -> Int +#if MIN_VERSION_ghc_prim(0,7,0) +addrLen addr# = I# (GHC.cstringLength# addr#) +#else +addrLen addr# = fromIntegral (c_strlen (Ptr addr#)) + +foreign import capi unsafe "string.h strlen" c_strlen :: CString -> CSize +#endif + +{-# RULES "TEXT literal" forall a. + pack (GHC.unpackCString# a) = unpackCStringAscii# a #-} + +{-# RULES "TEXT literal UTF8" forall a. + pack (GHC.unpackCStringUtf8# a) = unpackCString# a #-} + +{-# RULES "TEXT empty literal" + pack [] = empty #-} + +{-# RULES "TEXT singleton literal" forall a. + pack [a] = singleton a #-} + +-- | /O(1)/ Convert a character into a Text. +-- Performs replacement on invalid scalar values. +singleton :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> Text +singleton c = Text (A.run x) 0 len + where x :: ST s (A.MArray s) + x = do arr <- A.new len + _ <- unsafeWrite arr 0 d + return arr + len = utf8Length d + d = safe c +{-# NOINLINE singleton #-} diff --git a/src/Data/Text/Unsafe.hs b/src/Data/Text/Unsafe.hs new file mode 100644 --- /dev/null +++ b/src/Data/Text/Unsafe.hs @@ -0,0 +1,166 @@ +{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-} +-- | +-- Module : Data.Text.Unsafe +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- A module containing unsafe 'Text' operations, for very very careful +-- use in heavily tested code. +module Data.Text.Unsafe + ( + inlineInterleaveST + , inlinePerformIO + , unsafeDupablePerformIO + , Iter(..) + , iter + , iterArray + , iter_ + , reverseIter + , reverseIterArray + , reverseIter_ + , unsafeHead + , unsafeTail + , lengthWord8 + , takeWord8 + , dropWord8 + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader) +import Data.Text.Internal (Text(..)) +import Data.Text.Internal.Unsafe (inlineInterleaveST, inlinePerformIO) +import Data.Text.Internal.Unsafe.Char (unsafeChr8) +import qualified Data.Text.Array as A +import GHC.IO (unsafeDupablePerformIO) + +-- | /O(1)/ A variant of 'head' for non-empty 'Text'. 'unsafeHead' +-- omits the check for the empty case, so there is an obligation on +-- the programmer to provide a proof that the 'Text' is non-empty. +unsafeHead :: Text -> Char +unsafeHead (Text arr off _len) = case utf8LengthByLeader m0 of + 1 -> unsafeChr8 m0 + 2 -> chr2 m0 m1 + 3 -> chr3 m0 m1 m2 + _ -> chr4 m0 m1 m2 m3 + where m0 = A.unsafeIndex arr off + m1 = A.unsafeIndex arr (off+1) + m2 = A.unsafeIndex arr (off+2) + m3 = A.unsafeIndex arr (off+3) +{-# INLINE unsafeHead #-} + +-- | /O(1)/ A variant of 'tail' for non-empty 'Text'. 'unsafeTail' +-- omits the check for the empty case, so there is an obligation on +-- the programmer to provide a proof that the 'Text' is non-empty. +unsafeTail :: Text -> Text +unsafeTail t@(Text arr off len) = +#if defined(ASSERTS) + assert (d <= len) $ +#endif + Text arr (off+d) (len-d) + where d = iter_ t 0 +{-# INLINE unsafeTail #-} + +data Iter = Iter {-# UNPACK #-} !Char {-# UNPACK #-} !Int + deriving (Show) + +-- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-8 +-- array, returning the current character and the delta to add to give +-- the next offset to iterate at. +iter :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Int -> Iter +iter (Text arr off _len) i = iterArray arr (off + i) +{-# INLINE iter #-} + +-- | @since 2.0 +iterArray :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -> Int -> Iter +iterArray arr j = Iter chr l + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j+1) + m2 = A.unsafeIndex arr (j+2) + m3 = A.unsafeIndex arr (j+3) + l = utf8LengthByLeader m0 + chr = case l of + 1 -> unsafeChr8 m0 + 2 -> chr2 m0 m1 + 3 -> chr3 m0 m1 m2 + _ -> chr4 m0 m1 m2 m3 +{-# INLINE iterArray #-} + +-- | /O(1)/ Iterate one step through a UTF-8 array, returning the +-- delta to add to give the next offset to iterate at. +iter_ :: Text -> Int -> Int +iter_ (Text arr off _len) i = utf8LengthByLeader m + where m = A.unsafeIndex arr (off+i) +{-# INLINE iter_ #-} + +-- | /O(1)/ Iterate one step backwards through a UTF-8 array, +-- returning the current character and the delta to add (i.e. a +-- negative number) to give the next offset to iterate at. +reverseIter :: Text -> Int -> Iter +reverseIter (Text arr off _len) i = reverseIterArray arr (off + i) +{-# INLINE reverseIter #-} + +-- | @since 2.0 +reverseIterArray :: A.Array -> Int -> Iter +reverseIterArray arr j + | m0 < 0x80 = Iter (unsafeChr8 m0) (-1) + | m1 >= 0xC0 = Iter (chr2 m1 m0) (-2) + | m2 >= 0xC0 = Iter (chr3 m2 m1 m0) (-3) + | otherwise = Iter (chr4 m3 m2 m1 m0) (-4) + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j-1) + m2 = A.unsafeIndex arr (j-2) + m3 = A.unsafeIndex arr (j-3) +{-# INLINE reverseIterArray #-} + +-- | /O(1)/ Iterate one step backwards through a UTF-8 array, +-- returning the delta to add (i.e. a negative number) to give the +-- next offset to iterate at. +-- +-- @since 1.1.1.0 +reverseIter_ :: Text -> Int -> Int +reverseIter_ (Text arr off _len) i + | m0 < 0x80 = -1 + | m1 >= 0xC0 = -2 + | m2 >= 0xC0 = -3 + | otherwise = -4 + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j-1) + m2 = A.unsafeIndex arr (j-2) + j = off + i +{-# INLINE reverseIter_ #-} + +-- | /O(1)/ Return the length of a 'Text' in units of 'Word8'. This +-- is useful for sizing a target array appropriately before using +-- 'unsafeCopyToPtr'. +-- +-- @since 2.0 +lengthWord8 :: Text -> Int +lengthWord8 (Text _arr _off len) = len +{-# INLINE lengthWord8 #-} + +-- | /O(1)/ Unchecked take of 'k' 'Word8's from the front of a 'Text'. +-- +-- @since 2.0 +takeWord8 :: Int -> Text -> Text +takeWord8 k (Text arr off _len) = Text arr off k +{-# INLINE takeWord8 #-} + +-- | /O(1)/ Unchecked drop of 'k' 'Word8's from the front of a 'Text'. +-- +-- @since 2.0 +dropWord8 :: Int -> Text -> Text +dropWord8 k (Text arr off len) = Text arr (off+k) (len-k) +{-# INLINE dropWord8 #-} diff --git a/tests-and-benchmarks.markdown b/tests-and-benchmarks.markdown deleted file mode 100644 --- a/tests-and-benchmarks.markdown +++ /dev/null @@ -1,63 +0,0 @@ -Tests and benchmarks -==================== - -Prerequisites -------------- - -To run the tests and benchmarks, you will need the test data, which -you can clone from one of the following locations: - -* Mercurial master repository: - [bitbucket.org/bos/text-test-data](https://bitbucket.org/bos/text-test-data) - -* Git mirror repository: - [github.com/bos/text-test-data](https://github.com/bos/text-test-data) - -You should clone that repository into the `tests` subdirectory (your -clone must be named `text-test-data` locally), then run `make -C -tests/text-test-data` to uncompress the test files. Many tests and -benchmarks will fail if the test files are missing. - -Functional tests ----------------- - -The functional tests are located in the `tests` subdirectory. An overview of -what's in that directory: - - Makefile Has targets for common tasks - Tests Source files of the testing code - scripts Various utility scripts - text-tests.cabal Cabal file that compiles all benchmarks - -The `text-tests.cabal` builds: - -- A copy of the text library, sharing the source code, but exposing all internal - modules, for testing purposes -- The different test suites - -To compile, run all tests, and generate a coverage report, simply use `make`. - -Benchmarks ----------- - -The benchmarks are located in the `benchmarks` subdirectory. An overview of -what's in that directory: - - Makefile Has targets for common tasks - haskell Source files of the haskell benchmarks - python Python implementations of some benchmarks - ruby Ruby implementations of some benchmarks - text-benchmarks.cabal Cabal file which compiles all benchmarks - -To compile the benchmarks, navigate to the `benchmarks` subdirectory and run -`cabal configure && cabal build`. Then, you can run the benchmarks using: - - ./dist/build/text-benchmarks/text-benchmarks - -However, since there's quite a lot of benchmarks, you usually don't want to -run them all. Instead, use the `-l` flag to get a list of benchmarks: - - ./dist/build/text-benchmarks/text-benchmarks - -And run the ones you want to inspect. If you want to configure the benchmarks -further, the exact parameters can be changed in `Benchmarks.hs`. diff --git a/tests/.ghci b/tests/.ghci deleted file mode 100644 --- a/tests/.ghci +++ /dev/null @@ -1,1 +0,0 @@ -:set -DHAVE_DEEPSEQ -isrc -i../.. diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 --- a/tests/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -count = 1000 - -all: coverage literal-rule-test - -literal-rule-test: - ./literal-rule-test.sh - -coverage: build coverage/hpc_index.html - -build: text-test-data - cabal configure -fhpc - cabal build - -text-test-data: - hg clone https://bitbucket.org/bos/text-test-data - $(MAKE) -C text-test-data - -coverage/text-tests.tix: - -mkdir -p coverage - ./dist/build/text-tests/text-tests -a $(count) - mv text-tests.tix $@ - -coverage/text-tests-stdio.tix: - -mkdir -p coverage - ./scripts/cover-stdio.sh ./dist/build/text-tests-stdio/text-tests-stdio - mv text-tests-stdio.tix $@ - -coverage/coverage.tix: coverage/text-tests.tix coverage/text-tests-stdio.tix - hpc combine --output=$@ \ - --exclude=Main \ - coverage/text-tests.tix \ - coverage/text-tests-stdio.tix - -coverage/hpc_index.html: coverage/coverage.tix - hpc markup --destdir=coverage coverage/coverage.tix - -clean: - rm -rf dist coverage .hpc - -.PHONY: build coverage all literal-rule-test diff --git a/tests/Tests.hs b/tests/Tests.hs --- a/tests/Tests.hs +++ b/tests/Tests.hs @@ -1,13 +1,22 @@ --- | Provides a simple main function which runs all the tests --- +{-# LANGUAGE CPP #-} + module Main ( main ) where -import Test.Framework (defaultMain) +import Test.Tasty (defaultMain, testGroup) +import qualified Tests.Lift as Lift import qualified Tests.Properties as Properties import qualified Tests.Regressions as Regressions +import qualified Tests.ShareEmpty as ShareEmpty +import qualified Tests.RebindableSyntaxTest as RST main :: IO () -main = defaultMain [Properties.tests, Regressions.tests] +main = defaultMain $ testGroup "All" + [ Lift.tests + , Properties.tests + , Regressions.tests + , ShareEmpty.tests + , RST.tests + ] diff --git a/tests/Tests/IO.hs b/tests/Tests/IO.hs deleted file mode 100644 --- a/tests/Tests/IO.hs +++ /dev/null @@ -1,34 +0,0 @@ --- | Program which exposes some haskell functions as an exutable. The results --- and coverage of this module is meant to be checked using a shell script. --- -module Main - ( - main - ) where - -import System.Environment (getArgs) -import System.Exit (exitFailure) -import System.IO (hPutStrLn, stderr) -import qualified Data.Text as T -import qualified Data.Text.IO as T -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.IO as TL - -main :: IO () -main = do - args <- getArgs - case args of - ["T.readFile", name] -> T.putStr =<< T.readFile name - ["T.writeFile", name, t] -> T.writeFile name (T.pack t) - ["T.appendFile", name, t] -> T.appendFile name (T.pack t) - ["T.interact"] -> T.interact id - ["T.getContents"] -> T.putStr =<< T.getContents - ["T.getLine"] -> T.putStrLn =<< T.getLine - - ["TL.readFile", name] -> TL.putStr =<< TL.readFile name - ["TL.writeFile", name, t] -> TL.writeFile name (TL.pack t) - ["TL.appendFile", name, t] -> TL.appendFile name (TL.pack t) - ["TL.interact"] -> TL.interact id - ["TL.getContents"] -> TL.putStr =<< TL.getContents - ["TL.getLine"] -> TL.putStrLn =<< TL.getLine - _ -> hPutStrLn stderr "invalid directive!" >> exitFailure diff --git a/tests/Tests/Lift.hs b/tests/Tests/Lift.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Lift.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +module Tests.Lift + ( tests + ) + where + +import qualified Data.Text as S +import qualified Data.Text.Lazy as L +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) + +tests :: TestTree +tests = testGroup "TH lifting Text" + [ testCase "strict" $ assertEqual "strict" + $(lift ("foo" :: S.Text)) + ("foo" :: S.Text) + , testCase "strict0" $ assertEqual "strict0" + $(lift ("f\0o\1o\2" :: S.Text)) + ("f\0o\1o\2" :: S.Text) + , testCase "strict-nihao" $ assertEqual "strict-nihao" + $(lift ("\20320\22909" :: S.Text)) + ("\20320\22909" :: S.Text) + , testCase "lazy" $ assertEqual "lazy" + $(lift ("foo" :: L.Text)) + ("foo" :: L.Text) + , testCase "lazy0" $ assertEqual "lazy0" + $(lift ("f\0o\1o\2" :: L.Text)) + ("f\0o\1o\2" :: L.Text) + , testCase "lazy-nihao" $ assertEqual "lazy-nihao" + $(lift ("\20320\22909" :: L.Text)) + ("\20320\22909" :: L.Text) + ] diff --git a/tests/Tests/Properties.hs b/tests/Tests/Properties.hs --- a/tests/Tests/Properties.hs +++ b/tests/Tests/Properties.hs @@ -1,1380 +1,36 @@ -- | QuickCheck properties for the text library. -{-# LANGUAGE BangPatterns, FlexibleInstances, OverloadedStrings, - ScopedTypeVariables, TypeSynonymInstances #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} -module Tests.Properties - ( - tests - ) where - -import Control.Applicative ((<$>), (<*>)) -import Control.Arrow ((***), second) -import Data.Bits ((.&.)) -import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord) -import Data.Int (Int8, Int16, Int32, Int64) -import Data.Monoid (Monoid(..)) -import Data.String (IsString(fromString)) -import Data.Text.Encoding.Error -import Data.Text.Foreign -import Data.Text.Internal.Encoding.Utf8 -import Data.Text.Internal.Fusion.Size -import Data.Text.Internal.Search (indices) -import Data.Text.Lazy.Read as TL -import Data.Text.Read as T -import Data.Word (Word, Word8, Word16, Word32, Word64) -import Data.Maybe (mapMaybe) -import Numeric (showEFloat, showFFloat, showGFloat, showHex) -import Prelude hiding (replicate) -import Test.Framework (Test, testGroup) -import Test.Framework.Providers.QuickCheck2 (testProperty) -import Test.QuickCheck hiding ((.&.)) -import Test.QuickCheck.Monadic -import Test.QuickCheck.Property (Property(..)) -import Tests.QuickCheckUtils -import Tests.Utils -import Text.Show.Functions () -import qualified Control.Exception as Exception -import qualified Data.Bits as Bits (shiftL, shiftR) -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.Char as C -import qualified Data.List as L -import qualified Data.Text as T -import qualified Data.Text.Encoding as E -import qualified Data.Text.IO as T -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import qualified Data.Text.Internal.Lazy.Fusion as SL -import qualified Data.Text.Internal.Lazy.Search as S (indices) -import qualified Data.Text.Internal.Unsafe.Shift as U -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.Builder as TB -import qualified Data.Text.Lazy.Builder.Int as TB -import qualified Data.Text.Lazy.Builder.RealFloat as TB -import qualified Data.Text.Lazy.Encoding as EL -import qualified Data.Text.Lazy.IO as TL -import qualified System.IO as IO -import qualified Tests.Properties.Mul as Mul -import qualified Tests.SlowFunctions as Slow - -t_pack_unpack = (T.unpack . T.pack) `eq` id -tl_pack_unpack = (TL.unpack . TL.pack) `eq` id -t_stream_unstream = (S.unstream . S.stream) `eq` id -tl_stream_unstream = (SL.unstream . SL.stream) `eq` id -t_reverse_stream t = (S.reverse . S.reverseStream) t === t -t_singleton c = [c] === (T.unpack . T.singleton) c -tl_singleton c = [c] === (TL.unpack . TL.singleton) c -tl_unstreamChunks x = f 11 x === f 1000 x - where f n = SL.unstreamChunks n . S.streamList -tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id -tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id - --- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack' -encodeL1 :: T.Text -> B.ByteString -encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack -encodeLazyL1 :: TL.Text -> BL.ByteString -encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks - -t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a - where a = T.map (\c -> chr (ord c `mod` 128)) t -tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a - where a = TL.map (\c -> chr (ord c `mod` 128)) t -t_latin1 t = E.decodeLatin1 (encodeL1 a) === a - where a = T.map (\c -> chr (ord c `mod` 256)) t -tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a - where a = TL.map (\c -> chr (ord c `mod` 256)) t -t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id -t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) -tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id -tl_utf8' = forAll genUnicode $ (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right) -t_utf16LE = forAll genUnicode $ (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id -tl_utf16LE = forAll genUnicode $ (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id -t_utf16BE = forAll genUnicode $ (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id -tl_utf16BE = forAll genUnicode $ (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id -t_utf32LE = forAll genUnicode $ (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id -tl_utf32LE = forAll genUnicode $ (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id -t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id -tl_utf32BE = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id - -t_utf8_incr = forAll genUnicode $ \s (Positive n) -> (recode n `eq` id) s - where recode n = T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 . - E.encodeUtf8 - -feedChunksOf :: Int -> (B.ByteString -> E.Decoding) -> B.ByteString - -> [(T.Text, B.ByteString)] -feedChunksOf n f bs - | B.null bs = [] - | otherwise = let (x,y) = B.splitAt n bs - E.Some t b f' = f x - in (t,b) : feedChunksOf n f' y - -t_utf8_undecoded = forAll genUnicode $ \t -> - let b = E.encodeUtf8 t - ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t - leftover = (++ [B.empty]) . init . tail . B.inits - in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls - -data Badness = Solo | Leading | Trailing - deriving (Eq, Show) - -instance Arbitrary Badness where - arbitrary = elements [Solo, Leading, Trailing] - -t_utf8_err :: Badness -> DecodeErr -> Property -t_utf8_err bad de = do - let gen = case bad of - Solo -> genInvalidUTF8 - Leading -> B.append <$> genInvalidUTF8 <*> genUTF8 - Trailing -> B.append <$> genUTF8 <*> genInvalidUTF8 - genUTF8 = E.encodeUtf8 <$> genUnicode - forAll gen $ \bs -> MkProperty $ do - onErr <- genDecodeErr de - unProperty . monadicIO $ do - l <- run $ let len = T.length (E.decodeUtf8With onErr bs) - in (len `seq` return (Right len)) `Exception.catch` - (\(e::UnicodeException) -> return (Left e)) - assert $ case l of - Left err -> length (show err) >= 0 - Right _ -> de /= Strict - -t_utf8_err' :: B.ByteString -> Property -t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of - Left err -> length (show err) >= 0 - Right t -> T.length t >= 0 - -genInvalidUTF8 :: Gen B.ByteString -genInvalidUTF8 = B.pack <$> oneof [ - -- invalid leading byte of a 2-byte sequence - (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte - -- invalid leading byte of a 4-byte sequence - , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte - -- 4-byte sequence greater than U+10FFFF - , do k <- choose (0x11, 0x13) - let w0 = 0xF0 + (k `Bits.shiftR` 2) - w1 = 0x80 + ((k .&. 3) `Bits.shiftL` 4) - ([w0,w1]++) <$> vectorOf 2 contByte - -- continuation bytes without a start byte - , listOf1 contByte - -- short 2-byte sequence - , (:[]) <$> choose (0xC2, 0xDF) - -- short 3-byte sequence - , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte - -- short 4-byte sequence - , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte - -- overlong encoding - , do k <- choose (0,0xFFFF) - let c = chr k - case k of - _ | k < 0x80 -> oneof [ let (w,x) = ord2 c in return [w,x] - , let (w,x,y) = ord3 c in return [w,x,y] - , let (w,x,y,z) = ord4 c in return [w,x,y,z] ] - | k < 0x7FF -> oneof [ let (w,x,y) = ord3 c in return [w,x,y] - , let (w,x,y,z) = ord4 c in return [w,x,y,z] ] - | otherwise -> let (w,x,y,z) = ord4 c in return [w,x,y,z] - ] - where - contByte = (0x80 +) <$> choose (0, 0x3f) - upTo n gen = do - k <- choose (0,n) - vectorOf k gen - -s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList) - where _types = s :: String -sf_Eq p s = - ((L.filter p s==) . L.filter p) `eq` - (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList) -t_Eq s = (s==) `eq` ((T.pack s==) . T.pack) -tl_Eq s = (s==) `eq` ((TL.pack s==) . TL.pack) -s_Ord s = (compare s) `eq` (compare (S.streamList s) . S.streamList) - where _types = s :: String -sf_Ord p s = - ((compare $ L.filter p s) . L.filter p) `eq` - (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList) -t_Ord s = (compare s) `eq` (compare (T.pack s) . T.pack) -tl_Ord s = (compare s) `eq` (compare (TL.pack s) . TL.pack) -t_Read = id `eq` (T.unpack . read . show) -tl_Read = id `eq` (TL.unpack . read . show) -t_Show = show `eq` (show . T.pack) -tl_Show = show `eq` (show . TL.pack) -t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) -tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s)) -t_mconcat = unsquare $ - mconcat `eq` (unpackS . mconcat . L.map T.pack) -tl_mconcat = unsquare $ - mconcat `eq` (unpackS . mconcat . L.map TL.pack) -t_mempty = mempty === (unpackS (mempty :: T.Text)) -tl_mempty = mempty === (unpackS (mempty :: TL.Text)) -t_IsString = fromString `eqP` (T.unpack . fromString) -tl_IsString = fromString `eqP` (TL.unpack . fromString) - -s_cons x = (x:) `eqP` (unpackS . S.cons x) -s_cons_s x = (x:) `eqP` (unpackS . S.unstream . S.cons x) -sf_cons p x = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p) -t_cons x = (x:) `eqP` (unpackS . T.cons x) -tl_cons x = (x:) `eqP` (unpackS . TL.cons x) -s_snoc x = (++ [x]) `eqP` (unpackS . (flip S.snoc) x) -t_snoc x = (++ [x]) `eqP` (unpackS . (flip T.snoc) x) -tl_snoc x = (++ [x]) `eqP` (unpackS . (flip TL.snoc) x) -s_append s = (s++) `eqP` (unpackS . S.append (S.streamList s)) -s_append_s s = (s++) `eqP` - (unpackS . S.unstream . S.append (S.streamList s)) -sf_append p s = (L.filter p s++) `eqP` - (unpackS . S.append (S.filter p $ S.streamList s)) -t_append s = (s++) `eqP` (unpackS . T.append (packS s)) - -uncons (x:xs) = Just (x,xs) -uncons _ = Nothing - -s_uncons = uncons `eqP` (fmap (second unpackS) . S.uncons) -sf_uncons p = (uncons . L.filter p) `eqP` - (fmap (second unpackS) . S.uncons . S.filter p) -t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons) -tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons) -s_head = head `eqP` S.head -sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p) -t_head = head `eqP` T.head -tl_head = head `eqP` TL.head -s_last = last `eqP` S.last -sf_last p = (last . L.filter p) `eqP` (S.last . S.filter p) -t_last = last `eqP` T.last -tl_last = last `eqP` TL.last -s_tail = tail `eqP` (unpackS . S.tail) -s_tail_s = tail `eqP` (unpackS . S.unstream . S.tail) -sf_tail p = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p) -t_tail = tail `eqP` (unpackS . T.tail) -tl_tail = tail `eqP` (unpackS . TL.tail) -s_init = init `eqP` (unpackS . S.init) -s_init_s = init `eqP` (unpackS . S.unstream . S.init) -sf_init p = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p) -t_init = init `eqP` (unpackS . T.init) -tl_init = init `eqP` (unpackS . TL.init) -s_null = null `eqP` S.null -sf_null p = (null . L.filter p) `eqP` (S.null . S.filter p) -t_null = null `eqP` T.null -tl_null = null `eqP` TL.null -s_length = length `eqP` S.length -sf_length p = (length . L.filter p) `eqP` (S.length . S.filter p) -sl_length = (fromIntegral . length) `eqP` SL.length -t_length = length `eqP` T.length -tl_length = L.genericLength `eqP` TL.length -t_compareLength t = (compare (T.length t)) `eq` T.compareLength t -tl_compareLength t= (compare (TL.length t)) `eq` TL.compareLength t - -s_map f = map f `eqP` (unpackS . S.map f) -s_map_s f = map f `eqP` (unpackS . S.unstream . S.map f) -sf_map p f = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) -t_map f = map f `eqP` (unpackS . T.map f) -tl_map f = map f `eqP` (unpackS . TL.map f) -s_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . S.intercalate (packS c) . map packS) -t_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . T.intercalate (packS c) . map packS) -tl_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . TL.intercalate (TL.pack c) . map TL.pack) -s_intersperse c = L.intersperse c `eqP` - (unpackS . S.intersperse c) -s_intersperse_s c = L.intersperse c `eqP` - (unpackS . S.unstream . S.intersperse c) -sf_intersperse p c= (L.intersperse c . L.filter p) `eqP` - (unpackS . S.intersperse c . S.filter p) -t_intersperse c = unsquare $ - L.intersperse c `eqP` (unpackS . T.intersperse c) -tl_intersperse c = unsquare $ - L.intersperse c `eqP` (unpackS . TL.intersperse c) -t_transpose = unsquare $ - L.transpose `eq` (map unpackS . T.transpose . map packS) -tl_transpose = unsquare $ - L.transpose `eq` (map unpackS . TL.transpose . map TL.pack) -t_reverse = L.reverse `eqP` (unpackS . T.reverse) -tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) -t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) - -t_replace s d = (L.intercalate d . splitOn s) `eqP` - (unpackS . T.replace (T.pack s) (T.pack d)) -tl_replace s d = (L.intercalate d . splitOn s) `eqP` - (unpackS . TL.replace (TL.pack s) (TL.pack d)) - -splitOn :: (Eq a) => [a] -> [a] -> [[a]] -splitOn pat src0 - | l == 0 = error "splitOn: empty" - | otherwise = go src0 - where - l = length pat - go src = search 0 src - where - search _ [] = [src] - search !n s@(_:s') - | pat `L.isPrefixOf` s = take n src : go (drop l s) - | otherwise = search (n+1) s' - -s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs - where s = S.streamList xs -sf_toCaseFold_length p xs = - (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) - where s = S.streamList xs -t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t -tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t -t_toLower_length t = T.length (T.toLower t) >= T.length t -t_toLower_lower t = p (T.toLower t) >= p t - where p = T.length . T.filter isLower -tl_toLower_lower t = p (TL.toLower t) >= p t - where p = TL.length . TL.filter isLower -t_toUpper_length t = T.length (T.toUpper t) >= T.length t -t_toUpper_upper t = p (T.toUpper t) >= p t - where p = T.length . T.filter isUpper -tl_toUpper_upper t = p (TL.toUpper t) >= p t - where p = TL.length . TL.filter isUpper -t_toTitle_title t = all (<= 1) (caps w) - where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle - -- TIL: there exist uppercase-only letters - w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t -t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable - where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words - -- Surprise! The Spanish/Portuguese ordinal indicators changed - -- from category Ll (letter, lowercase) to Lo (letter, other) - -- in Unicode 7.0 - -- Oh, and there exist lowercase-only letters (see previous test) - stable c = if isLower c - then C.toUpper c /= c - else c /= '\170' && c /= '\186' - -justifyLeft k c xs = xs ++ L.replicate (k - length xs) c -justifyRight m n xs = L.replicate (m - length xs) n ++ xs -center k c xs - | len >= k = xs - | otherwise = L.replicate l c ++ xs ++ L.replicate r c - where len = length xs - d = k - len - r = d `div` 2 - l = d - r - -s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) - where j = fromIntegral (k :: Word8) -s_justifyLeft_s k c = justifyLeft j c `eqP` - (unpackS . S.unstream . S.justifyLeftI j c) - where j = fromIntegral (k :: Word8) -sf_justifyLeft p k c = (justifyLeft j c . L.filter p) `eqP` - (unpackS . S.justifyLeftI j c . S.filter p) - where j = fromIntegral (k :: Word8) -t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) - where j = fromIntegral (k :: Word8) -tl_justifyLeft k c = justifyLeft j c `eqP` - (unpackS . TL.justifyLeft (fromIntegral j) c) - where j = fromIntegral (k :: Word8) -t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) - where j = fromIntegral (k :: Word8) -tl_justifyRight k c = justifyRight j c `eqP` - (unpackS . TL.justifyRight (fromIntegral j) c) - where j = fromIntegral (k :: Word8) -t_center k c = center j c `eqP` (unpackS . T.center j c) - where j = fromIntegral (k :: Word8) -tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) - where j = fromIntegral (k :: Word8) - -sf_foldl p f z = (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldl f z = L.foldl f z `eqP` (T.foldl f z) - where _types = f :: Char -> Char -> Char -tl_foldl f z = L.foldl f z `eqP` (TL.foldl f z) - where _types = f :: Char -> Char -> Char -sf_foldl' p f z = (L.foldl' f z . L.filter p) `eqP` - (S.foldl' f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldl' f z = L.foldl' f z `eqP` T.foldl' f z - where _types = f :: Char -> Char -> Char -tl_foldl' f z = L.foldl' f z `eqP` TL.foldl' f z - where _types = f :: Char -> Char -> Char -sf_foldl1 p f = (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) -t_foldl1 f = L.foldl1 f `eqP` T.foldl1 f -tl_foldl1 f = L.foldl1 f `eqP` TL.foldl1 f -sf_foldl1' p f = (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) -t_foldl1' f = L.foldl1' f `eqP` T.foldl1' f -tl_foldl1' f = L.foldl1' f `eqP` TL.foldl1' f -sf_foldr p f z = (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldr f z = L.foldr f z `eqP` T.foldr f z - where _types = f :: Char -> Char -> Char -tl_foldr f z = unsquare $ - L.foldr f z `eqP` TL.foldr f z - where _types = f :: Char -> Char -> Char -sf_foldr1 p f = unsquare $ - (L.foldr1 f . L.filter p) `eqP` (S.foldr1 f . S.filter p) -t_foldr1 f = L.foldr1 f `eqP` T.foldr1 f -tl_foldr1 f = unsquare $ - L.foldr1 f `eqP` TL.foldr1 f - -s_concat_s = unsquare $ - L.concat `eq` (unpackS . S.unstream . S.concat . map packS) -sf_concat p = unsquare $ - (L.concat . map (L.filter p)) `eq` - (unpackS . S.concat . map (S.filter p . packS)) -t_concat = unsquare $ - L.concat `eq` (unpackS . T.concat . map packS) -tl_concat = unsquare $ - L.concat `eq` (unpackS . TL.concat . map TL.pack) -sf_concatMap p f = unsquare $ (L.concatMap f . L.filter p) `eqP` - (unpackS . S.concatMap (packS . f) . S.filter p) -t_concatMap f = unsquare $ - L.concatMap f `eqP` (unpackS . T.concatMap (packS . f)) -tl_concatMap f = unsquare $ - L.concatMap f `eqP` (unpackS . TL.concatMap (TL.pack . f)) -sf_any q p = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) -t_any p = L.any p `eqP` T.any p -tl_any p = L.any p `eqP` TL.any p -sf_all q p = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) -t_all p = L.all p `eqP` T.all p -tl_all p = L.all p `eqP` TL.all p -sf_maximum p = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) -t_maximum = L.maximum `eqP` T.maximum -tl_maximum = L.maximum `eqP` TL.maximum -sf_minimum p = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) -t_minimum = L.minimum `eqP` T.minimum -tl_minimum = L.minimum `eqP` TL.minimum - -sf_scanl p f z = (L.scanl f z . L.filter p) `eqP` - (unpackS . S.scanl f z . S.filter p) -t_scanl f z = L.scanl f z `eqP` (unpackS . T.scanl f z) -tl_scanl f z = L.scanl f z `eqP` (unpackS . TL.scanl f z) -t_scanl1 f = L.scanl1 f `eqP` (unpackS . T.scanl1 f) -tl_scanl1 f = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) -t_scanr f z = L.scanr f z `eqP` (unpackS . T.scanr f z) -tl_scanr f z = L.scanr f z `eqP` (unpackS . TL.scanr f z) -t_scanr1 f = L.scanr1 f `eqP` (unpackS . T.scanr1 f) -tl_scanr1 f = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) - -t_mapAccumL f z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) - where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumL f z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) - where _types = f :: Int -> Char -> (Int,Char) -t_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) - where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) - where _types = f :: Int -> Char -> (Int,Char) - -tl_repeat n = (L.take m . L.repeat) `eq` - (unpackS . TL.take (fromIntegral m) . TL.repeat) - where m = fromIntegral (n :: Word8) - -replicate n l = concat (L.replicate n l) - -s_replicate n = replicate m `eq` - (unpackS . S.replicateI (fromIntegral m) . packS) - where m = fromIntegral (n :: Word8) -t_replicate n = replicate m `eq` (unpackS . T.replicate m . packS) - where m = fromIntegral (n :: Word8) -tl_replicate n = replicate m `eq` - (unpackS . TL.replicate (fromIntegral m) . packS) - where m = fromIntegral (n :: Word8) - -tl_cycle n = (L.take m . L.cycle) `eq` - (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) - where m = fromIntegral (n :: Word8) - -tl_iterate f n = (L.take m . L.iterate f) `eq` - (unpackS . TL.take (fromIntegral m) . TL.iterate f) - where m = fromIntegral (n :: Word8) - -unf :: Int -> Char -> Maybe (Char, Char) -unf n c | fromEnum c * 100 > n = Nothing - | otherwise = Just (c, succ c) - -t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) - where m = fromIntegral (n :: Word16) -tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) - where m = fromIntegral (n :: Word16) -t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` - (unpackS . T.unfoldrN i (unf j)) - where i = fromIntegral (n :: Word16) - j = fromIntegral (m :: Word16) -tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` - (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) - where i = fromIntegral (n :: Word16) - j = fromIntegral (m :: Word16) - -unpack2 :: (Stringy s) => (s,s) -> (String,String) -unpack2 = unpackS *** unpackS - -s_take n = L.take n `eqP` (unpackS . S.take n) -s_take_s m = L.take n `eqP` (unpackS . S.unstream . S.take n) - where n = small m -sf_take p n = (L.take n . L.filter p) `eqP` - (unpackS . S.take n . S.filter p) -t_take n = L.take n `eqP` (unpackS . T.take n) -t_takeEnd n = (L.reverse . L.take n . L.reverse) `eqP` - (unpackS . T.takeEnd n) -tl_take n = L.take n `eqP` (unpackS . TL.take (fromIntegral n)) -tl_takeEnd n = (L.reverse . L.take (fromIntegral n) . L.reverse) `eqP` - (unpackS . TL.takeEnd n) -s_drop n = L.drop n `eqP` (unpackS . S.drop n) -s_drop_s m = L.drop n `eqP` (unpackS . S.unstream . S.drop n) - where n = small m -sf_drop p n = (L.drop n . L.filter p) `eqP` - (unpackS . S.drop n . S.filter p) -t_drop n = L.drop n `eqP` (unpackS . T.drop n) -t_dropEnd n = (L.reverse . L.drop n . L.reverse) `eqP` - (unpackS . T.dropEnd n) -tl_drop n = L.drop n `eqP` (unpackS . TL.drop (fromIntegral n)) -tl_dropEnd n = (L.reverse . L.drop n . L.reverse) `eqP` - (unpackS . TL.dropEnd (fromIntegral n)) -s_take_drop m = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n) - where n = small m -s_take_drop_s m = (L.take n . L.drop n) `eqP` - (unpackS . S.unstream . S.take n . S.drop n) - where n = small m -s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p) -s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p) -sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP` - (unpackS . S.takeWhile p . S.filter q) -t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p) -tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p) -t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP` - (unpackS . T.takeWhileEnd p) -tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP` - (unpackS . TL.takeWhileEnd p) -s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) -s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p) -sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP` - (unpackS . S.dropWhile p . S.filter q) -t_dropWhile p = L.dropWhile p `eqP` (unpackS . T.dropWhile p) -tl_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) -t_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` - (unpackS . T.dropWhileEnd p) -tl_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` - (unpackS . TL.dropWhileEnd p) -t_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) - `eqP` (unpackS . T.dropAround p) -tl_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) - `eqP` (unpackS . TL.dropAround p) -t_stripStart = T.dropWhile isSpace `eq` T.stripStart -tl_stripStart = TL.dropWhile isSpace `eq` TL.stripStart -t_stripEnd = T.dropWhileEnd isSpace `eq` T.stripEnd -tl_stripEnd = TL.dropWhileEnd isSpace `eq` TL.stripEnd -t_strip = T.dropAround isSpace `eq` T.strip -tl_strip = TL.dropAround isSpace `eq` TL.strip -t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) -tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n)) -t_span p = L.span p `eqP` (unpack2 . T.span p) -tl_span p = L.span p `eqP` (unpack2 . TL.span p) - -t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s) - where squid t | T.null s = error "empty" - | otherwise = t -tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s) - where squid t | TL.null s = error "empty" - | otherwise = t -t_breakOn_start (NotEmpty s) t = - let (k,m) = T.breakOn s t - in k `T.isPrefixOf` t && (T.null m || s `T.isPrefixOf` m) -tl_breakOn_start (NotEmpty s) t = - let (k,m) = TL.breakOn s t - in k `TL.isPrefixOf` t && TL.null m || s `TL.isPrefixOf` m -t_breakOnEnd_end (NotEmpty s) t = - let (m,k) = T.breakOnEnd s t - in k `T.isSuffixOf` t && (T.null m || s `T.isSuffixOf` m) -tl_breakOnEnd_end (NotEmpty s) t = - let (m,k) = TL.breakOnEnd s t - in k `TL.isSuffixOf` t && (TL.null m || s `TL.isSuffixOf` m) -t_break p = L.break p `eqP` (unpack2 . T.break p) -tl_break p = L.break p `eqP` (unpack2 . TL.break p) -t_group = L.group `eqP` (map unpackS . T.group) -tl_group = L.group `eqP` (map unpackS . TL.group) -t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p) -tl_groupBy p = L.groupBy p `eqP` (map unpackS . TL.groupBy p) -t_inits = L.inits `eqP` (map unpackS . T.inits) -tl_inits = L.inits `eqP` (map unpackS . TL.inits) -t_tails = L.tails `eqP` (map unpackS . T.tails) -tl_tails = unsquare $ - L.tails `eqP` (map unpackS . TL.tails) -t_findAppendId = unsquare $ \(NotEmpty s) ts -> - let t = T.intercalate s ts - in all (==t) $ map (uncurry T.append) (T.breakOnAll s t) -tl_findAppendId = unsquare $ \(NotEmpty s) ts -> - let t = TL.intercalate s ts - in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t) -t_findContains = unsquare $ \(NotEmpty s) -> - all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s -tl_findContains = unsquare $ \(NotEmpty s) -> all (TL.isPrefixOf s . snd) . - TL.breakOnAll s . TL.intercalate s -sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c -t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s -tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s - -t_splitOn_split s = unsquare $ - (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s -tl_splitOn_split s = unsquare $ - ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq` - (map TL.fromStrict . T.splitOn s)) . T.intercalate s -t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t) -tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) - -t_split p = split p `eqP` (map unpackS . T.split p) -t_split_count c = (L.length . T.split (==c)) `eq` - ((1+) . T.count (T.singleton c)) -t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c) -tl_split p = split p `eqP` (map unpackS . TL.split p) - -split :: (a -> Bool) -> [a] -> [[a]] -split _ [] = [[]] -split p xs = loop xs - where loop s | null s' = [l] - | otherwise = l : loop (tail s') - where (l, s') = break p s - -t_chunksOf_same_lengths k = all ((==k) . T.length) . ini . T.chunksOf k - where ini [] = [] - ini xs = init xs - -t_chunksOf_length k t = len == T.length t || (k <= 0 && len == 0) - where len = L.sum . L.map T.length $ T.chunksOf k t - -tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) . - TL.chunksOf (fromIntegral k) . TL.fromStrict) - -t_lines = L.lines `eqP` (map unpackS . T.lines) -tl_lines = L.lines `eqP` (map unpackS . TL.lines) -{- -t_lines' = lines' `eqP` (map unpackS . T.lines') - where lines' "" = [] - lines' s = let (l, s') = break eol s - in l : case s' of - [] -> [] - ('\r':'\n':s'') -> lines' s'' - (_:s'') -> lines' s'' - eol c = c == '\r' || c == '\n' --} -t_words = L.words `eqP` (map unpackS . T.words) - -tl_words = L.words `eqP` (map unpackS . TL.words) -t_unlines = unsquare $ - L.unlines `eq` (unpackS . T.unlines . map packS) -tl_unlines = unsquare $ - L.unlines `eq` (unpackS . TL.unlines . map packS) -t_unwords = unsquare $ - L.unwords `eq` (unpackS . T.unwords . map packS) -tl_unwords = unsquare $ - L.unwords `eq` (unpackS . TL.unwords . map packS) - -s_isPrefixOf s = L.isPrefixOf s `eqP` - (S.isPrefixOf (S.stream $ packS s) . S.stream) -sf_isPrefixOf p s = (L.isPrefixOf s . L.filter p) `eqP` - (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream) -t_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (packS s) -tl_isPrefixOf s = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s) -t_isSuffixOf s = L.isSuffixOf s`eqP` T.isSuffixOf (packS s) -tl_isSuffixOf s = L.isSuffixOf s`eqP` TL.isSuffixOf (packS s) -t_isInfixOf s = L.isInfixOf s `eqP` T.isInfixOf (packS s) -tl_isInfixOf s = L.isInfixOf s `eqP` TL.isInfixOf (packS s) - -t_stripPrefix s = (fmap packS . L.stripPrefix s) `eqP` T.stripPrefix (packS s) -tl_stripPrefix s = (fmap packS . L.stripPrefix s) `eqP` TL.stripPrefix (packS s) - -stripSuffix p t = reverse `fmap` L.stripPrefix (reverse p) (reverse t) - -t_stripSuffix s = (fmap packS . stripSuffix s) `eqP` T.stripSuffix (packS s) -tl_stripSuffix s = (fmap packS . stripSuffix s) `eqP` TL.stripSuffix (packS s) - -commonPrefixes a0@(_:_) b0@(_:_) = Just (go a0 b0 []) - where go (a:as) (b:bs) ps - | a == b = go as bs (a:ps) - go as bs ps = (reverse ps,as,bs) -commonPrefixes _ _ = Nothing - -t_commonPrefixes a b (NonEmpty p) - = commonPrefixes pa pb == - repack `fmap` T.commonPrefixes (packS pa) (packS pb) - where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) - pa = p ++ a - pb = p ++ b - -tl_commonPrefixes a b (NonEmpty p) - = commonPrefixes pa pb == - repack `fmap` TL.commonPrefixes (packS pa) (packS pb) - where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) - pa = p ++ a - pb = p ++ b - -sf_elem p c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) -sf_filter q p = (L.filter p . L.filter q) `eqP` - (unpackS . S.filter p . S.filter q) -t_filter p = L.filter p `eqP` (unpackS . T.filter p) -tl_filter p = L.filter p `eqP` (unpackS . TL.filter p) -sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) -t_find p = L.find p `eqP` T.find p -tl_find p = L.find p `eqP` TL.find p -t_partition p = L.partition p `eqP` (unpack2 . T.partition p) -tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p) - -sf_index p s = forAll (choose (-l,l*2)) - ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) - where l = L.length s -t_index s = forAll (choose (-l,l*2)) ((s L.!!) `eq` T.index (packS s)) - where l = L.length s - -tl_index s = forAll (choose (-l,l*2)) - ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) - where l = L.length s - -t_findIndex p = L.findIndex p `eqP` T.findIndex p -t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t -tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` - TL.count t -t_zip s = L.zip s `eqP` T.zip (packS s) -tl_zip s = L.zip s `eqP` TL.zip (packS s) -sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` - (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) -t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) -tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) - -t_indices (NotEmpty s) = Slow.indices s `eq` indices s -tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s - where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) - conc = T.concat . TL.toChunks -t_indices_occurs = unsquare $ \(NotEmpty t) ts -> - let s = T.intercalate t ts - in Slow.indices t s === indices t s - --- Bit shifts. -shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k - where width = round (log (fromIntegral m) / log 2 :: Double) - (m,_) = (maxBound, m == w) -shiftR w = forAll (choose (0,width-1)) $ \k -> Bits.shiftR w k == U.shiftR w k - where width = round (log (fromIntegral m) / log 2 :: Double) - (m,_) = (maxBound, m == w) - -shiftL_Int = shiftL :: Int -> Property -shiftL_Word16 = shiftL :: Word16 -> Property -shiftL_Word32 = shiftL :: Word32 -> Property -shiftR_Int = shiftR :: Int -> Property -shiftR_Word16 = shiftR :: Word16 -> Property -shiftR_Word32 = shiftR :: Word32 -> Property - --- Builder. - -tb_singleton = id `eqP` - (unpackS . TB.toLazyText . mconcat . map TB.singleton) -tb_fromText = L.concat `eq` (unpackS . TB.toLazyText . mconcat . - map (TB.fromText . packS)) -tb_associative s1 s2 s3 = - TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) == - TB.toLazyText ((b1 `mappend` b2) `mappend` b3) - where b1 = TB.fromText (packS s1) - b2 = TB.fromText (packS s2) - b3 = TB.fromText (packS s3) - --- Numeric builder stuff. - -tb_decimal :: (Integral a, Show a) => a -> Bool -tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show) - -tb_decimal_integer (a::Integer) = tb_decimal a -tb_decimal_integer_big (Big a) = tb_decimal a -tb_decimal_int (a::Int) = tb_decimal a -tb_decimal_int8 (a::Int8) = tb_decimal a -tb_decimal_int16 (a::Int16) = tb_decimal a -tb_decimal_int32 (a::Int32) = tb_decimal a -tb_decimal_int64 (a::Int64) = tb_decimal a -tb_decimal_word (a::Word) = tb_decimal a -tb_decimal_word8 (a::Word8) = tb_decimal a -tb_decimal_word16 (a::Word16) = tb_decimal a -tb_decimal_word32 (a::Word32) = tb_decimal a -tb_decimal_word64 (a::Word64) = tb_decimal a - -tb_decimal_big_int (BigBounded (a::Int)) = tb_decimal a -tb_decimal_big_int64 (BigBounded (a::Int64)) = tb_decimal a -tb_decimal_big_word (BigBounded (a::Word)) = tb_decimal a -tb_decimal_big_word64 (BigBounded (a::Word64)) = tb_decimal a - -tb_hex :: (Integral a, Show a) => a -> Bool -tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "") - -tb_hexadecimal_integer (a::Integer) = tb_hex a -tb_hexadecimal_int (a::Int) = tb_hex a -tb_hexadecimal_int8 (a::Int8) = tb_hex a -tb_hexadecimal_int16 (a::Int16) = tb_hex a -tb_hexadecimal_int32 (a::Int32) = tb_hex a -tb_hexadecimal_int64 (a::Int64) = tb_hex a -tb_hexadecimal_word (a::Word) = tb_hex a -tb_hexadecimal_word8 (a::Word8) = tb_hex a -tb_hexadecimal_word16 (a::Word16) = tb_hex a -tb_hexadecimal_word32 (a::Word32) = tb_hex a -tb_hexadecimal_word64 (a::Word64) = tb_hex a - -tb_realfloat :: (RealFloat a, Show a) => a -> Bool -tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show) - -tb_realfloat_float (a::Float) = tb_realfloat a -tb_realfloat_double (a::Double) = tb_realfloat a - -showFloat :: (RealFloat a) => TB.FPFormat -> Maybe Int -> a -> ShowS -showFloat TB.Exponent = showEFloat -showFloat TB.Fixed = showFFloat -showFloat TB.Generic = showGFloat - -tb_formatRealFloat :: (RealFloat a, Show a) => - a -> TB.FPFormat -> Precision a -> Property -tb_formatRealFloat a fmt prec = - TB.formatRealFloat fmt p a === - TB.fromString (showFloat fmt p a "") - where p = precision a prec - -tb_formatRealFloat_float (a::Float) = tb_formatRealFloat a -tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a - --- Reading. - -t_decimal (n::Int) s = - T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t) - where t = T.dropWhile isDigit s -tl_decimal (n::Int) s = - TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t) - where t = TL.dropWhile isDigit s -t_hexadecimal m s ox = - T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t) - where t = T.dropWhile isHexDigit s - p = if ox then "0x" else "" - n = getPositive m :: Int -tl_hexadecimal m s ox = - TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t) - where t = TL.dropWhile isHexDigit s - p = if ox then "0x" else "" - n = getPositive m :: Int - -isFloaty c = c `elem` ("+-.0123456789eE" :: String) - -t_read_rational p tol (n::Double) s = - case p (T.pack (show n) `T.append` t) of - Left _err -> False - Right (n',t') -> t == t' && abs (n-n') <= tol - where t = T.dropWhile isFloaty s - -tl_read_rational p tol (n::Double) s = - case p (TL.pack (show n) `TL.append` t) of - Left _err -> False - Right (n',t') -> t == t' && abs (n-n') <= tol - where t = TL.dropWhile isFloaty s - -t_double = t_read_rational T.double 1e-13 -tl_double = tl_read_rational TL.double 1e-13 -t_rational = t_read_rational T.rational 1e-16 -tl_rational = tl_read_rational TL.rational 1e-16 - --- Input and output. - -t_put_get = write_read T.unlines T.filter put get - where put h = withRedirect h IO.stdout . T.putStr - get h = withRedirect h IO.stdin T.getContents -tl_put_get = write_read TL.unlines TL.filter put get - where put h = withRedirect h IO.stdout . TL.putStr - get h = withRedirect h IO.stdin TL.getContents -t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents -tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents - -t_write_read_line e m b t = write_read head T.filter T.hPutStrLn - T.hGetLine e m b [t] -tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn - TL.hGetLine e m b [t] - --- Low-level. - -t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t -t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t -t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t - where n = small m -t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr) - -t_copy t = T.copy t === t - --- Regression tests. -s_filter_eq s = S.filter p t == S.streamList (filter p s) - where p = (/= S.last t) - t = S.streamList s - --- Make a stream appear shorter than it really is, to ensure that --- functions that consume inaccurately sized streams behave --- themselves. -shorten :: Int -> S.Stream a -> S.Stream a -shorten n t@(S.Stream arr off len) - | n > 0 = S.Stream arr off (smaller (exactSize n) len) - | otherwise = t - -tests :: Test -tests = - testGroup "Properties" [ - testGroup "creation/elimination" [ - testProperty "t_pack_unpack" t_pack_unpack, - testProperty "tl_pack_unpack" tl_pack_unpack, - testProperty "t_stream_unstream" t_stream_unstream, - testProperty "tl_stream_unstream" tl_stream_unstream, - testProperty "t_reverse_stream" t_reverse_stream, - testProperty "t_singleton" t_singleton, - testProperty "tl_singleton" tl_singleton, - testProperty "tl_unstreamChunks" tl_unstreamChunks, - testProperty "tl_chunk_unchunk" tl_chunk_unchunk, - testProperty "tl_from_to_strict" tl_from_to_strict - ], - - testGroup "transcoding" [ - testProperty "t_ascii" t_ascii, - testProperty "tl_ascii" tl_ascii, - testProperty "t_latin1" t_latin1, - testProperty "tl_latin1" tl_latin1, - testProperty "t_utf8" t_utf8, - testProperty "t_utf8'" t_utf8', - testProperty "t_utf8_incr" t_utf8_incr, - testProperty "t_utf8_undecoded" t_utf8_undecoded, - testProperty "tl_utf8" tl_utf8, - testProperty "tl_utf8'" tl_utf8', - testProperty "t_utf16LE" t_utf16LE, - testProperty "tl_utf16LE" tl_utf16LE, - testProperty "t_utf16BE" t_utf16BE, - testProperty "tl_utf16BE" tl_utf16BE, - testProperty "t_utf32LE" t_utf32LE, - testProperty "tl_utf32LE" tl_utf32LE, - testProperty "t_utf32BE" t_utf32BE, - testProperty "tl_utf32BE" tl_utf32BE, - testGroup "errors" [ - testProperty "t_utf8_err" t_utf8_err, - testProperty "t_utf8_err'" t_utf8_err' - ] - ], - - testGroup "instances" [ - testProperty "s_Eq" s_Eq, - testProperty "sf_Eq" sf_Eq, - testProperty "t_Eq" t_Eq, - testProperty "tl_Eq" tl_Eq, - testProperty "s_Ord" s_Ord, - testProperty "sf_Ord" sf_Ord, - testProperty "t_Ord" t_Ord, - testProperty "tl_Ord" tl_Ord, - testProperty "t_Read" t_Read, - testProperty "tl_Read" tl_Read, - testProperty "t_Show" t_Show, - testProperty "tl_Show" tl_Show, - testProperty "t_mappend" t_mappend, - testProperty "tl_mappend" tl_mappend, - testProperty "t_mconcat" t_mconcat, - testProperty "tl_mconcat" tl_mconcat, - testProperty "t_mempty" t_mempty, - testProperty "tl_mempty" tl_mempty, - testProperty "t_IsString" t_IsString, - testProperty "tl_IsString" tl_IsString - ], - - testGroup "basics" [ - testProperty "s_cons" s_cons, - testProperty "s_cons_s" s_cons_s, - testProperty "sf_cons" sf_cons, - testProperty "t_cons" t_cons, - testProperty "tl_cons" tl_cons, - testProperty "s_snoc" s_snoc, - testProperty "t_snoc" t_snoc, - testProperty "tl_snoc" tl_snoc, - testProperty "s_append" s_append, - testProperty "s_append_s" s_append_s, - testProperty "sf_append" sf_append, - testProperty "t_append" t_append, - testProperty "s_uncons" s_uncons, - testProperty "sf_uncons" sf_uncons, - testProperty "t_uncons" t_uncons, - testProperty "tl_uncons" tl_uncons, - testProperty "s_head" s_head, - testProperty "sf_head" sf_head, - testProperty "t_head" t_head, - testProperty "tl_head" tl_head, - testProperty "s_last" s_last, - testProperty "sf_last" sf_last, - testProperty "t_last" t_last, - testProperty "tl_last" tl_last, - testProperty "s_tail" s_tail, - testProperty "s_tail_s" s_tail_s, - testProperty "sf_tail" sf_tail, - testProperty "t_tail" t_tail, - testProperty "tl_tail" tl_tail, - testProperty "s_init" s_init, - testProperty "s_init_s" s_init_s, - testProperty "sf_init" sf_init, - testProperty "t_init" t_init, - testProperty "tl_init" tl_init, - testProperty "s_null" s_null, - testProperty "sf_null" sf_null, - testProperty "t_null" t_null, - testProperty "tl_null" tl_null, - testProperty "s_length" s_length, - testProperty "sf_length" sf_length, - testProperty "sl_length" sl_length, - testProperty "t_length" t_length, - testProperty "tl_length" tl_length, - testProperty "t_compareLength" t_compareLength, - testProperty "tl_compareLength" tl_compareLength - ], - - testGroup "transformations" [ - testProperty "s_map" s_map, - testProperty "s_map_s" s_map_s, - testProperty "sf_map" sf_map, - testProperty "t_map" t_map, - testProperty "tl_map" tl_map, - testProperty "s_intercalate" s_intercalate, - testProperty "t_intercalate" t_intercalate, - testProperty "tl_intercalate" tl_intercalate, - testProperty "s_intersperse" s_intersperse, - testProperty "s_intersperse_s" s_intersperse_s, - testProperty "sf_intersperse" sf_intersperse, - testProperty "t_intersperse" t_intersperse, - testProperty "tl_intersperse" tl_intersperse, - testProperty "t_transpose" t_transpose, - testProperty "tl_transpose" tl_transpose, - testProperty "t_reverse" t_reverse, - testProperty "tl_reverse" tl_reverse, - testProperty "t_reverse_short" t_reverse_short, - testProperty "t_replace" t_replace, - testProperty "tl_replace" tl_replace, - - testGroup "case conversion" [ - testProperty "s_toCaseFold_length" s_toCaseFold_length, - testProperty "sf_toCaseFold_length" sf_toCaseFold_length, - testProperty "t_toCaseFold_length" t_toCaseFold_length, - testProperty "tl_toCaseFold_length" tl_toCaseFold_length, - testProperty "t_toLower_length" t_toLower_length, - testProperty "t_toLower_lower" t_toLower_lower, - testProperty "tl_toLower_lower" tl_toLower_lower, - testProperty "t_toUpper_length" t_toUpper_length, - testProperty "t_toUpper_upper" t_toUpper_upper, - testProperty "tl_toUpper_upper" tl_toUpper_upper, - testProperty "t_toTitle_title" t_toTitle_title, - testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower - ], - - testGroup "justification" [ - testProperty "s_justifyLeft" s_justifyLeft, - testProperty "s_justifyLeft_s" s_justifyLeft_s, - testProperty "sf_justifyLeft" sf_justifyLeft, - testProperty "t_justifyLeft" t_justifyLeft, - testProperty "tl_justifyLeft" tl_justifyLeft, - testProperty "t_justifyRight" t_justifyRight, - testProperty "tl_justifyRight" tl_justifyRight, - testProperty "t_center" t_center, - testProperty "tl_center" tl_center - ] - ], - - testGroup "folds" [ - testProperty "sf_foldl" sf_foldl, - testProperty "t_foldl" t_foldl, - testProperty "tl_foldl" tl_foldl, - testProperty "sf_foldl'" sf_foldl', - testProperty "t_foldl'" t_foldl', - testProperty "tl_foldl'" tl_foldl', - testProperty "sf_foldl1" sf_foldl1, - testProperty "t_foldl1" t_foldl1, - testProperty "tl_foldl1" tl_foldl1, - testProperty "t_foldl1'" t_foldl1', - testProperty "sf_foldl1'" sf_foldl1', - testProperty "tl_foldl1'" tl_foldl1', - testProperty "sf_foldr" sf_foldr, - testProperty "t_foldr" t_foldr, - testProperty "tl_foldr" tl_foldr, - testProperty "sf_foldr1" sf_foldr1, - testProperty "t_foldr1" t_foldr1, - testProperty "tl_foldr1" tl_foldr1, - - testGroup "special" [ - testProperty "s_concat_s" s_concat_s, - testProperty "sf_concat" sf_concat, - testProperty "t_concat" t_concat, - testProperty "tl_concat" tl_concat, - testProperty "sf_concatMap" sf_concatMap, - testProperty "t_concatMap" t_concatMap, - testProperty "tl_concatMap" tl_concatMap, - testProperty "sf_any" sf_any, - testProperty "t_any" t_any, - testProperty "tl_any" tl_any, - testProperty "sf_all" sf_all, - testProperty "t_all" t_all, - testProperty "tl_all" tl_all, - testProperty "sf_maximum" sf_maximum, - testProperty "t_maximum" t_maximum, - testProperty "tl_maximum" tl_maximum, - testProperty "sf_minimum" sf_minimum, - testProperty "t_minimum" t_minimum, - testProperty "tl_minimum" tl_minimum - ] - ], - - testGroup "construction" [ - testGroup "scans" [ - testProperty "sf_scanl" sf_scanl, - testProperty "t_scanl" t_scanl, - testProperty "tl_scanl" tl_scanl, - testProperty "t_scanl1" t_scanl1, - testProperty "tl_scanl1" tl_scanl1, - testProperty "t_scanr" t_scanr, - testProperty "tl_scanr" tl_scanr, - testProperty "t_scanr1" t_scanr1, - testProperty "tl_scanr1" tl_scanr1 - ], - - testGroup "mapAccum" [ - testProperty "t_mapAccumL" t_mapAccumL, - testProperty "tl_mapAccumL" tl_mapAccumL, - testProperty "t_mapAccumR" t_mapAccumR, - testProperty "tl_mapAccumR" tl_mapAccumR - ], - - testGroup "unfolds" [ - testProperty "tl_repeat" tl_repeat, - testProperty "s_replicate" s_replicate, - testProperty "t_replicate" t_replicate, - testProperty "tl_replicate" tl_replicate, - testProperty "tl_cycle" tl_cycle, - testProperty "tl_iterate" tl_iterate, - testProperty "t_unfoldr" t_unfoldr, - testProperty "tl_unfoldr" tl_unfoldr, - testProperty "t_unfoldrN" t_unfoldrN, - testProperty "tl_unfoldrN" tl_unfoldrN - ] - ], - - testGroup "substrings" [ - testGroup "breaking" [ - testProperty "s_take" s_take, - testProperty "s_take_s" s_take_s, - testProperty "sf_take" sf_take, - testProperty "t_take" t_take, - testProperty "t_takeEnd" t_takeEnd, - testProperty "tl_take" tl_take, - testProperty "tl_takeEnd" tl_takeEnd, - testProperty "s_drop" s_drop, - testProperty "s_drop_s" s_drop_s, - testProperty "sf_drop" sf_drop, - testProperty "t_drop" t_drop, - testProperty "t_dropEnd" t_dropEnd, - testProperty "tl_drop" tl_drop, - testProperty "tl_dropEnd" tl_dropEnd, - testProperty "s_take_drop" s_take_drop, - testProperty "s_take_drop_s" s_take_drop_s, - testProperty "s_takeWhile" s_takeWhile, - testProperty "s_takeWhile_s" s_takeWhile_s, - testProperty "sf_takeWhile" sf_takeWhile, - testProperty "t_takeWhile" t_takeWhile, - testProperty "tl_takeWhile" tl_takeWhile, - testProperty "t_takeWhileEnd" t_takeWhileEnd, - testProperty "tl_takeWhileEnd" tl_takeWhileEnd, - testProperty "sf_dropWhile" sf_dropWhile, - testProperty "s_dropWhile" s_dropWhile, - testProperty "s_dropWhile_s" s_dropWhile_s, - testProperty "t_dropWhile" t_dropWhile, - testProperty "tl_dropWhile" tl_dropWhile, - testProperty "t_dropWhileEnd" t_dropWhileEnd, - testProperty "tl_dropWhileEnd" tl_dropWhileEnd, - testProperty "t_dropAround" t_dropAround, - testProperty "tl_dropAround" tl_dropAround, - testProperty "t_stripStart" t_stripStart, - testProperty "tl_stripStart" tl_stripStart, - testProperty "t_stripEnd" t_stripEnd, - testProperty "tl_stripEnd" tl_stripEnd, - testProperty "t_strip" t_strip, - testProperty "tl_strip" tl_strip, - testProperty "t_splitAt" t_splitAt, - testProperty "tl_splitAt" tl_splitAt, - testProperty "t_span" t_span, - testProperty "tl_span" tl_span, - testProperty "t_breakOn_id" t_breakOn_id, - testProperty "tl_breakOn_id" tl_breakOn_id, - testProperty "t_breakOn_start" t_breakOn_start, - testProperty "tl_breakOn_start" tl_breakOn_start, - testProperty "t_breakOnEnd_end" t_breakOnEnd_end, - testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end, - testProperty "t_break" t_break, - testProperty "tl_break" tl_break, - testProperty "t_group" t_group, - testProperty "tl_group" tl_group, - testProperty "t_groupBy" t_groupBy, - testProperty "tl_groupBy" tl_groupBy, - testProperty "t_inits" t_inits, - testProperty "tl_inits" tl_inits, - testProperty "t_tails" t_tails, - testProperty "tl_tails" tl_tails - ], - - testGroup "breaking many" [ - testProperty "t_findAppendId" t_findAppendId, - testProperty "tl_findAppendId" tl_findAppendId, - testProperty "t_findContains" t_findContains, - testProperty "tl_findContains" tl_findContains, - testProperty "sl_filterCount" sl_filterCount, - testProperty "t_findCount" t_findCount, - testProperty "tl_findCount" tl_findCount, - testProperty "t_splitOn_split" t_splitOn_split, - testProperty "tl_splitOn_split" tl_splitOn_split, - testProperty "t_splitOn_i" t_splitOn_i, - testProperty "tl_splitOn_i" tl_splitOn_i, - testProperty "t_split" t_split, - testProperty "t_split_count" t_split_count, - testProperty "t_split_splitOn" t_split_splitOn, - testProperty "tl_split" tl_split, - testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths, - testProperty "t_chunksOf_length" t_chunksOf_length, - testProperty "tl_chunksOf" tl_chunksOf - ], - - testGroup "lines and words" [ - testProperty "t_lines" t_lines, - testProperty "tl_lines" tl_lines, - --testProperty "t_lines'" t_lines', - testProperty "t_words" t_words, - testProperty "tl_words" tl_words, - testProperty "t_unlines" t_unlines, - testProperty "tl_unlines" tl_unlines, - testProperty "t_unwords" t_unwords, - testProperty "tl_unwords" tl_unwords - ] - ], - - testGroup "predicates" [ - testProperty "s_isPrefixOf" s_isPrefixOf, - testProperty "sf_isPrefixOf" sf_isPrefixOf, - testProperty "t_isPrefixOf" t_isPrefixOf, - testProperty "tl_isPrefixOf" tl_isPrefixOf, - testProperty "t_isSuffixOf" t_isSuffixOf, - testProperty "tl_isSuffixOf" tl_isSuffixOf, - testProperty "t_isInfixOf" t_isInfixOf, - testProperty "tl_isInfixOf" tl_isInfixOf, - - testGroup "view" [ - testProperty "t_stripPrefix" t_stripPrefix, - testProperty "tl_stripPrefix" tl_stripPrefix, - testProperty "t_stripSuffix" t_stripSuffix, - testProperty "tl_stripSuffix" tl_stripSuffix, - testProperty "t_commonPrefixes" t_commonPrefixes, - testProperty "tl_commonPrefixes" tl_commonPrefixes - ] - ], - - testGroup "searching" [ - testProperty "sf_elem" sf_elem, - testProperty "sf_filter" sf_filter, - testProperty "t_filter" t_filter, - testProperty "tl_filter" tl_filter, - testProperty "sf_findBy" sf_findBy, - testProperty "t_find" t_find, - testProperty "tl_find" tl_find, - testProperty "t_partition" t_partition, - testProperty "tl_partition" tl_partition - ], - - testGroup "indexing" [ - testProperty "sf_index" sf_index, - testProperty "t_index" t_index, - testProperty "tl_index" tl_index, - testProperty "t_findIndex" t_findIndex, - testProperty "t_count" t_count, - testProperty "tl_count" tl_count, - testProperty "t_indices" t_indices, - testProperty "tl_indices" tl_indices, - testProperty "t_indices_occurs" t_indices_occurs - ], - - testGroup "zips" [ - testProperty "t_zip" t_zip, - testProperty "tl_zip" tl_zip, - testProperty "sf_zipWith" sf_zipWith, - testProperty "t_zipWith" t_zipWith, - testProperty "tl_zipWith" tl_zipWith - ], - - testGroup "regressions" [ - testProperty "s_filter_eq" s_filter_eq - ], - - testGroup "shifts" [ - testProperty "shiftL_Int" shiftL_Int, - testProperty "shiftL_Word16" shiftL_Word16, - testProperty "shiftL_Word32" shiftL_Word32, - testProperty "shiftR_Int" shiftR_Int, - testProperty "shiftR_Word16" shiftR_Word16, - testProperty "shiftR_Word32" shiftR_Word32 - ], - - testGroup "builder" [ - testProperty "tb_associative" tb_associative, - testGroup "decimal" [ - testProperty "tb_decimal_int" tb_decimal_int, - testProperty "tb_decimal_int8" tb_decimal_int8, - testProperty "tb_decimal_int16" tb_decimal_int16, - testProperty "tb_decimal_int32" tb_decimal_int32, - testProperty "tb_decimal_int64" tb_decimal_int64, - testProperty "tb_decimal_integer" tb_decimal_integer, - testProperty "tb_decimal_integer_big" tb_decimal_integer_big, - testProperty "tb_decimal_word" tb_decimal_word, - testProperty "tb_decimal_word8" tb_decimal_word8, - testProperty "tb_decimal_word16" tb_decimal_word16, - testProperty "tb_decimal_word32" tb_decimal_word32, - testProperty "tb_decimal_word64" tb_decimal_word64, - testProperty "tb_decimal_big_int" tb_decimal_big_int, - testProperty "tb_decimal_big_word" tb_decimal_big_word, - testProperty "tb_decimal_big_int64" tb_decimal_big_int64, - testProperty "tb_decimal_big_word64" tb_decimal_big_word64 - ], - testGroup "hexadecimal" [ - testProperty "tb_hexadecimal_int" tb_hexadecimal_int, - testProperty "tb_hexadecimal_int8" tb_hexadecimal_int8, - testProperty "tb_hexadecimal_int16" tb_hexadecimal_int16, - testProperty "tb_hexadecimal_int32" tb_hexadecimal_int32, - testProperty "tb_hexadecimal_int64" tb_hexadecimal_int64, - testProperty "tb_hexadecimal_integer" tb_hexadecimal_integer, - testProperty "tb_hexadecimal_word" tb_hexadecimal_word, - testProperty "tb_hexadecimal_word8" tb_hexadecimal_word8, - testProperty "tb_hexadecimal_word16" tb_hexadecimal_word16, - testProperty "tb_hexadecimal_word32" tb_hexadecimal_word32, - testProperty "tb_hexadecimal_word64" tb_hexadecimal_word64 - ], - testGroup "realfloat" [ - testProperty "tb_realfloat_double" tb_realfloat_double, - testProperty "tb_realfloat_float" tb_realfloat_float, - testProperty "tb_formatRealFloat_float" tb_formatRealFloat_float, - testProperty "tb_formatRealFloat_double" tb_formatRealFloat_double - ], - testProperty "tb_fromText" tb_fromText, - testProperty "tb_singleton" tb_singleton - ], - - testGroup "read" [ - testProperty "t_decimal" t_decimal, - testProperty "tl_decimal" tl_decimal, - testProperty "t_hexadecimal" t_hexadecimal, - testProperty "tl_hexadecimal" tl_hexadecimal, - testProperty "t_double" t_double, - testProperty "tl_double" tl_double, - testProperty "t_rational" t_rational, - testProperty "tl_rational" tl_rational - ], - - {- - testGroup "input-output" [ - testProperty "t_write_read" t_write_read, - testProperty "tl_write_read" tl_write_read, - testProperty "t_write_read_line" t_write_read_line, - testProperty "tl_write_read_line" tl_write_read_line - -- These tests are subject to I/O race conditions when run under - -- test-framework-quickcheck2. - -- testProperty "t_put_get" t_put_get - -- testProperty "tl_put_get" tl_put_get - ], - -} - - testGroup "lowlevel" [ - testProperty "t_dropWord16" t_dropWord16, - testProperty "t_takeWord16" t_takeWord16, - testProperty "t_take_drop_16" t_take_drop_16, - testProperty "t_use_from" t_use_from, - testProperty "t_copy" t_copy - ], - - testGroup "mul" Mul.tests +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +module Tests.Properties + ( + tests + ) where + +import Test.Tasty (TestTree, testGroup) +import Tests.Properties.Basics (testBasics) +import Tests.Properties.Builder (testBuilder) +import Tests.Properties.Folds (testFolds) +import Tests.Properties.LowLevel (testLowLevel) +import Tests.Properties.Instances (testInstances) +import Tests.Properties.Substrings (testSubstrings) +import Tests.Properties.Read (testRead) +import Tests.Properties.Text (testText) +import Tests.Properties.Transcoding (testTranscoding) +import Tests.Properties.Validate (testValidate) +import Tests.Properties.CornerCases (testCornerCases) + +tests :: TestTree +tests = + testGroup "Properties" [ + testTranscoding, + testInstances, + testBasics, + testFolds, + testText, + testSubstrings, + testBuilder, + testLowLevel, + testRead, + testCornerCases, + testValidate ] diff --git a/tests/Tests/Properties/Basics.hs b/tests/Tests/Properties/Basics.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Basics.hs @@ -0,0 +1,157 @@ +-- | Test basic text functions + +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -Wno-missing-signatures #-} +{-# OPTIONS_GHC -Wno-warnings-deprecations #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.Properties.Basics + ( testBasics + ) where + +import Control.Arrow (first, second) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty, applyFun) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Lazy as TL + +s_cons x = (x:) `eqP` (unpackS . S.cons x) +s_cons_s x = (x:) `eqP` (unpackS . S.unstream . S.cons x) +sf_cons (applyFun -> p) x + = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p) +t_cons x = (x:) `eqP` (unpackS . T.cons x) +tl_cons x = (x:) `eqP` (unpackS . TL.cons x) +t_length_cons x = (L.length . (x:)) `eqP` (T.length . T.cons x) +tl_length_cons x = (L.genericLength . (x:)) `eqP` (TL.length . TL.cons x) + +s_snoc x = (++ [x]) `eqP` (unpackS . flip S.snoc x) +t_snoc x = (++ [x]) `eqP` (unpackS . flip T.snoc x) +tl_snoc x = (++ [x]) `eqP` (unpackS . flip TL.snoc x) +t_length_snoc x = (L.length . (++ [x])) `eqP` (T.length . flip T.snoc x) +tl_length_snoc x = (L.genericLength . (++ [x])) `eqP` (TL.length . flip TL.snoc x) + +s_append s = (s++) `eqP` (unpackS . S.append (S.streamList s)) +s_append_s s = (s++) `eqP` + (unpackS . S.unstream . S.append (S.streamList s)) +sf_append (applyFun -> p) s + = (L.filter p s++) `eqP` + (unpackS . S.append (S.filter p $ S.streamList s)) +t_append s = (s++) `eqP` (unpackS . T.append (packS s)) + +uncons (x:xs) = Just (x,xs) +uncons _ = Nothing + +s_uncons = uncons `eqP` (fmap (second unpackS) . S.uncons) +sf_uncons (applyFun -> p) + = (uncons . L.filter p) `eqP` + (fmap (second unpackS) . S.uncons . S.filter p) +t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons) +tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons) + +unsnoc xs@(_:_) = Just (init xs, last xs) +unsnoc [] = Nothing + +t_unsnoc = unsnoc `eqP` (fmap (first unpackS) . T.unsnoc) +tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc) + +s_head = head `eqP` S.head +sf_head (applyFun -> p) = (head . L.filter p) `eqP` (S.head . S.filter p) +t_head = head `eqP` T.head +tl_head = head `eqP` TL.head +s_last = last `eqP` S.last +sf_last (applyFun -> p) = (last . L.filter p) `eqP` (S.last . S.filter p) +t_last = last `eqP` T.last +tl_last = last `eqP` TL.last +s_tail = tail `eqP` (unpackS . S.tail) +s_tail_s = tail `eqP` (unpackS . S.unstream . S.tail) +sf_tail (applyFun -> p) = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p) +t_tail = tail `eqP` (unpackS . T.tail) +tl_tail = tail `eqP` (unpackS . TL.tail) +s_init = init `eqP` (unpackS . S.init) +s_init_s = init `eqP` (unpackS . S.unstream . S.init) +sf_init (applyFun -> p) = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p) +t_init = init `eqP` (unpackS . T.init) +tl_init = init `eqP` (unpackS . TL.init) +s_null = null `eqP` S.null +sf_null (applyFun -> p) = (null . L.filter p) `eqP` (S.null . S.filter p) +t_null = null `eqP` T.null +tl_null = null `eqP` TL.null +s_length = length `eqP` S.length +sf_length (applyFun -> p) = (length . L.filter p) `eqP` (S.length . S.filter p) +sl_length = (fromIntegral . length) `eqP` SL.length +t_length = length `eqP` T.length +tl_length = L.genericLength `eqP` TL.length +t_compareLength t = (compare (T.length t)) `eq` T.compareLength t +tl_compareLength t= (compare (TL.length t)) `eq` TL.compareLength t + +-- Regression tests. +s_filter_eq s = S.filter p t == S.streamList (filter p s) + where p = (/= S.last t) + t = S.streamList s + +testBasics :: TestTree +testBasics = + testGroup "basics" [ + testProperty "s_cons" s_cons, + testProperty "s_cons_s" s_cons_s, + testProperty "sf_cons" sf_cons, + testProperty "t_cons" t_cons, + testProperty "tl_cons" tl_cons, + testProperty "t_length_cons" t_length_cons, + testProperty "tl_length_cons" tl_length_cons, + testProperty "s_snoc" s_snoc, + testProperty "t_snoc" t_snoc, + testProperty "tl_snoc" tl_snoc, + testProperty "t_length_snoc" t_length_snoc, + testProperty "tl_length_snoc" tl_length_snoc, + testProperty "s_append" s_append, + testProperty "s_append_s" s_append_s, + testProperty "sf_append" sf_append, + testProperty "t_append" t_append, + testProperty "s_uncons" s_uncons, + testProperty "sf_uncons" sf_uncons, + testProperty "t_uncons" t_uncons, + testProperty "tl_uncons" tl_uncons, + testProperty "t_unsnoc" t_unsnoc, + testProperty "tl_unsnoc" tl_unsnoc, + testProperty "s_head" s_head, + testProperty "sf_head" sf_head, + testProperty "t_head" t_head, + testProperty "tl_head" tl_head, + testProperty "s_last" s_last, + testProperty "sf_last" sf_last, + testProperty "t_last" t_last, + testProperty "tl_last" tl_last, + testProperty "s_tail" s_tail, + testProperty "s_tail_s" s_tail_s, + testProperty "sf_tail" sf_tail, + testProperty "t_tail" t_tail, + testProperty "tl_tail" tl_tail, + testProperty "s_init" s_init, + testProperty "s_init_s" s_init_s, + testProperty "sf_init" sf_init, + testProperty "t_init" t_init, + testProperty "tl_init" tl_init, + testProperty "s_null" s_null, + testProperty "sf_null" sf_null, + testProperty "t_null" t_null, + testProperty "tl_null" tl_null, + testProperty "s_length" s_length, + testProperty "sf_length" sf_length, + testProperty "sl_length" sl_length, + testProperty "t_length" t_length, + testProperty "tl_length" tl_length, + testProperty "t_compareLength" t_compareLength, + testProperty "tl_compareLength" tl_compareLength, + + testGroup "regressions" [ + testProperty "s_filter_eq" s_filter_eq + ] + ] diff --git a/tests/Tests/Properties/Builder.hs b/tests/Tests/Properties/Builder.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Builder.hs @@ -0,0 +1,149 @@ +-- | Test @Builder@ + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +module Tests.Properties.Builder + ( testBuilder + ) where + +import Data.Int (Int8, Int16, Int32, Int64) +import Data.Word +import Numeric (showEFloat, showFFloat, showGFloat, showHex) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Text.Lazy.Builder.Int as TB +import qualified Data.Text.Lazy.Builder.RealFloat as TB + +-- Builder. + +tb_singleton = id `eqP` (unpackS . TB.toLazyText . mconcat . map TB.singleton) +tb_fromString = id `eq` (TL.unpack . TB.toLazyText . TB.fromString) +tb_fromText = id `eqP` (unpackS . TB.toLazyText . TB.fromText) +tb_fromStrings = L.concat `eq` (TL.unpack . TB.toLazyText . mconcat . map TB.fromString) +tb_fromTexts = L.concat `eq` (unpackS . TB.toLazyText . mconcat . map (TB.fromText . packS)) + +tb_associative s1 s2 s3 = + TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) === + TB.toLazyText ((b1 `mappend` b2) `mappend` b3) + where b1 = TB.fromText (packS s1) + b2 = TB.fromText (packS s2) + b3 = TB.fromText (packS s3) + +-- Numeric builder stuff. + +tb_decimal :: (Integral a, Show a) => a -> Property +tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show) + +tb_decimal_integer (a::Integer) = tb_decimal a +tb_decimal_integer_big (Big a) = tb_decimal a +tb_decimal_int (a::Int) = tb_decimal a +tb_decimal_int8 (a::Int8) = tb_decimal a +tb_decimal_int16 (a::Int16) = tb_decimal a +tb_decimal_int32 (a::Int32) = tb_decimal a +tb_decimal_int64 (a::Int64) = tb_decimal a +tb_decimal_word (a::Word) = tb_decimal a +tb_decimal_word8 (a::Word8) = tb_decimal a +tb_decimal_word16 (a::Word16) = tb_decimal a +tb_decimal_word32 (a::Word32) = tb_decimal a +tb_decimal_word64 (a::Word64) = tb_decimal a + +tb_decimal_big_int (Large (a::Int)) = tb_decimal a +tb_decimal_big_int64 (Large (a::Int64)) = tb_decimal a +tb_decimal_big_word (Large (a::Word)) = tb_decimal a +tb_decimal_big_word64 (Large (a::Word64)) = tb_decimal a + +tb_hex :: (Integral a, Show a) => a -> Property +tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "") + +tb_hexadecimal_integer (a::Integer) = tb_hex a +tb_hexadecimal_int (a::Int) = tb_hex a +tb_hexadecimal_int8 (a::Int8) = tb_hex a +tb_hexadecimal_int16 (a::Int16) = tb_hex a +tb_hexadecimal_int32 (a::Int32) = tb_hex a +tb_hexadecimal_int64 (a::Int64) = tb_hex a +tb_hexadecimal_word (a::Word) = tb_hex a +tb_hexadecimal_word8 (a::Word8) = tb_hex a +tb_hexadecimal_word16 (a::Word16) = tb_hex a +tb_hexadecimal_word32 (a::Word32) = tb_hex a +tb_hexadecimal_word64 (a::Word64) = tb_hex a + +tb_realfloat :: (RealFloat a, Show a) => a -> Property +tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show) + +tb_realfloat_float (a::Float) = tb_realfloat a +tb_realfloat_double (a::Double) = tb_realfloat a + +showFloat :: (RealFloat a) => TB.FPFormat -> Maybe Int -> a -> ShowS +showFloat TB.Exponent (Just 0) = showEFloat (Just 1) -- see gh-231 +showFloat TB.Exponent p = showEFloat p +showFloat TB.Fixed p = showFFloat p +showFloat TB.Generic p = showGFloat p + +tb_formatRealFloat :: (RealFloat a, Show a) => + a -> TB.FPFormat -> Precision a -> Property +tb_formatRealFloat a fmt prec = cond ==> + TB.formatRealFloat fmt p a === + TB.fromString (showFloat fmt p a "") + where p = precision a prec + cond = case (p,fmt) of +#if MIN_VERSION_base(4,12,0) + (Just 0, TB.Generic) -> False -- skipping due to gh-231 +#endif + _ -> True + +tb_formatRealFloat_float (a::Float) = tb_formatRealFloat a +tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a + +testBuilder :: TestTree +testBuilder = + testGroup "builder" [ + testProperty "tb_singleton" tb_singleton, + testProperty "tb_fromString" tb_fromString, + testProperty "tb_fromText" tb_fromText, + testProperty "tb_fromStrings" tb_fromStrings, + testProperty "tb_fromTexts" tb_fromTexts, + testProperty "tb_associative" tb_associative, + testGroup "decimal" [ + testProperty "tb_decimal_int" tb_decimal_int, + testProperty "tb_decimal_int8" tb_decimal_int8, + testProperty "tb_decimal_int16" tb_decimal_int16, + testProperty "tb_decimal_int32" tb_decimal_int32, + testProperty "tb_decimal_int64" tb_decimal_int64, + testProperty "tb_decimal_integer" tb_decimal_integer, + testProperty "tb_decimal_integer_big" tb_decimal_integer_big, + testProperty "tb_decimal_word" tb_decimal_word, + testProperty "tb_decimal_word8" tb_decimal_word8, + testProperty "tb_decimal_word16" tb_decimal_word16, + testProperty "tb_decimal_word32" tb_decimal_word32, + testProperty "tb_decimal_word64" tb_decimal_word64, + testProperty "tb_decimal_big_int" tb_decimal_big_int, + testProperty "tb_decimal_big_word" tb_decimal_big_word, + testProperty "tb_decimal_big_int64" tb_decimal_big_int64, + testProperty "tb_decimal_big_word64" tb_decimal_big_word64 + ], + testGroup "hexadecimal" [ + testProperty "tb_hexadecimal_int" tb_hexadecimal_int, + testProperty "tb_hexadecimal_int8" tb_hexadecimal_int8, + testProperty "tb_hexadecimal_int16" tb_hexadecimal_int16, + testProperty "tb_hexadecimal_int32" tb_hexadecimal_int32, + testProperty "tb_hexadecimal_int64" tb_hexadecimal_int64, + testProperty "tb_hexadecimal_integer" tb_hexadecimal_integer, + testProperty "tb_hexadecimal_word" tb_hexadecimal_word, + testProperty "tb_hexadecimal_word8" tb_hexadecimal_word8, + testProperty "tb_hexadecimal_word16" tb_hexadecimal_word16, + testProperty "tb_hexadecimal_word32" tb_hexadecimal_word32, + testProperty "tb_hexadecimal_word64" tb_hexadecimal_word64 + ], + testGroup "realfloat" [ + testProperty "tb_realfloat_double" tb_realfloat_double, + testProperty "tb_realfloat_float" tb_realfloat_float, + testProperty "tb_formatRealFloat_float" tb_formatRealFloat_float, + testProperty "tb_formatRealFloat_double" tb_formatRealFloat_double + ] + ] diff --git a/tests/Tests/Properties/CornerCases.hs b/tests/Tests/Properties/CornerCases.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/CornerCases.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +-- | Check that the definitions that are partial crash in the expected ways or +-- return sensible defaults. +module Tests.Properties.CornerCases (testCornerCases) where + +import Control.Exception +import Data.Either +import Data.Semigroup +import Data.Text +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils () + +testCornerCases :: TestTree +testCornerCases = + testGroup + "corner cases" + [ testGroup + "stimes" + $ let specimen = stimes :: Integer -> Text -> Text + in [ testProperty + "given a negative number, evaluate to error call" + $ \(Negative number) text -> + (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $ + specimen + (fromIntegral (number :: Int)) + text + , testProperty + "given a number that does not fit into Int, evaluate to error call" + $ \(NonNegative number) text -> + (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $ + specimen + (fromIntegral (number :: Int) + fromIntegral (maxBound :: Int) + 1) + text + ] + ] diff --git a/tests/Tests/Properties/Folds.hs b/tests/Tests/Properties/Folds.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Folds.hs @@ -0,0 +1,357 @@ +-- | Test folds, scans, and unfolds + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.Folds + ( testFolds + ) where + +import Control.Arrow (second) +import Control.Exception (ErrorCall, evaluate, try) +import Data.Functor.Identity (Identity(..)) +import Control.Monad.Trans.State (runState, state) +import Data.Word (Word8, Word16) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertFailure, assertBool) +import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Lazy as TL +import qualified Data.Char as Char + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectTest, (==~)) +import GHC.Exts (inline) +#endif + +-- Folds + +sf_foldl (applyFun -> p) (applyFun2 -> f) z = + (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) + where _types = f :: Char -> Char -> Char +tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) + where _types = f :: Char -> Char -> Char +sf_foldl' (applyFun -> p) (applyFun2 -> f) z = + (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z + where _types = f :: Char -> Char -> Char +tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z + where _types = f :: Char -> Char -> Char +sf_foldl1 (applyFun -> p) (applyFun2 -> f) = + (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' (applyFun -> p) (applyFun2 -> f) = + (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr (applyFun -> p) (applyFun2 -> f) z = + (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z + where _types = f :: Char -> Char -> Char +t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z + where _types = f :: Char -> Char -> Char +tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z + where _types = f :: Char -> Char -> Char +sf_foldr1 (applyFun -> p) (applyFun2 -> f) = + (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) +t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f + +-- Distinguish foldl/foldr from foldl'/foldr' + +fold_apart :: IO () +fold_apart = do + ok (T.foldr f () (T.pack "az")) + ko (T.foldr' f () (T.pack "az")) + ok (T.foldl (flip f) () (T.pack "za")) + ko (T.foldl' (flip f) () (T.pack "za")) + where + f c _ = if c == 'z' then error "catchme" else () + ok = evaluate + ko t = do + x <- try (evaluate t) + case x :: Either ErrorCall () of + Left _ -> pure () + Right _ -> assertFailure "test should have failed but didn't" + +-- Special folds + +s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) +sf_concat (applyFun -> p) + = (L.concat . map (L.filter p) . unSqrt) `eq` + (unpackS . S.concat . map (S.filter p . packS) . unSqrt) +t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) +tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) +sf_concatMap (applyFun -> p) (applyFun -> f) = + (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) +t_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) +tl_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) +sf_any (applyFun -> q) (applyFun -> p) + = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any (applyFun -> p) + = L.any p `eqP` T.any p +tl_any (applyFun -> p) + = L.any p `eqP` TL.any p +sf_all (applyFun -> q) (applyFun -> p) + = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all (applyFun -> p) + = L.all p `eqP` T.all p +tl_all (applyFun -> p) + = L.all p `eqP` TL.all p +sf_maximum (applyFun -> p) + = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +t_maximum = L.maximum `eqP` T.maximum +tl_maximum = L.maximum `eqP` TL.maximum +sf_minimum (applyFun -> p) + = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +t_minimum = L.minimum `eqP` T.minimum +tl_minimum = L.minimum `eqP` TL.minimum +t_isAscii = L.all Char.isAscii `eqP` T.isAscii +tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii + +-- Scans + +sf_scanl (applyFun -> p) (applyFun2 -> f) z = + (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) + +t_scanl_is_safe = let c = '\55296' in + T.scanl undefined c mempty === T.singleton c +tl_scanl_is_safe = let c = '\55296' in + TL.scanl undefined c mempty === TL.singleton c +t_scanr_is_safe = let c = '\55296' in + T.scanr undefined c mempty === T.singleton c +tl_scanr_is_safe = let c = '\55296' in + TL.scanr undefined c mempty === TL.singleton c + +t_mapAccumL_char c t = + snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumL_char c t = + snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +t_mapAccumR_char c t = + snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumR_char c t = + snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) + +-- Unfolds + +tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) + +s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) + +t_replicate_char (Small n) c = + L.replicate n c === T.unpack (T.replicate n (T.singleton c)) +tl_replicate_char (Small n) c = + L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) +t_length_replicate_char (Small n) c = + L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) +tl_length_replicate_char (Small n) c = + L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) + +t_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) +tl_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) +t_length_replicate (Small n) = + (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) +tl_length_replicate (Small n) = + (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) + +tl_cycle n = (L.take m . L.cycle) `eq` + (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) + where m = fromIntegral (n :: Word8) + +tl_iterate (applyFun -> f) n + = (L.take m . L.iterate f) `eq` + (unpackS . TL.take (fromIntegral m) . TL.iterate f) + where m = fromIntegral (n :: Word8) + +unf :: Int -> Char -> Maybe (Char, Char) +unf n c | fromEnum c * 100 > n = Nothing + | otherwise = Just (c, succ c) + +t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . T.unfoldrN i (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) +tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) + +-- Monadic folds + +-- Parametric polymorphism allows us to only test foldlM' specialized to +-- one function in the state monad (called @logger@ in the following tests) +-- that just logs the arguments it was applied to and produces a fresh +-- accumulator. That alone determines the general behavior of foldlM' with an +-- arbitrary function in any monad. +-- Reference: "Testing Polymorphic Properties" by Bernardy et al. +-- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf + +t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order +tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order + +#ifdef MIN_VERSION_tasty_inspection_testing +-- As a sanity check for performance, the simplified Core +-- foldlM' specialized to Identity is the same as foldl'. + +_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x + +_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl' = inline S.foldl' +#endif + +isAscii_border :: IO () +isAscii_border = do + let text = T.drop 2 $ T.pack "XX1234五" + assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text + +testFolds :: TestTree +testFolds = + testGroup "folds-unfolds" [ + testGroup "folds" [ + testProperty "sf_foldl" sf_foldl, + testProperty "t_foldl" t_foldl, + testProperty "tl_foldl" tl_foldl, + testProperty "sf_foldl'" sf_foldl', + testProperty "t_foldl'" t_foldl', + testProperty "tl_foldl'" tl_foldl', + testProperty "sf_foldl1" sf_foldl1, + testProperty "t_foldl1" t_foldl1, + testProperty "tl_foldl1" tl_foldl1, + testProperty "t_foldl1'" t_foldl1', + testProperty "sf_foldl1'" sf_foldl1', + testProperty "tl_foldl1'" tl_foldl1', + testProperty "sf_foldr" sf_foldr, + testProperty "t_foldr" t_foldr, + testProperty "t_foldr'" t_foldr', + testProperty "tl_foldr" tl_foldr, + testProperty "sf_foldr1" sf_foldr1, + testProperty "t_foldr1" t_foldr1, + testProperty "tl_foldr1" tl_foldr1, + testProperty "t_foldlM'" t_foldlM', + testProperty "tl_foldlM'" tl_foldlM', +#ifdef MIN_VERSION_tasty_inspection_testing + let _unused = ['_S_foldl'_from_foldlM', '_S_foldl'] in + $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')), +#endif + testCase "fold_apart" fold_apart, + + testGroup "special" [ + testProperty "s_concat_s" s_concat_s, + testProperty "sf_concat" sf_concat, + testProperty "t_concat" t_concat, + testProperty "tl_concat" tl_concat, + testProperty "sf_concatMap" sf_concatMap, + testProperty "t_concatMap" t_concatMap, + testProperty "tl_concatMap" tl_concatMap, + testProperty "sf_any" sf_any, + testProperty "t_any" t_any, + testProperty "tl_any" tl_any, + testProperty "sf_all" sf_all, + testProperty "t_all" t_all, + testProperty "tl_all" tl_all, + testProperty "sf_maximum" sf_maximum, + testProperty "t_maximum" t_maximum, + testProperty "tl_maximum" tl_maximum, + testProperty "sf_minimum" sf_minimum, + testProperty "t_minimum" t_minimum, + testProperty "tl_minimum" tl_minimum, + testProperty "t_isAscii " t_isAscii, + testProperty "tl_isAscii " tl_isAscii, + testCase "isAscii_border" isAscii_border + ] + ], + + testGroup "scans" [ + testProperty "sf_scanl" sf_scanl, + testProperty "t_scanl" t_scanl, + testProperty "tl_scanl" tl_scanl, + testProperty "t_scanl1" t_scanl1, + testProperty "tl_scanl1" tl_scanl1, + testProperty "t_scanr" t_scanr, + testProperty "tl_scanr" tl_scanr, + testProperty "t_scanr1" t_scanr1, + testProperty "tl_scanr1" tl_scanr1, + + testProperty "t_scanl_is_safe" t_scanl_is_safe, + testProperty "tl_scanl_is_safe" tl_scanl_is_safe, + testProperty "t_scanr_is_safe" t_scanr_is_safe, + testProperty "tl_scanr_is_safe" tl_scanr_is_safe + ], + + testGroup "mapAccum" [ + testProperty "t_mapAccumL_char" t_mapAccumL_char, + testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL_char" tl_mapAccumL_char, + testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR_char" t_mapAccumR_char, + testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR_char" tl_mapAccumR_char, + testProperty "tl_mapAccumR" tl_mapAccumR + ], + + testGroup "unfolds" [ + testProperty "tl_cycle" tl_cycle, + testProperty "tl_iterate" tl_iterate, + testProperty "t_unfoldr" t_unfoldr, + testProperty "tl_unfoldr" tl_unfoldr, + testProperty "t_unfoldrN" t_unfoldrN, + testProperty "tl_unfoldrN" tl_unfoldrN + ], + + testGroup "replicate" [ + testProperty "tl_repeat" tl_repeat, + testProperty "s_replicate" s_replicate, + testProperty "t_replicate_char" t_replicate_char, + testProperty "tl_replicate_char" tl_replicate_char, + testProperty "t_length_replicate_char" t_length_replicate_char, + testProperty "tl_length_replicate_char" tl_length_replicate_char, + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_length_replicate" t_length_replicate, + testProperty "tl_length_replicate" tl_length_replicate + ] + + ] diff --git a/tests/Tests/Properties/Instances.hs b/tests/Tests/Properties/Instances.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Instances.hs @@ -0,0 +1,91 @@ +-- | Test instances + +{-# LANGUAGE ViewPatterns #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +module Tests.Properties.Instances + ( testInstances + ) where + +import Data.Binary (encode, decodeOrFail) +import Data.Semigroup +import Data.String (IsString(fromString)) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Lazy as TL + +s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList) + where _types = s :: String +sf_Eq (applyFun -> p) s = + ((L.filter p s==) . L.filter p) `eq` + (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList) +t_Eq s = (s==) `eq` ((T.pack s==) . T.pack) +tl_Eq s = (s==) `eq` ((TL.pack s==) . TL.pack) +s_Ord s = (compare s) `eq` (compare (S.streamList s) . S.streamList) + where _types = s :: String +sf_Ord (applyFun -> p) s = + ((compare $ L.filter p s) . L.filter p) `eq` + (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList) +t_Ord s = (compare s) `eq` (compare (T.pack s) . T.pack) +tl_Ord s = (compare s) `eq` (compare (TL.pack s) . TL.pack) +t_Read = id `eq` (T.unpack . read . show) +tl_Read = id `eq` (TL.unpack . read . show) +t_Show = show `eq` (show . T.pack) +tl_Show = show `eq` (show . TL.pack) +t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) +tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s)) +t_stimes = \ number -> eq + ((stimes :: Int -> String -> String) number . unSqrt) + (unpackS . (stimes :: Int -> T.Text -> T.Text) number . T.pack . unSqrt) +tl_stimes = \ number -> eq + ((stimes :: Int -> String -> String) number . unSqrt) + (unpackS . (stimes :: Int -> TL.Text -> TL.Text) number . TL.pack . unSqrt) +t_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt) +tl_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt) +t_mempty = mempty === (unpackS (mempty :: T.Text)) +tl_mempty = mempty === (unpackS (mempty :: TL.Text)) +t_IsString = fromString `eqP` (T.unpack . fromString) +tl_IsString = fromString `eqP` (TL.unpack . fromString) + +t_Binary s = + case decodeOrFail . encode $ (s :: T.Text) of + Left _ -> counterexample (show (T.unpack s)) (property False) + Right (_, _, s') -> s === s' + +tl_Binary s = + case decodeOrFail . encode $ (s :: TL.Text) of + Left _ -> counterexample (show (TL.unpack s)) (property False) + Right (_, _, s') -> s === s' + +testInstances :: TestTree +testInstances = + testGroup "instances" [ + testProperty "s_Eq" s_Eq, + testProperty "sf_Eq" sf_Eq, + testProperty "t_Eq" t_Eq, + testProperty "tl_Eq" tl_Eq, + testProperty "s_Ord" s_Ord, + testProperty "sf_Ord" sf_Ord, + testProperty "t_Ord" t_Ord, + testProperty "tl_Ord" tl_Ord, + testProperty "t_Read" t_Read, + testProperty "tl_Read" tl_Read, + testProperty "t_Show" t_Show, + testProperty "tl_Show" tl_Show, + testProperty "t_mappend" t_mappend, + testProperty "tl_mappend" tl_mappend, + testProperty "t_stimes" t_stimes, + testProperty "tl_stimes" tl_stimes, + testProperty "t_mconcat" t_mconcat, + testProperty "tl_mconcat" tl_mconcat, + testProperty "t_mempty" t_mempty, + testProperty "tl_mempty" tl_mempty, + testProperty "t_IsString" t_IsString, + testProperty "tl_IsString" tl_IsString, + testProperty "t_Binary" t_Binary, + testProperty "tl_Binary" tl_Binary + ] diff --git a/tests/Tests/Properties/LowLevel.hs b/tests/Tests/Properties/LowLevel.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/LowLevel.hs @@ -0,0 +1,176 @@ +-- | Test low-level operations + +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.LowLevel (testLowLevel) where + +import Prelude hiding (head, tail) +import Control.Applicative ((<$>), pure) +import Control.Exception as E (SomeException, catch, evaluate) +import Data.Functor.Identity (Identity(..)) +import Data.Int (Int32, Int64) +import Data.Text.Foreign +import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) +import Data.Word (Word8, Word16, Word32) +import System.IO.Unsafe (unsafePerformIO) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty.QuickCheck (testProperty) +import Test.QuickCheck hiding ((.&.)) +import Tests.QuickCheckUtils +import Tests.Utils +import qualified Data.Text as T +import qualified Data.Text.Foreign as T +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.IO.Utf8 as TU +import qualified System.IO as IO + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified GHC.CString as GHC +#endif + +mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a +mulRef a b + | ab < bot || ab > top = Nothing + | otherwise = Just (fromIntegral ab) + where ab = fromIntegral a * fromIntegral b + top = fromIntegral (maxBound `asTypeOf` a) :: Integer + bot = fromIntegral (minBound `asTypeOf` a) :: Integer + +eval :: (a -> b -> c) -> a -> b -> Maybe c +eval f a b = unsafePerformIO $ + (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) + +t_mul32 :: Int32 -> Int32 -> Property +t_mul32 a b = mulRef a b === eval mul32 a b + +t_mul64 :: Int64 -> Int64 -> Property +t_mul64 a b = mulRef a b === eval mul64 a b + +t_mul :: Int -> Int -> Property +t_mul a b = mulRef a b === eval mul a b + +-- Misc. + +t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t +t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t +t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t +t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr +t_use_from0 t = ioProperty $ do + let t' = t `T.snoc` '\0' + (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0) + +t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do + roundTrip <- T.withCString t T.peekCString + assertEqual "cstring" t roundTrip + +t_peek_cstring_len t = ioProperty $ do + roundTrip <- T.withCStringLen t T.peekCStringLen + assertEqual "cstring_len" t roundTrip + +t_copy t = T.copy t === t + +t_literal_length1 = assertEqual xs (length xs) byteLen + where + xs = "\0\1\0\1\0" + Text _ _ byteLen = T.pack xs +t_literal_length2 = assertEqual xs (length xs) byteLen + where + xs = "\1\2\3\4\5" + Text _ _ byteLen = T.pack xs +t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) + where + ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" + xs = map safe ys + +#ifdef MIN_VERSION_tasty_inspection_testing +t_literal_foo :: Text +t_literal_foo = T.pack "foo" +#endif + +-- Input and output. + +-- t_put_get = write_read T.unlines T.filter put get +-- where put h = withRedirect h IO.stdout . T.putStr +-- get h = withRedirect h IO.stdin T.getContents +-- tl_put_get = write_read TL.unlines TL.filter put get +-- where put h = withRedirect h IO.stdout . TL.putStr +-- get h = withRedirect h IO.stdin TL.getContents + +inputOutput :: TestTree +inputOutput = testGroup "input-output" [ + testProperty "t_write_read" $ write_read arbitrary shrink (T.replace "\n" "\r\n") T.hPutStr T.hGetContents, + testProperty "tl_write_read" $ write_read arbitrary shrink (TL.replace "\n" "\r\n") TL.hPutStr TL.hGetContents, + testProperty "t_write_read_line" $ write_read genTLine shrinkTLine (`T.append` "\r") T.hPutStrLn T.hGetLine, + testProperty "tl_write_read_line" $ write_read genTLLine shrinkTLLine (`TL.append` "\r") TL.hPutStrLn TL.hGetLine, + -- Note: Data.Text.IO.Utf8 does NO newline translation + testProperty "utf8_write_read" $ write_read arbitrary shrink id TU.hPutStr TU.hGetContents, + testProperty "utf8_write_read_line" $ write_read genTLine shrinkTLine id TU.hPutStrLn TU.hGetLine + -- These tests are subject to I/O race conditions + -- testProperty "t_put_get" t_put_get, + -- testProperty "tl_put_get" tl_put_get + ] + +genTLine :: Gen T.Text +genTLine = T.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +genTLLine :: Gen TL.Text +genTLLine = TL.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +shrinkTLine :: T.Text -> [T.Text] +shrinkTLine = filter (T.all (/= '\n')) . shrink + +shrinkTLLine :: TL.Text -> [TL.Text] +shrinkTLLine = filter (TL.all (/= '\n')) . shrink + +testLowLevel :: TestTree +testLowLevel = + testGroup "lowlevel" [ + testGroup "mul" [ + testProperty "t_mul" t_mul, + testProperty "t_mul32" t_mul32, + testProperty "t_mul64" t_mul64 + ], + + testGroup "misc" [ + testProperty "t_dropWord8" t_dropWord8, + testProperty "t_takeWord8" t_takeWord8, + testProperty "t_take_drop_8" t_take_drop_8, + testProperty "t_use_from" t_use_from, + testProperty "t_use_from0" t_use_from0, + testProperty "t_copy" t_copy, + testProperty "t_peek_cstring" t_peek_cstring, + testProperty "t_peek_cstring_len" t_peek_cstring_len, + testCase "t_literal_length1" t_literal_length1, + testCase "t_literal_length2" t_literal_length2, + testCase "t_literal_surrogates" t_literal_surrogates +#ifdef MIN_VERSION_tasty_inspection_testing + -- Hack to force GHC to keep the binding around until this is fixed: https://gitlab.haskell.org/ghc/ghc/-/issues/26436 + , let _unused = 't_literal_foo in + $(inspectObligations + [ (`hasNoTypes` [''Char, ''[]]) + , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) + , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) + , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) + ] + 't_literal_foo) +#endif + ], + + inputOutput + ] diff --git a/tests/Tests/Properties/Mul.hs b/tests/Tests/Properties/Mul.hs deleted file mode 100644 --- a/tests/Tests/Properties/Mul.hs +++ /dev/null @@ -1,40 +0,0 @@ -{-# LANGUAGE ScopedTypeVariables #-} - -module Tests.Properties.Mul (tests) where - -import Control.Applicative ((<$>), pure) -import Control.Exception as E (SomeException, catch, evaluate) -import Data.Int (Int32, Int64) -import Data.Text.Internal (mul, mul32, mul64) -import System.IO.Unsafe (unsafePerformIO) -import Test.Framework (Test) -import Test.Framework.Providers.QuickCheck2 (testProperty) -import Test.QuickCheck hiding ((.&.)) - -mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a -mulRef a b - | ab < bot || ab > top = Nothing - | otherwise = Just (fromIntegral ab) - where ab = fromIntegral a * fromIntegral b - top = fromIntegral (maxBound `asTypeOf` a) :: Integer - bot = fromIntegral (minBound `asTypeOf` a) :: Integer - -eval :: (a -> b -> c) -> a -> b -> Maybe c -eval f a b = unsafePerformIO $ - (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) - -t_mul32 :: Int32 -> Int32 -> Property -t_mul32 a b = mulRef a b === eval mul32 a b - -t_mul64 :: Int64 -> Int64 -> Property -t_mul64 a b = mulRef a b === eval mul64 a b - -t_mul :: Int -> Int -> Property -t_mul a b = mulRef a b === eval mul a b - -tests :: [Test] -tests = [ - testProperty "t_mul" t_mul - , testProperty "t_mul32" t_mul32 - , testProperty "t_mul64" t_mul64 - ] diff --git a/tests/Tests/Properties/Read.hs b/tests/Tests/Properties/Read.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Read.hs @@ -0,0 +1,111 @@ +-- | Tests for readers + +{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +module Tests.Properties.Read + ( testRead + ) where + +import Data.Char (isDigit, isHexDigit) +import Numeric (showHex) +import Test.Tasty (TestTree, testGroup, localOption, mkTimeout) +import Test.Tasty.QuickCheck (testProperty) +import Test.QuickCheck +import Tests.QuickCheckUtils () +import qualified Data.Text as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Read as TL +import qualified Data.Text.Read as T + +-- Reading. + +t_decimal (n::Int) s = + T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t) + where t = T.dropWhile isDigit s +tl_decimal (n::Int) s = + TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t) + where t = TL.dropWhile isDigit s +t_hexadecimal m s ox = + T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t) + where t = T.dropWhile isHexDigit s + p = if ox then "0x" else "" + n = getPositive m :: Int +tl_hexadecimal m s ox = + TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t) + where t = TL.dropWhile isHexDigit s + p = if ox then "0x" else "" + n = getPositive m :: Int + +isFloaty c = c `elem` ("+-.0123456789eE" :: String) + +t_read_rational :: Double -> T.Text -> Property +t_read_rational n s = + case T.rational (T.pack (show n) `T.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. n' === n'' + where + t = T.dropWhile isFloaty s + n'' = read (show n) :: Double + +t_read_double :: Double -> Double -> T.Text -> Property +t_read_double tol n s = + case T.double (T.pack (show n) `T.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol) + where + t = T.dropWhile isFloaty s + +tl_read_rational :: Double -> TL.Text -> Property +tl_read_rational n s = + case TL.rational (TL.pack (show n) `TL.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. n' === n'' + where + t = TL.dropWhile isFloaty s + n'' = read (show n) :: Double + +tl_read_double :: Double -> Double -> TL.Text -> Property +tl_read_double tol n s = + case TL.rational (TL.pack (show n) `TL.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol) + where + t = TL.dropWhile isFloaty s + +testRead :: TestTree +testRead = + testGroup "read" [ + testProperty "t_decimal" t_decimal, + testProperty "tl_decimal" tl_decimal, + testProperty "t_hexadecimal" t_hexadecimal, + testProperty "tl_hexadecimal" tl_hexadecimal, + + testProperty "t_double" $ t_read_double 1e-13, + testProperty "tl_double" $ tl_read_double 1e-13, + + testProperty "t_rational" t_read_rational, + testProperty "t_rational 1.3e-2" (t_read_rational 1.3e-2), + testProperty "tl_rational" tl_read_rational, + testProperty "tl_rational 9e-3" (tl_read_rational 9e-3), + + localOption (mkTimeout 100000) $ testGroup "DDoS attacks" [ + testProperty "t_double large positive exponent" $ + T.double (T.pack "1.1e1000000000") === Right (1 / 0, mempty), + testProperty "t_double large negative exponent" $ + T.double (T.pack "1.1e-1000000000") === Right (0.0, mempty), + testProperty "tl_double large positive exponent" $ + TL.double (TL.pack "1.1e1000000000") === Right (1 / 0, mempty), + testProperty "tl_double large negative exponent" $ + TL.double (TL.pack "1.1e-1000000000") === Right (0.0, mempty), + + testProperty "t_rational large positive exponent" $ + T.rational (T.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty), + testProperty "t_rational large negative exponent" $ + T.rational (T.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty), + testProperty "tl_rational large positive exponent" $ + TL.rational (TL.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty), + testProperty "tl_rational large negative exponent" $ + TL.rational (TL.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty) + ] + + ] diff --git a/tests/Tests/Properties/Substrings.hs b/tests/Tests/Properties/Substrings.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Substrings.hs @@ -0,0 +1,438 @@ +-- | Tests for substring functions (@take@, @split@, @isInfixOf@, etc.) + +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +module Tests.Properties.Substrings + ( testSubstrings + ) where + +import Prelude hiding (head, tail) +import Control.Monad.Trans.State (State, state, runState) +import Data.Char (isSpace) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy as TL (Text(..)) +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Lazy as TL +import qualified Tests.SlowFunctions as Slow + +s_take n = L.take n `eqP` (unpackS . S.take n) +s_take_s (Small n) = L.take n `eqP` (unpackS . S.unstream . S.take n) +sf_take (applyFun -> p) n + = (L.take n . L.filter p) `eqP` + (unpackS . S.take n . S.filter p) +t_take n = L.take n `eqP` (unpackS . T.take n) +t_takeEnd n = (L.reverse . L.take n . L.reverse) `eqP` + (unpackS . T.takeEnd n) +tl_take n = L.genericTake n `eqP` (unpackS . TL.take n) +tl_take_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + L.genericTake n `eqP` (unpackS . TL.take n) +tl_takeEnd n = (L.reverse . L.genericTake n . L.reverse) `eqP` + (unpackS . TL.takeEnd n) +tl_takeEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + (L.reverse . L.genericTake n . L.reverse) `eqP` + (unpackS . TL.takeEnd n) + +s_drop n = L.drop n `eqP` (unpackS . S.drop n) +s_drop_s (Small n) = L.drop n `eqP` (unpackS . S.unstream . S.drop n) +sf_drop (applyFun -> p) n + = (L.drop n . L.filter p) `eqP` (unpackS . S.drop n . S.filter p) +t_drop n = L.drop n `eqP` (unpackS . T.drop n) +t_dropEnd n = (L.reverse . L.drop n . L.reverse) `eqP` + (unpackS . T.dropEnd n) +tl_drop n = L.genericDrop n `eqP` (unpackS . TL.drop n) +tl_drop_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in L.genericDrop n `eqP` (unpackS . TL.drop n) +tl_dropEnd n = (L.reverse . L.genericDrop n . L.reverse) `eqP` + (unpackS . TL.dropEnd n) +tl_dropEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in (L.reverse . L.genericDrop n . L.reverse) `eqP` + (unpackS . TL.dropEnd n) + +s_take_drop (Small n) = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n) +s_take_drop_s (Small n) = (L.take n . L.drop n) `eqP` + (unpackS . S.unstream . S.take n . S.drop n) +s_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . S.takeWhile p) +s_takeWhile_s (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p) +sf_takeWhile (applyFun -> q) (applyFun -> p) + = (L.takeWhile p . L.filter q) `eqP` (unpackS . S.takeWhile p . S.filter q) + +data NoMatch = NoMatch Char Char + deriving (Eq, Show) + +instance Arbitrary NoMatch where + arbitrary = do + c <- arbitraryUnicodeChar + d <- suchThat arbitraryUnicodeChar (/= c) + return $ NoMatch c d + shrink (NoMatch c d) = fmap (NoMatch c) (filter (/= c) (shrink d)) + ++ fmap (`NoMatch` d) (filter (/= d) (shrink c)) + +t_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . T.takeWhile p) +tl_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . TL.takeWhile p) +t_takeWhileEnd (applyFun -> p) + = (L.reverse . L.takeWhile p . L.reverse) `eqP` + (unpackS . T.takeWhileEnd p) +t_takeWhileEnd_null t (NoMatch c d) + = T.null $ T.takeWhileEnd (==d) (T.snoc t c) +tl_takeWhileEnd (applyFun -> p) + = (L.reverse . L.takeWhile p . L.reverse) `eqP` + (unpackS . TL.takeWhileEnd p) +tl_takeWhileEnd_null t (NoMatch c d) + = TL.null $ TL.takeWhileEnd (==d) (TL.snoc t c) +s_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +s_dropWhile_s (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p) +sf_dropWhile (applyFun -> q) (applyFun -> p) + = (L.dropWhile p . L.filter q) `eqP` + (unpackS . S.dropWhile p . S.filter q) +t_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . T.dropWhile p) +tl_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +t_dropWhileEnd (applyFun -> p) + = (L.reverse . L.dropWhile p . L.reverse) `eqP` + (unpackS . T.dropWhileEnd p) +tl_dropWhileEnd (applyFun -> p) + = (L.reverse . L.dropWhile p . L.reverse) `eqP` + (unpackS . TL.dropWhileEnd p) +t_dropAround (applyFun -> p) + = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) + `eqP` (unpackS . T.dropAround p) +tl_dropAround (applyFun -> p) + = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) + `eqP` (unpackS . TL.dropAround p) +t_stripStart = T.dropWhile isSpace `eq` T.stripStart +tl_stripStart = TL.dropWhile isSpace `eq` TL.stripStart +t_stripEnd = T.dropWhileEnd isSpace `eq` T.stripEnd +tl_stripEnd = TL.dropWhileEnd isSpace `eq` TL.stripEnd +t_strip = T.dropAround isSpace `eq` T.strip +tl_strip = TL.dropAround isSpace `eq` TL.strip + +t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) +tl_splitAt n = L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n) +tl_splitAt_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n) + +t_span (applyFun -> p) = L.span p `eqP` (unpack2 . T.span p) +tl_span (applyFun -> p) = L.span p `eqP` (unpack2 . TL.span p) + +t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s) + where squid t | T.null s = error "empty" + | otherwise = t +tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s) + where squid t | TL.null s = error "empty" + | otherwise = t +t_breakOn_start (NotEmpty s) t = + let (k,m) = T.breakOn s t + in k `T.isPrefixOf` t && (T.null m || s `T.isPrefixOf` m) +tl_breakOn_start (NotEmpty s) t = + let (k,m) = TL.breakOn s t + in k `TL.isPrefixOf` t && TL.null m || s `TL.isPrefixOf` m +t_breakOnEnd_end (NotEmpty s) t = + let (m,k) = T.breakOnEnd s t + in k `T.isSuffixOf` t && (T.null m || s `T.isSuffixOf` m) +tl_breakOnEnd_end (NotEmpty s) t = + let (m,k) = TL.breakOnEnd s t + in k `TL.isSuffixOf` t && (TL.null m || s `TL.isSuffixOf` m) +t_break (applyFun -> p) + = L.break p `eqP` (unpack2 . T.break p) +tl_break (applyFun -> p) + = L.break p `eqP` (unpack2 . TL.break p) +t_group = L.group `eqP` (map unpackS . T.group) +tl_group = L.group `eqP` (map unpackS . TL.group) +t_groupBy (applyFun2 -> p) + = L.groupBy p `eqP` (map unpackS . T.groupBy p) +tl_groupBy (applyFun2 -> p) + = L.groupBy p `eqP` (map unpackS . TL.groupBy p) +t_inits = L.inits `eqP` (map unpackS . T.inits) +t_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . T.initsNE) +tl_inits = L.inits `eqP` (map unpackS . TL.inits) +tl_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . TL.initsNE) +t_tails = L.tails `eqP` (map unpackS . T.tails) +t_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . T.tailsNE) +tl_tails = L.tails `eqPSqrt` (map unpackS . TL.tails) +tl_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . TL.tailsNE) + +spanML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b]) +spanML p s = go [] s + where + go acc [] = pure (reverse acc, []) + go acc ccs@(c : cs) = do + continue <- p c + if continue then go (c : acc) cs else pure (reverse acc, ccs) + +spanEndML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b]) +spanEndML p s = (\(s1, s2) -> (reverse s2, reverse s1)) <$> spanML p (reverse s) + +eqSP :: Stringy s => (String -> State Int (String, String)) -> (s -> State Int (s, s)) -> Property +eqSP a b = property $ ((`runState` 0) . a) `eqP` ((`runState` 0) . (fmap unpack2 . b)) + +t_spanM (applyFun2 -> f') + = let f c = state $ \i -> (getSkewed (f' i c), i+1) in + spanML f `eqSP` T.spanM f + +t_spanEndM (applyFun2 -> f') + = let f c = state $ \i -> (getSkewed (f' i c), i+1) in + spanEndML f `eqSP` T.spanEndM f + +tl_spanM (applyFun2 -> f') + = let f c = state $ \i -> (getSkewed (f' i c), i+1) in + spanML f `eqSP` TL.spanM f + +tl_spanEndM (applyFun2 -> f') + = let f c = state $ \i -> (getSkewed (f' i c), i+1) in + spanEndML f `eqSP` TL.spanEndM f + +t_findAppendId = \(Sqrt (NotEmpty s)) ts -> + let t = T.intercalate s ts + in conjoin $ map (=== t) $ map (uncurry T.append) (T.breakOnAll s t) +tl_findAppendId = \(Sqrt (NotEmpty s)) ts -> + let t = TL.intercalate s ts + in conjoin $ map (=== t) $ map (uncurry TL.append) (TL.breakOnAll s t) +t_findContains = \(Sqrt (NotEmpty s)) -> + all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s +tl_findContains = \(Sqrt (NotEmpty s)) -> all (TL.isPrefixOf s . snd) . + TL.breakOnAll s . TL.intercalate s +sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c +t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s +tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s + +t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s . unSqrt +tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq` + (map TL.fromStrict . T.splitOn s)) . T.intercalate s . unSqrt +t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t) +tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) + +t_split (applyFun -> p) = split p `eqP` (map unpackS . T.split p) +t_split_count c = (L.length . T.split (==c)) `eq` + ((1+) . T.count (T.singleton c)) +t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c) +tl_split (applyFun -> p) = split p `eqP` (map unpackS . TL.split p) + +split :: (a -> Bool) -> [a] -> [[a]] +split _ [] = [[]] +split p xs = loop xs + where + loop s = case break p s of + (l, []) -> [l] + (l, _ : s') -> l : loop s' + +t_chunksOf_same_lengths k = + conjoin . map ((===k) . T.length) . drop 1 . reverse . T.chunksOf k + +t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0) + where len = L.sum . L.map T.length $ T.chunksOf k t + +tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) . + TL.chunksOf (fromIntegral k) . TL.fromStrict) + +t_lines = L.lines `eqP` (map unpackS . T.lines) +tl_lines = L.lines `eqP` (map unpackS . TL.lines) +t_lines_spacy = (L.lines `eqP` (map unpackS . T.lines)) . getSpacyString +tl_lines_spacy = (L.lines `eqP` (map unpackS . TL.lines)) . getSpacyString + +tl_lines_laziness = case TL.lines (TL.replicate 1000000000000000 (TL.singleton 'a')) of + [] -> property False + hd : _ -> TL.head hd === 'a' + +tl_lines_specialCase = TL.lines (TL.Chunk (T.pack "foo") $ TL.Chunk (T.pack "bar\nbaz\n") $ TL.Empty) === [TL.pack "foobar", TL.pack "baz"] + +t_words = L.words `eqP` (map unpackS . T.words) +tl_words = L.words `eqP` (map unpackS . TL.words) +t_words_spacy = (L.words `eqP` (map unpackS . T.words)) . getSpacyString +tl_words_spacy = (L.words `eqP` (map unpackS . TL.words)) . getSpacyString + +t_unlines = (L.unlines . unSqrt) `eq` (unpackS . T.unlines . map packS . unSqrt) +tl_unlines = (L.unlines . unSqrt) `eq` (unpackS . TL.unlines . map packS . unSqrt) +t_unwords = (L.unwords . unSqrt) `eq` (unpackS . T.unwords . map packS . unSqrt) +tl_unwords = (L.unwords . unSqrt) `eq` (unpackS . TL.unwords . map packS . unSqrt) + +s_isPrefixOf s = L.isPrefixOf s `eqP` + (S.isPrefixOf (S.stream $ packS s) . S.stream) +sf_isPrefixOf (applyFun -> p) s + = (L.isPrefixOf s . L.filter p) `eqP` + (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream) +t_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (packS s) +tl_isPrefixOf s = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s) +t_isSuffixOf s = L.isSuffixOf s`eqP` T.isSuffixOf (packS s) +tl_isSuffixOf s = L.isSuffixOf s`eqP` TL.isSuffixOf (packS s) +t_isInfixOf s = L.isInfixOf s `eqP` T.isInfixOf (packS s) +tl_isInfixOf s = L.isInfixOf s `eqP` TL.isInfixOf (packS s) + +t_stripPrefix s = (fmap packS . L.stripPrefix s) `eqP` T.stripPrefix (packS s) +tl_stripPrefix s = (fmap packS . L.stripPrefix s) `eqP` TL.stripPrefix (packS s) + +stripSuffix p t = reverse `fmap` L.stripPrefix (reverse p) (reverse t) + +t_stripSuffix s = (fmap packS . stripSuffix s) `eqP` T.stripSuffix (packS s) +tl_stripSuffix s = (fmap packS . stripSuffix s) `eqP` TL.stripSuffix (packS s) + +commonPrefixes a0@(_:_) b0@(_:_) = Just (go a0 b0 []) + where go (a:as) (b:bs) ps + | a == b = go as bs (a:ps) + go as bs ps = (reverse ps,as,bs) +commonPrefixes _ _ = Nothing + +t_commonPrefixes a b (NonEmpty p) + = commonPrefixes pa pb === + repack `fmap` T.commonPrefixes (packS pa) (packS pb) + where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) + pa = p ++ a + pb = p ++ b + +tl_commonPrefixes a b (NonEmpty p) + = commonPrefixes pa pb === + repack `fmap` TL.commonPrefixes (packS pa) (packS pb) + where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) + pa = p ++ a + pb = p ++ b + +testSubstrings :: TestTree +testSubstrings = + testGroup "substrings" [ + testGroup "breaking" [ + testProperty "s_take" s_take, + testProperty "s_take_s" s_take_s, + testProperty "sf_take" sf_take, + testProperty "t_take" t_take, + testProperty "t_takeEnd" t_takeEnd, + testProperty "tl_take" tl_take, + testProperty "tl_take_maxBound" tl_take_maxBound, + testProperty "tl_takeEnd" tl_takeEnd, + testProperty "tl_takeEnd_maxBound" tl_takeEnd_maxBound, + testProperty "s_drop" s_drop, + testProperty "s_drop_s" s_drop_s, + testProperty "sf_drop" sf_drop, + testProperty "t_drop" t_drop, + testProperty "t_dropEnd" t_dropEnd, + testProperty "tl_drop" tl_drop, + testProperty "tl_drop_maxBound" tl_drop_maxBound, + testProperty "tl_dropEnd" tl_dropEnd, + testProperty "tl_dropEnd_maxBound" tl_dropEnd_maxBound, + testProperty "s_take_drop" s_take_drop, + testProperty "s_take_drop_s" s_take_drop_s, + testProperty "s_takeWhile" s_takeWhile, + testProperty "s_takeWhile_s" s_takeWhile_s, + testProperty "sf_takeWhile" sf_takeWhile, + testProperty "t_takeWhile" t_takeWhile, + testProperty "tl_takeWhile" tl_takeWhile, + testProperty "t_takeWhileEnd" t_takeWhileEnd, + testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null, + testProperty "tl_takeWhileEnd" tl_takeWhileEnd, + testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null, + testProperty "sf_dropWhile" sf_dropWhile, + testProperty "s_dropWhile" s_dropWhile, + testProperty "s_dropWhile_s" s_dropWhile_s, + testProperty "t_dropWhile" t_dropWhile, + testProperty "tl_dropWhile" tl_dropWhile, + testProperty "t_dropWhileEnd" t_dropWhileEnd, + testProperty "tl_dropWhileEnd" tl_dropWhileEnd, + testProperty "t_dropAround" t_dropAround, + testProperty "tl_dropAround" tl_dropAround, + testProperty "t_stripStart" t_stripStart, + testProperty "tl_stripStart" tl_stripStart, + testProperty "t_stripEnd" t_stripEnd, + testProperty "tl_stripEnd" tl_stripEnd, + testProperty "t_strip" t_strip, + testProperty "tl_strip" tl_strip, + testProperty "t_splitAt" t_splitAt, + testProperty "tl_splitAt" tl_splitAt, + testProperty "tl_splitAt_maxBound" tl_splitAt_maxBound, + testProperty "t_span" t_span, + testProperty "tl_span" tl_span, + testProperty "t_breakOn_id" t_breakOn_id, + testProperty "tl_breakOn_id" tl_breakOn_id, + testProperty "t_breakOn_start" t_breakOn_start, + testProperty "tl_breakOn_start" tl_breakOn_start, + testProperty "t_breakOnEnd_end" t_breakOnEnd_end, + testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end, + testProperty "t_break" t_break, + testProperty "tl_break" tl_break, + testProperty "t_group" t_group, + testProperty "tl_group" tl_group, + testProperty "t_groupBy" t_groupBy, + testProperty "tl_groupBy" tl_groupBy, + testProperty "t_inits" t_inits, + testProperty "t_initsNE" t_initsNE, + testProperty "tl_inits" tl_inits, + testProperty "tl_initsNE" tl_initsNE, + testProperty "t_tails" t_tails, + testProperty "t_tailsNE" t_tailsNE, + testProperty "tl_tails" tl_tails, + testProperty "tl_tailsNE" tl_tailsNE, + testProperty "t_spanM" t_spanM, + testProperty "t_spanEndM" t_spanEndM, + testProperty "tl_spanM" tl_spanM, + testProperty "tl_spanEndM" tl_spanEndM + ], + + testGroup "breaking many" [ + testProperty "t_findAppendId" t_findAppendId, + testProperty "tl_findAppendId" tl_findAppendId, + testProperty "t_findContains" t_findContains, + testProperty "tl_findContains" tl_findContains, + testProperty "sl_filterCount" sl_filterCount, + testProperty "t_findCount" t_findCount, + testProperty "tl_findCount" tl_findCount, + testProperty "t_splitOn_split" t_splitOn_split, + testProperty "tl_splitOn_split" tl_splitOn_split, + testProperty "t_splitOn_i" t_splitOn_i, + testProperty "tl_splitOn_i" tl_splitOn_i, + testProperty "t_split" t_split, + testProperty "t_split_count" t_split_count, + testProperty "t_split_splitOn" t_split_splitOn, + testProperty "tl_split" tl_split, + testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths, + testProperty "t_chunksOf_length" t_chunksOf_length, + testProperty "tl_chunksOf" tl_chunksOf + ], + + testGroup "lines and words" [ + testProperty "t_lines" t_lines, + testProperty "tl_lines" tl_lines, + testProperty "t_lines_spacy" t_lines_spacy, + testProperty "tl_lines_spacy" tl_lines_spacy, + testProperty "tl_lines_laziness" tl_lines_laziness, + testProperty "tl_lines_specialCase" tl_lines_specialCase, + testProperty "t_words" t_words, + testProperty "tl_words" tl_words, + testProperty "t_words_spacy" t_words_spacy, + testProperty "tl_words_spacy" tl_words_spacy, + testProperty "t_unlines" t_unlines, + testProperty "tl_unlines" tl_unlines, + testProperty "t_unwords" t_unwords, + testProperty "tl_unwords" tl_unwords + ], + + testGroup "predicates" [ + testProperty "s_isPrefixOf" s_isPrefixOf, + testProperty "sf_isPrefixOf" sf_isPrefixOf, + testProperty "t_isPrefixOf" t_isPrefixOf, + testProperty "tl_isPrefixOf" tl_isPrefixOf, + testProperty "t_isSuffixOf" t_isSuffixOf, + testProperty "tl_isSuffixOf" tl_isSuffixOf, + testProperty "t_isInfixOf" t_isInfixOf, + testProperty "tl_isInfixOf" tl_isInfixOf, + + testGroup "view" [ + testProperty "t_stripPrefix" t_stripPrefix, + testProperty "tl_stripPrefix" tl_stripPrefix, + testProperty "t_stripSuffix" t_stripSuffix, + testProperty "tl_stripSuffix" tl_stripSuffix, + testProperty "t_commonPrefixes" t_commonPrefixes, + testProperty "tl_commonPrefixes" tl_commonPrefixes + ] + ] + ] diff --git a/tests/Tests/Properties/Text.hs b/tests/Tests/Properties/Text.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Text.hs @@ -0,0 +1,491 @@ +-- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. + +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Avoid restricted function" #-} + +module Tests.Properties.Text + ( testText + ) where + +import Control.Exception (SomeException, evaluate, try) +import Data.Char (isLower, isLetter, isUpper) +import Data.Maybe (mapMaybe) +import Data.Text.Internal.Fusion.Size +import Data.Word (Word8) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.Char as C +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Internal.Lazy.Search as S (indices) +import qualified Data.Text.Internal.Search as T (indices) +import qualified Data.Text.Lazy as TL +import qualified Tests.SlowFunctions as Slow +#if MIN_VERSION_base(4, 15, 0) +import qualified GHC.Unicode as G (unicodeVersion) +import qualified Data.Text.Internal.Fusion.CaseMapping as T (unicodeVersion) +#endif + +t_pack_unpack = (T.unpack . T.pack) `eq` id +tl_pack_unpack = (TL.unpack . TL.pack) `eq` id +t_stream_unstream = (S.unstream . S.stream) `eq` id +tl_stream_unstream = (SL.unstream . SL.stream) `eq` id +t_reverse_stream t = (S.reverse . S.reverseStream) t === t +t_singleton c = [c] === (T.unpack . T.singleton) c +tl_singleton c = [c] === (TL.unpack . TL.singleton) c +tl_unstreamChunks x = f 11 x === f 1000 x + where f n = SL.unstreamChunks n . S.streamList +tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id +tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id + +s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) +s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) + +t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) +tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) +t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) +tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) +t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) +tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) + +s_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . S.intercalate (packS c) . map packS . unSqrt) +t_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . T.intercalate (packS c) . map packS . unSqrt) +tl_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` + (T.length . T.intercalate (packS c) . map packS . unSqrt) +tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` + (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +s_intersperse c = L.intersperse c `eqP` + (unpackS . S.intersperse c) +s_intersperse_s c = L.intersperse c `eqP` + (unpackS . S.unstream . S.intersperse c) +sf_intersperse (applyFun -> p) c + = (L.intersperse c . L.filter p) `eqP` + (unpackS . S.intersperse c . S.filter p) +t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) +t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) +tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) +t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) +tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) +t_reverse = L.reverse `eqP` (unpackS . T.reverse) +tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) +t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) + +t_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . T.replace (T.pack s) (T.pack d)) +tl_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . TL.replace (TL.pack s) (TL.pack d)) + +splitOn :: (Eq a) => [a] -> [a] -> [[a]] +splitOn pat src0 + | l == 0 = error "splitOn: empty" + | otherwise = go src0 + where + l = length pat + go src = search 0 src + where + search _ [] = [src] + search !n s@(_:s') + | pat `L.isPrefixOf` s = take n src : go (drop l s) + | otherwise = search (n+1) s' + +s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs + where s = S.streamList xs +sf_toCaseFold_length (applyFun -> p) xs = + (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) + where s = S.streamList xs +t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t + +tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t +t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==> + T.toCaseFold (T.singleton c) === T.singleton (C.toLower c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389" +t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower)) +t_toCaseFold_cherokeeUpper = conjoin $ + map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $ +-- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c)) +-- [minBound .. maxBound] +toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" +cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase +cherokeeLower = ['\xAB70'..'\xABBF'] + +t_toLower_length t = T.length (T.toLower t) >= T.length t +t_toLower_lower t = p (T.toLower t) >= p t + where p = T.length . T.filter isLower +tl_toLower_lower t = p (TL.toLower t) >= p t + where p = TL.length . TL.filter isLower +t_toLower_char c = c /= '\304' ==> + T.toLower (T.singleton c) === T.singleton (C.toLower c) +t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775" + +t_toUpper_length t = T.length (T.toUpper t) >= T.length t +t_toUpper_upper t = p (T.toUpper t) >= p t + where p = T.length . T.filter isUpper +tl_toUpper_upper t = p (TL.toUpper t) >= p t + where p = TL.length . TL.filter isUpper +t_toUpper_char c = c `notElem` toUpperExceptions ==> + T.toUpper (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toTitle_title t = all (<= 1) (caps w) + where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle + -- TIL: there exist uppercase-only letters + w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t +t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian) + where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words + -- Surprise! The Spanish/Portuguese ordinal indicators changed + -- from category Ll (letter, lowercase) to Lo (letter, other) + -- in Unicode 7.0 + -- Oh, and there exist lowercase-only letters (see previous test) + stable c = if isLower c + then C.toUpper c /= c + else c /= '\170' && c /= '\186' + -- Georgian text does not have a concept of title case + -- https://en.wikipedia.org/wiki/Georgian_Extended + isGeorgian c = c >= '\4256' && c < '\4352' +t_toTitle_char c = c `notElem` toTitleExceptions ==> + T.toTitle (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t +t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t +t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t + +ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) +ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) +ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) + +ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) + where + referenceToTitle _ [] = [] + referenceToTitle False (y : ys) + | C.isLetter y = C.toUpper y : referenceToTitle True ys + | otherwise = y : referenceToTitle False ys + referenceToTitle True (y : ys) + | C.isLetter y = C.toLower y : referenceToTitle True ys + | otherwise = y : referenceToTitle (not (C.isSpace y)) ys + +justifyLeft k c xs = xs ++ L.replicate (k - length xs) c +justifyRight m n xs = L.replicate (m - length xs) n ++ xs +center k c xs + | len >= k = xs + | otherwise = L.replicate l c ++ xs ++ L.replicate r c + where len = length xs + d = k - len + r = d `div` 2 + l = d - r + +s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +s_justifyLeft_s k c = justifyLeft j c `eqP` + (unpackS . S.unstream . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +sf_justifyLeft (applyFun -> p) k c + = (justifyLeft j c . L.filter p) `eqP` + (unpackS . S.justifyLeftI j c . S.filter p) + where j = fromIntegral (k :: Word8) +t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) + where j = fromIntegral (k :: Word8) +tl_justifyLeft k c = justifyLeft j c `eqP` + (unpackS . TL.justifyLeft (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) + where j = fromIntegral (k :: Word8) +tl_justifyRight k c = justifyRight j c `eqP` + (unpackS . TL.justifyRight (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_center k c = center j c `eqP` (unpackS . T.center j c) + where j = fromIntegral (k :: Word8) +tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) + where j = fromIntegral (k :: Word8) + +t_elem c = L.elem c `eqP` T.elem c +tl_elem c = L.elem c `eqP` TL.elem c +sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter (applyFun -> q) (applyFun -> p) + = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) + +t_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . T.filter p) +tl_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . TL.filter p) +t_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) +tl_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) +t_length_filter (applyFun -> p) + = (L.length . L.filter p) `eqP` (T.length . T.filter p) +tl_length_filter (applyFun -> p) + = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) + +sf_findBy (applyFun -> q) (applyFun -> p) + = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_find (applyFun -> p) = L.find p `eqP` T.find p +tl_find (applyFun -> p) = L.find p `eqP` TL.find p +t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) + +sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j + where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l + +t_index :: T.Text -> Int -> Property +t_index xs i = ioProperty $ do + ch <- try (evaluate (T.index xs i)) + pure $ case ch of + Left (_ :: SomeException) -> i < 0 .||. i >= T.length xs + Right c -> i >= 0 .&&. i < T.length xs .&&. c === T.unpack xs L.!! i + +tl_index :: TL.Text -> Int -> Property +tl_index xs i = ioProperty $ do + let i' = fromIntegral i + ch <- try (evaluate (TL.index xs i')) + pure $ case ch of + Left (_ :: SomeException) -> i' < 0 .||. i' >= TL.length xs + Right c -> i >= 0 .&&. i' < TL.length xs .&&. c === TL.unpack xs L.!! i + +t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p +t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t +tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` + TL.count t +t_zip s = L.zip s `eqP` T.zip (packS s) +tl_zip s = L.zip s `eqP` TL.zip (packS s) +sf_zipWith (applyFun -> p) (applyFun2 -> c) s + = (L.zipWith c (L.filter p s) . L.filter p) `eqP` + (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) +t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) +tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) + +t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s +tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s + where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) + conc = T.concat . TL.toChunks +t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> + let s = T.intercalate t ts + in Slow.indices t s === T.indices t s + +t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] +tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] + +t_indices_drop n s pref suff = T.indices s t === Slow.indices s t + where + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_drop n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] +tl_indices_drop_chunked n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + -- constructing a pathologically chunked haystack + t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff + +t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t + where + s = T.singleton c + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + s = TL.singleton c + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +-- Make a stream appear shorter than it really is, to ensure that +-- functions that consume inaccurately sized streams behave +-- themselves. +shorten :: Int -> S.Stream a -> S.Stream a +shorten n t@(S.Stream arr off len) + | n > 0 = S.Stream arr off (smaller (exactSize n) len) + | otherwise = t + +testText :: TestTree +testText = + testGroup "Text" [ + testGroup "creation/elimination" [ + testProperty "t_pack_unpack" t_pack_unpack, + testProperty "tl_pack_unpack" tl_pack_unpack, + testProperty "t_stream_unstream" t_stream_unstream, + testProperty "tl_stream_unstream" tl_stream_unstream, + testProperty "t_reverse_stream" t_reverse_stream, + testProperty "t_singleton" t_singleton, + testProperty "tl_singleton" tl_singleton, + testProperty "tl_unstreamChunks" tl_unstreamChunks, + testProperty "tl_chunk_unchunk" tl_chunk_unchunk, + testProperty "tl_from_to_strict" tl_from_to_strict + ], + + testGroup "transformations" [ + testProperty "s_map" s_map, + testProperty "s_map_s" s_map_s, + testProperty "sf_map" sf_map, + + testProperty "t_map" t_map, + testProperty "tl_map" tl_map, + testProperty "t_map_map" t_map_map, + testProperty "tl_map_map" tl_map_map, + testProperty "t_length_map" t_length_map, + testProperty "tl_length_map" tl_length_map, + + testProperty "s_intercalate" s_intercalate, + testProperty "t_intercalate" t_intercalate, + testProperty "tl_intercalate" tl_intercalate, + testProperty "t_length_intercalate" t_length_intercalate, + testProperty "tl_length_intercalate" tl_length_intercalate, + testProperty "s_intersperse" s_intersperse, + testProperty "s_intersperse_s" s_intersperse_s, + testProperty "sf_intersperse" sf_intersperse, + testProperty "t_intersperse" t_intersperse, + testProperty "tl_intersperse" tl_intersperse, + testProperty "t_length_intersperse" t_length_intersperse, + testProperty "tl_length_intersperse" tl_length_intersperse, + testProperty "t_transpose" t_transpose, + testProperty "tl_transpose" tl_transpose, + testProperty "t_reverse" t_reverse, + testProperty "tl_reverse" tl_reverse, + testProperty "t_reverse_short" t_reverse_short, + testProperty "t_replace" t_replace, + testProperty "tl_replace" tl_replace, + + testGroup "case conversion" [ + testProperty "s_toCaseFold_length" s_toCaseFold_length, + testProperty "sf_toCaseFold_length" sf_toCaseFold_length, + testProperty "t_toCaseFold_length" t_toCaseFold_length, + testProperty "tl_toCaseFold_length" tl_toCaseFold_length, + testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions, + testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower, + testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper, + + testProperty "t_toLower_length" t_toLower_length, + testProperty "t_toLower_lower" t_toLower_lower, + testProperty "tl_toLower_lower" tl_toLower_lower, + testProperty "t_toLower_dotted_i" t_toLower_dotted_i, + + testProperty "t_toUpper_length" t_toUpper_length, + testProperty "t_toUpper_upper" t_toUpper_upper, + testProperty "tl_toUpper_upper" tl_toUpper_upper, + testProperty "t_toUpper_exceptions" t_toUpper_exceptions, + + testProperty "t_toTitle_title" t_toTitle_title, + testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, + testProperty "t_toTitle_exceptions" t_toTitle_exceptions, + + testProperty "t_toUpper_idempotent" t_toUpper_idempotent, + testProperty "t_toLower_idempotent" t_toLower_idempotent, + testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent, + + testProperty "ascii_toLower" ascii_toLower, + testProperty "ascii_toUpper" ascii_toUpper, + testProperty "ascii_toTitle" ascii_toTitle, + testProperty "ascii_toCaseFold" ascii_toCaseFold + ], + +#if MIN_VERSION_base(4, 15, 0) + -- Requires matching version of Unicode in base and text + testGroup "char case conversion" $ if T.unicodeVersion == G.unicodeVersion then [ + testProperty "t_toCaseFold_char" t_toCaseFold_char, + testProperty "t_toLower_char" t_toLower_char, + testProperty "t_toUpper_char" t_toUpper_char, + testProperty "t_toTitle_char" t_toTitle_char + ] else [], +#endif + + testGroup "justification" [ + testProperty "s_justifyLeft" s_justifyLeft, + testProperty "s_justifyLeft_s" s_justifyLeft_s, + testProperty "sf_justifyLeft" sf_justifyLeft, + testProperty "t_justifyLeft" t_justifyLeft, + testProperty "tl_justifyLeft" tl_justifyLeft, + testProperty "t_justifyRight" t_justifyRight, + testProperty "tl_justifyRight" tl_justifyRight, + testProperty "t_center" t_center, + testProperty "tl_center" tl_center + ] + ], + + testGroup "searching" [ + testProperty "t_elem" t_elem, + testProperty "tl_elem" tl_elem, + testProperty "sf_elem" sf_elem, + testProperty "sf_filter" sf_filter, + testProperty "t_filter" t_filter, + testProperty "tl_filter" tl_filter, + testProperty "t_filter_filter" t_filter_filter, + testProperty "tl_filter_filter" tl_filter_filter, + testProperty "t_length_filter" t_length_filter, + testProperty "tl_length_filter" tl_length_filter, + testProperty "sf_findBy" sf_findBy, + testProperty "t_find" t_find, + testProperty "tl_find" tl_find, + testProperty "t_partition" t_partition, + testProperty "tl_partition" tl_partition + ], + + testGroup "indexing" [ + testProperty "sf_index" sf_index, + testProperty "t_index" t_index, + testProperty "tl_index" tl_index, + testProperty "t_findIndex" t_findIndex, + testProperty "t_count" t_count, + testProperty "tl_count" tl_count, + testProperty "t_indices" t_indices, + testProperty "tl_indices" tl_indices, + testProperty "t_indices_occurs" t_indices_occurs, + + testProperty "t_indices_drop5" t_indices_drop5, + testProperty "tl_indices_drop5" tl_indices_drop5, + testProperty "t_indices_drop" t_indices_drop, + testProperty "tl_indices_drop" tl_indices_drop, + testProperty "tl_indices_chunked" tl_indices_chunked, + testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, + testProperty "t_indices_char_drop" t_indices_char_drop, + testProperty "tl_indices_char_drop" tl_indices_char_drop + ], + + testGroup "zips" [ + testProperty "t_zip" t_zip, + testProperty "tl_zip" tl_zip, + testProperty "sf_zipWith" sf_zipWith, + testProperty "t_zipWith" t_zipWith, + testProperty "tl_zipWith" tl_zipWith, + testProperty "t_length_zipWith" t_length_zipWith, + testProperty "tl_length_zipWith" tl_length_zipWith + ] + ] diff --git a/tests/Tests/Properties/Transcoding.hs b/tests/Tests/Properties/Transcoding.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Transcoding.hs @@ -0,0 +1,560 @@ +-- | Tests for encoding and decoding + +{-# LANGUAGE CPP, OverloadedStrings, ScopedTypeVariables #-} +{-# OPTIONS_GHC -Wno-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.Properties.Transcoding + ( testTranscoding + ) where + +import Prelude hiding (head, tail) +import Data.Bits ((.&.), shiftR) +import Data.Char (chr, ord) +import Data.Functor (void) +import Data.Maybe (isNothing) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif +import Data.Word (Word8) +import Test.QuickCheck hiding ((.&.)) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Test.Tasty.HUnit ((@?=), assertBool, assertFailure, testCase) +import Tests.QuickCheckUtils +import qualified Control.Exception as Exception +import qualified Data.Bits as Bits (shiftL, shiftR) +import qualified Data.ByteString as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Extra as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Char8 as BC +import qualified Data.ByteString.Lazy as BL +import qualified Data.ByteString.Lazy.Char8 as BLC +import qualified Data.Text as T +import qualified Data.Text.Encoding as E +import qualified Data.Text.Encoding.Error as E +import qualified Data.Text.Internal.Encoding as E +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Encoding as EL + +t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a + where a = T.map (\c -> chr (ord c `mod` 128)) t +tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a + where a = TL.map (\c -> chr (ord c `mod` 128)) t + +t_latin1 = E.decodeLatin1 `eq` (T.pack . BC.unpack) +tl_latin1 = EL.decodeLatin1 `eq` (TL.pack . BLC.unpack) + +t_p_utf8_1 = testValidateUtf8_ [0x63] 1 +t_p_utf8_2 = testValidateUtf8_ [0x63, 0x63, 0x63] 3 +t_p_utf8_3 = testValidateUtf8_ [0x63, 0x63, 0xc2, 0x80, 0x63] 5 +t_p_utf8_4 = testValidateUtf8_ [0x63, 0xe1, 0x80, 0x80, 0x63] 5 +t_p_utf8_5 = testValidateUtf8_ [0xF0, 0x90, 0x80, 0x80, 0x63] 5 +t_p_utf8_6 = testValidateUtf8_ [0x63, 0x63, 0xF0, 0x90, 0x80] 2 +t_p_utf8_7 = testValidateUtf8_ [0x63, 0x63, 0x63, 0xF0, 0x90] 3 +t_p_utf8_8 = testValidateUtf8Fail [0xF0, 0x90, 0x80, 0x63, 0x63] 0 +t_p_utf8_9 = testValidateUtf8Fail [0x63, 0x63, 0x80, 0x63, 0x63] 2 +t_p_utf8_0 = testValidateUtf8Fail [0x63, 0x63, 0xe1, 0x63, 0x63] 2 + +testValidateUtf8With :: + (B.ByteString -> (Int, Maybe E.Utf8State)) -> + (Maybe E.Utf8State -> IO r) -> + [Word8] -> Int -> IO r +testValidateUtf8With validate k xs expectedLen = case validate (B.pack xs) of + (len, s) -> do + len @?= expectedLen + k s + +expectJust :: Maybe a -> IO a +expectJust Nothing = assertFailure "Unexpected Nothing" +expectJust (Just s) = pure s + +expectNothing :: Maybe a -> IO () +expectNothing Nothing = pure () +expectNothing (Just _) = assertFailure "Unexpected Just" + +testValidateUtf8 :: [Word8] -> Int -> IO E.Utf8State +testValidateUtf8 = testValidateUtf8With E.validateUtf8Chunk expectJust + +testValidateUtf8_ :: [Word8] -> Int -> IO () +testValidateUtf8_ = testValidateUtf8With E.validateUtf8Chunk (void . expectJust) + +testValidateUtf8Fail :: [Word8] -> Int -> IO () +testValidateUtf8Fail = testValidateUtf8With E.validateUtf8Chunk expectNothing + +testValidateUtf8More :: E.Utf8State -> [Word8] -> Int -> IO E.Utf8State +testValidateUtf8More s = testValidateUtf8With (E.validateUtf8More s) expectJust + +testValidateUtf8MoreFail :: E.Utf8State -> [Word8] -> Int -> IO () +testValidateUtf8MoreFail s = testValidateUtf8With (E.validateUtf8More s) expectNothing + +t_pn_utf8_1 = do + s <- testValidateUtf8 [0xF0, 0x90, 0x80] 0 + _ <- testValidateUtf8More s [0x80] 1 + testValidateUtf8MoreFail s [0x7f] (-3) +t_pn_utf8_2 = do + s0 <- testValidateUtf8 [0xF0] 0 + testValidateUtf8MoreFail s0 [0x7f] (-1) + s1 <- testValidateUtf8More s0 [0x90] (-1) + testValidateUtf8MoreFail s1 [0x7f] (-2) + s2 <- testValidateUtf8More s1 [0x80] (-2) + testValidateUtf8MoreFail s2 [0x7f] (-3) + _ <- testValidateUtf8More s2 [0x80] 1 + pure () +t_pn_utf8_3 = do + s1 <- testValidateUtf8 [0xc2] 0 + assertBool "PartialUtf8 must be partial" $ B.length (E.getPartialUtf8 s1) < E.getCompleteLen s1 + testValidateUtf8MoreFail s1 [0x80, 0x80] 1 + +-- Precondition: (i, ms1) = E.validateUtf8More s chunk +-- +-- The index points to the end of the longest valid prefix +-- of prechunk `B.append` chunk +pre_validateUtf8More_validPrefix s chunk i = + let prechunk = E.getPartialUtf8 s in + -- Note: i <= 0 implies take i = id + let (j, ms2) = E.validateUtf8Chunk (B.take (B.length prechunk + i) (prechunk `B.append` chunk)) in + counterexample (show prechunk) $ + (B.length prechunk + i, ms2) === (j, Just E.startUtf8State) + +-- Precondition: (i, Nothing) = E.validateUtf8More s chunk +-- +-- Appending to an invalid chunk yields another invalid chunk. +pre_validateUtf8More_maximalPrefix s chunk i more = + E.validateUtf8More s (chunk `B.append` more) === (i, Nothing) + +-- Precondition: (i, Just s1) = E.validateUtf8More s chunk +pre_validateUtf8More_suffix s chunk i s1 = + if 0 <= i + then B.drop i chunk === p2b s1 -- The state s1 contains a suffix of the chunk. + else p2b s `B.append` chunk === p2b s1 -- Or the chunk extends the incomplete code point in s1. + +-- Precondition: (i, Just s1) = E.validateUtf8More s chunk1 +-- +-- Validating two chunks sequentially is equivalent to validating them at once. +pre_validateUtf8More_append s chunk1 s1 chunk2 = + let (j, ms2) = E.validateUtf8More s1 chunk2 in + (B.length chunk1 + j, ms2) === E.validateUtf8More s (chunk1 `B.append` chunk2) + +-- These wrappers use custom generators to satisfy the above properties. + +t_validateUtf8More_validPrefix = property $ do + cex@(s, chunk, i, _ms1) <- randomMoreChunk + pure $ counterexample (show cex) $ + pre_validateUtf8More_validPrefix s chunk i + +t_validateUtf8More_maximalPrefix = property $ do + -- We want chunks that fail validation: force their size to be big,.. + cex@(s, chunk, i, ms1) <- scale (* 3) arbitraryMoreChunk + pure $ counterexample (show cex) $ + -- ... and just use rejection sampling + isNothing ms1 ==> + pre_validateUtf8More_maximalPrefix s chunk i + +t_validateUtf8More_valid = property $ do + cex@(s, chunk1, i, s1, chunk2) <- validMoreChunks + pure $ counterexample (show cex) $ + pre_validateUtf8More_suffix s chunk1 i s1 .&&. + pre_validateUtf8More_append s chunk1 s1 chunk2 + +randomMoreChunk, arbitraryMoreChunk, validMoreChunk :: Gen (E.Utf8State, B.ByteString, Int, Maybe E.Utf8State) +randomMoreChunk = oneof [arbitraryMoreChunk, validMoreChunk] + +arbitraryMoreChunk = do + s <- randomUtf8State + chunk <- arbitrary + let (i, ms1) = E.validateUtf8More s chunk + pure (s, chunk, i, ms1) + +-- | Generate a random state by parsing a prefix of a Char +randomUtf8State :: Gen E.Utf8State +randomUtf8State = do + c <- arbitrary + chunk <- elements (B.inits (E.encodeUtf8 (T.singleton c))) + case E.validateUtf8Chunk chunk of + (_, Just s) -> pure s + (_, Nothing) -> error "should not happen" + +-- | Make a valid chunk, i.e., (s, chunk) such that +-- +-- validateUtf8More s chunk = (i, Just s1) +-- +-- Also returning i and s1 to not repeat work. +validMoreChunk = do + (s, chunk, i, s1, _chunk2) <- validMoreChunks + pure (s, chunk, i, Just s1) + +-- | Make a valid chunk by slicing a valid UTF8 bs, +-- and also provide a second chunk which is a valid extension +-- with 0.5 probability. +validMoreChunks :: Gen (E.Utf8State, B.ByteString, Int, E.Utf8State, B.ByteString) +validMoreChunks = do + bs <- E.encodeUtf8 <$> scale (* 3) arbitrary + -- Take an intermediate state. + -- No need to go too far since code points are at most 4 bytes long + i <- choose (0, 3) + let (bs0, bs1) = B.splitAt i bs + case E.validateUtf8Chunk bs0 of + (_, Just s) -> do + j <- choose (0, B.length bs1) + let (chunk1, chunk2') = B.splitAt j bs1 + case E.validateUtf8More s chunk1 of + (n1, Just s1) -> do + chunk2 <- oneof [pure chunk2', arbitrary] + pure (s, chunk1, n1, s1, chunk2) + (_, Nothing) -> error "should not happen" + (_, Nothing) -> error "should not happen" + +t_utf8_c = (E.strictBuilderToText . fst3 . E.decodeUtf8Chunk . E.encodeUtf8) `eq` id +t_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id +t_utf8' = (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) +tl_utf8 = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id +tl_utf8' = (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right) +t_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id +tl_utf16LE = (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id +t_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id +tl_utf16BE = (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id +t_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id +tl_utf32LE = (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id +t_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id +tl_utf32BE = (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id + +fst3 :: (a, b, c) -> a +fst3 (a, _, _) = a + +runBuilder :: B.Builder -> B.ByteString +runBuilder = + -- Use smallish buffers to exercise bufferFull case as well + BL.toStrict . B.toLazyByteStringWith (B.safeStrategy 5 5) "" + +t_encodeUtf8Builder_ toBuilder = (runBuilder . toBuilder) `eq` E.encodeUtf8 + +t_encodeUtf8Builder_nonZeroOffset_ toBuilder (Positive n) = + (runBuilder . toBuilder . T.drop n) `eq` (E.encodeUtf8 . T.drop n) + +t_encodeUtf8Builder = t_encodeUtf8Builder_ E.encodeUtf8Builder +t_encodeUtf8Builder_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ E.encodeUtf8Builder + +t_encodeUtf8BuilderEscaped = t_encodeUtf8Builder_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) +t_encodeUtf8BuilderEscaped_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) + +t_encodeUtf8Builder_sanity t = + (runBuilder . E.encodeUtf8Builder) t === + (runBuilder . E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) t + +t_utf8_incr (Positive n) = + (T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 . E.encodeUtf8) `eq` id + +feedChunksOf :: Int -> (B.ByteString -> E.Decoding) -> B.ByteString + -> [(T.Text, B.ByteString)] +feedChunksOf n f bs + | B.null bs = [] + | otherwise = let (x,y) = B.splitAt n bs + E.Some t b f' = f x + in (t,b) : feedChunksOf n f' y + +t_utf8_undecoded t = + let b = E.encodeUtf8 t + ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t + leftover = (++ [B.empty]) . init . drop 1 . B.inits + in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls + +data InvalidUtf8 = InvalidUtf8 + { iu8Prefix :: T.Text + , iu8Invalid :: B.ByteString + , iu8Suffix :: T.Text + } deriving (Eq) + +instance Show InvalidUtf8 where + show i = "InvalidUtf8 {prefix = " ++ show (iu8Prefix i) + ++ ", invalid = " ++ show (iu8Invalid i) + ++ ", suffix = " ++ show (iu8Suffix i) + ++ ", asBS = " ++ show (toByteString i) + ++ ", length = " ++ show (B.length (toByteString i)) + ++ "}" + +toByteString :: InvalidUtf8 -> B.ByteString +toByteString (InvalidUtf8 a b c) = + E.encodeUtf8 a `B.append` b `B.append` E.encodeUtf8 c + +instance Arbitrary InvalidUtf8 where + arbitrary = oneof + [ InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> pure mempty + , InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> arbitrary + , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> pure mempty + , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> arbitrary + ] + shrink (InvalidUtf8 a b c) + = map (\c' -> InvalidUtf8 a b c') (shrink c) + ++ map (\a' -> InvalidUtf8 a' b c) (shrink a) + +t_utf8_err :: InvalidUtf8 -> DecodeErr -> Property +t_utf8_err bad de = forAll (Blind <$> genDecodeErr de) $ \(Blind onErr) -> ioProperty $ do + let decoded = E.decodeUtf8With onErr (toByteString bad) + len = T.length (E.decodeUtf8With onErr (toByteString bad)) + l <- Exception.try (Exception.evaluate len) + pure $ case l of + Left (err :: Exception.SomeException) -> counterexample (show err) $ + length (show err) >= 0 + Right _ -> counterexample (show (decoded, l)) $ de /= Strict + +t_utf8_err' :: B.ByteString -> Bool +t_utf8_err' bs = case E.decodeUtf8' bs of + Left err -> length (show err) >= 0 + Right t -> T.length t >= 0 + +genInvalidUTF8 :: Gen B.ByteString +genInvalidUTF8 = B.pack <$> oneof [ + -- invalid leading byte of a 2-byte sequence + (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte + -- invalid leading byte of a 4-byte sequence + , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte + -- 4-byte sequence greater than U+10FFFF + , do k <- choose (0x11, 0x13) + let w0 = 0xF0 + (k `Bits.shiftR` 2) + w1 = 0x80 + ((k .&. 3) `Bits.shiftL` 4) + ([w0,w1]++) <$> vectorOf 2 contByte + -- continuation bytes without a start byte + , listOf1 contByte + -- short 2-byte sequence + , (:[]) <$> choose (0xC2, 0xDF) + -- short 3-byte sequence + , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte + -- short 4-byte sequence + , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte + -- overlong encoding + , do k <- choose (0 :: Int, 0xFFFF) + case k of + _ | k < 0x80 -> elements [ord2_ k, ord3_ k, ord4_ k] + | k < 0x7FF -> elements [ord3_ k, ord4_ k] + | otherwise -> return (ord4_ k) + ] + where + contByte = (0x80 +) <$> choose (0, 0x3f) + upTo n gen = do + k <- choose (0,n) + vectorOf k gen + -- Data.Text.Internal.Encoding.Utf8.ord{2,3,4} without sanity checks + ord2_ n = map fromIntegral [(n `shiftR` 6) + 0xC0, (n .&. 0x3F) + 0x80] + ord3_ n = map fromIntegral [(n `shiftR` 12) + 0xE0, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80] + ord4_ n = map fromIntegral [(n `shiftR` 18) + 0xF0, ((n `shiftR` 12) .&. 0x3F) + 0x80, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80] + +decodeLL :: BL.ByteString -> TL.Text +decodeLL = EL.decodeUtf8With E.lenientDecode + +decodeL :: B.ByteString -> T.Text +decodeL = E.decodeUtf8With E.lenientDecode + +-- The lenient decoding of lazy bytestrings should not depend on how they are chunked, +-- and it should behave the same as decoding of strict bytestrings. +t_decode_utf8_lenient :: Property +t_decode_utf8_lenient = forAllShrinkShow arbitrary shrink (show . BL.toChunks) $ \bs -> + decodeLL bs === (TL.fromStrict . decodeL . B.concat . BL.toChunks) bs + +-- See http://unicode.org/faq/utf_bom.html#gen8 +-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ... +-- When faced with this illegal byte sequence ... a UTF-8 conformant process +-- must treat the first byte 110xxxxx2 as an illegal termination error +-- (e.g. filter it out or replace by 0xFFFD) ... +-- ... and continue processing at the second byte 0xxxxxxx2 +t_decode_with_error2 = + E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa" +t_decode_with_error3 = + E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa" +t_decode_with_error4 = + E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa" + +t_decode_with_error1' = do + E.Some x1 bs1 f1 <- pure $ E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xc2]) + x1 @?= "" + bs1 @?= B.pack [0xc2] + E.Some x2 bs2 _ <- pure $ f1 $ B.pack [0x80, 0x80] + x2 @?= "\x80x" + bs2 @?= mempty +t_decode_with_error2' = + case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of + E.Some x _ _ -> x @?= "xa" +t_decode_with_error3' = + case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of + E.Some x _ _ -> x @?= "xaa" +t_decode_with_error4' = + case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of + E.Some x _ _ -> x @?= "xaaa" +t_decode_with_error5' = do + ret <- Exception.try $ Exception.evaluate $ E.streamDecodeUtf8 (B.pack [0x81]) + case ret of + Left (_ :: E.UnicodeException) -> pure () + Right{} -> assertFailure "Unexpected success" + +testDecodeUtf8With :: (Maybe E.Utf8State -> IO r) -> E.Utf8State -> [Word8] -> T.Text -> IO r +testDecodeUtf8With k s xs expected = + let xs' = B.pack xs in + case E.decodeUtf8More s xs' of + (prefix, bs, s') -> do + let txt = E.strictBuilderToText prefix + txt @?= expected + if T.null txt then + bs @?= xs' + else + E.encodeUtf8 txt `B.append` bs @?= E.getPartialUtf8 s `B.append` xs' + k s' + +testDecodeUtf8 :: E.Utf8State -> [Word8] -> T.Text -> IO E.Utf8State +testDecodeUtf8 = testDecodeUtf8With (\ms -> case ms of + Just s -> pure s + Nothing -> assertFailure "Unexpected failure") + +testDecodeUtf8Fail :: E.Utf8State -> [Word8] -> T.Text -> IO () +testDecodeUtf8Fail = testDecodeUtf8With (\ms -> case ms of + Just _ -> assertFailure "Unexpected failure" + Nothing -> pure ()) + +t_decode_chunk1 = do + s1 <- testDecodeUtf8 E.startUtf8State [0xc2] "" + B.length (E.getPartialUtf8 s1) @?= 1 + testDecodeUtf8Fail s1 [0x80, 0x80] "\128" + +t_decode_chunk2 = do + s1 <- testDecodeUtf8 E.startUtf8State [0xf0] "" + s2 <- testDecodeUtf8 s1 [0x90, 0x80] "" + _ <- testDecodeUtf8 s2 [0x80, 0x41] "\65536A" + pure () + +t_infix_concat bs1 text bs2 = + forAll (Blind <$> genDecodeErr Replace) $ \(Blind onErr) -> + text `T.isInfixOf` + E.decodeUtf8With onErr (B.concat [bs1, E.encodeUtf8 text, bs2]) + +t_textToStrictBuilder = + (E.strictBuilderToText . E.textToStrictBuilder) `eq` id + +-- decodeUtf8Chunk splits the input bytestring +t_decodeUtf8Chunk_split chunk = + let (pre, suf, _ms) = E.decodeUtf8Chunk chunk + in s2b pre `B.append` suf === chunk + +-- decodeUtf8More mostly splits the input bytestring, +-- also inserting bytes from the partial code point in s. +-- +-- This is wrapped by t_decodeUtf8More_split to have more +-- likely valid chunks. +t_decodeUtf8More_split' s chunk = + let (pre, suf, _ms) = E.decodeUtf8More s chunk + in if B.length chunk > B.length suf + then s2b pre `B.append` suf === p2b s `B.append` chunk + else suf === chunk + +-- The output state of decodeUtf8More contains the suffix. +-- +-- Precondition (valid chunk): ms = Just s' +pre_decodeUtf8More_suffix s chunk = + let (_pre, suf, ms) = E.decodeUtf8More s chunk + in case ms of + Nothing -> discard + Just s' -> if B.length chunk > B.length suf + then p2b s' === suf + else p2b s' === p2b s `B.append` suf + +-- Decoding chunks separately is equivalent to decoding their concatenation. +pre_decodeUtf8More_append s chunk1 chunk2 = + let (pre1, _, ms1) = E.decodeUtf8More s chunk1 in + case ms1 of + Nothing -> discard + Just s1 -> + let (pre2, _, ms2) = E.decodeUtf8More s1 chunk2 in + let (pre3, _, ms3) = E.decodeUtf8More s (chunk1 `B.append` chunk2) in + (s2b (pre1 <> pre2), ms2) === (s2b pre3, ms3) + +-- Properties for any chunk +-- (but do try to generate valid chunks often enough) +t_decodeUtf8More1 = property $ do + cex@(s, chunk, _, _) <- randomMoreChunk + pure $ counterexample (show cex) $ + t_decodeUtf8More_split' s chunk + +-- Properties that require valid chunks +t_decodeUtf8More2 = property $ do + cex@(s, chunk, _, _, chunk2) <- validMoreChunks + pure $ counterexample (show cex) $ + pre_decodeUtf8More_suffix s chunk .&&. + pre_decodeUtf8More_append s chunk chunk2 + +s2b = E.encodeUtf8 . E.strictBuilderToText +p2b = E.getPartialUtf8 + +testTranscoding :: TestTree +testTranscoding = + testGroup "transcoding" [ + testProperty "t_ascii" t_ascii, + testProperty "tl_ascii" tl_ascii, + testProperty "t_latin1" t_latin1, + testProperty "tl_latin1" tl_latin1, + testProperty "t_utf8" t_utf8, + testProperty "t_utf8'" t_utf8', + testProperty "t_utf8_undecoded" t_utf8_undecoded, + testProperty "t_utf8_incr" t_utf8_incr, + testProperty "tl_utf8" tl_utf8, + testProperty "tl_utf8'" tl_utf8', + testProperty "t_utf16LE" t_utf16LE, + testProperty "tl_utf16LE" tl_utf16LE, + testProperty "t_utf16BE" t_utf16BE, + testProperty "tl_utf16BE" tl_utf16BE, + testProperty "t_utf32LE" t_utf32LE, + testProperty "tl_utf32LE" tl_utf32LE, + testProperty "t_utf32BE" t_utf32BE, + testProperty "tl_utf32BE" tl_utf32BE, + testGroup "builder" [ + testProperty "t_encodeUtf8Builder" t_encodeUtf8Builder, + testProperty "t_encodeUtf8Builder_nonZeroOffset" t_encodeUtf8Builder_nonZeroOffset, + testProperty "t_encodeUtf8BuilderEscaped" t_encodeUtf8BuilderEscaped, + testProperty "t_encodeUtf8BuilderEscaped_nonZeroOffset" t_encodeUtf8BuilderEscaped_nonZeroOffset, + testProperty "t_encodeUtf8Builder_sanity" t_encodeUtf8Builder_sanity + ], + testGroup "errors" [ + testProperty "t_utf8_err" t_utf8_err, + testProperty "t_utf8_err'" t_utf8_err' + ], + testGroup "error recovery" [ + testProperty "t_decode_utf8_lenient" t_decode_utf8_lenient, + testProperty "t_decode_with_error2" t_decode_with_error2, + testProperty "t_decode_with_error3" t_decode_with_error3, + testProperty "t_decode_with_error4" t_decode_with_error4, + testCase "t_decode_with_error1'" t_decode_with_error1', + testCase "t_decode_with_error2'" t_decode_with_error2', + testCase "t_decode_with_error3'" t_decode_with_error3', + testCase "t_decode_with_error4'" t_decode_with_error4', + testCase "t_decode_with_error5'" t_decode_with_error5', + testProperty "t_infix_concat" t_infix_concat + ], + testGroup "validate" [ + testProperty "t_validateUtf8More_validPrefix" t_validateUtf8More_validPrefix, + testProperty "t_validateUtf8More_maximalPrefix" t_validateUtf8More_maximalPrefix, + testProperty "t_validateUtf8More_valid" t_validateUtf8More_valid + ], + testGroup "streaming" [ + testProperty "t_utf8_c" t_utf8_c, + testCase "t_p_utf8_1" t_p_utf8_1, + testCase "t_p_utf8_2" t_p_utf8_2, + testCase "t_p_utf8_3" t_p_utf8_3, + testCase "t_p_utf8_4" t_p_utf8_4, + testCase "t_p_utf8_5" t_p_utf8_5, + testCase "t_p_utf8_6" t_p_utf8_6, + testCase "t_p_utf8_7" t_p_utf8_7, + testCase "t_p_utf8_8" t_p_utf8_8, + testCase "t_p_utf8_9" t_p_utf8_9, + testCase "t_p_utf8_0" t_p_utf8_0, + testCase "t_pn_utf8_1" t_pn_utf8_1, + testCase "t_pn_utf8_2" t_pn_utf8_2, + testCase "t_pn_utf8_3" t_pn_utf8_3, + testCase "t_decode_chunk1" t_decode_chunk1, + testCase "t_decode_chunk2" t_decode_chunk2, + testProperty "t_decodeUtf8Chunk_split" t_decodeUtf8Chunk_split, + testProperty "t_decodeUtf8More1" t_decodeUtf8More1, + testProperty "t_decodeUtf8More2" t_decodeUtf8More2 + ], + testGroup "strictBuilder" [ + testProperty "textToStrictBuilder" t_textToStrictBuilder + ] + ] diff --git a/tests/Tests/Properties/Validate.hs b/tests/Tests/Properties/Validate.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/Validate.hs @@ -0,0 +1,50 @@ +{-# LANGUAGE CPP #-} +module Tests.Properties.Validate (testValidate) where + +import Data.Array.Byte (ByteArray) +import Data.ByteString (ByteString) +import qualified Data.ByteString as B +import Data.ByteString.Short (toShort) +import Data.Either (isRight) +import Data.Text.Encoding (decodeUtf8', encodeUtf8) +import Data.Text.Internal.Validate (isValidUtf8ByteString, isValidUtf8ByteArray) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck ((===), Gen, Property, + testProperty, arbitrary, forAllShrink, oneof, shrink) +import Tests.QuickCheckUtils () +#if MIN_VERSION_bytestring(0,12,0) +import Data.ByteString.Short (unShortByteString) +#else +#if MIN_VERSION_bytestring(0,11,1) +import Data.ByteString.Short (ShortByteString(SBS)) +#else +import Data.ByteString.Short.Internal (ShortByteString(SBS)) +#endif +import Data.Array.Byte (ByteArray(ByteArray)) + +unShortByteString :: ShortByteString -> ByteArray +unShortByteString (SBS ba) = ByteArray ba +#endif + +testValidate :: TestTree +testValidate = testGroup "validate" + [ testProperty "bytestring" $ forAllShrink genByteString shrink $ \bs -> + isValidUtf8ByteString bs === isRight (decodeUtf8' bs) + , testProperty "bytearray" $ forAllByteArray $ \ba off len bs -> + isValidUtf8ByteArray ba off len === isRight (decodeUtf8' bs) + ] + +genByteString :: Gen ByteString +genByteString = oneof + [ arbitrary + , encodeUtf8 <$> arbitrary + ] + +-- | We want to test 'isValidUtf8ByteArray' with various offsets, so we insert a random +-- prefix and remember its length. +forAllByteArray :: (ByteArray -> Int -> Int -> ByteString -> Property) -> Property +forAllByteArray prop = + forAllShrink genByteString shrink $ \mainSlice -> + forAllShrink arbitrary shrink $ \prefix -> + let bs2ba = unShortByteString . toShort in + prop (bs2ba (prefix `B.append` mainSlice)) (B.length prefix) (B.length mainSlice) mainSlice diff --git a/tests/Tests/QuickCheckUtils.hs b/tests/Tests/QuickCheckUtils.hs --- a/tests/Tests/QuickCheckUtils.hs +++ b/tests/Tests/QuickCheckUtils.hs @@ -1,360 +1,326 @@ --- | This module provides quickcheck utilities, e.g. arbitrary and show --- instances, and comparison functions, so we can focus on the actual properties --- in the 'Tests.Properties' module. --- -{-# LANGUAGE CPP, FlexibleInstances, TypeSynonymInstances #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} -module Tests.QuickCheckUtils - ( - genUnicode - , unsquare - , smallArbitrary - - , BigBounded(..) - , BigInt(..) - , NotEmpty(..) - - , Small(..) - , small - - , Precision(..) - , precision - - , integralRandomR - - , DecodeErr(..) - , genDecodeErr - - , Stringy(..) - , eq - , eqP - - , Encoding(..) - - , write_read - ) where - -import Control.Applicative ((<$>)) -import Control.Arrow (first, (***)) -import Control.DeepSeq (NFData (..), deepseq) -import Control.Exception (bracket) -import Data.String (IsString, fromString) -import Data.Text.Foreign (I16) -import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) -import Data.Word (Word8, Word16) -import Debug.Trace (trace) -import System.Random (Random(..), RandomGen) -import Test.QuickCheck hiding (Fixed(..), Small (..), (.&.)) -import Test.QuickCheck.Monadic (assert, monadicIO, run) -import Test.QuickCheck.Unicode (string) -import Tests.Utils -import qualified Data.ByteString as B -import qualified Data.Text as T -import qualified Data.Text.Encoding.Error as T -import qualified Data.Text.Internal.Fusion as TF -import qualified Data.Text.Internal.Fusion.Common as TF -import qualified Data.Text.Internal.Lazy as TL -import qualified Data.Text.Internal.Lazy.Fusion as TLF -import qualified Data.Text.Lazy as TL -import qualified System.IO as IO - -#if !MIN_VERSION_base(4,4,0) -import Data.Int (Int64) -import Data.Word (Word, Word64) -#endif - -genUnicode :: IsString a => Gen a -genUnicode = fromString <$> string - -instance Random I16 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary I16 where - arbitrary = arbitrarySizedIntegral - shrink = shrinkIntegral - -instance Arbitrary B.ByteString where - arbitrary = B.pack `fmap` arbitrary - shrink = map B.pack . shrink . B.unpack - -#if !MIN_VERSION_base(4,4,0) -instance Random Int64 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Random Word where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Random Word8 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Random Word64 where - randomR = integralRandomR - random = randomR (minBound,maxBound) -#endif - --- For tests that have O(n^2) running times or input sizes, resize --- their inputs to the square root of the originals. -unsquare :: (Arbitrary a, Show a, Testable b) => (a -> b) -> Property -unsquare = forAll smallArbitrary - -smallArbitrary :: (Arbitrary a, Show a) => Gen a -smallArbitrary = sized $ \n -> resize (smallish n) arbitrary - where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs - -instance Arbitrary T.Text where - arbitrary = T.pack `fmap` string - shrink = map T.pack . shrink . T.unpack - -instance Arbitrary TL.Text where - arbitrary = (TL.fromChunks . map notEmpty) `fmap` smallArbitrary - shrink = map TL.pack . shrink . TL.unpack - -newtype BigInt = Big Integer - deriving (Eq, Show) - -instance Arbitrary BigInt where - arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) - shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] - where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer - -newtype BigBounded a = BigBounded a - deriving (Eq, Show) - -instance (Bounded a, Random a, Arbitrary a) => Arbitrary (BigBounded a) where - arbitrary = BigBounded <$> choose (minBound, maxBound) - -newtype NotEmpty a = NotEmpty { notEmpty :: a } - deriving (Eq, Ord) - -instance Show a => Show (NotEmpty a) where - show (NotEmpty a) = show a - -instance Functor NotEmpty where - fmap f (NotEmpty a) = NotEmpty (f a) - -instance Arbitrary a => Arbitrary (NotEmpty [a]) where - arbitrary = sized (\n -> NotEmpty `fmap` (choose (1,n+1) >>= vector)) - shrink = shrinkNotEmpty null - -instance Arbitrary (NotEmpty T.Text) where - arbitrary = (fmap T.pack) `fmap` arbitrary - shrink = shrinkNotEmpty T.null - -instance Arbitrary (NotEmpty TL.Text) where - arbitrary = (fmap TL.pack) `fmap` arbitrary - shrink = shrinkNotEmpty TL.null - -instance Arbitrary (NotEmpty B.ByteString) where - arbitrary = (fmap B.pack) `fmap` arbitrary - shrink = shrinkNotEmpty B.null - -shrinkNotEmpty :: Arbitrary a => (a -> Bool) -> NotEmpty a -> [NotEmpty a] -shrinkNotEmpty isNull (NotEmpty xs) = - [ NotEmpty xs' | xs' <- shrink xs, not (isNull xs') ] - -data Small = S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 - | S8 | S9 | S10 | S11 | S12 | S13 | S14 | S15 - | S16 | S17 | S18 | S19 | S20 | S21 | S22 | S23 - | S24 | S25 | S26 | S27 | S28 | S29 | S30 | S31 - deriving (Eq, Ord, Enum, Bounded) - -small :: Integral a => Small -> a -small = fromIntegral . fromEnum - -intf :: (Int -> Int -> Int) -> Small -> Small -> Small -intf f a b = toEnum ((fromEnum a `f` fromEnum b) `mod` 32) - -instance Show Small where - show = show . fromEnum - -instance Read Small where - readsPrec n = map (first toEnum) . readsPrec n - -instance Num Small where - fromInteger = toEnum . fromIntegral - signum _ = 1 - abs = id - (+) = intf (+) - (-) = intf (-) - (*) = intf (*) - -instance Real Small where - toRational = toRational . fromEnum - -instance Integral Small where - toInteger = toInteger . fromEnum - quotRem a b = (toEnum x, toEnum y) - where (x, y) = fromEnum a `quotRem` fromEnum b - -instance Random Small where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Small where - arbitrary = choose (minBound, maxBound) - shrink = shrinkIntegral - -integralRandomR :: (Integral a, RandomGen g) => (a,a) -> g -> (a,g) -integralRandomR (a,b) g = case randomR (fromIntegral a :: Integer, - fromIntegral b :: Integer) g of - (x,h) -> (fromIntegral x, h) - -data DecodeErr = Lenient | Ignore | Strict | Replace - deriving (Show, Eq) - -genDecodeErr :: DecodeErr -> Gen T.OnDecodeError -genDecodeErr Lenient = return T.lenientDecode -genDecodeErr Ignore = return T.ignore -genDecodeErr Strict = return T.strictDecode -genDecodeErr Replace = arbitrary - -instance Arbitrary DecodeErr where - arbitrary = elements [Lenient, Ignore, Strict, Replace] - -class Stringy s where - packS :: String -> s - unpackS :: s -> String - splitAtS :: Int -> s -> (s,s) - packSChunkSize :: Int -> String -> s - packSChunkSize _ = packS - -instance Stringy String where - packS = id - unpackS = id - splitAtS = splitAt - -instance Stringy (TF.Stream Char) where - packS = TF.streamList - unpackS = TF.unstreamList - splitAtS n s = (TF.take n s, TF.drop n s) - -instance Stringy T.Text where - packS = T.pack - unpackS = T.unpack - splitAtS = T.splitAt - -instance Stringy TL.Text where - packSChunkSize k = TLF.unstreamChunks k . TF.streamList - packS = TL.pack - unpackS = TL.unpack - splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . - TL.splitAt . fromIntegral - --- Do two functions give the same answer? -eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Bool -eq a b s = a s =^= b s - --- What about with the RHS packed? -eqP :: (Eq a, Show a, Stringy s) => - (String -> a) -> (s -> a) -> String -> Word8 -> Bool -eqP f g s w = eql "orig" (f s) (g t) && - eql "mini" (f s) (g mini) && - eql "head" (f sa) (g ta) && - eql "tail" (f sb) (g tb) - where t = packS s - mini = packSChunkSize 10 s - (sa,sb) = splitAt m s - (ta,tb) = splitAtS m t - l = length s - m | l == 0 = n - | otherwise = n `mod` l - n = fromIntegral w - eql d a b - | a =^= b = True - | otherwise = trace (d ++ ": " ++ show a ++ " /= " ++ show b) False - -instance Arbitrary FPFormat where - arbitrary = elements [Exponent, Fixed, Generic] - -newtype Precision a = Precision (Maybe Int) - deriving (Eq, Show) - -precision :: a -> Precision a -> Maybe Int -precision _ (Precision prec) = prec - -arbitraryPrecision :: Int -> Gen (Precision a) -arbitraryPrecision maxDigits = Precision <$> do - n <- choose (-1,maxDigits) - return $ if n == -1 - then Nothing - else Just n - -instance Arbitrary (Precision Float) where - arbitrary = arbitraryPrecision 11 - shrink = map Precision . shrink . precision undefined - -instance Arbitrary (Precision Double) where - arbitrary = arbitraryPrecision 22 - shrink = map Precision . shrink . precision undefined - --- Work around lack of Show instance for TextEncoding. -data Encoding = E String IO.TextEncoding - -instance Show Encoding where show (E n _) = "utf" ++ n - -instance Arbitrary Encoding where - arbitrary = oneof . map return $ - [ E "8" IO.utf8, E "8_bom" IO.utf8_bom, E "16" IO.utf16 - , E "16le" IO.utf16le, E "16be" IO.utf16be, E "32" IO.utf32 - , E "32le" IO.utf32le, E "32be" IO.utf32be - ] - -windowsNewlineMode :: IO.NewlineMode -windowsNewlineMode = IO.NewlineMode - { IO.inputNL = IO.CRLF, IO.outputNL = IO.CRLF - } - -instance Arbitrary IO.NewlineMode where - arbitrary = oneof . map return $ - [ IO.noNewlineTranslation, IO.universalNewlineMode, IO.nativeNewlineMode - , windowsNewlineMode - ] - -instance Arbitrary IO.BufferMode where - arbitrary = oneof [ return IO.NoBuffering, - return IO.LineBuffering, - return (IO.BlockBuffering Nothing), - (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` - (arbitrary :: Gen Word16) ] - --- This test harness is complex! What property are we checking? --- --- Reading after writing a multi-line file should give the same --- results as were written. --- --- What do we vary while checking this property? --- * The lines themselves, scrubbed to contain neither CR nor LF. (By --- working with a list of lines, we ensure that the data will --- sometimes contain line endings.) --- * Encoding. --- * Newline translation mode. --- * Buffering. -write_read :: (NFData a, Eq a) - => ([b] -> a) - -> ((Char -> Bool) -> a -> b) - -> (IO.Handle -> a -> IO ()) - -> (IO.Handle -> IO a) - -> Encoding - -> IO.NewlineMode - -> IO.BufferMode - -> [a] - -> Property -write_read unline filt writer reader (E _ _) nl buf ts = - monadicIO $ assert . (==t) =<< run act - where t = unline . map (filt (not . (`elem` "\r\n"))) $ ts - act = withTempFile $ \path h -> do - -- hSetEncoding h enc - IO.hSetNewlineMode h nl - IO.hSetBuffering h buf - () <- writer h t - IO.hClose h - bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do - -- hSetEncoding h' enc - IO.hSetNewlineMode h' nl - IO.hSetBuffering h' buf - r <- reader h' - r `deepseq` return r +-- | This module provides quickcheck utilities, e.g. arbitrary and show +-- instances, and comparison functions, so we can focus on the actual properties +-- in the 'Tests.Properties' module. +-- +{-# LANGUAGE CPP #-} +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Tests.QuickCheckUtils + ( BigInt(..) + , NotEmpty(..) + , Sqrt(..) + , SpacyString(..) + , SkewedBool(..) + + , Precision(..) + , precision + + , DecodeErr(..) + , genDecodeErr + + , Stringy(..) + , unpack2 + , eq + , eqP + , eqPSqrt + + , write_read + ) where + +import Control.Arrow ((***)) +import Control.Monad (when) +import Data.Char (isSpace) +import Data.IORef (writeIORef) +import Data.Text.Foreign (I8) +import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) +import Data.Word (Word8) +#if !MIN_VERSION_QuickCheck(2,17,0) +import Data.Word (Word16) +#endif +import qualified GHC.IO.Buffer as GIO +import qualified GHC.IO.Handle.Internals as GIO +import qualified GHC.IO.Handle.Types as GIO +import GHC.IO.Encoding.Types (TextEncoding(textEncodingName)) +import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, counterexample, scale, (.&&.), NonEmptyList(..), forAllShrink) +import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) +import Tests.Utils +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import qualified Data.Text as T +import qualified Data.Text.Encoding.Error as T +import qualified Data.Text.Internal.Fusion as TF +import qualified Data.Text.Internal.Fusion.Common as TF +import qualified Data.Text.Internal.Lazy as TL +import qualified Data.Text.Internal.Lazy.Fusion as TLF +import qualified Data.Text.Lazy as TL +import qualified System.IO as IO + +genWord8 :: Gen Word8 +genWord8 = chooseAny + +instance Arbitrary I8 where + arbitrary = arbitrarySizedIntegral + shrink = shrinkIntegral + +instance Arbitrary B.ByteString where + arbitrary = B.pack `fmap` listOf genWord8 + shrink = map B.pack . shrink . B.unpack + +instance Arbitrary BL.ByteString where + arbitrary = oneof + [ BL.fromChunks <$> arbitrary + -- so that a single utf8 code point could appear split over up to 4 chunks + , BL.fromChunks . map B.singleton <$> listOf genWord8 + -- so that a code point with 4 byte long utf8 representation + -- could appear split over 3 non-singleton chunks + , (\a b c -> BL.fromChunks [a, b, c]) + <$> arbitrary + <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8) + <*> arbitrary + ] + shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) + +-- | For tests that have O(n^2) running times or input sizes, resize +-- their inputs to the square root of the originals. +newtype Sqrt a = Sqrt { unSqrt :: a } + deriving (Eq, Show) + +instance Arbitrary a => Arbitrary (Sqrt a) where + arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary + where + smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs + shrink = map Sqrt . shrink . unSqrt + +instance Arbitrary T.Text where + arbitrary = do + t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary + -- Generate chunks that start in the middle of their buffers. + (\i -> T.drop i t) <$> choose (0, T.length t) + shrink = map T.pack . shrink . T.unpack + +instance Arbitrary TL.Text where + arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary + shrink = map TL.pack . shrink . TL.unpack + +newtype BigInt = Big Integer + deriving (Eq, Show) + +instance Arbitrary BigInt where + arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) + shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] + where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer + +newtype NotEmpty a = NotEmpty { notEmpty :: a } + deriving (Eq, Ord, Show) + +instance Arbitrary (NotEmpty T.Text) where + arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . T.pack . getNonEmpty) + . shrink . NonEmpty . T.unpack . notEmpty + +instance Arbitrary (NotEmpty TL.Text) where + arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . TL.pack . getNonEmpty) + . shrink . NonEmpty . TL.unpack . notEmpty + +data DecodeErr = Lenient | Ignore | Strict | Replace + deriving (Show, Eq, Bounded, Enum) + +genDecodeErr :: DecodeErr -> Gen T.OnDecodeError +genDecodeErr Lenient = return T.lenientDecode +genDecodeErr Ignore = return T.ignore +genDecodeErr Strict = return T.strictDecode +genDecodeErr Replace = (\c _ _ -> c) <$> frequency + [ (1, return Nothing) + , (50, Just <$> arbitraryUnicodeChar) + ] + +instance Arbitrary DecodeErr where + arbitrary = arbitraryBoundedEnum + +class Stringy s where + packS :: String -> s + unpackS :: s -> String + splitAtS :: Int -> s -> (s,s) + packSChunkSize :: Int -> String -> s + packSChunkSize _ = packS + +instance Stringy String where + packS = id + unpackS = id + splitAtS = splitAt + +instance Stringy (TF.Stream Char) where + packS = TF.streamList + unpackS = TF.unstreamList + splitAtS n s = (TF.take n s, TF.drop n s) + +instance Stringy T.Text where + packS = T.pack + unpackS = T.unpack + splitAtS = T.splitAt + +instance Stringy TL.Text where + packSChunkSize k = TLF.unstreamChunks k . TF.streamList + packS = TL.pack + unpackS = TL.unpack + splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . + TL.splitAt . fromIntegral + +unpack2 :: (Stringy s) => (s,s) -> (String,String) +unpack2 = unpackS *** unpackS + +-- Do two functions give the same answer? +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property +eq a b s = a s =^= b s + +-- What about with the RHS packed? +eqP :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> String -> Word8 -> Property +eqP f g s w = counterexample "orig" (f s =^= g t) .&&. + counterexample "mini" (f s =^= g mini) .&&. + counterexample "head" (f sa =^= g ta) .&&. + counterexample "tail" (f sb =^= g tb) + where t = packS s + mini = packSChunkSize 10 s + (sa,sb) = splitAt m s + (ta,tb) = splitAtS m t + l = length s + m | l == 0 = n + | otherwise = n `mod` l + n = fromIntegral w + +eqPSqrt :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property +eqPSqrt f g s = eqP f g (unSqrt s) + +instance Arbitrary FPFormat where + arbitrary = arbitraryBoundedEnum + +newtype Precision a = Precision (Maybe Int) + deriving (Eq, Show) + +precision :: a -> Precision a -> Maybe Int +precision _ (Precision prec) = prec + +arbitraryPrecision :: Int -> Gen (Precision a) +arbitraryPrecision maxDigits = Precision <$> do + n <- choose (-1,maxDigits) + return $ if n == -1 + then Nothing + else Just n + +instance Arbitrary (Precision Float) where + arbitrary = arbitraryPrecision 11 + shrink = map Precision . shrink . precision undefined + +instance Arbitrary (Precision Double) where + arbitrary = arbitraryPrecision 22 + shrink = map Precision . shrink . precision undefined + +#if !MIN_VERSION_QuickCheck(2,14,3) +instance Arbitrary IO.Newline where + arbitrary = oneof [return IO.LF, return IO.CRLF] + +instance Arbitrary IO.NewlineMode where + arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary +#endif + +#if !MIN_VERSION_QuickCheck(2,17,0) +instance Arbitrary IO.BufferMode where + arbitrary = oneof [ return IO.NoBuffering, + return IO.LineBuffering, + return (IO.BlockBuffering Nothing), + (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` + genWord16 ] + +genWord16 :: Gen Word16 +genWord16 = chooseAny +#endif + +-- This test harness is complex! What property are we checking? +-- +-- Reading after writing a multi-line file should give the same +-- results as were written. +-- +-- What do we vary while checking this property? +-- * The lines themselves, scrubbed to contain neither CR nor LF. (By +-- working with a list of lines, we ensure that the data will +-- sometimes contain line endings.) +-- * Newline translation mode. +-- * Buffering. +write_read :: forall a. + (Eq a, Show a) + => Gen a + -> (a -> [a]) + -> (a -> a) -- ^ replace '\n' with '\r\n' (for multiline tests) or append '\r' (for single-line tests) + -> (IO.Handle -> a -> IO ()) + -> (IO.Handle -> IO a) + -> Property +write_read genTxt shrinkTxt expandNl writer reader + = forAllShrink genEncoding shrinkEncoding propTest + where + propTest :: TextEncoding -> IO.BufferMode -> Property + propTest enc mode = forAllShrink genTxt shrinkTxt $ \txt -> ioProperty $ do + file <- emptyTempFile + let with nl k = IO.withFile file IO.ReadWriteMode $ \h -> do + IO.hSetEncoding h enc + IO.hSetBuffering h mode + IO.hSetNewlineMode h nl + setSmallBuffer h + k h + -- Put a very small buffer in Handle to easily test boundary conditions in `writeBlocks` + setSmallBuffer h = GIO.withHandle_ "setSmallBuffer" h $ \h_ -> do + buf <- GIO.newCharBuffer 9 GIO.WriteBuffer + writeIORef (GIO.haCharBuffer h_) buf + readExpecting h txt' msg = do + out <- reader h + when (txt' /= out) $ error (show txt' ++ " /= " ++ show out ++ msg) + -- 'reader' may be 'hGetContents', which closes the handle + -- So we reopen a new file every time. + + -- Test with CRLF encoding + with (IO.NewlineMode IO.CRLF IO.CRLF) $ \h -> do + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 1)" + + -- Re-read without CRLF decoding to check that we did encode CRLF correctly + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + readExpecting h (expandNl txt) " (at location 2)" + + -- Test without CRLF encoding + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + IO.hSetFileSize h 0 + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 3)" + + genEncoding = elements [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be] + shrinkEncoding enc = if textEncodingName enc == textEncodingName IO.utf8 then [] else [IO.utf8] + +-- Generate various Unicode space characters with high probability +arbitrarySpacyChar :: Gen Char +arbitrarySpacyChar = oneof + [ arbitraryUnicodeChar + , elements $ filter isSpace [minBound..maxBound] + ] + +newtype SpacyString = SpacyString { getSpacyString :: String } + deriving (Eq, Ord, Show, Read) + +instance Arbitrary SpacyString where + arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar + shrink (SpacyString xs) = SpacyString `fmap` shrink xs + +newtype SkewedBool = Skewed { getSkewed :: Bool } + deriving Show + +instance Arbitrary SkewedBool where + arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)] diff --git a/tests/Tests/RebindableSyntaxTest.hs b/tests/Tests/RebindableSyntaxTest.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/RebindableSyntaxTest.hs @@ -0,0 +1,18 @@ +{-# LANGUAGE CPP, RebindableSyntax, TemplateHaskell #-} + +module Tests.RebindableSyntaxTest where + +import qualified Data.Text as Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) +import Prelude (($)) + +tests :: TestTree +tests = testGroup "RebindableSyntax" + [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a") + ] diff --git a/tests/Tests/Regressions.hs b/tests/Tests/Regressions.hs --- a/tests/Tests/Regressions.hs +++ b/tests/Tests/Regressions.hs @@ -1,82 +1,236 @@ --- | Regression tests for specific bugs. --- -{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} -module Tests.Regressions - ( - tests - ) where - -import Control.Exception (SomeException, handle) -import System.IO -import Test.HUnit (assertBool, assertEqual, assertFailure) -import qualified Data.ByteString as B -import Data.ByteString.Char8 () -import qualified Data.ByteString.Lazy as LB -import qualified Data.Text as T -import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as T -import qualified Data.Text.Lazy as LT -import qualified Data.Text.Lazy.Encoding as LE -import qualified Data.Text.Unsafe as T -import qualified Test.Framework as F -import qualified Test.Framework.Providers.HUnit as F - -import Tests.Utils (withTempFile) - --- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring --- caused either a segfault or attempt to allocate a negative number --- of bytes. -lazy_encode_crash :: IO () -lazy_encode_crash = withTempFile $ \ _ h -> - LB.hPut h . LE.encodeUtf8 . LT.pack . replicate 100000 $ 'a' - --- Reported by Pieter Laeremans: attempting to read an incorrectly --- encoded file can result in a crash in the RTS (i.e. not merely an --- exception). -hGetContents_crash :: IO () -hGetContents_crash = withTempFile $ \ path h -> do - B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) >> hClose h - h' <- openFile path ReadMode - hSetEncoding h' utf8 - handle (\(_::SomeException) -> return ()) $ - T.hGetContents h' >> assertFailure "T.hGetContents should crash" - --- Reported by Ian Lynagh: attempting to allocate a sufficiently large --- string (via either Array.new or Text.replicate) could result in an --- integer overflow. -replicate_crash :: IO () -replicate_crash = handle (\(_::SomeException) -> return ()) $ - T.replicate (2^power) "0123456789abcdef" `seq` - assertFailure "T.replicate should crash" - where - power | maxBound == (2147483647::Int) = 28 - | otherwise = 60 :: Int - --- Reported by John Millikin: a UTF-8 decode error handler could --- return a bogus substitution character, which we would write without --- checking. -utf8_decode_unsafe :: IO () -utf8_decode_unsafe = do - let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" - assertBool "broken error recovery shouldn't break us" (t == "\xfffd") - --- Reported by Eric Seidel: we mishandled mapping Chars that fit in a --- single Word16 to Chars that require two. -mapAccumL_resize :: IO () -mapAccumL_resize = do - let f a _ = (a, '\65536') - count = 5 - val = T.mapAccumL f (0::Int) (T.replicate count "a") - assertEqual "mapAccumL should correctly fill buffers for two-word results" - (0, T.replicate count "\65536") val - assertEqual "mapAccumL should correctly size buffers for two-word results" - (count * 2) (T.lengthWord16 (snd val)) - -tests :: F.Test -tests = F.testGroup "Regressions" - [ F.testCase "hGetContents_crash" hGetContents_crash - , F.testCase "lazy_encode_crash" lazy_encode_crash - , F.testCase "mapAccumL_resize" mapAccumL_resize - , F.testCase "replicate_crash" replicate_crash - , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe - ] +-- | Regression tests for specific bugs. +-- +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Tests.Regressions + ( + tests + ) where + +import Control.Exception (ErrorCall, SomeException, handle, evaluate, displayException, try) +import Data.Char (isLetter, chr) +import GHC.Exts (Int(..), sizeofByteArray#) +import System.IO +import System.IO.Temp (withSystemTempFile) +import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure, (@?=)) +import qualified Data.ByteString as B +import Data.ByteString.Char8 () +import qualified Data.ByteString.Lazy as LB +import Data.Semigroup (stimes) +import qualified Data.Text as T +import qualified Data.Text.Array as TA +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as E +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as LF +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as LT +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Text.Lazy.Encoding as LE +import qualified Data.Text.Unsafe as T +import qualified Test.Tasty as F +import qualified Test.Tasty.HUnit as F +import Tests.Utils (withTempFile) +import System.IO.Error (isFullError) + +-- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring +-- caused either a segfault or attempt to allocate a negative number +-- of bytes. +lazy_encode_crash :: IO () +lazy_encode_crash = withTempFile $ \ _ h -> do + putRes <- try $ LB.hPut h $ LE.encodeUtf8 $ LT.pack $ replicate 100000 'a' + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> pure () + +-- Reported by Pieter Laeremans: attempting to read an incorrectly +-- encoded file can result in a crash in the RTS (i.e. not merely an +-- exception). +hGetContents_crash :: IO () +hGetContents_crash = withSystemTempFile "crashy.txt" $ \path h -> do + putRes <- try $ B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> do + hClose h + h' <- openFile path ReadMode + hSetEncoding h' utf8 + handle (\(_::SomeException) -> pure ()) $ + T.hGetContents h' >> assertFailure "T.hGetContents should crash" + hClose h' + +-- Reported by Ian Lynagh: attempting to allocate a sufficiently large +-- string (via either Array.new or Text.replicate) could result in an +-- integer overflow. +replicate_crash :: IO () +replicate_crash = handle (\(_::SomeException) -> return ()) $ + T.replicate (2^power) "0123456789abcdef" `seq` + assertFailure "T.replicate should crash" + where + power | maxBound == (2147483647::Int) = 28 + | otherwise = 60 :: Int + +-- Reported by John Millikin: a UTF-8 decode error handler could +-- return a bogus substitution character, which we would write without +-- checking. +utf8_decode_unsafe :: IO () +utf8_decode_unsafe = do + let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" + assertBool "broken error recovery shouldn't break us" (t == "\xfffd") + +-- Reported by Eric Seidel: we mishandled mapping Chars that fit in a +-- single Word16 to Chars that require two. +mapAccumL_resize :: IO () +mapAccumL_resize = do + let f a _ = (a, '\65536') + count = 5 + val = T.mapAccumL f (0::Int) (T.replicate count "a") + assertEqual "mapAccumL should correctly fill buffers for four-byte results" + (0, T.replicate count "\65536") val + assertEqual "mapAccumL should correctly size buffers for four-byte results" + (count * 4) (T.lengthWord8 (snd val)) + +-- See GitHub #197 +t197 :: IO () +t197 = + assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00") + where + currencyParser x = cond == 1 + where + cond = length fltr + fltr = filter (== ',') x + +t221 :: IO () +t221 = + assertEqual "toLower of large input shouldn't crash" + (T.toLower (T.replicate 200000 "0") `seq` ()) + () + +t227 :: IO () +t227 = + assertEqual "take (-3) shouldn't crash with overflow" + (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?") + 0 + +t280_fromString :: IO () +t280_fromString = + assertEqual "TB.fromString performs replacement on invalid scalar values" + (TB.toLazyText (TB.fromString "\xD800")) + (LT.pack "\xFFFD") + +t280_singleton :: IO () +t280_singleton = + assertEqual "TB.singleton performs replacement on invalid scalar values" + (TB.toLazyText (TB.singleton '\xD800')) + (LT.pack "\xFFFD") + +-- See GitHub issue #301 +-- This tests whether the "TEXT take . drop -> unfused" rule is applied to the +-- slice function. When the slice function is fused, a new array will be +-- constructed that is shorter than the original array. Without fusion the +-- array remains unmodified. +t301 :: IO () +t301 = do + assertEqual "The length of the array remains the same despite slicing" + (I# (sizeofByteArray# originalArr)) + (I# (sizeofByteArray# newArr)) + + assertEqual "The new array still contains the original value" + (T.Text (TA.ByteArray newArr) originalOff originalLen) + original + where + !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" + !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original + +t330 :: IO () +t330 = do + let decodeL = LE.decodeUtf8With E.lenientDecode + assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked" + (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]])) + (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]])) + +-- Stream decoders should not loop on incomplete code points +t525 :: IO () +t525 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0" @?= "\65533" + LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533" + +-- Stream decoders skip one invalid byte at a time +t528 :: IO () +t528 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536" + LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536" + LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536" + LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0" + LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280" + +t529 :: IO () +t529 = do + let decode = TE.decodeUtf8With E.lenientDecode + -- https://github.com/haskell/bytestring/issues/575 + assertEqual "Data.ByteString.isValidUtf8 should work correctly" + (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0])) + (decode (B.pack (33 : replicate 31 0 ++ [128, 0]))) + +-- See Github #559 +-- filter/filter fusion rules should apply predicates in the right order. +t559 :: IO () +t559 = do + T.filter undefined (T.filter (const False) "a") @?= "" + LT.filter undefined (LT.filter (const False) "a") @?= "" + +-- Github #633 +-- stimes checked for an `a` to `Int` to `a` roundtrip, but the `a` and `Int` values could represent different integers. +t633 :: IO () +t633 = + handle (\(_ :: ErrorCall) -> return ()) $ do + _ <- evaluate (stimes (maxBound :: Word) "a" :: T.Text) + assertFailure "should fail" + +t648 :: IO () +t648 = withTempFile $ \_ h -> do + hSetEncoding h utf8 + hSetNewlineMode h (NewlineMode LF CRLF) + hSetBuffering h (BlockBuffering $ Just 4) + let line = T.replicate 2047 "_" + T.hPutStrLn h line + hSeek h AbsoluteSeek 0 + line' <- T.hGetLine h + T.append line "\r" @?= line' + +tests :: F.TestTree +tests = F.testGroup "Regressions" + [ F.testCase "hGetContents_crash" hGetContents_crash + , F.testCase "lazy_encode_crash" lazy_encode_crash + , F.testCase "mapAccumL_resize" mapAccumL_resize + , F.testCase "replicate_crash" replicate_crash + , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe + , F.testCase "t197" t197 + , F.testCase "t221" t221 + , F.testCase "t227" t227 + , F.testCase "t280/fromString" t280_fromString + , F.testCase "t280/singleton" t280_singleton + , F.testCase "t301" t301 + , F.testCase "t330" t330 + , F.testCase "t525" t525 + , F.testCase "t528" t528 + , F.testCase "t529" t529 + , F.testCase "t559" t559 + , F.testCase "t633" t633 + , F.testCase "t648" t648 + ] diff --git a/tests/Tests/ShareEmpty.hs b/tests/Tests/ShareEmpty.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/ShareEmpty.hs @@ -0,0 +1,138 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE BangPatterns #-} + +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.ShareEmpty + ( tests + ) where + +import Control.Exception (evaluate) +import Data.Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertFailure, assertEqual) +import Test.Tasty (TestTree, testGroup) +import GHC.Exts +import GHC.Stack +import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList +import qualified Data.Text as T + + +-- | assert that a text value is represented by the same pointer +-- as the 'empty' value. +assertPtrEqEmpty :: HasCallStack => Text -> IO () +assertPtrEqEmpty t = do + t' <- evaluate t + empty' <- evaluate empty + assertEqual "" empty' t' + case reallyUnsafePtrEquality# empty' t' of + 1# -> pure () + _ -> assertFailure "Pointers are not equal" +{-# NOINLINE assertPtrEqEmpty #-} + +tests :: TestTree +tests = testGroup "empty Text values are shared" + [ testCase "empty = empty" $ assertPtrEqEmpty T.empty + , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack "" + , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString "" + , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack "")) + , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a" + , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b" + , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty + , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " [] + , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' "" + , testCase "reverse empty = empty" $ assertPtrEqEmpty $ + T.reverse empty + , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $ + T.replace "needle" "replacement" empty + , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold "" + , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower "" + , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper "" + , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle "" + , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyLeft 0 ' ' empty + , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyRight 0 ' ' empty + , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $ + T.center 0 ' ' empty + , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $ + T.transpose [empty] + , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat [] + , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty] + , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x" + , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty + , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $ + T.unfoldr (const Nothing) () + , testCase "take 0 _ = empty" $ assertPtrEqEmpty $ + T.take 0 "xyz" + , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $ + T.takeEnd 0 "xyz" + , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhile (const False) "xyz" + , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhileEnd (const False) "xyz" + , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $ + T.drop 5 "xyz" + , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $ + T.dropEnd 5 "xyz" + , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $ + T.dropWhile (const True) "xyz" + , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $ + dropWhileEnd (const True) "xyz" + , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $ + dropAround (const True) empty + , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty + , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty + , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty + , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123" + , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $ + snd $ T.splitAt 5 "123" + , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (spanM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123" + , testCase "snd (spanM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123" + , testCase "fst (spanEndM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123" + , testCase "snd (spanEndM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123" + , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty + , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty + , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty + , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123" + , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123" + , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty + , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty + , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" + , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123" + , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') "" + , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234" + , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" "" + , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines [] + , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords [] + , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripPrefix empty empty + , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripSuffix empty empty + , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $ + mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123" + , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $ + mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz" + , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy "" + ] diff --git a/tests/Tests/Utils.hs b/tests/Tests/Utils.hs --- a/tests/Tests/Utils.hs +++ b/tests/Tests/Utils.hs @@ -1,52 +1,50 @@ --- | Miscellaneous testing utilities --- -{-# LANGUAGE ScopedTypeVariables #-} -module Tests.Utils - ( - (=^=) - , withRedirect - , withTempFile - ) where - -import Control.Exception (SomeException, bracket, bracket_, evaluate, try) -import Control.Monad (when) -import Debug.Trace (trace) -import GHC.IO.Handle.Internals (withHandle) -import System.Directory (removeFile) -import System.IO (Handle, hClose, hFlush, hIsOpen, hIsWritable, openTempFile) -import System.IO.Unsafe (unsafePerformIO) - --- Ensure that two potentially bottom values (in the sense of crashing --- for some inputs, not looping infinitely) either both crash, or both --- give comparable results for some input. -(=^=) :: (Eq a, Show a) => a -> a -> Bool -i =^= j = unsafePerformIO $ do - x <- try (evaluate i) - y <- try (evaluate j) - case (x,y) of - (Left (_ :: SomeException), Left (_ :: SomeException)) - -> return True - (Right a, Right b) -> return (a == b) - e -> trace ("*** Divergence: " ++ show e) return False -infix 4 =^= -{-# NOINLINE (=^=) #-} - -withTempFile :: (FilePath -> Handle -> IO a) -> IO a -withTempFile = bracket (openTempFile "." "crashy.txt") cleanupTemp . uncurry - where - cleanupTemp (path,h) = do - open <- hIsOpen h - when open (hClose h) - removeFile path - -withRedirect :: Handle -> Handle -> IO a -> IO a -withRedirect tmp h = bracket_ swap swap - where - whenM p a = p >>= (`when` a) - swap = do - whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp - whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h - withHandle "spam" tmp $ \tmph -> do - hh <- withHandle "spam" h $ \hh -> - return (tmph,hh) - return (hh,()) +-- | Miscellaneous testing utilities +-- +{-# LANGUAGE ScopedTypeVariables #-} +module Tests.Utils + ( + (=^=) + , withRedirect + , withTempFile + , emptyTempFile + ) where + +import Control.Exception (SomeException, bracket_, evaluate, try) +import Control.Monad (when) +import System.IO.Temp (withSystemTempFile, emptySystemTempFile) +import GHC.IO.Handle.Internals (withHandle) +import System.IO (Handle, hFlush, hIsOpen, hIsWritable) +import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) + +-- Ensure that two potentially bottom values (in the sense of crashing +-- for some inputs, not looping infinitely) either both crash, or both +-- give comparable results for some input. +(=^=) :: (Eq a, Show a) => a -> a -> Property +i =^= j = ioProperty $ do + x <- try (evaluate i) + y <- try (evaluate j) + return $ case (x, y) of + (Left (_ :: SomeException), Left (_ :: SomeException)) + -> property True + (Right a, Right b) -> a === b + e -> counterexample ("Divergence: " ++ show e) $ property False +infix 4 =^= +{-# NOINLINE (=^=) #-} + +withTempFile :: (FilePath -> Handle -> IO a) -> IO a +withTempFile = withSystemTempFile "crashy.txt" + +emptyTempFile :: IO FilePath +emptyTempFile = emptySystemTempFile "crashy.txt" + +withRedirect :: Handle -> Handle -> IO a -> IO a +withRedirect tmp h = bracket_ swap swap + where + whenM p a = p >>= (`when` a) + swap = do + whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp + whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h + withHandle "spam" tmp $ \tmph -> do + hh <- withHandle "spam" h $ \hh -> + return (tmph,hh) + return (hh,()) diff --git a/tests/cabal.config b/tests/cabal.config deleted file mode 100644 --- a/tests/cabal.config +++ /dev/null @@ -1,6 +0,0 @@ --- These flags help to speed up building the test suite. - -documentation: False -executable-stripping: False -flags: developer -library-profiling: False diff --git a/tests/literal-rule-test.sh b/tests/literal-rule-test.sh new file mode 100644 --- /dev/null +++ b/tests/literal-rule-test.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +failed=0 + +function check_firings() { + rule=$1 + expected=$2 + build="ghc -O -ddump-rule-firings LiteralRuleTest.hs" + build="$build -i.. -I../include" + touch LiteralRuleTest.hs + echo -n "Want to see $expected firings of rule $rule... " >&2 + firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l) + rm -f LiteralRuleTest.{o.hi} + + if [ $firings != $expected ]; then + echo "failed, saw $firings" >&2 + failed=1 + else + echo "pass" >&2 + fi +} + +check_firings "TEXT literal" 8 +check_firings "TEXT literal UTF8" 7 +check_firings "TEXT empty literal" 4 +# This is broken at the moment. "TEXT literal" rule fires instead. +#check_firings "TEXT singleton literal" 5 + +exit $failed diff --git a/tests/scripts/cover-stdio.sh b/tests/scripts/cover-stdio.sh deleted file mode 100644 --- a/tests/scripts/cover-stdio.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -if [[ $# < 1 ]]; then - echo "Usage: $0 " - exit 1 -fi - -exe=$1 - -rm -f $exe.tix - -f=$(mktemp stdio-f.XXXXXX) -g=$(mktemp stdio-g.XXXXXX) - -for t in T TL; do - echo $t.readFile > $f - $exe $t.readFile $f > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.readFile 1>&2 - fi - - $exe $t.writeFile $f $t.writeFile - echo -n $t.writeFile > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.writeFile 1>&2 - fi - - echo -n quux > $f - $exe $t.appendFile $f $t.appendFile - echo -n quux$t.appendFile > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.appendFile 1>&2 - fi - - echo $t.interact | $exe $t.interact > $f - echo $t.interact > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.interact 1>&2 - fi - - echo $t.getContents | $exe $t.getContents > $f - echo $t.getContents > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.getContents 1>&2 - fi - - echo $t.getLine | $exe $t.getLine > $f - echo $t.getLine > $g - if ! diff -u $f $g; then - errs=$((errs+1)) - echo FAIL: $t.getLine 1>&2 - fi -done - -rm -f $f $g - -exit $errs diff --git a/tests/text-tests.cabal b/tests/text-tests.cabal deleted file mode 100644 --- a/tests/text-tests.cabal +++ /dev/null @@ -1,158 +0,0 @@ -name: text-tests -version: 0.0.0.0 -synopsis: Functional tests for the text package -description: Functional tests for the text package -homepage: https://github.com/bos/text -license: BSD2 -license-file: ../LICENSE -author: Jasper Van der Jeugt , - Bryan O'Sullivan , - Tom Harper , - Duncan Coutts -maintainer: Bryan O'Sullivan -category: Text -build-type: Simple - -cabal-version: >=1.8 - -flag hpc - description: Enable HPC to generate coverage reports - default: False - manual: True - -flag bytestring-builder - description: Depend on the bytestring-builder package for backwards compatibility. - default: False - manual: False - -executable text-tests - main-is: Tests.hs - - other-modules: - Tests.IO - Tests.Properties - Tests.Properties.Mul - Tests.QuickCheckUtils - Tests.Regressions - Tests.SlowFunctions - Tests.Utils - - ghc-options: - -Wall -threaded -O0 -rtsopts - - if flag(hpc) - ghc-options: - -fhpc - - cpp-options: - -DTEST_SUITE - -DASSERTS - -DHAVE_DEEPSEQ - - build-depends: - HUnit >= 1.2, - QuickCheck >= 2.7, - base == 4.*, - deepseq, - directory, - quickcheck-unicode >= 1.0.1.0, - random, - test-framework >= 0.4, - test-framework-hunit >= 0.2, - test-framework-quickcheck2 >= 0.2, - text-tests - - if flag(bytestring-builder) - build-depends: bytestring >= 0.9 && < 0.10.4, - bytestring-builder >= 0.10.4 - else - build-depends: bytestring >= 0.10.4 - -executable text-tests-stdio - main-is: Tests/IO.hs - - ghc-options: - -Wall -threaded -rtsopts - - -- Optional HPC support - if flag(hpc) - ghc-options: - -fhpc - - build-depends: - text-tests, - base >= 4 && < 5 - -library - hs-source-dirs: .. - c-sources: ../cbits/cbits.c - include-dirs: ../include - ghc-options: -Wall - exposed-modules: - Data.Text - Data.Text.Array - Data.Text.Encoding - Data.Text.Encoding.Error - Data.Text.Internal.Encoding.Fusion - Data.Text.Internal.Encoding.Fusion.Common - Data.Text.Internal.Encoding.Utf16 - Data.Text.Internal.Encoding.Utf32 - Data.Text.Internal.Encoding.Utf8 - Data.Text.Foreign - Data.Text.Internal.Fusion - Data.Text.Internal.Fusion.CaseMapping - Data.Text.Internal.Fusion.Common - Data.Text.Internal.Fusion.Size - Data.Text.Internal.Fusion.Types - Data.Text.IO - Data.Text.Internal.IO - Data.Text.Internal - Data.Text.Lazy - Data.Text.Lazy.Builder - Data.Text.Internal.Builder.Functions - Data.Text.Lazy.Builder.Int - Data.Text.Internal.Builder.Int.Digits - Data.Text.Internal.Builder - Data.Text.Lazy.Builder.RealFloat - Data.Text.Internal.Builder.RealFloat.Functions - Data.Text.Lazy.Encoding - Data.Text.Internal.Lazy.Encoding.Fusion - Data.Text.Internal.Lazy.Fusion - Data.Text.Lazy.IO - Data.Text.Internal.Lazy - Data.Text.Lazy.Read - Data.Text.Internal.Lazy.Search - Data.Text.Internal.Private - Data.Text.Read - Data.Text.Show - Data.Text.Internal.Read - Data.Text.Internal.Search - Data.Text.Unsafe - Data.Text.Internal.Unsafe - Data.Text.Internal.Unsafe.Char - Data.Text.Internal.Unsafe.Shift - Data.Text.Internal.Functions - - if flag(hpc) - ghc-options: - -fhpc - - cpp-options: - -DTEST_SUITE - -DHAVE_DEEPSEQ - -DASSERTS - -DINTEGER_GMP - - build-depends: - array, - base == 4.*, - binary, - deepseq, - ghc-prim, - integer-gmp - - if flag(bytestring-builder) - build-depends: bytestring >= 0.9 && < 0.10.4, - bytestring-builder >= 0.10.4 - else - build-depends: bytestring >= 0.10.4 diff --git a/text.cabal b/text.cabal --- a/text.cabal +++ b/text.cabal @@ -1,208 +1,385 @@ -name: text -version: 1.2.2.2 -homepage: https://github.com/bos/text -bug-reports: https://github.com/bos/text/issues -synopsis: An efficient packed Unicode text type. -description: - . - An efficient packed, immutable Unicode text type (both strict and - lazy), with a powerful loop fusion optimization framework. - . - The 'Text' type represents Unicode character strings, in a time and - space-efficient manner. This package provides text processing - capabilities that are optimized for performance critical use, both - in terms of large data quantities and high speed. - . - The 'Text' type provides character-encoding, type-safe case - conversion via whole-string case conversion functions. It also - provides a range of functions for converting 'Text' values to and from - 'ByteStrings', using several standard encodings. - . - Efficient locale-sensitive support for text IO is also supported. - . - These modules are intended to be imported qualified, to avoid name - clashes with Prelude functions, e.g. - . - > import qualified Data.Text as T - . - To use an extended and very rich family of functions for working - with Unicode text (including normalization, regular expressions, - non-standard encodings, text breaking, and locales), see - the @text-icu@ package: - - -license: BSD2 -license-file: LICENSE -author: Bryan O'Sullivan -maintainer: Bryan O'Sullivan -copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper -category: Data, Text -build-type: Simple -cabal-version: >= 1.8 -extra-source-files: - -- scripts/CaseFolding.txt - -- scripts/SpecialCasing.txt - README.markdown - benchmarks/Setup.hs - benchmarks/cbits/*.c - benchmarks/haskell/*.hs - benchmarks/haskell/Benchmarks/*.hs - benchmarks/haskell/Benchmarks/Programs/*.hs - benchmarks/python/*.py - benchmarks/ruby/*.rb - benchmarks/text-benchmarks.cabal - changelog.md - include/*.h - scripts/*.hs - tests-and-benchmarks.markdown - tests/*.hs - tests/.ghci - tests/Makefile - tests/Tests/*.hs - tests/Tests/Properties/*.hs - tests/cabal.config - tests/scripts/*.sh - tests/text-tests.cabal - -flag bytestring-builder - description: Depend on the bytestring-builder package for backwards compatibility. - default: False - manual: False - -flag developer - description: operate in developer mode - default: False - manual: True - -flag integer-simple - description: Use the simple integer library instead of GMP - default: False - manual: False - -library - c-sources: cbits/cbits.c - include-dirs: include - - exposed-modules: - Data.Text - Data.Text.Array - Data.Text.Encoding - Data.Text.Encoding.Error - Data.Text.Foreign - Data.Text.IO - Data.Text.Internal - Data.Text.Internal.Builder - Data.Text.Internal.Builder.Functions - Data.Text.Internal.Builder.Int.Digits - Data.Text.Internal.Builder.RealFloat.Functions - Data.Text.Internal.Encoding.Fusion - Data.Text.Internal.Encoding.Fusion.Common - Data.Text.Internal.Encoding.Utf16 - Data.Text.Internal.Encoding.Utf32 - Data.Text.Internal.Encoding.Utf8 - Data.Text.Internal.Functions - Data.Text.Internal.Fusion - Data.Text.Internal.Fusion.CaseMapping - Data.Text.Internal.Fusion.Common - Data.Text.Internal.Fusion.Size - Data.Text.Internal.Fusion.Types - Data.Text.Internal.IO - Data.Text.Internal.Lazy - Data.Text.Internal.Lazy.Encoding.Fusion - Data.Text.Internal.Lazy.Fusion - Data.Text.Internal.Lazy.Search - Data.Text.Internal.Private - Data.Text.Internal.Read - Data.Text.Internal.Search - Data.Text.Internal.Unsafe - Data.Text.Internal.Unsafe.Char - Data.Text.Internal.Unsafe.Shift - Data.Text.Lazy - Data.Text.Lazy.Builder - Data.Text.Lazy.Builder.Int - Data.Text.Lazy.Builder.RealFloat - Data.Text.Lazy.Encoding - Data.Text.Lazy.IO - Data.Text.Lazy.Internal - Data.Text.Lazy.Read - Data.Text.Read - Data.Text.Unsafe - - other-modules: - Data.Text.Show - - build-depends: - array >= 0.3, - base >= 4.2 && < 5, - binary, - deepseq >= 1.1.0.0, - ghc-prim >= 0.2 - - if flag(bytestring-builder) - build-depends: bytestring >= 0.9 && < 0.10.4, - bytestring-builder >= 0.10.4 - else - build-depends: bytestring >= 0.10.4 - - cpp-options: -DHAVE_DEEPSEQ - ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 - if flag(developer) - ghc-prof-options: -auto-all - ghc-options: -Werror - cpp-options: -DASSERTS - - if flag(integer-simple) - cpp-options: -DINTEGER_SIMPLE - build-depends: integer-simple >= 0.1 && < 0.5 - else - cpp-options: -DINTEGER_GMP - build-depends: integer-gmp >= 0.2 - -test-suite tests - type: exitcode-stdio-1.0 - hs-source-dirs: tests . - main-is: Tests.hs - c-sources: cbits/cbits.c - include-dirs: include - - ghc-options: - -Wall -threaded -O0 -rtsopts - - cpp-options: - -DASSERTS -DHAVE_DEEPSEQ -DTEST_SUITE - - build-depends: - HUnit >= 1.2, - QuickCheck >= 2.7, - array, - base, - binary, - deepseq, - directory, - ghc-prim, - quickcheck-unicode >= 1.0.1.0, - random, - test-framework >= 0.4, - test-framework-hunit >= 0.2, - test-framework-quickcheck2 >= 0.2 - - if flag(bytestring-builder) - build-depends: bytestring >= 0.9 && < 0.10.4, - bytestring-builder >= 0.10.4 - else - build-depends: bytestring >= 0.10.4 - - if flag(integer-simple) - cpp-options: -DINTEGER_SIMPLE - build-depends: integer-simple >= 0.1 && < 0.5 - else - cpp-options: -DINTEGER_GMP - build-depends: integer-gmp >= 0.2 - -source-repository head - type: git - location: https://github.com/bos/text - -source-repository head - type: mercurial - location: https://bitbucket.org/bos/text +cabal-version: 2.2 +name: text +version: 2.1.4 + +homepage: https://github.com/haskell/text +bug-reports: https://github.com/haskell/text/issues +synopsis: An efficient packed Unicode text type. +description: + . + An efficient packed, immutable Unicode text type (both strict and + lazy). + . + The 'Text' type represents Unicode character strings, in a time and + space-efficient manner. This package provides text processing + capabilities that are optimized for performance critical use, both + in terms of large data quantities and high speed. + . + The 'Text' type provides character-encoding, type-safe case + conversion via whole-string case conversion functions (see "Data.Text"). + It also provides a range of functions for converting 'Text' values to + and from 'ByteStrings', using several standard encodings + (see "Data.Text.Encoding"). + . + Efficient locale-sensitive support for text IO is also supported + (see "Data.Text.IO"). + . + These modules are intended to be imported qualified, to avoid name + clashes with Prelude functions, e.g. + . + > import qualified Data.Text as T + . + == ICU Support + . + To use an extended and very rich family of functions for working + with Unicode text (including normalization, regular expressions, + non-standard encodings, text breaking, and locales), see + the [text-icu package](https://hackage.haskell.org/package/text-icu) + based on the well-respected and liberally + licensed [ICU library](http://site.icu-project.org/). + +license: BSD-2-Clause +license-file: LICENSE +author: Bryan O'Sullivan +maintainer: Haskell Text Team , Core Libraries Committee +copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko +category: Data, Text +build-type: Simple +tested-with: + GHC == 8.4.4 + GHC == 8.6.5 + GHC == 8.8.4 + GHC == 8.10.7 + GHC == 9.0.2 + GHC == 9.2.8 + GHC == 9.4.8 + GHC == 9.6.7 + GHC == 9.8.4 + GHC == 9.10.1 + GHC == 9.12.2 + +extra-source-files: + -- scripts/CaseFolding.txt + -- scripts/SpecialCasing.txt + scripts/*.hs + simdutf/LICENSE-APACHE + simdutf/LICENSE-MIT + simdutf/simdutf_c.h + simdutf/simdutf.h + tests/literal-rule-test.sh + tests/LiteralRuleTest.hs +extra-doc-files: + README.md + changelog.md + +flag developer + description: operate in developer mode + default: False + manual: True + +flag simdutf + description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed + default: True + manual: True + +flag pure-haskell + description: Don't use text's standard C routines + NB: This feature is not fully implemented. Several C routines are still in + use. + + When this flag is true, text will use pure Haskell variants of the + routines. This is not recommended except for use with GHC's JavaScript + backend. + + This flag also disables simdutf. + + default: False + manual: True + +flag ExtendedBenchmarks + description: Runs extra benchmarks which can be very slow. + default: False + manual: True + +library + if arch(javascript) || flag(pure-haskell) + cpp-options: -DPURE_HASKELL + else + c-sources: cbits/is_ascii.c + cbits/reverse.c + cbits/utils.c + if (arch(aarch64)) + c-sources: cbits/aarch64/measure_off.c + else + c-sources: cbits/measure_off.c + + hs-source-dirs: src + + if flag(simdutf) && !(arch(javascript) || flag(pure-haskell)) + exposed-modules: Data.Text.Internal.Validate.Simd + include-dirs: simdutf + c-sources: simdutf/hs_simdutf.c + cxx-sources: simdutf/simdutf.cpp + cxx-options: -std=c++17 + cpp-options: -DSIMDUTF + if impl(ghc >= 9.4) + build-depends: system-cxx-std-lib == 1.0 + elif os(darwin) || os(freebsd) + extra-libraries: c++ + elif os(openbsd) + extra-libraries: c++ c++abi pthread + elif os(windows) + -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later + if impl(ghc < 9.3) + extra-libraries: stdc++ + else + extra-libraries: c++ c++abi + elif arch(wasm32) + cpp-options: -DSIMDUTF_NO_THREADS + cxx-options: -fno-exceptions + extra-libraries: c++ c++abi + else + extra-libraries: stdc++ + + -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417 + if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1) + build-depends: base < 0 + + -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker. + if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1) + build-depends: base < 0 + + -- GHC 8.10 has linking issues (probably TH-related) on ARM. + if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*) + build-depends: base < 0 + + -- Subword primitives in GHC 9.2.1 are broken on ARM platforms. + if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1) + build-depends: base < 0 + + -- NetBSD + GHC 9.2.1 + TH + C++ does not work together. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577 + if flag(simdutf) && os(netbsd) && impl(ghc < 9.4) + build-depends: base < 0 + + exposed-modules: + Data.Text + Data.Text.Array + Data.Text.Encoding + Data.Text.Encoding.Error + Data.Text.Foreign + Data.Text.IO + Data.Text.IO.Utf8 + Data.Text.Internal + Data.Text.Internal.ArrayUtils + Data.Text.Internal.Builder + Data.Text.Internal.Builder.Functions + Data.Text.Internal.Builder.Int.Digits + Data.Text.Internal.Builder.RealFloat.Functions + Data.Text.Internal.ByteStringCompat + Data.Text.Internal.PrimCompat + Data.Text.Internal.Encoding + Data.Text.Internal.Encoding.Fusion + Data.Text.Internal.Encoding.Fusion.Common + Data.Text.Internal.Encoding.Utf16 + Data.Text.Internal.Encoding.Utf32 + Data.Text.Internal.Encoding.Utf8 + Data.Text.Internal.Fusion + Data.Text.Internal.Fusion.CaseMapping + Data.Text.Internal.Fusion.Common + Data.Text.Internal.Fusion.Size + Data.Text.Internal.Fusion.Types + Data.Text.Internal.IO + Data.Text.Internal.Lazy + Data.Text.Internal.Lazy.Encoding.Fusion + Data.Text.Internal.Lazy.Fusion + Data.Text.Internal.Lazy.Search + Data.Text.Internal.Private + Data.Text.Internal.Read + Data.Text.Internal.Search + Data.Text.Internal.StrictBuilder + Data.Text.Internal.Unsafe + Data.Text.Internal.Unsafe.Char + Data.Text.Internal.Validate + Data.Text.Internal.Validate.Native + Data.Text.Lazy + Data.Text.Lazy.Builder + Data.Text.Lazy.Builder.Int + Data.Text.Lazy.Builder.RealFloat + Data.Text.Lazy.Encoding + Data.Text.Lazy.IO + Data.Text.Lazy.Internal + Data.Text.Lazy.Read + Data.Text.Read + Data.Text.Unsafe + + other-modules: + Data.Text.Show + Data.Text.Internal.Measure + Data.Text.Internal.Reverse + Data.Text.Internal.Transformation + Data.Text.Internal.IsAscii + + build-depends: + array >= 0.3 && < 0.6, + base >= 4.11 && < 5, + binary >= 0.8.3 && < 0.9, + bytestring >= 0.10.4 && < 0.13, + deepseq >= 1.1 && < 1.6, + ghc-prim >= 0.2 && < 0.15, + + -- template-haskell-lift was added as a boot library in GHC-9.14 + -- once we no longer wish to backport releases to older major releases of GHC, + -- this conditional can be dropped + if impl(ghc < 9.14) + build-depends: template-haskell >= 2.5 && < 3 + else + build-depends: template-haskell-lift >= 0.1 && <0.2 + + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + + ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 + if flag(developer) + ghc-options: -fno-ignore-asserts + cpp-options: -DASSERTS + if impl(ghc >= 9.2.2) + ghc-options: -fcheck-prim-bounds + + default-language: Haskell2010 + default-extensions: + NondecreasingIndentation + other-extensions: + BangPatterns + CPP + DeriveDataTypeable + ExistentialQuantification + ForeignFunctionInterface + GeneralizedNewtypeDeriving + MagicHash + OverloadedStrings + Rank2Types + RankNTypes + RecordWildCards + Safe + ScopedTypeVariables + TemplateHaskellQuotes + Trustworthy + TypeFamilies + UnboxedTuples + UnliftedFFITypes + +source-repository head + type: git + location: https://github.com/haskell/text + +test-suite tests + type: exitcode-stdio-1.0 + ghc-options: + -Wall -threaded -rtsopts -with-rtsopts=-N + + hs-source-dirs: tests + main-is: Tests.hs + other-modules: + Tests.Lift + Tests.Properties + Tests.Properties.Basics + Tests.Properties.Builder + Tests.Properties.Folds + Tests.Properties.Instances + Tests.Properties.LowLevel + Tests.Properties.Read + Tests.Properties.Substrings + Tests.Properties.Text + Tests.Properties.Transcoding + Tests.Properties.CornerCases + Tests.Properties.Validate + Tests.QuickCheckUtils + Tests.RebindableSyntaxTest + Tests.Regressions + Tests.SlowFunctions + Tests.ShareEmpty + Tests.Utils + + build-depends: + QuickCheck >= 2.12.6 && < 2.18, + base <5, + binary, + bytestring, + deepseq, + ghc-prim, + tasty, + tasty-hunit, + tasty-quickcheck, + temporary, + transformers, + text + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + if impl(ghc < 9.14) + build-depends: template-haskell + else + build-depends: template-haskell-lift + + -- Plugin infrastructure does not work properly in 8.6.1, and + -- ghc-9.2.1 library depends on parsec, which causes a circular dependency. + if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2) + build-depends: tasty-inspection-testing + + -- https://github.com/haskellari/splitmix/issues/101 + if os(openbsd) + build-depends: splitmix < 0.1.3 || > 0.1.3.1 + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + +benchmark text-benchmarks + type: exitcode-stdio-1.0 + + ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m" + if impl(ghc >= 8.6) + ghc-options: -fproc-alignment=64 + if flag(ExtendedBenchmarks) + cpp-options: -DExtendedBenchmarks + + build-depends: base, + bytestring >= 0.10.4, + containers, + deepseq, + directory, + filepath, + tasty-bench >= 0.2, + temporary, + text, + transformers + + hs-source-dirs: benchmarks/haskell + main-is: Benchmarks.hs + other-modules: + Benchmarks.Builder + Benchmarks.Concat + Benchmarks.DecodeUtf8 + Benchmarks.EncodeUtf8 + Benchmarks.Equality + Benchmarks.FileRead + Benchmarks.FileWrite + Benchmarks.FoldLines + Benchmarks.Micro + Benchmarks.Multilang + Benchmarks.Programs.BigTable + Benchmarks.Programs.Cut + Benchmarks.Programs.Fold + Benchmarks.Programs.Sort + Benchmarks.Programs.StripTags + Benchmarks.Programs.Throughput + Benchmarks.Pure + Benchmarks.ReadNumbers + Benchmarks.Replace + Benchmarks.Search + Benchmarks.Stream + Benchmarks.WordFrequencies + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + other-extensions: DeriveGeneric