packages feed

text 1.2.3.2 → 2.1.4

raw patch · 188 files changed

Files

− Data/Text.hs 
@@ -1,1882 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types, UnboxedTuples #-}-{-# OPTIONS_GHC -fno-warn-orphans #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif-#if __GLASGOW_HASKELL__ >= 708-{-# LANGUAGE TypeFamilies #-}-#endif---- |--- Module      : Data.Text--- Copyright   : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts,---               (c) 2008, 2009 Tom Harper------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text.--- Suitable for performance critical use, both in terms of large data--- quantities and high speed.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text as T------ To use an extended and very rich family of functions for working--- with Unicode text (including normalization, regular expressions,--- non-standard encodings, text breaking, and locales), see the--- <http://hackage.haskell.org/package/text-icu text-icu package >.-----module Data.Text-    (-    -- * Strict vs lazy types-    -- $strict--    -- * Acceptable data-    -- $replacement--    -- * Definition of character-    -- $character_definition--    -- * Fusion-    -- $fusion--    -- * Types-      Text--    -- * Creation and elimination-    , pack-    , unpack-    , singleton-    , empty--    -- * Basic interface-    , cons-    , snoc-    , append-    , uncons-    , unsnoc-    , head-    , last-    , tail-    , init-    , null-    , length-    , compareLength--    -- * Transformations-    , map-    , intercalate-    , intersperse-    , transpose-    , reverse-    , replace--    -- ** Case conversion-    -- $case-    , toCaseFold-    , toLower-    , toUpper-    , toTitle--    -- ** Justification-    , justifyLeft-    , justifyRight-    , center--    -- * Folds-    , foldl-    , foldl'-    , foldl1-    , foldl1'-    , foldr-    , foldr1--    -- ** Special folds-    , concat-    , concatMap-    , any-    , all-    , maximum-    , minimum--    -- * Construction--    -- ** Scans-    , scanl-    , scanl1-    , scanr-    , scanr1--    -- ** Accumulating maps-    , mapAccumL-    , mapAccumR--    -- ** Generation and unfolding-    , replicate-    , unfoldr-    , unfoldrN--    -- * Substrings--    -- ** Breaking strings-    , take-    , takeEnd-    , drop-    , dropEnd-    , takeWhile-    , takeWhileEnd-    , dropWhile-    , dropWhileEnd-    , dropAround-    , strip-    , stripStart-    , stripEnd-    , splitAt-    , breakOn-    , breakOnEnd-    , break-    , span-    , group-    , groupBy-    , inits-    , tails--    -- ** Breaking into many substrings-    -- $split-    , splitOn-    , split-    , chunksOf--    -- ** Breaking into lines and words-    , lines-    --, lines'-    , words-    , unlines-    , unwords--    -- * Predicates-    , isPrefixOf-    , isSuffixOf-    , isInfixOf--    -- ** View patterns-    , stripPrefix-    , stripSuffix-    , commonPrefixes--    -- * Searching-    , filter-    , breakOnAll-    , find-    , partition--    -- , findSubstring--    -- * Indexing-    -- $index-    , index-    , findIndex-    , count--    -- * Zipping-    , zip-    , zipWith--    -- -* Ordered text-    -- , sort--    -- * Low level operations-    , copy-    , unpackCString#-    ) where--import Prelude (Char, Bool(..), Int, Maybe(..), String,-                Eq(..), Ord(..), Ordering(..), (++),-                Read(..),-                (&&), (||), (+), (-), (.), ($), ($!), (>>),-                not, return, otherwise, quot)-import Control.DeepSeq (NFData(rnf))-#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Char (isSpace)-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,-                  Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Control.Monad (foldM)-import Control.Monad.ST (ST)-import qualified Data.Text.Array as A-import qualified Data.List as L-import Data.Binary (Binary(get, put))-import Data.Monoid (Monoid(..))-#if MIN_VERSION_base(4,9,0)-import Data.Semigroup (Semigroup(..))-#endif-import Data.String (IsString(..))-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import Data.Text.Encoding (decodeUtf8', encodeUtf8)-import Data.Text.Internal.Fusion (stream, reverseStream, unstream)-import Data.Text.Internal.Private (span_)-import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text)-import Data.Text.Show (singleton, unpack, unpackCString#)-import qualified Prelude as P-import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,-                         reverseIter_, unsafeHead, unsafeTail)-import Data.Text.Internal.Unsafe.Char (unsafeChr)-import qualified Data.Text.Internal.Functions as F-import qualified Data.Text.Internal.Encoding.Utf16 as U16-import Data.Text.Internal.Search (indices)-#if defined(__HADDOCK__)-import Data.ByteString (ByteString)-import qualified Data.Text.Lazy as L-import Data.Int (Int64)-#endif-import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)-#if __GLASGOW_HASKELL__ >= 708-import qualified GHC.Exts as Exts-#endif-#if MIN_VERSION_base(4,7,0)-import Text.Printf (PrintfArg, formatArg, formatString)-#endif---- $character_definition------ This package uses the term /character/ to denote Unicode /code points/.------ Note that this is not the same thing as a grapheme (e.g. a--- composition of code points that form one visual symbol). For--- instance, consider the grapheme \"&#x00e4;\". This symbol has two--- Unicode representations: a single code-point representation--- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),--- and a two code point representation @U+0061@ (the \"@A@\" code--- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).---- $strict------ This package provides both strict and lazy 'Text' types.  The--- strict type is provided by the "Data.Text" module, while the lazy--- type is provided by the "Data.Text.Lazy" module. Internally, the--- lazy @Text@ type consists of a list of strict chunks.------ The strict 'Text' type requires that an entire string fit into--- memory at once.  The lazy 'Data.Text.Lazy.Text' type is capable of--- streaming strings that are larger than memory using a small memory--- footprint.  In many cases, the overhead of chunked streaming makes--- the lazy 'Data.Text.Lazy.Text' type slower than its strict--- counterpart, but this is not always the case.  Sometimes, the time--- complexity of a function in one module may be different from the--- other, due to their differing internal structures.------ Each module provides an almost identical API, with the main--- difference being that the strict module uses 'Int' values for--- lengths and counts, while the lazy module uses 'Data.Int.Int64'--- lengths.---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"&#xfffd;\" (U+FFFD).  Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\".------ (One reason for this policy of replacement is that internally, a--- 'Text' value is represented as packed UTF-16 data. Values in the--- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate--- code points, and so cannot be represented. The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)---- $fusion------ Most of the functions in this module are subject to /fusion/,--- meaning that a pipeline of such functions will usually allocate at--- most one 'Text' value.------ As an example, consider the following pipeline:------ > import Data.Text as T--- > import Data.Text.Encoding as E--- > import Data.ByteString (ByteString)--- >--- > countChars :: ByteString -> Int--- > countChars = T.length . T.toUpper . E.decodeUtf8------ From the type signatures involved, this looks like it should--- allocate one 'Data.ByteString.ByteString' value, and two 'Text'--- values. However, when a module is compiled with optimisation--- enabled under GHC, the two intermediate 'Text' values will be--- optimised away, and the function will be compiled down to a single--- loop over the source 'Data.ByteString.ByteString'.------ Functions that can be fused by the compiler are documented with the--- phrase \"Subject to fusion\".--instance Eq Text where-    Text arrA offA lenA == Text arrB offB lenB-        | lenA == lenB = A.equal arrA offA arrB offB lenA-        | otherwise    = False-    {-# INLINE (==) #-}--instance Ord Text where-    compare = compareText--instance Read Text where-    readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]--#if MIN_VERSION_base(4,9,0)--- | Non-orphan 'Semigroup' instance only defined for--- @base-4.9.0.0@ and later; orphan instances for older GHCs are--- provided by--- the [semigroups](http://hackage.haskell.org/package/semigroups)--- package------ @since 1.2.2.0-instance Semigroup Text where-    (<>) = append-#endif--instance Monoid Text where-    mempty  = empty-#if MIN_VERSION_base(4,9,0)-    mappend = (<>) -- future-proof definition-#else-    mappend = append-#endif-    mconcat = concat--instance IsString Text where-    fromString = pack--#if __GLASGOW_HASKELL__ >= 708--- | @since 1.2.0.0-instance Exts.IsList Text where-    type Item Text = Char-    fromList       = pack-    toList         = unpack-#endif--instance NFData Text where rnf !_ = ()---- | @since 1.2.1.0-instance Binary Text where-    put t = put (encodeUtf8 t)-    get   = do-      bs <- get-      case decodeUtf8' bs of-        P.Left exn -> P.fail (P.show exn)-        P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you--- feel a mistake has been made, please feel free to submit--- improvements.------ The original discussion is archived here:--- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? >------ The followup discussion that changed the behavior of 'Data.Set.Set'--- and 'Data.Map.Map' is archived here:--- <http://markmail.org/message/trovdc6zkphyi3cr#query:+page:1+mid:a46der3iacwjcf6n+state:results Proposal: Allow gunfold for Data.Map, ... >--instance Data Text where-  gfoldl f z txt = z pack `f` (unpack txt)-  toConstr _ = packConstr-  gunfold k z c = case constrIndex c of-    1 -> k (z pack)-    _ -> P.error "gunfold"-  dataTypeOf _ = textDataType--#if MIN_VERSION_base(4,7,0)--- | Only defined for @base-4.7.0.0@ and later------ @since 1.2.2.0-instance PrintfArg Text where-  formatArg txt = formatString $ unpack txt-#endif--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Text" [packConstr]---- | /O(n)/ Compare two 'Text' values lexicographically.-compareText :: Text -> Text -> Ordering-compareText ta@(Text _arrA _offA lenA) tb@(Text _arrB _offB lenB)-    | lenA == 0 && lenB == 0 = EQ-    | otherwise              = go 0 0-  where-    go !i !j-        | i >= lenA || j >= lenB = compare lenA lenB-        | a < b                  = LT-        | a > b                  = GT-        | otherwise              = go (i+di) (j+dj)-      where Iter a di = iter ta i-            Iter b dj = iter tb j---- -------------------------------------------------------------------------------- * Conversion to/from 'Text'---- | /O(n)/ Convert a 'String' into a 'Text'.  Subject to--- fusion.  Performs replacement on invalid scalar values.-pack :: String -> Text-pack = unstream . S.map safe . S.streamList-{-# INLINE [1] pack #-}---- -------------------------------------------------------------------------------- * Basic functions---- | /O(n)/ Adds a character to the front of a 'Text'.  This function--- is more costly than its 'List' counterpart because it requires--- copying a new array.  Subject to fusion.  Performs replacement on--- invalid scalar values.-cons :: Char -> Text -> Text-cons c t = unstream (S.cons (safe c) (stream t))-{-# INLINE cons #-}--infixr 5 `cons`---- | /O(n)/ Adds a character to the end of a 'Text'.  This copies the--- entire array in the process, unless fused.  Subject to fusion.--- Performs replacement on invalid scalar values.-snoc :: Text -> Char -> Text-snoc t c = unstream (S.snoc (stream t) (safe c))-{-# INLINE snoc #-}---- | /O(n)/ Appends one 'Text' to the other by copying both of them--- into a new 'Text'.  Subject to fusion.-append :: Text -> Text -> Text-append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2)-    | len1 == 0 = b-    | len2 == 0 = a-    | len > 0   = Text (A.run x) 0 len-    | otherwise = overflowError "append"-    where-      len = len1+len2-      x :: ST s (A.MArray s)-      x = do-        arr <- A.new len-        A.copyI arr 0 arr1 off1 len1-        A.copyI arr len1 arr2 off2 len-        return arr-{-# NOINLINE append #-}--{-# RULES-"TEXT append -> fused" [~1] forall t1 t2.-    append t1 t2 = unstream (S.append (stream t1) (stream t2))-"TEXT append -> unfused" [1] forall t1 t2.-    unstream (S.append (stream t1) (stream t2)) = append t1 t2- #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty.  Subject to fusion.-head :: Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty. Subject to fusion.-uncons :: Text -> Maybe (Char, Text)-uncons t@(Text arr off len)-    | len <= 0  = Nothing-    | otherwise = Just $ let !(Iter c d) = iter t 0-                         in (c, text arr (off+d) (len-d))-{-# INLINE [1] uncons #-}---- | Lifted from Control.Arrow and specialized.-second :: (b -> c) -> (a,b) -> (a,c)-second f (a, b) = (a, f b)---- | /O(1)/ Returns the last character of a 'Text', which must be--- non-empty.  Subject to fusion.-last :: Text -> Char-last (Text arr off len)-    | len <= 0                 = emptyError "last"-    | n < 0xDC00 || n > 0xDFFF = unsafeChr n-    | otherwise                = U16.chr2 n0 n-    where n  = A.unsafeIndex arr (off+len-1)-          n0 = A.unsafeIndex arr (off+len-2)-{-# INLINE [1] last #-}--{-# RULES-"TEXT last -> fused" [~1] forall t.-    last t = S.last (stream t)-"TEXT last -> unfused" [1] forall t.-    S.last (stream t) = last t-  #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty.  Subject to fusion.-tail :: Text -> Text-tail t@(Text arr off len)-    | len <= 0  = emptyError "tail"-    | otherwise = text arr (off+d) (len-d)-    where d = iter_ t 0-{-# INLINE [1] tail #-}--{-# RULES-"TEXT tail -> fused" [~1] forall t.-    tail t = unstream (S.tail (stream t))-"TEXT tail -> unfused" [1] forall t.-    unstream (S.tail (stream t)) = tail t- #-}---- | /O(1)/ Returns all but the last character of a 'Text', which must--- be non-empty.  Subject to fusion.-init :: Text -> Text-init (Text arr off len) | len <= 0                   = emptyError "init"-                        | n >= 0xDC00 && n <= 0xDFFF = text arr off (len-2)-                        | otherwise                  = text arr off (len-1)-    where-      n = A.unsafeIndex arr (off+len-1)-{-# INLINE [1] init #-}--{-# RULES-"TEXT init -> fused" [~1] forall t.-    init t = unstream (S.init (stream t))-"TEXT init -> unfused" [1] forall t.-    unstream (S.init (stream t)) = init t- #-}---- | /O(1)/ Returns all but the last character and the last character of a--- 'Text', or 'Nothing' if empty.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc (Text arr off len)-    | len <= 0                 = Nothing-    | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n)-    | otherwise                = Just (text arr off (len-2), U16.chr2 n0 n)-    where n  = A.unsafeIndex arr (off+len-1)-          n0 = A.unsafeIndex arr (off+len-2)-{-# INLINE [1] unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.  Subject to--- fusion.-null :: Text -> Bool-null (Text _arr _off len) =-#if defined(ASSERTS)-    assert (len >= 0) $-#endif-    len <= 0-{-# INLINE [1] null #-}--{-# RULES-"TEXT null -> fused" [~1] forall t.-    null t = S.null (stream t)-"TEXT null -> unfused" [1] forall t.-    S.null (stream t) = null t- #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.--- Subject to fusion.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n)/ Returns the number of characters in a 'Text'.--- Subject to fusion.-length :: Text -> Int-length t = S.length (stream t)-{-# INLINE [0] length #-}--- length needs to be phased after the compareN/length rules otherwise--- it may inline before the rules have an opportunity to fire.---- | /O(n)/ Compare the count of characters in a 'Text' to a number.--- Subject to fusion.------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int -> Ordering-compareLength t n = S.compareLengthI (stream t) n-{-# INLINE [1] compareLength #-}--{-# RULES-"TEXT compareN/length -> compareLength" [~1] forall t n.-    compare (length t) n = compareLength t n-  #-}--{-# RULES-"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.-    eqInt (length t) n = compareLength t n == EQ-  #-}--{-# RULES-"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.-    neInt (length t) n = compareLength t n /= EQ-  #-}--{-# RULES-"TEXT <N/length -> compareLength/==LT" [~1] forall t n.-    ltInt (length t) n = compareLength t n == LT-  #-}--{-# RULES-"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.-    leInt (length t) n = compareLength t n /= GT-  #-}--{-# RULES-"TEXT >N/length -> compareLength/==GT" [~1] forall t n.-    gtInt (length t) n = compareLength t n == GT-  #-}--{-# RULES-"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.-    geInt (length t) n = compareLength t n /= LT-  #-}---- -------------------------------------------------------------------------------- * Transformations--- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.------ Example:------ >>> let message = pack "I am not angry. Not at all."--- >>> T.map (\c -> if c == '.' then '!' else c) message--- "I am not angry! Not at all!"------ Subject to fusion.  Performs replacement on invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f t = unstream (S.map (safe . f) (stream t))-{-# INLINE [1] map #-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.------ Example:------ >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]--- "WeNI!seekNI!theNI!HolyNI!Grail"-intercalate :: Text -> [Text] -> Text-intercalate t = concat . (F.intersperse t)-{-# INLINE intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.------ Example:------ >>> T.intersperse '.' "SHIELD"--- "S.H.I.E.L.D"------ Subject to fusion.  Performs replacement on invalid scalar values.-intersperse     :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse (safe c) (stream t))-{-# INLINE intersperse #-}---- | /O(n)/ Reverse the characters of a string.------ Example:------ >>> T.reverse "desrever"--- "reversed"------ Subject to fusion.-reverse :: Text -> Text-reverse t = S.reverse (stream t)-{-# INLINE reverse #-}---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =---   'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once.  So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ >>> replace "oo" "foo" "oo"--- "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ >>> replace "ofo" "bar" "ofofo"--- "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: Text-        -- ^ @needle@ to search for.  If this string is empty, an-        -- error will occur.-        -> Text-        -- ^ @replacement@ to replace @needle@ with.-        -> Text-        -- ^ @haystack@ in which to search.-        -> Text-replace needle@(Text _      _      neeLen)-               (Text repArr repOff repLen)-      haystack@(Text hayArr hayOff hayLen)-  | neeLen == 0 = emptyError "replace"-  | L.null ixs  = haystack-  | len > 0     = Text (A.run x) 0 len-  | otherwise   = empty-  where-    ixs = indices needle haystack-    len = hayLen - (neeLen - repLen) `mul` L.length ixs-    x :: ST s (A.MArray s)-    x = do-      marr <- A.new len-      let loop (i:is) o d = do-            let d0 = d + i - o-                d1 = d0 + repLen-            A.copyI marr d  hayArr (hayOff+o) d0-            A.copyI marr d0 repArr repOff d1-            loop is (i + neeLen) d1-          loop []     o d = A.copyI marr d hayArr (hayOff+o) len-      loop ixs 0 0-      return marr---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ When case converting 'Text' values, do not use combinators like--- @map toUpper@ to case convert each character of a string--- individually, as this gives incorrect results according to the--- rules of some writing systems.  The whole-string case conversion--- functions from this module, such as @toUpper@, obey the correct--- case conversion rules.  As a result, these functions may map one--- input character to two or three output characters. For examples,--- see the documentation of each function.------ /Note/: In some languages, case conversion is a locale- and--- context-dependent operation. The case conversion functions in this--- module are /not/ locale sensitive. Programs that require locale--- sensitivity should use appropriate versions of the--- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >.---- | /O(n)/ Convert a string to folded case.  Subject to fusion.------ This function is mainly useful for performing caseless (also known--- as case insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string.  For instance,--- the Armenian small ligature \"&#xfb13;\" (men now, U+FB13) is case--- folded to the sequence \"&#x574;\" (men, U+0574) followed by--- \"&#x576;\" (now, U+0576), while the Greek \"&#xb5;\" (micro sign,--- U+00B5) is case folded to \"&#x3bc;\" (small letter mu, U+03BC)--- instead of itself.-toCaseFold :: Text -> Text-toCaseFold t = unstream (S.toCaseFold (stream t))-{-# INLINE toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.  Subject to fusion.------ The result string may be longer than the input string.  For--- instance, \"&#x130;\" (Latin capital letter I with dot above,--- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069)--- followed by \" &#x307;\" (combining dot above, U+0307).-toLower :: Text -> Text-toLower t = unstream (S.toLower (stream t))-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.  Subject to fusion.------ The result string may be longer than the input string.  For--- instance, the German \"&#xdf;\" (eszett, U+00DF) maps to the--- two-letter sequence \"SS\".-toUpper :: Text -> Text-toUpper t = unstream (S.toUpper (stream t))-{-# INLINE toUpper #-}---- | /O(n)/ Convert a string to title case, using simple case--- conversion. Subject to fusion.------ The first letter of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature &#xfb02; (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased&#x2014;but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle t = unstream (S.toTitle (stream t))-{-# INLINE toTitle #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Subject to fusion.--- Performs replacement on invalid scalar values.------ Examples:------ >>> justifyLeft 7 'x' "foo"--- "fooxxxx"------ >>> justifyLeft 3 'x' "foobar"--- "foobar"-justifyLeft :: Int -> Char -> Text -> Text-justifyLeft k c t-    | len >= k  = t-    | otherwise = t `append` replicateChar (k-len) c-  where len = length t-{-# INLINE [1] justifyLeft #-}--{-# RULES-"TEXT justifyLeft -> fused" [~1] forall k c t.-    justifyLeft k c t = unstream (S.justifyLeftI k c (stream t))-"TEXT justifyLeft -> unfused" [1] forall k c t.-    unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t-  #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left.  Performs replacement on--- invalid scalar values.------ Examples:------ >>> justifyRight 7 'x' "bar"--- "xxxxbar"------ >>> justifyRight 3 'x' "foobar"--- "foobar"-justifyRight :: Int -> Char -> Text -> Text-justifyRight k c t-    | len >= k  = t-    | otherwise = replicateChar (k-len) c `append` t-  where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side.  Performs replacement on invalid--- scalar values.------ Examples:------ >>> center 8 'x' "HS"--- "xxxHSxxx"-center :: Int -> Char -> Text -> Text-center k c t-    | len >= k  = t-    | otherwise = replicateChar l c `append` t `append` replicateChar r c-  where len = length t-        d   = k - len-        r   = d `quot` 2-        l   = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument.  Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.------ Examples:------ >>> transpose ["green","orange"]--- ["go","rr","ea","en","ng","e"]------ >>> transpose ["blue","red"]--- ["br","le","ud","e"]-transpose :: [Text] -> [Text]-transpose ts = P.map pack (L.transpose (P.map unpack ts))---- -------------------------------------------------------------------------------- * Reducing 'Text's (folds)---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.--- Subject to fusion.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.  Subject to fusion.-foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.  Subject to fusion.-foldl1 :: (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.  Subject to fusion.-foldl1' :: (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.--- Subject to fusion.-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.  Subject to--- fusion.-foldr1 :: (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- -------------------------------------------------------------------------------- ** Special folds---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat ts = case ts' of-              [] -> empty-              [t] -> t-              _ -> Text (A.run go) 0 len-  where-    ts' = L.filter (not . null) ts-    len = sumP "concat" $ L.map lengthWord16 ts'-    go :: ST s (A.MArray s)-    go = do-      arr <- A.new len-      let step i (Text a o l) =-            let !j = i + l in A.copyI arr i a o j >> return j-      foldM step 0 ts' >> return arr---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty. Subject to fusion.-maximum :: Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty. Subject to fusion.-minimum :: Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- -------------------------------------------------------------------------------- * Building 'Text's---- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left. Subject to fusion.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ Note that------ > last (scanl f z xs) == foldl f z xs.-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))-    where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument.  Subject to fusion.  Performs replacement on--- invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t | null t    = empty-           | otherwise = scanl f (unsafeHead t) (unsafeTail t)-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f z = S.reverse . S.reverseScanr g z . reverseStream-    where g a b = safe (f a b)-{-# INLINE scanr #-}---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument.  Subject to fusion.  Performs replacement on--- invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t    = empty-           | otherwise = scanr f (last t) (init t)-{-# INLINE scanr1 #-}---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'.  Performs--- replacement on invalid scalar values.-mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumL f z0 = S.mapAccumL g z0 . stream-    where g a b = second safe (f a b)-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'.--- Performs replacement on invalid scalar values.-mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream-    where g a b = second safe (f a b)-{-# INLINE mapAccumR #-}---- -------------------------------------------------------------------------------- ** Generating and unfolding 'Text's---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int -> Text -> Text-replicate n t@(Text a o l)-    | n <= 0 || l <= 0       = empty-    | n == 1                 = t-    | isSingleton t          = replicateChar n (unsafeHead t)-    | otherwise              = Text (A.run x) 0 len-  where-    len = l `mul` n-    x :: ST s (A.MArray s)-    x = do-      arr <- A.new len-      let loop !d !i | i >= n    = return arr-                     | otherwise = let m = d + l-                                   in A.copyI arr d a o m >> loop m (i+1)-      loop 0 0-{-# INLINE [1] replicate #-}--{-# RULES-"TEXT replicate/singleton -> replicateChar" [~1] forall n c.-    replicate n (singleton c) = replicateChar n c-  #-}---- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the--- value of every element. Subject to fusion.-replicateChar :: Int -> Char -> Text-replicateChar n c = unstream (S.replicateCharI n (safe c))-{-# INLINE replicateChar #-}---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production. Subject--- to fusion.  Performs replacement on invalid scalar values.-unfoldr     :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'. Subject--- to fusion.  Performs replacement on invalid scalar values.-unfoldrN     :: Int -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- -------------------------------------------------------------------------------- * Substrings---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text. Subject to fusion.-take :: Int -> Text -> Text-take n t@(Text arr off len)-    | n <= 0    = empty-    | n >= len  = t-    | otherwise = text arr off (iterN n t)-{-# INLINE [1] take #-}--iterN :: Int -> Text -> Int-iterN n t@(Text _arr _off len) = loop 0 0-  where loop !i !cnt-            | i >= len || cnt >= n = i-            | otherwise            = loop (i+d) (cnt+1)-          where d = iter_ t i--{-# RULES-"TEXT take -> fused" [~1] forall n t.-    take n t = unstream (S.take n (stream t))-"TEXT take -> unfused" [1] forall n t.-    unstream (S.take n (stream t)) = take n t-  #-}---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ >>> takeEnd 3 "foobar"--- "bar"------ @since 1.1.1.0-takeEnd :: Int -> Text -> Text-takeEnd n t@(Text arr off len)-    | n <= 0    = empty-    | n >= len  = t-    | otherwise = text arr (off+i) (len-i)-  where i = iterNEnd n t--iterNEnd :: Int -> Text -> Int-iterNEnd n t@(Text _arr _off len) = loop (len-1) n-  where loop i !m-          | m <= 0    = i+1-          | i <= 0    = 0-          | otherwise = loop (i+d) (m-1)-          where d = reverseIter_ t i---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'. Subject to fusion.-drop :: Int -> Text -> Text-drop n t@(Text arr off len)-    | n <= 0    = t-    | n >= len  = empty-    | otherwise = text arr (off+i) (len-i)-  where i = iterN n t-{-# INLINE [1] drop #-}--{-# RULES-"TEXT drop -> fused" [~1] forall n t.-    drop n t = unstream (S.drop n (stream t))-"TEXT drop -> unfused" [1] forall n t.-    unstream (S.drop n (stream t)) = drop n t-  #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ >>> dropEnd 3 "foobar"--- "foo"------ @since 1.1.1.0-dropEnd :: Int -> Text -> Text-dropEnd n t@(Text arr off len)-    | n <= 0    = t-    | n >= len  = empty-    | otherwise = text arr off (iterNEnd n t)---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.  Subject to fusion.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t@(Text arr off len) = loop 0-  where loop !i | i >= len    = t-                | p c         = loop (i+d)-                | otherwise   = text arr off i-            where Iter c d    = iter t i-{-# INLINE [1] takeWhile #-}--{-# RULES-"TEXT takeWhile -> fused" [~1] forall p t.-    takeWhile p t = unstream (S.takeWhile p (stream t))-"TEXT takeWhile -> unfused" [1] forall p t.-    unstream (S.takeWhile p (stream t)) = takeWhile p t-  #-}---- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.  Subject to fusion.--- Examples:------ >>> takeWhileEnd (=='o') "foo"--- "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p t@(Text arr off len) = loop (len-1) len-  where loop !i !l | l <= 0    = t-                   | p c       = loop (i+d) (l+d)-                   | otherwise = text arr (off+l) (len-l)-            where (c,d)        = reverseIter t i-{-# INLINE [1] takeWhileEnd #-}--{-# RULES-"TEXT takeWhileEnd -> fused" [~1] forall p t.-    takeWhileEnd p t = S.reverse (S.takeWhile p (S.reverseStream t))-"TEXT takeWhileEnd -> unfused" [1] forall p t.-    S.reverse (S.takeWhile p (S.reverseStream t)) = takeWhileEnd p t-  #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@. Subject to fusion.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t@(Text arr off len) = loop 0 0-  where loop !i !l | l >= len  = empty-                   | p c       = loop (i+d) (l+d)-                   | otherwise = Text arr (off+i) (len-l)-            where Iter c d     = iter t i-{-# INLINE [1] dropWhile #-}--{-# RULES-"TEXT dropWhile -> fused" [~1] forall p t.-    dropWhile p t = unstream (S.dropWhile p (stream t))-"TEXT dropWhile -> unfused" [1] forall p t.-    unstream (S.dropWhile p (stream t)) = dropWhile p t-  #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.  Subject to fusion.------ Examples:------ >>> dropWhileEnd (=='.') "foo..."--- "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p t@(Text arr off len) = loop (len-1) len-  where loop !i !l | l <= 0    = empty-                   | p c       = loop (i+d) (l+d)-                   | otherwise = Text arr off l-            where (c,d)        = reverseIter t i-{-# INLINE [1] dropWhileEnd #-}--{-# RULES-"TEXT dropWhileEnd -> fused" [~1] forall p t.-    dropWhileEnd p t = S.reverse (S.dropWhile p (S.reverseStream t))-"TEXT dropWhileEnd -> unfused" [1] forall p t.-    S.reverse (S.dropWhile p (S.reverseStream t)) = dropWhileEnd p t-  #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.  Subject to fusion.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string.  Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile isSpace-{-# INLINE [1] stripStart #-}---- | /O(n)/ Remove trailing white space from a string.  Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int -> Text -> (Text, Text)-splitAt n t@(Text arr off len)-    | n <= 0    = (empty, t)-    | n >= len  = (t, empty)-    | otherwise = let k = iterN n t-                  in (text arr off k, text arr (off+k) (len-k))---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the list.-span :: (Char -> Bool) -> Text -> (Text, Text)-span p t = case span_ p t of-             (# hd,tl #) -> (hd,tl)-{-# INLINE span #-}---- | /O(n)/ 'break' is like 'span', but the prefix returned is--- over elements that fail the predicate @p@.-break :: (Char -> Bool) -> Text -> (Text, Text)-break p = span (not . p)-{-# INLINE break #-}---- | /O(n)/ Group characters in a string according to a predicate.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy p = loop-  where-    loop t@(Text arr off len)-        | null t    = []-        | otherwise = text arr off n : loop (text arr (off+n) (len-n))-        where Iter c d = iter t 0-              n     = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d))---- | Returns the /array/ index (in units of 'Word16') at which a--- character may be found.  This is /not/ the same as the logical--- index returned by e.g. 'findIndex'.-findAIndexOrEnd :: (Char -> Bool) -> Text -> Int-findAIndexOrEnd q t@(Text _arr _off len) = go 0-    where go !i | i >= len || q c       = i-                | otherwise             = go (i+d)-                where Iter c d          = iter t i---- | /O(n)/ Group characters in a string by equality.-group :: Text -> [Text]-group = groupBy (==)---- | /O(n)/ Return all initial segments of the given 'Text', shortest--- first.-inits :: Text -> [Text]-inits t@(Text arr off len) = loop 0-    where loop i | i >= len = [t]-                 | otherwise = Text arr off i : loop (i + iter_ t i)---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails t | null t    = [empty]-        | otherwise = t : tails (unsafeTail t)---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be empty), consuming the delimiter. An empty--- delimiter is invalid, and will cause an error to be raised.------ Examples:------ >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"--- ["a","b","d","e"]------ >>> splitOn "aaa"  "aaaXaaaXaaaXaaa"--- ["","X","X","X",""]------ >>> splitOn "x"    "x"--- ["",""]------ and------ > intercalate s . splitOn s         == id--- > splitOn (singleton c)             == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: Text-        -- ^ String to split on. If this string is empty, an error-        -- will occur.-        -> Text-        -- ^ Input text.-        -> [Text]-splitOn pat@(Text _ _ l) src@(Text arr off len)-    | l <= 0          = emptyError "splitOn"-    | isSingleton pat = split (== unsafeHead pat) src-    | otherwise       = go 0 (indices pat src)-  where-    go !s (x:xs) =  text arr (s+off) (x-s) : go (x+l) xs-    go  s _      = [text arr (s+off) (len-s)]-{-# INLINE [1] splitOn #-}--{-# RULES-"TEXT splitOn/singleton -> split/==" [~1] forall c t.-    splitOn (singleton c) t = split (==c) t-  #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element.  The--- resulting components do not contain the separators.  Two adjacent--- separators result in an empty component in the output.  eg.------ >>> split (=='a') "aabbaca"--- ["","","bb","c",""]------ >>> split (=='a') ""--- [""]-split :: (Char -> Bool) -> Text -> [Text]-split _ t@(Text _off _arr 0) = [t]-split p t = loop t-    where loop s | null s'   = [l]-                 | otherwise = l : loop (unsafeTail s')-              where (# l, s' #) = span_ (not . p) s-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@.  The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ >>> chunksOf 3 "foobarbaz"--- ["foo","bar","baz"]------ >>> chunksOf 4 "haskell.org"--- ["hask","ell.","org"]-chunksOf :: Int -> Text -> [Text]-chunksOf k = go-  where-    go t = case splitAt k t of-             (a,b) | null a    -> []-                   | otherwise -> a : go b-{-# INLINE chunksOf #-}---- ------------------------------------------------------------------------------- * Searching------------------------------------------------------------------------------------ ** Searching with a predicate---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element matching the predicate, or 'Nothing' if--- there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p t = unstream (S.filter p (stream t))-{-# INLINE filter #-}---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@.  The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched.  The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ >>> breakOn "::" "a::b::c"--- ("a","::b::c")------ >>> breakOn "/" "foobar"--- ("foobar","")------ Laws:------ > append prefix match == haystack--- >   where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: Text -> Text -> (Text, Text)-breakOn pat src@(Text arr off len)-    | null pat  = emptyError "breakOn"-    | otherwise = case indices pat src of-                    []    -> (src, empty)-                    (x:_) -> (text arr off x, text arr (off+x) (len-x))-{-# INLINE breakOn #-}---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the--- string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@.  The second is the--- remainder of @haystack@, following the match.------ >>> breakOnEnd "::" "a::b::c"--- ("a::b::","c")-breakOnEnd :: Text -> Text -> (Text, Text)-breakOnEnd pat src = (reverse b, reverse a)-    where (a,b) = breakOn (reverse pat) (reverse src)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@.  Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ >>> breakOnAll "::" ""--- []------ >>> breakOnAll "/" "a/b/c/"--- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: Text              -- ^ @needle@ to search for-           -> Text              -- ^ @haystack@ in which to search-           -> [(Text, Text)]-breakOnAll pat src@(Text arr off slen)-    | null pat  = emptyError "breakOnAll"-    | otherwise = L.map step (indices pat src)-  where-    step       x = (chunk 0 x, chunk x (slen-x))-    chunk !n !l  = text arr (n+off) l-{-# INLINE breakOnAll #-}------------------------------------------------------------------------------------ ** Indexing 'Text's---- $index------ If you think of a 'Text' value as an array of 'Char' values (which--- it is not), you run the risk of writing inefficient code.------ An idiom that is common in some languages is to find the numeric--- offset of a character or substring, then use that number to split--- or trim the searched string.  With a 'Text' value, this approach--- would require two /O(n)/ operations: one to perform the search, and--- one to operate from wherever the search ended.------ For example, suppose you have a string that you want to split on--- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of--- searching for the index of @\"::\"@ and taking the substrings--- before and after that index, you would instead use @breakOnAll \"::\"@.---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: Text -> Int -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text'--- and returns the index of the first element in the 'Text' satisfying--- the predicate. Subject to fusion.-findIndex :: (Char -> Bool) -> Text -> Maybe Int-findIndex p t = S.findIndex p (stream t)-{-# INLINE findIndex #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: Text -> Text -> Int-count pat src-    | null pat        = emptyError "count"-    | isSingleton pat = countChar (unsafeHead pat) src-    | otherwise       = L.length (indices pat src)-{-# INLINE [1] count #-}--{-# RULES-"TEXT count/singleton -> countChar" [~1] forall c t.-    count (singleton c) t = countChar c t-  #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'. Subject to fusion.-countChar :: Char -> Text -> Int-countChar c t = S.countChar c (stream t)-{-# INLINE countChar #-}------------------------------------------------------------------------------------ * Zipping---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))-    where g a b = safe (f a b)-{-# INLINE zipWith #-}---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words t@(Text arr off len) = loop 0 0-  where-    loop !start !n-        | n >= len = if start == n-                     then []-                     else [Text arr (start+off) (n-start)]-        | isSpace c =-            if start == n-            then loop (start+1) (start+1)-            else Text arr (start+off) (n-start) : loop (n+d) (n+d)-        | otherwise = loop start (n+d)-        where Iter c d = iter t n-{-# INLINE words #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at--- newline 'Char's. The resulting strings do not contain newlines.-lines :: Text -> [Text]-lines ps | null ps   = []-         | otherwise = h : if null t-                           then []-                           else lines (unsafeTail t)-    where (# h,t #) = span_ (/= '\n') ps-{-# INLINE lines #-}--{---- | /O(n)/ Portably breaks a 'Text' up into a list of 'Text's at line--- boundaries.------ A line boundary is considered to be either a line feed, a carriage--- return immediately followed by a line feed, or a carriage return.--- This accounts for both Unix and Windows line ending conventions,--- and for the old convention used on Mac OS 9 and earlier.-lines' :: Text -> [Text]-lines' ps | null ps   = []-          | otherwise = h : case uncons t of-                              Nothing -> []-                              Just (c,t')-                                  | c == '\n' -> lines t'-                                  | c == '\r' -> case uncons t' of-                                                   Just ('\n',t'') -> lines t''-                                                   _               -> lines t'-    where (h,t)    = span notEOL ps-          notEOL c = c /= '\n' && c /= '\r'-{-# INLINE lines' #-}--}---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.map (`snoc` '\n')-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.  Subject to fusion.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) =-    alen <= blen && S.isPrefixOf (stream a) (stream b)-{-# INLINE [1] isPrefixOf #-}--{-# RULES-"TEXT isPrefixOf -> fused" [~1] forall s t.-    isPrefixOf s t = S.isPrefixOf (stream s) (stream t)-  #-}---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' iff the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf a@(Text _aarr _aoff alen) b@(Text barr boff blen) =-    d >= 0 && a == b'-  where d              = blen - alen-        b' | d == 0    = b-           | otherwise = Text barr (boff+d) alen-{-# INLINE isSuffixOf #-}---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' iff the first is contained, wholly and intact, anywhere--- within the second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf :: Text -> Text -> Bool-isInfixOf needle haystack-    | null needle        = True-    | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack-    | otherwise          = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}--{-# RULES-"TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h.-    isInfixOf (singleton n) h = S.elem n (S.stream h)-  #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ >>> stripPrefix "foo" "foobar"--- Just "bar"------ >>> stripPrefix ""    "baz"--- Just "baz"------ >>> stripPrefix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _                                 = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p@(Text _arr _off plen) t@(Text arr off len)-    | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen)-    | otherwise        = Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ >>> commonPrefixes "foobar" "fooquux"--- Just ("foo","bar","quux")------ >>> commonPrefixes "veeble" "fetzer"--- Nothing------ >>> commonPrefixes "" "baz"--- Nothing-commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)-commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0-  where-    go !i !j | i < len0 && j < len1 && a == b = go (i+d0) (j+d1)-             | i > 0     = Just (Text arr0 off0 i,-                                 text arr0 (off0+i) (len0-i),-                                 text arr1 (off1+j) (len1-j))-             | otherwise = Nothing-      where Iter a d0 = iter t0 i-            Iter b d1 = iter t1 j---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ >>> stripSuffix "bar" "foobar"--- Just "foo"------ >>> stripSuffix ""    "baz"--- Just "baz"------ >>> stripSuffix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _                                = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p@(Text _arr _off plen) t@(Text arr off len)-    | p `isSuffixOf` t = Just $! text arr off (len-plen)-    | otherwise        = Nothing---- | Add a list of non-negative numbers.  Errors out on overflow.-sumP :: String -> [Int] -> Int-sumP fun = go 0-  where go !a (x:xs)-            | ax >= 0   = go ax xs-            | otherwise = overflowError fun-          where ax = a + x-        go a  _         = a--emptyError :: String -> a-emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input"--overflowError :: String -> a-overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow"---- | /O(n)/ Make a distinct copy of the given string, sharing no--- storage with the original string.------ As an example, suppose you read a large string, of which you need--- only a small portion.  If you do not use 'copy', the entire original--- array will be kept alive in memory by the smaller string. Making a--- copy \"breaks the link\" to the original array, allowing it to be--- garbage collected if there are no other live references to it.-copy :: Text -> Text-copy (Text arr off len) = Text (A.run go) 0 len-  where-    go :: ST s (A.MArray s)-    go = do-      marr <- A.new len-      A.copyI marr 0 arr off len-      return marr------------------------------------------------------- NOTE: the named chunk below used by doctest;---       verify the doctests via `doctest -fobject-code Data/Text.hs`---- $setup--- >>> :set -XOverloadedStrings--- >>> import qualified Data.Text as T
− Data/Text/Array.hs 
@@ -1,249 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, MagicHash, Rank2Types,-    RecordWildCards, UnboxedTuples, UnliftedFFITypes #-}-{-# OPTIONS_GHC -fno-warn-unused-matches #-}--- |--- Module      : Data.Text.Array--- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : portable------ Packed, unboxed, heap-resident arrays.  Suitable for performance--- critical use, both in terms of large data quantities and high--- speed.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text.Array as A------ The names in this module resemble those in the 'Data.Array' family--- of modules, but are shorter due to the assumption of qualified--- naming.-module Data.Text.Array-    (-    -- * Types-      Array(Array, aBA)-    , MArray(MArray, maBA)--    -- * Functions-    , copyM-    , copyI-    , empty-    , equal-#if defined(ASSERTS)-    , length-#endif-    , run-    , run2-    , toList-    , unsafeFreeze-    , unsafeIndex-    , new-    , unsafeWrite-    ) where--#if defined(ASSERTS)--- This fugly hack is brought by GHC's apparent reluctance to deal--- with MagicHash and UnboxedTuples when inferring types. Eek!-# define CHECK_BOUNDS(_func_,_len_,_k_) \-if (_k_) < 0 || (_k_) >= (_len_) then error ("Data.Text.Array." ++ (_func_) ++ ": bounds error, offset " ++ show (_k_) ++ ", length " ++ show (_len_)) else-#else-# define CHECK_BOUNDS(_func_,_len_,_k_)-#endif--#include "MachDeps.h"--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-#if __GLASGOW_HASKELL__ >= 702-import Control.Monad.ST.Unsafe (unsafeIOToST)-#else-import Control.Monad.ST (unsafeIOToST)-#endif-import Data.Bits ((.&.), xor)-import Data.Text.Internal.Unsafe (inlinePerformIO)-import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)-#if __GLASGOW_HASKELL__ >= 703-import Foreign.C.Types (CInt(CInt), CSize(CSize))-#else-import Foreign.C.Types (CInt, CSize)-#endif-import GHC.Base (ByteArray#, MutableByteArray#, Int(..),-                 indexWord16Array#, newByteArray#,-                 unsafeFreezeByteArray#, writeWord16Array#)-import GHC.ST (ST(..), runST)-import GHC.Word (Word16(..))-import Prelude hiding (length, read)---- | Immutable array type.------ The 'Array' constructor is exposed since @text-1.1.1.3@-data Array = Array {-      aBA :: ByteArray#-#if defined(ASSERTS)-    , aLen :: {-# UNPACK #-} !Int -- length (in units of Word16, not bytes)-#endif-    }---- | Mutable array type, for use in the ST monad.------ The 'MArray' constructor is exposed since @text-1.1.1.3@-data MArray s = MArray {-      maBA :: MutableByteArray# s-#if defined(ASSERTS)-    , maLen :: {-# UNPACK #-} !Int -- length (in units of Word16, not bytes)-#endif-    }--#if defined(ASSERTS)--- | Operations supported by all arrays.-class IArray a where-    -- | Return the length of an array.-    length :: a -> Int--instance IArray Array where-    length = aLen-    {-# INLINE length #-}--instance IArray (MArray s) where-    length = maLen-    {-# INLINE length #-}-#endif---- | Create an uninitialized mutable array.-new :: forall s. Int -> ST s (MArray s)-new n-  | n < 0 || n .&. highBit /= 0 = array_size_error-  | otherwise = ST $ \s1# ->-       case newByteArray# len# s1# of-         (# s2#, marr# #) -> (# s2#, MArray marr#-#if defined(ASSERTS)-                                n-#endif-                                #)-  where !(I# len#) = bytesInArray n-        highBit    = maxBound `xor` (maxBound `shiftR` 1)-{-# INLINE new #-}--array_size_error :: a-array_size_error = error "Data.Text.Array.new: size overflow"---- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!-unsafeFreeze :: MArray s -> ST s Array-unsafeFreeze MArray{..} = ST $ \s1# ->-    case unsafeFreezeByteArray# maBA s1# of-        (# s2#, ba# #) -> (# s2#, Array ba#-#if defined(ASSERTS)-                             maLen-#endif-                             #)-{-# INLINE unsafeFreeze #-}---- | Indicate how many bytes would be used for an array of the given--- size.-bytesInArray :: Int -> Int-bytesInArray n = n `shiftL` 1-{-# INLINE bytesInArray #-}---- | Unchecked read of an immutable array.  May return garbage or--- crash on an out-of-bounds access.-unsafeIndex :: Array -> Int -> Word16-unsafeIndex Array{..} i@(I# i#) =-  CHECK_BOUNDS("unsafeIndex",aLen,i)-    case indexWord16Array# aBA i# of r# -> (W16# r#)-{-# INLINE unsafeIndex #-}---- | Unchecked write of a mutable array.  May return garbage or crash--- on an out-of-bounds access.-unsafeWrite :: MArray s -> Int -> Word16 -> ST s ()-unsafeWrite MArray{..} i@(I# i#) (W16# e#) = ST $ \s1# ->-  CHECK_BOUNDS("unsafeWrite",maLen,i)-  case writeWord16Array# maBA i# e# s1# of-    s2# -> (# s2#, () #)-{-# INLINE unsafeWrite #-}---- | Convert an immutable array to a list.-toList :: Array -> Int -> Int -> [Word16]-toList ary off len = loop 0-    where loop i | i < len   = unsafeIndex ary (off+i) : loop (i+1)-                 | otherwise = []---- | An empty immutable array.-empty :: Array-empty = runST (new 0 >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result.-run :: (forall s. ST s (MArray s)) -> Array-run k = runST (k >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result paired with whatever else the action returns.-run2 :: (forall s. ST s (MArray s, a)) -> (Array, a)-run2 k = runST (do-                 (marr,b) <- k-                 arr <- unsafeFreeze marr-                 return (arr,b))-{-# INLINE run2 #-}---- | Copy some elements of a mutable array.-copyM :: MArray s               -- ^ Destination-      -> Int                    -- ^ Destination offset-      -> MArray s               -- ^ Source-      -> Int                    -- ^ Source offset-      -> Int                    -- ^ Count-      -> ST s ()-copyM dest didx src sidx count-    | count <= 0 = return ()-    | otherwise =-#if defined(ASSERTS)-    assert (sidx + count <= length src) .-    assert (didx + count <= length dest) .-#endif-    unsafeIOToST $ memcpyM (maBA dest) (fromIntegral didx)-                           (maBA src) (fromIntegral sidx)-                           (fromIntegral count)-{-# INLINE copyM #-}---- | Copy some elements of an immutable array.-copyI :: MArray s               -- ^ Destination-      -> Int                    -- ^ Destination offset-      -> Array                  -- ^ Source-      -> Int                    -- ^ Source offset-      -> Int                    -- ^ First offset in destination /not/ to-                                -- copy (i.e. /not/ length)-      -> ST s ()-copyI dest i0 src j0 top-    | i0 >= top = return ()-    | otherwise = unsafeIOToST $-                  memcpyI (maBA dest) (fromIntegral i0)-                          (aBA src) (fromIntegral j0)-                          (fromIntegral (top-i0))-{-# INLINE copyI #-}---- | Compare portions of two arrays for equality.  No bounds checking--- is performed.-equal :: Array                  -- ^ First-      -> Int                    -- ^ Offset into first-      -> Array                  -- ^ Second-      -> Int                    -- ^ Offset into second-      -> Int                    -- ^ Count-      -> Bool-equal arrA offA arrB offB count = inlinePerformIO $ do-  i <- memcmp (aBA arrA) (fromIntegral offA)-                     (aBA arrB) (fromIntegral offB) (fromIntegral count)-  return $! i == 0-{-# INLINE equal #-}--foreign import ccall unsafe "_hs_text_memcpy" memcpyI-    :: MutableByteArray# s -> CSize -> ByteArray# -> CSize -> CSize -> IO ()--foreign import ccall unsafe "_hs_text_memcmp" memcmp-    :: ByteArray# -> CSize -> ByteArray# -> CSize -> CSize -> IO CInt--foreign import ccall unsafe "_hs_text_memcpy" memcpyM-    :: MutableByteArray# s -> CSize -> MutableByteArray# s -> CSize -> CSize-    -> IO ()
− Data/Text/Encoding.hs 
@@ -1,535 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, GeneralizedNewtypeDeriving, MagicHash,-    UnliftedFFITypes #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif--- |--- Module      : Data.Text.Encoding--- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts,---               (c) 2008, 2009 Tom Harper------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : portable------ Functions for converting 'Text' values to and from 'ByteString',--- using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Encoding-    (-    -- * Decoding ByteStrings to Text-    -- $strict-      decodeASCII-    , decodeLatin1-    , decodeUtf8-    , decodeUtf16LE-    , decodeUtf16BE-    , decodeUtf32LE-    , decodeUtf32BE--    -- ** Catchable failure-    , decodeUtf8'--    -- ** Controllable error handling-    , decodeUtf8With-    , decodeUtf16LEWith-    , decodeUtf16BEWith-    , decodeUtf32LEWith-    , decodeUtf32BEWith--    -- ** Stream oriented decoding-    -- $stream-    , streamDecodeUtf8-    , streamDecodeUtf8With-    , Decoding(..)--    -- * Encoding Text to ByteStrings-    , encodeUtf8-    , encodeUtf16LE-    , encodeUtf16BE-    , encodeUtf32LE-    , encodeUtf32BE--    -- * Encoding Text using ByteString Builders-    , encodeUtf8Builder-    , encodeUtf8BuilderEscaped-    ) where--#if __GLASGOW_HASKELL__ >= 702-import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)-#else-import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)-#endif--import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall))-import Control.Monad.ST (runST)-import Data.Bits ((.&.))-import Data.ByteString as B-import Data.ByteString.Internal as B hiding (c2w)-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)-import Data.Text.Internal (Text(..), safe, text)-import Data.Text.Internal.Private (runText)-import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)-import Data.Text.Internal.Unsafe.Shift (shiftR)-import Data.Text.Show ()-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Data.Word (Word8, Word32)-#if __GLASGOW_HASKELL__ >= 703-import Foreign.C.Types (CSize(CSize))-#else-import Foreign.C.Types (CSize)-#endif-import Foreign.ForeignPtr (withForeignPtr)-import Foreign.Marshal.Utils (with)-import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)-import Foreign.Storable (Storable, peek, poke)-import GHC.Base (ByteArray#, MutableByteArray#)-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Builder.Prim.Internal as BP-import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Encoding.Fusion as E-import qualified Data.Text.Internal.Encoding.Utf16 as U16-import qualified Data.Text.Internal.Fusion as F--#include "text_cbits.h"---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- | /Deprecated/.  Decode a 'ByteString' containing 7-bit ASCII--- encoded text.-decodeASCII :: ByteString -> Text-decodeASCII = decodeUtf8-{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.------ 'decodeLatin1' is semantically equivalent to---  @Data.Text.pack . Data.ByteString.Char8.unpack@-decodeLatin1 :: ByteString -> Text-decodeLatin1 (PS fp off len) = text a 0 len- where-  a = A.run (A.new len >>= unsafeIOToST . go)-  go dest = withForeignPtr fp $ \ptr -> do-    c_decode_latin1 (A.maBA dest) (ptr `plusPtr` off) (ptr `plusPtr` (off+len))-    return dest---- | Decode a 'ByteString' containing UTF-8 encoded text.------ __NOTE__: The replacement character returned by 'OnDecodeError'--- MUST be within the BMP plane; surrogate code points will--- automatically be remapped to the replacement char @U+FFFD@--- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an--- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using--- those unsupported code points would result in undefined behavior.-decodeUtf8With :: OnDecodeError -> ByteString -> Text-decodeUtf8With onErr (PS fp off len) = runText $ \done -> do-  let go dest = withForeignPtr fp $ \ptr ->-        with (0::CSize) $ \destOffPtr -> do-          let end = ptr `plusPtr` (off + len)-              loop curPtr = do-                curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end-                if curPtr' == end-                  then do-                    n <- peek destOffPtr-                    unsafeSTToIO (done dest (fromIntegral n))-                  else do-                    x <- peek curPtr'-                    case onErr desc (Just x) of-                      Nothing -> loop $ curPtr' `plusPtr` 1-                      Just c-                        | c > '\xFFFF' -> throwUnsupportedReplChar-                        | otherwise -> do-                            destOff <- peek destOffPtr-                            w <- unsafeSTToIO $-                                 unsafeWrite dest (fromIntegral destOff)-                                             (safe c)-                            poke destOffPtr (destOff + fromIntegral w)-                            loop $ curPtr' `plusPtr` 1-          loop (ptr `plusPtr` off)-  (unsafeIOToST . go) =<< A.new len- where-  desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream"--  throwUnsupportedReplChar = throwIO $-    ErrorCall "decodeUtf8With: non-BMP replacement characters not supported"-  -- TODO: The code currently assumes that the transcoded UTF-16-  -- stream is at most twice as long (in bytes) as the input UTF-8-  -- stream. To justify this assumption one has to assume that the-  -- error handler replacement character also satisfies this-  -- invariant, by emitting at most one UTF16 code unit.-  ---  -- One easy way to support the full range of code-points for-  -- replacement characters in the error handler is to simply change-  -- the (over-)allocation to `A.new (2*len)` and then shrink back the-  -- `ByteArray#` to the real size (recent GHCs have a cheap-  -- `ByteArray#` resize-primop for that which allow the GC to reclaim-  -- the overallocation). However, this would require 4 times as much-  -- (temporary) storage as the original UTF-8 required.-  ---  -- Another strategy would be to optimistically assume that-  -- replacement characters are within the BMP, and if the case of a-  -- non-BMP replacement occurs reallocate the target buffer (or throw-  -- an exception, and fallback to a pessimistic codepath, like e.g.-  -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`)-  ---  -- Alternatively, `OnDecodeError` could become a datastructure which-  -- statically encodes the replacement-character range,-  -- e.g. something isomorphic to-  ---  --   Either (... -> Maybe Word16) (... -> Maybe Char)-  ---  -- And allow to statically switch between the BMP/non-BMP-  -- replacement-character codepaths. There's multiple ways to address-  -- this with different tradeoffs; but ideally we should optimise for-  -- the optimistic/error-free case.-{- INLINE[0] decodeUtf8With #-}---- $stream------ The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept--- a 'ByteString' that represents a possibly incomplete input (e.g. a--- packet from a network stream) that may not end on a UTF-8 boundary.------ 1. The maximal prefix of 'Text' that could be decoded from the---    given input.------ 2. The suffix of the 'ByteString' that could not be decoded due to---    insufficient input.------ 3. A function that accepts another 'ByteString'.  That string will---    be assumed to directly follow the string that was passed as---    input to the original function, and it will in turn be decoded.------ To help understand the use of these functions, consider the Unicode--- string @\"hi &#9731;\"@. If encoded as UTF-8, this becomes @\"hi--- \\xe2\\x98\\x83\"@; the final @\'&#9731;\'@ is encoded as 3 bytes.------ Now suppose that we receive this encoded string as 3 packets that--- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",--- \"\\x83\"]@. We cannot decode the entire Unicode string until we--- have received all three packets, but we would like to make progress--- as we receive each one.------ @--- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\"--- ghci> s0--- 'Some' \"hi \" \"\\xe2\" _--- @------ We use the continuation @f0@ to decode our second packet.------ @--- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\"--- ghci> s1--- 'Some' \"\" \"\\xe2\\x98\"--- @------ We could not give @f0@ enough input to decode anything, so it--- returned an empty string. Once we feed our second continuation @f1@--- the last byte of input, it will make progress.------ @--- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\"--- ghci> s2--- 'Some' \"\\x2603\" \"\" _--- @------ If given invalid input, an exception will be thrown by the function--- or continuation where it is encountered.---- | A stream oriented decoding result.------ @since 1.0.0.0-data Decoding = Some Text ByteString (ByteString -> Decoding)--instance Show Decoding where-    showsPrec d (Some t bs _) = showParen (d > prec) $-                                showString "Some " . showsPrec prec' t .-                                showChar ' ' . showsPrec prec' bs .-                                showString " _"-      where prec = 10; prec' = prec + 1--newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable)-newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable)---- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8--- encoded text that is known to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown (either by this function or a continuation) that cannot be--- caught in pure code.  For more control over the handling of invalid--- data, use 'streamDecodeUtf8With'.------ @since 1.0.0.0-streamDecodeUtf8 :: ByteString -> Decoding-streamDecodeUtf8 = streamDecodeUtf8With strictDecode---- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8--- encoded text.------ @since 1.0.0.0-streamDecodeUtf8With :: OnDecodeError -> ByteString -> Decoding-streamDecodeUtf8With onErr = decodeChunk B.empty 0 0- where-  -- We create a slightly larger than necessary buffer to accommodate a-  -- potential surrogate pair started in the last buffer-  decodeChunk :: ByteString -> CodePoint -> DecoderState -> ByteString-              -> Decoding-  decodeChunk undecoded0 codepoint0 state0 bs@(PS fp off len) =-    runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+1)-   where-    decodeChunkToBuffer :: A.MArray s -> IO Decoding-    decodeChunkToBuffer dest = withForeignPtr fp $ \ptr ->-      with (0::CSize) $ \destOffPtr ->-      with codepoint0 $ \codepointPtr ->-      with state0 $ \statePtr ->-      with nullPtr $ \curPtrPtr ->-        let end = ptr `plusPtr` (off + len)-            loop curPtr = do-              poke curPtrPtr curPtr-              curPtr' <- c_decode_utf8_with_state (A.maBA dest) destOffPtr-                         curPtrPtr end codepointPtr statePtr-              state <- peek statePtr-              case state of-                UTF8_REJECT -> do-                  -- We encountered an encoding error-                  x <- peek curPtr'-                  poke statePtr 0-                  case onErr desc (Just x) of-                    Nothing -> loop $ curPtr' `plusPtr` 1-                    Just c -> do-                      destOff <- peek destOffPtr-                      w <- unsafeSTToIO $-                           unsafeWrite dest (fromIntegral destOff) (safe c)-                      poke destOffPtr (destOff + fromIntegral w)-                      loop $ curPtr' `plusPtr` 1--                _ -> do-                  -- We encountered the end of the buffer while decoding-                  n <- peek destOffPtr-                  codepoint <- peek codepointPtr-                  chunkText <- unsafeSTToIO $ do-                      arr <- A.unsafeFreeze dest-                      return $! text arr 0 (fromIntegral n)-                  lastPtr <- peek curPtrPtr-                  let left = lastPtr `minusPtr` curPtr-                      !undecoded = case state of-                        UTF8_ACCEPT -> B.empty-                        _           -> B.append undecoded0 (B.drop left bs)-                  return $ Some chunkText undecoded-                           (decodeChunk undecoded codepoint state)-        in loop (ptr `plusPtr` off)-  desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream"---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code.  For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.-decodeUtf8 :: ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}-{-# RULES "STREAM stream/decodeUtf8 fusion" [1]-    forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-}---- | Decode a 'ByteString' containing UTF-8 encoded text.------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.-decodeUtf8' :: ByteString -> Either UnicodeException Text-decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode-{-# INLINE decodeUtf8' #-}---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}--- TODO: Extend documentation with references to source code in @blaze-html@--- or @aeson@ that uses this function.-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped be =-    -- manual eta-expansion to ensure inlining works as expected-    \txt -> B.builder (mkBuildstep txt)-  where-    bound = max 4 $ BP.sizeBound be--    mkBuildstep (Text arr off len) !k =-        outerLoop off-      where-        iend = off + len--        outerLoop !i0 !br@(B.BufferRange op0 ope)-          | i0 >= iend       = k br-          | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)-          -- TODO: Use a loop with an integrated bound's check if outRemaining-          -- is smaller than 8, as this will save on divisions.-          | otherwise        = return $ B.bufferFull bound op0 (outerLoop i0)-          where-            outRemaining = (ope `minusPtr` op0) `div` bound-            inpRemaining = iend - i0--            goPartial !iendTmp = go i0 op0-              where-                go !i !op-                  | i < iendTmp = case A.unsafeIndex arr i of-                      w | w <= 0x7F -> do-                            BP.runB be (fromIntegral w) op >>= go (i + 1)-                        | w <= 0x7FF -> do-                            poke8 0 $ (w `shiftR` 6) + 0xC0-                            poke8 1 $ (w .&. 0x3f) + 0x80-                            go (i + 1) (op `plusPtr` 2)-                        | 0xD800 <= w && w <= 0xDBFF -> do-                            let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1))-                            poke8 0 $ (c `shiftR` 18) + 0xF0-                            poke8 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80-                            poke8 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80-                            poke8 3 $ (c .&. 0x3F) + 0x80-                            go (i + 2) (op `plusPtr` 4)-                        | otherwise -> do-                            poke8 0 $ (w `shiftR` 12) + 0xE0-                            poke8 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80-                            poke8 2 $ (w .&. 0x3F) + 0x80-                            go (i + 1) (op `plusPtr` 3)-                  | otherwise =-                      outerLoop i (B.BufferRange op ope)-                  where-                    poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> ByteString-encodeUtf8 (Text arr off len)-  | len == 0  = B.empty-  | otherwise = unsafeDupablePerformIO $ do-  fp <- mallocByteString (len*3) -- see https://github.com/haskell/text/issues/194 for why len*3 is enough-  withForeignPtr fp $ \ptr ->-    with ptr $ \destPtr -> do-      c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)-      newDest <- peek destPtr-      let utf8len = newDest `minusPtr` ptr-      if utf8len >= len `shiftR` 1-        then return (PS fp 0 utf8len)-        else do-          fp' <- mallocByteString utf8len-          withForeignPtr fp' $ \ptr' -> do-            memcpy ptr' ptr (fromIntegral utf8len)-            return (PS fp' 0 utf8len)---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> ByteString-encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> ByteString-encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> ByteString-encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> ByteString-encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))-{-# INLINE encodeUtf32BE #-}--foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8-    :: MutableByteArray# s -> Ptr CSize-    -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8)--foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state-    :: MutableByteArray# s -> Ptr CSize-    -> Ptr (Ptr Word8) -> Ptr Word8-    -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8)--foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1-    :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()--foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8-    :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()
− Data/Text/Encoding/Error.hs 
@@ -1,124 +0,0 @@-{-# LANGUAGE CPP, DeriveDataTypeable #-}-#if __GLASGOW_HASKELL__ >= 704-{-# LANGUAGE Safe #-}-#elif __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif--- |--- Module      : Data.Text.Encoding.Error--- Copyright   : (c) Bryan O'Sullivan 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Types and functions for dealing with encoding and decoding errors--- in Unicode text.------ The standard functions for encoding and decoding text are strict,--- which is to say that they throw exceptions on invalid input.  This--- is often unhelpful on real world input, so alternative functions--- exist that accept custom handlers for dealing with invalid inputs.--- These 'OnError' handlers are normal Haskell functions.  You can use--- one of the presupplied functions in this module, or you can write a--- custom handler of your own.--module Data.Text.Encoding.Error-    (-    -- * Error handling types-      UnicodeException(..)-    , OnError-    , OnDecodeError-    , OnEncodeError-    -- * Useful error handling functions-    , lenientDecode-    , strictDecode-    , strictEncode-    , ignore-    , replace-    ) where--import Control.DeepSeq (NFData (..))-import Control.Exception (Exception, throw)-import Data.Typeable (Typeable)-import Data.Word (Word8)-import Numeric (showHex)---- | Function type for handling a coding error.  It is supplied with--- two inputs:------ * A 'String' that describes the error.------ * The input value that caused the error.  If the error arose---   because the end of input was reached or could not be identified---   precisely, this value will be 'Nothing'.------ If the handler returns a value wrapped with 'Just', that value will--- be used in the output as the replacement for the invalid input.  If--- it returns 'Nothing', no value will be used in the output.------ Should the handler need to abort processing, it should use 'error'--- or 'throw' an exception (preferably a 'UnicodeException').  It may--- use the description provided to construct a more helpful error--- report.-type OnError a b = String -> Maybe a -> Maybe b---- | A handler for a decoding error.-type OnDecodeError = OnError Word8 Char---- | A handler for an encoding error.-{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-}-type OnEncodeError = OnError Char Word8---- | An exception type for representing Unicode encoding errors.-data UnicodeException =-    DecodeError String (Maybe Word8)-    -- ^ Could not decode a byte sequence because it was invalid under-    -- the given encoding, or ran out of input in mid-decode.-  | EncodeError String (Maybe Char)-    -- ^ Tried to encode a character that could not be represented-    -- under the given encoding, or ran out of input in mid-encode.-    deriving (Eq, Typeable)--{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-}--showUnicodeException :: UnicodeException -> String-showUnicodeException (DecodeError desc (Just w))-    = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)-showUnicodeException (DecodeError desc Nothing)-    = "Cannot decode input: " ++ desc-showUnicodeException (EncodeError desc (Just c))-    = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc)-showUnicodeException (EncodeError desc Nothing)-    = "Cannot encode input: " ++ desc--instance Show UnicodeException where-    show = showUnicodeException--instance Exception UnicodeException--instance NFData UnicodeException where-    rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` ()-    rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` ()---- | Throw a 'UnicodeException' if decoding fails.-strictDecode :: OnDecodeError-strictDecode desc c = throw (DecodeError desc c)---- | Replace an invalid input byte with the Unicode replacement--- character U+FFFD.-lenientDecode :: OnDecodeError-lenientDecode _ _ = Just '\xfffd'---- | Throw a 'UnicodeException' if encoding fails.-{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-}-strictEncode :: OnEncodeError-strictEncode desc c = throw (EncodeError desc c)---- | Ignore an invalid input, substituting nothing in the output.-ignore :: OnError a b-ignore _ _ = Nothing---- | Replace an invalid input with a valid output.-replace :: b -> OnError a b-replace c _ _ = Just c
− Data/Text/Foreign.hs 
@@ -1,176 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving #-}--- |--- Module      : Data.Text.Foreign--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Support for using 'Text' data with native code via the Haskell--- foreign function interface.--module Data.Text.Foreign-    (-    -- * Interoperability with native code-    -- $interop-      I16-    -- * Safe conversion functions-    , fromPtr-    , useAsPtr-    , asForeignPtr-    -- ** Encoding as UTF-8-    , peekCStringLen-    , withCStringLen-    -- * Unsafe conversion code-    , lengthWord16-    , unsafeCopyToPtr-    -- * Low-level manipulation-    -- $lowlevel-    , dropWord16-    , takeWord16-    ) where--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-#if __GLASGOW_HASKELL__ >= 702-import Control.Monad.ST.Unsafe (unsafeIOToST)-#else-import Control.Monad.ST (unsafeIOToST)-#endif-import Data.ByteString.Unsafe (unsafePackCStringLen, unsafeUseAsCStringLen)-import Data.Text.Encoding (decodeUtf8, encodeUtf8)-import Data.Text.Internal (Text(..), empty)-import Data.Text.Unsafe (lengthWord16)-import Data.Word (Word16)-import Foreign.C.String (CStringLen)-import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray, withForeignPtr)-import Foreign.Marshal.Alloc (allocaBytes)-import Foreign.Ptr (Ptr, castPtr, plusPtr)-import Foreign.Storable (peek, poke)-import qualified Data.Text.Array as A---- $interop------ The 'Text' type is implemented using arrays that are not guaranteed--- to have a fixed address in the Haskell heap. All communication with--- native code must thus occur by copying data back and forth.------ The 'Text' type's internal representation is UTF-16, using the--- platform's native endianness.  This makes copied data suitable for--- use with native libraries that use a similar representation, such--- as ICU.  To interoperate with native libraries that use different--- internal representations, such as UTF-8 or UTF-32, consider using--- the functions in the 'Data.Text.Encoding' module.---- | A type representing a number of UTF-16 code units.-newtype I16 = I16 Int-    deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show)---- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word16' by copying the--- contents of the array.-fromPtr :: Ptr Word16           -- ^ source array-        -> I16                  -- ^ length of source array (in 'Word16' units)-        -> IO Text-fromPtr _   (I16 0)   = return empty-fromPtr ptr (I16 len) =-#if defined(ASSERTS)-    assert (len > 0) $-#endif-    return $! Text arr 0 len-  where-    arr = A.run (A.new len >>= copy)-    copy marr = loop ptr 0-      where-        loop !p !i | i == len = return marr-                   | otherwise = do-          A.unsafeWrite marr i =<< unsafeIOToST (peek p)-          loop (p `plusPtr` 2) (i + 1)---- $lowlevel------ Foreign functions that use UTF-16 internally may return indices in--- units of 'Word16' instead of characters.  These functions may--- safely be used with such indices, as they will adjust offsets if--- necessary to preserve the validity of a Unicode string.---- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word16' units in--- length.------ If @n@ would cause the 'Text' to end inside a surrogate pair, the--- end of the prefix will be advanced by one additional 'Word16' unit--- to maintain its validity.-takeWord16 :: I16 -> Text -> Text-takeWord16 (I16 n) t@(Text arr off len)-    | n <= 0               = empty-    | n >= len || m >= len = t-    | otherwise            = Text arr off m-  where-    m | w < 0xD800 || w > 0xDBFF = n-      | otherwise                = n+1-    w = A.unsafeIndex arr (off+n-1)---- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word16' units--- dropped from its beginning.------ If @n@ would cause the 'Text' to begin inside a surrogate pair, the--- beginning of the suffix will be advanced by one additional 'Word16'--- unit to maintain its validity.-dropWord16 :: I16 -> Text -> Text-dropWord16 (I16 n) t@(Text arr off len)-    | n <= 0               = t-    | n >= len || m >= len = empty-    | otherwise            = Text arr (off+m) (len-m)-  where-    m | w < 0xD800 || w > 0xDBFF = n-      | otherwise                = n+1-    w = A.unsafeIndex arr (off+n-1)---- | /O(n)/ Copy a 'Text' to an array.  The array is assumed to be big--- enough to hold the contents of the entire 'Text'.-unsafeCopyToPtr :: Text -> Ptr Word16 -> IO ()-unsafeCopyToPtr (Text arr off len) ptr = loop ptr off-  where-    end = off + len-    loop !p !i | i == end  = return ()-               | otherwise = do-      poke p (A.unsafeIndex arr i)-      loop (p `plusPtr` 2) (i + 1)---- | /O(n)/ Perform an action on a temporary, mutable copy of a--- 'Text'.  The copy is freed as soon as the action returns.-useAsPtr :: Text -> (Ptr Word16 -> I16 -> IO a) -> IO a-useAsPtr t@(Text _arr _off len) action =-    allocaBytes (len * 2) $ \buf -> do-      unsafeCopyToPtr t buf-      action (castPtr buf) (fromIntegral len)---- | /O(n)/ Make a mutable copy of a 'Text'.-asForeignPtr :: Text -> IO (ForeignPtr Word16, I16)-asForeignPtr t@(Text _arr _off len) = do-  fp <- mallocForeignPtrArray len-  withForeignPtr fp $ unsafeCopyToPtr t-  return (fp, I16 len)---- | /O(n)/ Decode a C string with explicit length, which is assumed--- to have been encoded as UTF-8. If decoding fails, a--- 'UnicodeException' is thrown.------ @since 1.0.0.0-peekCStringLen :: CStringLen -> IO Text-peekCStringLen cs = do-  bs <- unsafePackCStringLen cs-  return $! decodeUtf8 bs---- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary--- storage, with explicit length information. The encoded string may--- contain NUL bytes, and is not followed by a trailing NUL byte.------ The temporary storage is freed when the subcomputation terminates--- (either normally or via an exception), so the pointer to the--- temporary storage must /not/ be used after this function returns.------ @since 1.0.0.0-withCStringLen :: Text -> (CStringLen -> IO a) -> IO a-withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act
− Data/Text/IO.hs 
@@ -1,350 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif--- |--- Module      : Data.Text.IO--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Simon Marlow--- License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Efficient locale-sensitive support for text I\/O.------ Skip past the synopsis for some important notes on performance and--- portability across different versions of GHC.--module Data.Text.IO-    (-    -- * Performance-    -- $performance--    -- * Locale support-    -- $locale-    -- * File-at-a-time operations-      readFile-    , writeFile-    , appendFile-    -- * Operations on handles-    , hGetContents-    , hGetChunk-    , hGetLine-    , hPutStr-    , hPutStrLn-    -- * Special cases for standard input and output-    , interact-    , getContents-    , getLine-    , putStr-    , putStrLn-    ) where--import Data.Text (Text)-import Prelude hiding (appendFile, getContents, getLine, interact,-                       putStr, putStrLn, readFile, writeFile)-import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout,-                  withFile)-import qualified Control.Exception as E-import Control.Monad (liftM2, when)-import Data.IORef (readIORef, writeIORef)-import qualified Data.Text as T-import Data.Text.Internal.Fusion (stream)-import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))-import Data.Text.Internal.IO (hGetLineWith, readChunk)-import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBufElem, CharBuffer,-                      RawCharBuffer, emptyBuffer, isEmptyBuffer, newCharBuffer,-                      writeCharBuf)-import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType))-import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle,-                                wantWritableHandle)-import GHC.IO.Handle.Text (commitBuffer')-import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..),-                            HandleType(..), Newline(..))-import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)-import System.IO.Error (isEOFError)---- $performance--- #performance#------ The functions in this module obey the runtime system's locale,--- character set encoding, and line ending conversion settings.------ If you know in advance that you will be working with data that has--- a specific encoding (e.g. UTF-8), and your application is highly--- performance sensitive, you may find that it is faster to perform--- I\/O with bytestrings and to encode and decode yourself than to use--- the functions in this module.------ Whether this will hold depends on the version of GHC you are using,--- the platform you are working on, the data you are working with, and--- the encodings you are using, so be sure to test for yourself.---- | The 'readFile' function reads a file and returns the contents of--- the file as a string.  The entire file is read strictly, as with--- 'getContents'.-readFile :: FilePath -> IO Text-readFile name = openFile name ReadMode >>= hGetContents---- | Write a string to a file.  The file is truncated to zero length--- before writing begins.-writeFile :: FilePath -> Text -> IO ()-writeFile p = withFile p WriteMode . flip hPutStr---- | Write a string the end of a file.-appendFile :: FilePath -> Text -> IO ()-appendFile p = withFile p AppendMode . flip hPutStr--catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool)-catchError caller h Handle__{..} err-    | isEOFError err = do-        buf <- readIORef haCharBuffer-        return $ if isEmptyBuffer buf-                 then (T.empty, True)-                 else (T.singleton '\r', True)-    | otherwise = E.throwIO (augmentIOError err caller h)---- | Wrap readChunk and return a value indicating if we're reached the EOF.--- This is needed because unpack_nl is unable to discern the difference--- between a buffer with just \r due to EOF or because not enough data was left--- for decoding. e.g. the final character decoded from the byte buffer was \r.-readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool)-readChunkEof hh buf = do t <- readChunk hh buf-                         return (t, False)---- | /Experimental./ Read a single chunk of strict text from a--- 'Handle'. The size of the chunk depends on the amount of input--- currently buffered.------ This function blocks only if there is no data available, and EOF--- has not yet been reached. Once EOF is reached, this function--- returns an empty string instead of throwing an exception.-hGetChunk :: Handle -> IO Text-hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk- where-  readSingleChunk hh@Handle__{..} = do-    buf <- readIORef haCharBuffer-    (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh-    return (hh, t)---- | Read the remaining contents of a 'Handle' as a string.  The--- 'Handle' is closed once the contents have been read, or if an--- exception is thrown.------ Internally, this function reads a chunk at a time from the--- lower-level buffering abstraction, and concatenates the chunks into--- a single string once the entire file has been read.------ As a result, it requires approximately twice as much memory as its--- result to construct its result.  For files more than a half of--- available RAM in size, this may result in memory exhaustion.-hGetContents :: Handle -> IO Text-hGetContents h = do-  chooseGoodBuffering h-  wantReadableHandle "hGetContents" h readAll- where-  readAll hh@Handle__{..} = do-    let readChunks = do-          buf <- readIORef haCharBuffer-          (t, eof) <- readChunkEof hh buf-                         `E.catch` catchError "hGetContents" h hh-          if eof-            then return [t]-            else (t:) `fmap` readChunks-    ts <- readChunks-    (hh', _) <- hClose_help hh-    return (hh'{haType=ClosedHandle}, T.concat ts)---- | Use a more efficient buffer size if we're reading in--- block-buffered mode with the default buffer size.  When we can--- determine the size of the handle we're reading, set the buffer size--- to that, so that we can read the entire file in one chunk.--- Otherwise, use a buffer size of at least 16KB.-chooseGoodBuffering :: Handle -> IO ()-chooseGoodBuffering h = do-  bufMode <- hGetBuffering h-  case bufMode of-    BlockBuffering Nothing -> do-      d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) ->-           if ioe_type e == InappropriateType-           then return 16384 -- faster than the 2KB default-           else E.throwIO e-      when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromIntegral $ d-    _ -> return ()---- | Read a single line from a handle.-hGetLine :: Handle -> IO Text-hGetLine = hGetLineWith T.concat---- | Write a string to a handle.-hPutStr :: Handle -> Text -> IO ()--- This function is lifted almost verbatim from GHC.IO.Handle.Text.-hPutStr h t = do-  (buffer_mode, nl) <--       wantWritableHandle "hPutStr" h $ \h_ -> do-                     bmode <- getSpareBuffer h_-                     return (bmode, haOutputNL h_)-  let str = stream t-  case buffer_mode of-     (NoBuffering, _)        -> hPutChars h str-     (LineBuffering, buf)    -> writeLines h nl buf str-     (BlockBuffering _, buf)-         | nl == CRLF        -> writeBlocksCRLF h buf str-         | otherwise         -> writeBlocksRaw h buf str--hPutChars :: Handle -> Stream Char -> IO ()-hPutChars h (Stream next0 s0 _len) = loop s0-  where-    loop !s = case next0 s of-                Done       -> return ()-                Skip s'    -> loop s'-                Yield x s' -> hPutChar h x >> loop s'---- The following functions are largely lifted from GHC.IO.Handle.Text,--- but adapted to a coinductive stream of data instead of an inductive--- list.------ We have several variations of more or less the same code for--- performance reasons.  Splitting the original buffered write--- function into line- and block-oriented versions gave us a 2.1x--- performance improvement.  Lifting out the raw/cooked newline--- handling gave a few more percent on top.--writeLines :: Handle -> Newline -> Buffer CharBufElem -> Stream Char -> IO ()-writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0- where-  outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)-   where-    inner !s !n =-      case next0 s of-        Done -> commit n False{-no flush-} True{-release-} >> return ()-        Skip s' -> inner s' n-        Yield x s'-          | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s-          | x == '\n'    -> do-                   n' <- if nl == CRLF-                         then do n1 <- writeCharBuf raw n '\r'-                                 writeCharBuf raw n1 '\n'-                         else writeCharBuf raw n x-                   commit n' True{-needs flush-} False >>= outer s'-          | otherwise    -> writeCharBuf raw n x >>= inner s'-    commit = commitBuffer h raw len--writeBlocksCRLF :: Handle -> Buffer CharBufElem -> Stream Char -> IO ()-writeBlocksCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0- where-  outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)-   where-    inner !s !n =-      case next0 s of-        Done -> commit n False{-no flush-} True{-release-} >> return ()-        Skip s' -> inner s' n-        Yield x s'-          | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s-          | x == '\n'    -> do n1 <- writeCharBuf raw n '\r'-                               writeCharBuf raw n1 '\n' >>= inner s'-          | otherwise    -> writeCharBuf raw n x >>= inner s'-    commit = commitBuffer h raw len--writeBlocksRaw :: Handle -> Buffer CharBufElem -> Stream Char -> IO ()-writeBlocksRaw h buf0 (Stream next0 s0 _len) = outer s0 buf0- where-  outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)-   where-    inner !s !n =-      case next0 s of-        Done -> commit n False{-no flush-} True{-release-} >> return ()-        Skip s' -> inner s' n-        Yield x s'-          | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s-          | otherwise    -> writeCharBuf raw n x >>= inner s'-    commit = commitBuffer h raw len---- This function is completely lifted from GHC.IO.Handle.Text.-getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer)-getSpareBuffer Handle__{haCharBuffer=ref,-                        haBuffers=spare_ref,-                        haBufferMode=mode}- = do-   case mode of-     NoBuffering -> return (mode, error "no buffer!")-     _ -> do-          bufs <- readIORef spare_ref-          buf  <- readIORef ref-          case bufs of-            BufferListCons b rest -> do-                writeIORef spare_ref rest-                return ( mode, emptyBuffer b (bufSize buf) WriteBuffer)-            BufferListNil -> do-                new_buf <- newCharBuffer (bufSize buf) WriteBuffer-                return (mode, new_buf)----- This function is completely lifted from GHC.IO.Handle.Text.-commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool-             -> IO CharBuffer-commitBuffer hdl !raw !sz !count flush release =-  wantWritableHandle "commitAndReleaseBuffer" hdl $-     commitBuffer' raw sz count flush release-{-# INLINE commitBuffer #-}---- | Write a string to a handle, followed by a newline.-hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h t = hPutStr h t >> hPutChar h '\n'---- | The 'interact' function takes a function of type @Text -> Text@--- as its argument. The entire input from the standard input device is--- passed to this function as its argument, and the resulting string--- is output on the standard output device.-interact :: (Text -> Text) -> IO ()-interact f = putStr . f =<< getContents---- | Read all user input on 'stdin' as a single string.-getContents :: IO Text-getContents = hGetContents stdin---- | Read a single line of user input from 'stdin'.-getLine :: IO Text-getLine = hGetLine stdin---- | Write a string to 'stdout'.-putStr :: Text -> IO ()-putStr = hPutStr stdout---- | Write a string to 'stdout', followed by a newline.-putStrLn :: Text -> IO ()-putStrLn = hPutStrLn stdout---- $locale------ /Note/: The behaviour of functions in this module depends on the--- version of GHC you are using.------ Beginning with GHC 6.12, text I\/O is performed using the system or--- handle's current locale and line ending conventions.------ Under GHC 6.10 and earlier, the system I\/O libraries do not--- support locale-sensitive I\/O or line ending conversion.  On these--- versions of GHC, functions in this library all use UTF-8.  What--- does this mean in practice?------ * All data that is read will be decoded as UTF-8.------ * Before data is written, it is first encoded as UTF-8.------ * On both reading and writing, the platform's native newline---   conversion is performed.------ If you must use a non-UTF-8 locale on an older version of GHC, you--- will have to perform the transcoding yourself, e.g. as follows:------ > import qualified Data.ByteString as B--- > import Data.Text (Text)--- > import Data.Text.Encoding (encodeUtf16)--- >--- > putStr_Utf16LE :: Text -> IO ()--- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)------ On transcoding errors, an 'IOError' exception is thrown. You can--- use the API in "Data.Text.Encoding" if you need more control over--- error handling or transcoding.
− Data/Text/Internal.hs 
@@ -1,188 +0,0 @@-{-# LANGUAGE CPP, DeriveDataTypeable, UnboxedTuples #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module      : Data.Text.Internal--- Copyright   : (c) 2008, 2009 Tom Harper,---               (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.------ You should not use this module unless you are determined to monkey--- with the internals, as the functions here do just about nothing to--- preserve data invariants.  You have been warned!--#if defined(__GLASGOW_HASKELL__) && !defined(__HADDOCK__)-#include "MachDeps.h"-#endif--module Data.Text.Internal-    (-    -- * Types-    -- $internals-      Text(..)-    -- * Construction-    , text-    , textP-    -- * Safety-    , safe-    -- * Code that must be here for accessibility-    , empty-    , empty_-    -- * Utilities-    , firstf-    -- * Checked multiplication-    , mul-    , mul32-    , mul64-    -- * Debugging-    , showText-    ) where--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Bits-import Data.Int (Int32, Int64)-import Data.Text.Internal.Unsafe.Char (ord)-import Data.Typeable (Typeable)-import qualified Data.Text.Array as A---- | A space efficient, packed, unboxed Unicode text type.-data Text = Text-    {-# UNPACK #-} !A.Array          -- payload (Word16 elements)-    {-# UNPACK #-} !Int              -- offset (units of Word16, not Char)-    {-# UNPACK #-} !Int              -- length (units of Word16, not Char)-    deriving (Typeable)---- | Smart constructor.-text_ :: A.Array -> Int -> Int -> Text-text_ arr off len =-#if defined(ASSERTS)-  let c    = A.unsafeIndex arr off-      alen = A.length arr-  in assert (len >= 0) .-     assert (off >= 0) .-     assert (alen == 0 || len == 0 || off < alen) .-     assert (len == 0 || c < 0xDC00 || c > 0xDFFF) $-#endif-     Text arr off len-{-# INLINE text_ #-}---- | /O(1)/ The empty 'Text'.-empty :: Text-empty = Text A.empty 0 0-{-# INLINE [1] empty #-}---- | A non-inlined version of 'empty'.-empty_ :: Text-empty_ = Text A.empty 0 0-{-# NOINLINE empty_ #-}---- | Construct a 'Text' without invisibly pinning its byte array in--- memory if its length has dwindled to zero.-text :: A.Array -> Int -> Int -> Text-text arr off len | len == 0  = empty-                 | otherwise = text_ arr off len-{-# INLINE text #-}--textP :: A.Array -> Int -> Int -> Text-{-# DEPRECATED textP "Use text instead" #-}-textP = text---- | A useful 'show'-like function for debugging purposes.-showText :: Text -> String-showText (Text arr off len) =-    "Text " ++ show (A.toList arr off len) ++ ' ' :-            show off ++ ' ' : show len---- | Map a 'Char' to a 'Text'-safe value.------ UTF-16 surrogate code points are not included in the set of Unicode--- scalar values, but are unfortunately admitted as valid 'Char'--- values by Haskell.  They cannot be represented in a 'Text'.  This--- function remaps those code points to the Unicode replacement--- character (U+FFFD, \'&#xfffd;\'), and leaves other code points--- unchanged.-safe :: Char -> Char-safe c-    | ord c .&. 0x1ff800 /= 0xd800 = c-    | otherwise                    = '\xfffd'-{-# INLINE [0] safe #-}---- | Apply a function to the first element of an optional pair.-firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)-firstf f (Just (a, b)) = Just (f a, b)-firstf _  Nothing      = Nothing---- | Checked multiplication.  Calls 'error' if the result would--- overflow.-mul :: Int -> Int -> Int-#if WORD_SIZE_IN_BITS == 64-mul a b = fromIntegral $ fromIntegral a `mul64` fromIntegral b-#else-mul a b = fromIntegral $ fromIntegral a `mul32` fromIntegral b-#endif-{-# INLINE mul #-}-infixl 7 `mul`---- | Checked multiplication.  Calls 'error' if the result would--- overflow.-mul64 :: Int64 -> Int64 -> Int64-mul64 a b-  | a >= 0 && b >= 0 =  mul64_ a b-  | a >= 0           = -mul64_ a (-b)-  | b >= 0           = -mul64_ (-a) b-  | otherwise        =  mul64_ (-a) (-b)-{-# INLINE mul64 #-}-infixl 7 `mul64`--mul64_ :: Int64 -> Int64 -> Int64-mul64_ a b-  | ahi > 0 && bhi > 0 = error "overflow"-  | top > 0x7fffffff   = error "overflow"-  | total < 0          = error "overflow"-  | otherwise          = total-  where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #)-        (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #)-        top            = ahi * blo + alo * bhi-        total          = (top `shiftL` 32) + alo * blo-{-# INLINE mul64_ #-}---- | Checked multiplication.  Calls 'error' if the result would--- overflow.-mul32 :: Int32 -> Int32 -> Int32-mul32 a b = case fromIntegral a * fromIntegral b of-              ab | ab < min32 || ab > max32 -> error "overflow"-                 | otherwise                -> fromIntegral ab-  where min32 = -0x80000000 :: Int64-        max32 =  0x7fffffff-{-# INLINE mul32 #-}-infixl 7 `mul32`---- $internals------ Internally, the 'Text' type is represented as an array of 'Word16'--- UTF-16 code units. The offset and length fields in the constructor--- are in these units, /not/ units of 'Char'.------ Invariants that all functions must maintain:------ * Since the 'Text' type uses UTF-16 internally, it cannot represent---   characters in the reserved surrogate code point range U+D800 to---   U+DFFF. To maintain this invariant, the 'safe' function maps---   'Char' values in this range to the replacement character (U+FFFD,---   \'&#xfffd;\').------ * A leading (or \"high\") surrogate code unit (0xD800–0xDBFF) must---   always be followed by a trailing (or \"low\") surrogate code unit---   (0xDC00-0xDFFF). A trailing surrogate code unit must always be---   preceded by a leading surrogate code unit.
− Data/Text/Internal/Builder.hs 
@@ -1,329 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}-{-# OPTIONS_HADDOCK not-home #-}---------------------------------------------------------------------------------- |--- Module      : Data.Text.Internal.Builder--- Copyright   : (c) 2013 Bryan O'Sullivan---               (c) 2010 Johan Tibell--- License     : BSD-style (see LICENSE)------ Maintainer  : Johan Tibell <johan.tibell@gmail.com>--- Stability   : experimental--- Portability : portable to Hugs and GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Efficient construction of lazy @Text@ values.  The principal--- operations on a @Builder@ are @singleton@, @fromText@, and--- @fromLazyText@, which construct new builders, and 'mappend', which--- concatenates two builders.------ To get maximum performance when building lazy @Text@ values using a--- builder, associate @mappend@ calls to the right.  For example,--- prefer------ > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')------ to------ > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'------ as the latter associates @mappend@ to the left.-----------------------------------------------------------------------------------module Data.Text.Internal.Builder-   ( -- * Public API-     -- ** The Builder type-     Builder-   , toLazyText-   , toLazyTextWith--     -- ** Constructing Builders-   , singleton-   , fromText-   , fromLazyText-   , fromString--     -- ** Flushing the buffer state-   , flush--     -- * Internal functions-   , append'-   , ensureFree-   , writeN-   ) where--import Control.Monad.ST (ST, runST)-import Data.Monoid (Monoid(..))-#if !MIN_VERSION_base(4,11,0) && MIN_VERSION_base(4,9,0)-import Data.Semigroup (Semigroup(..))-#endif-import Data.Text.Internal (Text(..))-import Data.Text.Internal.Lazy (smallChunkSize)-import Data.Text.Unsafe (inlineInterleaveST)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Prelude hiding (map, putChar)--import qualified Data.String as String-import qualified Data.Text as S-import qualified Data.Text.Array as A-import qualified Data.Text.Lazy as L------------------------------------------------------------------------------ | A @Builder@ is an efficient way to build lazy @Text@ values.--- There are several functions for constructing builders, but only one--- to inspect them: to extract any data, you have to turn them into--- lazy @Text@ values using @toLazyText@.------ Internally, a builder constructs a lazy @Text@ by filling arrays--- piece by piece.  As each buffer is filled, it is \'popped\' off, to--- become a new chunk of the resulting lazy @Text@.  All this is--- hidden from the user of the @Builder@.-newtype Builder = Builder {-     -- Invariant (from Data.Text.Lazy):-     --      The lists include no null Texts.-     runBuilder :: forall s. (Buffer s -> ST s [S.Text])-                -> Buffer s-                -> ST s [S.Text]-   }--#if MIN_VERSION_base(4,9,0)-instance Semigroup Builder where-   (<>) = append-   {-# INLINE (<>) #-}-#endif--instance Monoid Builder where-   mempty  = empty-   {-# INLINE mempty #-}-#if MIN_VERSION_base(4,9,0)-   mappend = (<>) -- future-proof definition-#else-   mappend = append-#endif-   {-# INLINE mappend #-}-   mconcat = foldr mappend Data.Monoid.mempty-   {-# INLINE mconcat #-}--instance String.IsString Builder where-    fromString = fromString-    {-# INLINE fromString #-}--instance Show Builder where-    show = show . toLazyText--instance Eq Builder where-    a == b = toLazyText a == toLazyText b--instance Ord Builder where-    a <= b = toLazyText a <= toLazyText b------------------------------------------------------------------------------ | /O(1)./ The empty @Builder@, satisfying------  * @'toLazyText' 'empty' = 'L.empty'@----empty :: Builder-empty = Builder (\ k buf -> k buf)-{-# INLINE empty #-}---- | /O(1)./ A @Builder@ taking a single character, satisfying------  * @'toLazyText' ('singleton' c) = 'L.singleton' c@----singleton :: Char -> Builder-singleton c = writeAtMost 2 $ \ marr o -> unsafeWrite marr o c-{-# INLINE singleton #-}------------------------------------------------------------------------------ | /O(1)./ The concatenation of two builders, an associative--- operation with identity 'empty', satisfying------  * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@----append :: Builder -> Builder -> Builder-append (Builder f) (Builder g) = Builder (f . g)-{-# INLINE [0] append #-}---- TODO: Experiment to find the right threshold.-copyLimit :: Int-copyLimit = 128---- This function attempts to merge small @Text@ values instead of--- treating each value as its own chunk.  We may not always want this.---- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying------  * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@----fromText :: S.Text -> Builder-fromText t@(Text arr off l)-    | S.null t       = empty-    | l <= copyLimit = writeN l $ \marr o -> A.copyI marr o arr off (l+o)-    | otherwise      = flush `append` mapBuilder (t :)-{-# INLINE [1] fromText #-}--{-# RULES-"fromText/pack" forall s .-        fromText (S.pack s) = fromString s- #-}---- | /O(1)./ A Builder taking a @String@, satisfying------  * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@----fromString :: String -> Builder-fromString str = Builder $ \k (Buffer p0 o0 u0 l0) ->-    let loop !marr !o !u !l [] = k (Buffer marr o u l)-        loop marr o u l s@(c:cs)-            | l <= 1 = do-                arr <- A.unsafeFreeze marr-                let !t = Text arr o u-                marr' <- A.new chunkSize-                ts <- inlineInterleaveST (loop marr' 0 0 chunkSize s)-                return $ t : ts-            | otherwise = do-                n <- unsafeWrite marr (o+u) c-                loop marr o (u+n) (l-n) cs-    in loop p0 o0 u0 l0 str-  where-    chunkSize = smallChunkSize-{-# INLINE fromString #-}---- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying------  * @'toLazyText' ('fromLazyText' t) = t@----fromLazyText :: L.Text -> Builder-fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++)-{-# INLINE fromLazyText #-}------------------------------------------------------------------------------ Our internal buffer type-data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s)-                       {-# UNPACK #-} !Int  -- offset-                       {-# UNPACK #-} !Int  -- used units-                       {-# UNPACK #-} !Int  -- length left------------------------------------------------------------------------------ | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default--- buffer size.  The construction work takes place if and when the--- relevant part of the lazy @Text@ is demanded.-toLazyText :: Builder -> L.Text-toLazyText = toLazyTextWith smallChunkSize---- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given--- size for the initial buffer.  The construction work takes place if--- and when the relevant part of the lazy @Text@ is demanded.------ If the initial buffer is too small to hold all data, subsequent--- buffers will be the default buffer size.-toLazyTextWith :: Int -> Builder -> L.Text-toLazyTextWith chunkSize m = L.fromChunks (runST $-  newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return [])))---- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any,--- yielding a new chunk in the result lazy @Text@.-flush :: Builder-flush = Builder $ \ k buf@(Buffer p o u l) ->-    if u == 0-    then k buf-    else do arr <- A.unsafeFreeze p-            let !b = Buffer p (o+u) 0 l-                !t = Text arr o u-            ts <- inlineInterleaveST (k b)-            return $! t : ts-{-# INLINE [1] flush #-}--- defer inlining so that flush/flush rule may fire.------------------------------------------------------------------------------ | Sequence an ST operation on the buffer-withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder-withBuffer f = Builder $ \k buf -> f buf >>= k-{-# INLINE withBuffer #-}---- | Get the size of the buffer-withSize :: (Int -> Builder) -> Builder-withSize f = Builder $ \ k buf@(Buffer _ _ _ l) ->-    runBuilder (f l) k buf-{-# INLINE withSize #-}---- | Map the resulting list of texts.-mapBuilder :: ([S.Text] -> [S.Text]) -> Builder-mapBuilder f = Builder (fmap f .)------------------------------------------------------------------------------ | Ensure that there are at least @n@ many elements available.-ensureFree :: Int -> Builder-ensureFree !n = withSize $ \ l ->-    if n <= l-    then empty-    else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize)))-{-# INLINE [0] ensureFree #-}--writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder-writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f)-{-# INLINE [0] writeAtMost #-}---- | Ensure that @n@ many elements are available, and then use @f@ to--- write some elements into the memory.-writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder-writeN n f = writeAtMost n (\ p o -> f p o >> return n)-{-# INLINE writeN #-}--writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s)-writeBuffer f (Buffer p o u l) = do-    n <- f p (o+u)-    return $! Buffer p o (u+n) (l-n)-{-# INLINE writeBuffer #-}--newBuffer :: Int -> ST s (Buffer s)-newBuffer size = do-    arr <- A.new size-    return $! Buffer arr 0 0 size-{-# INLINE newBuffer #-}----------------------------------------------------------------------------- Some nice rules for Builder---- This function makes GHC understand that 'writeN' and 'ensureFree'--- are *not* recursive in the precense of the rewrite rules below.--- This is not needed with GHC 7+.-append' :: Builder -> Builder -> Builder-append' (Builder f) (Builder g) = Builder (f . g)-{-# INLINE append' #-}--{-# RULES--"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)-                           (g::forall s. A.MArray s -> Int -> ST s Int) ws.-    append (writeAtMost a f) (append (writeAtMost b g) ws) =-        append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->-                                    g marr (o+n) >>= \ m ->-                                    let s = n+m in s `seq` return s)) ws--"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)-                           (g::forall s. A.MArray s -> Int -> ST s Int).-    append (writeAtMost a f) (writeAtMost b g) =-        writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->-                            g marr (o+n) >>= \ m ->-                            let s = n+m in s `seq` return s)--"ensureFree/ensureFree" forall a b .-    append (ensureFree a) (ensureFree b) = ensureFree (max a b)--"flush/flush"-    append flush flush = flush-- #-}
− Data/Text/Internal/Builder/Functions.hs 
@@ -1,40 +0,0 @@-{-# LANGUAGE MagicHash #-}---- |--- Module      : Data.Text.Internal.Builder.Functions--- Copyright   : (c) 2011 MailRank, Inc.------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Useful functions and combinators.--module Data.Text.Internal.Builder.Functions-    (-      (<>)-    , i2d-    ) where--import Data.Monoid (mappend)-import Data.Text.Lazy.Builder (Builder)-import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#))-import Prelude ()---- | Unsafe conversion for decimal digits.-{-# INLINE i2d #-}-i2d :: Int -> Char-i2d (I# i#) = C# (chr# (ord# '0'# +# i#))---- | The normal 'mappend' function with right associativity instead of--- left.-(<>) :: Builder -> Builder -> Builder-(<>) = mappend-{-# INLINE (<>) #-}--infixr 4 <>
− Data/Text/Internal/Builder/Int/Digits.hs 
@@ -1,26 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}---- Module:      Data.Text.Internal.Builder.Int.Digits--- Copyright:   (c) 2013 Bryan O'Sullivan--- License:     BSD-style--- Maintainer:  Bryan O'Sullivan <bos@serpentine.com>--- Stability:   experimental--- Portability: portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ This module exists because the C preprocessor does things that we--- shall not speak of when confronted with Haskell multiline strings.--module Data.Text.Internal.Builder.Int.Digits (digits) where--import Data.ByteString.Char8 (ByteString)--digits :: ByteString-digits = "0001020304050607080910111213141516171819\-         \2021222324252627282930313233343536373839\-         \4041424344454647484950515253545556575859\-         \6061626364656667686970717273747576777879\-         \8081828384858687888990919293949596979899"
− Data/Text/Internal/Builder/RealFloat/Functions.hs 
@@ -1,57 +0,0 @@-{-# LANGUAGE CPP #-}---- |--- Module:    Data.Text.Internal.Builder.RealFloat.Functions--- Copyright: (c) The University of Glasgow 1994-2002--- License:   see libraries/base/LICENSE------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!--module Data.Text.Internal.Builder.RealFloat.Functions-    (-      roundTo-    ) where--roundTo :: Int -> [Int] -> (Int,[Int])--#if MIN_VERSION_base(4,6,0)--roundTo d is =-  case f d True is of-    x@(0,_) -> x-    (1,xs)  -> (1, 1:xs)-    _       -> error "roundTo: bad Value"- where-  b2 = base `quot` 2--  f n _ []     = (0, replicate n 0)-  f 0 e (x:xs) | x == b2 && e && all (== 0) xs = (0, [])   -- Round to even when at exactly half the base-               | otherwise = (if x >= b2 then 1 else 0, [])-  f n _ (i:xs)-     | i' == base = (1,0:ds)-     | otherwise  = (0,i':ds)-      where-       (c,ds) = f (n-1) (even i) xs-       i'     = c + i-  base = 10--#else--roundTo d is =-  case f d is of-    x@(0,_) -> x-    (1,xs)  -> (1, 1:xs)-    _       -> error "roundTo: bad Value"- where-  f n []     = (0, replicate n 0)-  f 0 (x:_)  = (if x >= 5 then 1 else 0, [])-  f n (i:xs)-     | i' == 10  = (1,0:ds)-     | otherwise = (0,i':ds)-      where-       (c,ds) = f (n-1) xs-       i'     = c + i--#endif
− Data/Text/Internal/Encoding/Fusion.hs 
@@ -1,208 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}---- |--- Module      : Data.Text.Internal.Encoding.Fusion--- Copyright   : (c) Tom Harper 2008-2009,---               (c) Bryan O'Sullivan 2009,---               (c) Duncan Coutts 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Fusible 'Stream'-oriented functions for converting between 'Text'--- and several common encodings.--module Data.Text.Internal.Encoding.Fusion-    (-    -- * Streaming-      streamASCII-    , streamUtf8-    , streamUtf16LE-    , streamUtf16BE-    , streamUtf32LE-    , streamUtf32BE--    -- * Unstreaming-    , unstream--    , module Data.Text.Internal.Encoding.Fusion.Common-    ) where--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)-import Data.Text.Internal.Fusion (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Size-import Data.Text.Encoding.Error-import Data.Text.Internal.Encoding.Fusion.Common-import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32)-import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)-import Data.Word (Word8, Word16, Word32)-import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)-import Foreign.Storable (pokeByteOff)-import qualified Data.ByteString as B-import qualified Data.ByteString.Unsafe as B-import qualified Data.Text.Internal.Encoding.Utf8 as U8-import qualified Data.Text.Internal.Encoding.Utf16 as U16-import qualified Data.Text.Internal.Encoding.Utf32 as U32-import Data.Text.Unsafe (unsafeDupablePerformIO)--streamASCII :: ByteString -> Stream Char-streamASCII bs = Stream next 0 (maxSize l)-    where-      l = B.length bs-      {-# INLINE next #-}-      next i-          | i >= l    = Done-          | otherwise = Yield (unsafeChr8 x1) (i+1)-          where-            x1 = B.unsafeIndex bs i-{-# DEPRECATED streamASCII "Do not use this function" #-}-{-# INLINE [0] streamASCII #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8--- encoding.-streamUtf8 :: OnDecodeError -> ByteString -> Stream Char-streamUtf8 onErr bs = Stream next 0 (maxSize l)-    where-      l = B.length bs-      next i-          | i >= l = Done-          | U8.validate1 x1 = Yield (unsafeChr8 x1) (i+1)-          | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (i+2)-          | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (i+3)-          | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (i+4)-          | otherwise = decodeError "streamUtf8" "UTF-8" onErr (Just x1) (i+1)-          where-            x1 = idx i-            x2 = idx (i + 1)-            x3 = idx (i + 2)-            x4 = idx (i + 3)-            idx = B.unsafeIndex bs-{-# INLINE [0] streamUtf8 #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little--- endian UTF-16 encoding.-streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char-streamUtf16LE onErr bs = Stream next 0 (maxSize (l `shiftR` 1))-    where-      l = B.length bs-      {-# INLINE next #-}-      next i-          | i >= l                         = Done-          | i+1 < l && U16.validate1 x1    = Yield (unsafeChr x1) (i+2)-          | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)-          | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1)-          where-            x1    = idx i       + (idx (i + 1) `shiftL` 8)-            x2    = idx (i + 2) + (idx (i + 3) `shiftL` 8)-            idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16-{-# INLINE [0] streamUtf16LE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big--- endian UTF-16 encoding.-streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char-streamUtf16BE onErr bs = Stream next 0 (maxSize (l `shiftR` 1))-    where-      l = B.length bs-      {-# INLINE next #-}-      next i-          | i >= l                         = Done-          | i+1 < l && U16.validate1 x1    = Yield (unsafeChr x1) (i+2)-          | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)-          | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1)-          where-            x1    = (idx i `shiftL` 8)       + idx (i + 1)-            x2    = (idx (i + 2) `shiftL` 8) + idx (i + 3)-            idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16-{-# INLINE [0] streamUtf16BE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big--- endian UTF-32 encoding.-streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char-streamUtf32BE onErr bs = Stream next 0 (maxSize (l `shiftR` 2))-    where-      l = B.length bs-      {-# INLINE next #-}-      next i-          | i >= l                    = Done-          | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)-          | otherwise = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (i+1)-          where-            x     = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4-            x1    = idx i-            x2    = idx (i+1)-            x3    = idx (i+2)-            x4    = idx (i+3)-            idx = fromIntegral . B.unsafeIndex bs :: Int -> Word32-{-# INLINE [0] streamUtf32BE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little--- endian UTF-32 encoding.-streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char-streamUtf32LE onErr bs = Stream next 0 (maxSize (l `shiftR` 2))-    where-      l = B.length bs-      {-# INLINE next #-}-      next i-          | i >= l                    = Done-          | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)-          | otherwise = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (i+1)-          where-            x     = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1-            x1    = idx i-            x2    = idx $ i+1-            x3    = idx $ i+2-            x4    = idx $ i+3-            idx = fromIntegral . B.unsafeIndex bs :: Int -> Word32-{-# INLINE [0] streamUtf32LE #-}---- | /O(n)/ Convert a 'Stream' 'Word8' to a 'ByteString'.-unstream :: Stream Word8 -> ByteString-unstream (Stream next s0 len) = unsafeDupablePerformIO $ do-    let mlen = upperBound 4 len-    mallocByteString mlen >>= loop mlen 0 s0-    where-      loop !n !off !s fp = case next s of-          Done -> trimUp fp n off-          Skip s' -> loop n off s' fp-          Yield x s'-              | off == n -> realloc fp n off s' x-              | otherwise -> do-            withForeignPtr fp $ \p -> pokeByteOff p off x-            loop n (off+1) s' fp-      {-# NOINLINE realloc #-}-      realloc fp n off s x = do-        let n' = n+n-        fp' <- copy0 fp n n'-        withForeignPtr fp' $ \p -> pokeByteOff p off x-        loop n' (off+1) s fp'-      {-# NOINLINE trimUp #-}-      trimUp fp _ off = return $! PS fp 0 off-      copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)-      copy0 !src !srcLen !destLen =-#if defined(ASSERTS)-        assert (srcLen <= destLen) $-#endif-        do-          dest <- mallocByteString destLen-          withForeignPtr src  $ \src'  ->-              withForeignPtr dest $ \dest' ->-                  memcpy dest' src' (fromIntegral srcLen)-          return dest--decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8-            -> s -> Step s Char-decodeError func kind onErr mb i =-    case onErr desc mb of-      Nothing -> Skip i-      Just c  -> Yield c i-    where desc = "Data.Text.Internal.Encoding.Fusion." ++ func ++ ": Invalid " ++-                 kind ++ " stream"
− Data/Text/Internal/Encoding/Fusion/Common.hs 
@@ -1,123 +0,0 @@-{-# LANGUAGE BangPatterns #-}---- |--- Module      : Data.Text.Internal.Encoding.Fusion.Common--- Copyright   : (c) Tom Harper 2008-2009,---               (c) Bryan O'Sullivan 2009,---               (c) Duncan Coutts 2009,---               (c) Jasper Van der Jeugt 2011------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Use at your own risk!------ Fusible 'Stream'-oriented functions for converting between 'Text'--- and several common encodings.--module Data.Text.Internal.Encoding.Fusion.Common-    (-    -- * Restreaming-    -- Restreaming is the act of converting from one 'Stream'-    -- representation to another.-      restreamUtf16LE-    , restreamUtf16BE-    , restreamUtf32LE-    , restreamUtf32BE-    ) where--import Data.Bits ((.&.))-import Data.Text.Internal.Fusion (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Types (RS(..))-import Data.Text.Internal.Unsafe.Char (ord)-import Data.Text.Internal.Unsafe.Shift (shiftR)-import Data.Word (Word8)--restreamUtf16BE :: Stream Char -> Stream Word8-restreamUtf16BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)-  where-    next (RS0 s) = case next0 s of-        Done -> Done-        Skip s' -> Skip (RS0 s')-        Yield x s'-            | n < 0x10000 -> Yield (fromIntegral $ n `shiftR` 8) $-                             RS1 s' (fromIntegral n)-            | otherwise   -> Yield c1 $ RS3 s' c2 c3 c4-            where-              n  = ord x-              n1 = n - 0x10000-              c1 = fromIntegral (n1 `shiftR` 18 + 0xD8)-              c2 = fromIntegral (n1 `shiftR` 10)-              n2 = n1 .&. 0x3FF-              c3 = fromIntegral (n2 `shiftR` 8 + 0xDC)-              c4 = fromIntegral n2-    next (RS1 s x2)       = Yield x2 (RS0 s)-    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)-    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)-    {-# INLINE next #-}-{-# INLINE restreamUtf16BE #-}--restreamUtf16LE :: Stream Char -> Stream Word8-restreamUtf16LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)-  where-    next (RS0 s) = case next0 s of-        Done -> Done-        Skip s' -> Skip (RS0 s')-        Yield x s'-            | n < 0x10000 -> Yield (fromIntegral n) $-                             RS1 s' (fromIntegral $ shiftR n 8)-            | otherwise   -> Yield c1 $ RS3 s' c2 c3 c4-          where-            n  = ord x-            n1 = n - 0x10000-            c2 = fromIntegral (shiftR n1 18 + 0xD8)-            c1 = fromIntegral (shiftR n1 10)-            n2 = n1 .&. 0x3FF-            c4 = fromIntegral (shiftR n2 8 + 0xDC)-            c3 = fromIntegral n2-    next (RS1 s x2)       = Yield x2 (RS0 s)-    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)-    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)-    {-# INLINE next #-}-{-# INLINE restreamUtf16LE #-}--restreamUtf32BE :: Stream Char -> Stream Word8-restreamUtf32BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)-  where-    next (RS0 s) = case next0 s of-        Done       -> Done-        Skip s'    -> Skip (RS0 s')-        Yield x s' -> Yield c1 (RS3 s' c2 c3 c4)-          where-            n  = ord x-            c1 = fromIntegral $ shiftR n 24-            c2 = fromIntegral $ shiftR n 16-            c3 = fromIntegral $ shiftR n 8-            c4 = fromIntegral n-    next (RS1 s x2)       = Yield x2 (RS0 s)-    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)-    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)-    {-# INLINE next #-}-{-# INLINE restreamUtf32BE #-}--restreamUtf32LE :: Stream Char -> Stream Word8-restreamUtf32LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)-  where-    next (RS0 s) = case next0 s of-        Done       -> Done-        Skip s'    -> Skip (RS0 s')-        Yield x s' -> Yield c1 (RS3 s' c2 c3 c4)-          where-            n  = ord x-            c4 = fromIntegral $ shiftR n 24-            c3 = fromIntegral $ shiftR n 16-            c2 = fromIntegral $ shiftR n 8-            c1 = fromIntegral n-    next (RS1 s x2)       = Yield x2 (RS0 s)-    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)-    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)-    {-# INLINE next #-}-{-# INLINE restreamUtf32LE #-}
− Data/Text/Internal/Encoding/Utf16.hs 
@@ -1,45 +0,0 @@-{-# LANGUAGE MagicHash, BangPatterns #-}---- |--- Module      : Data.Text.Internal.Encoding.Utf16--- Copyright   : (c) 2008, 2009 Tom Harper,---               (c) 2009 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Basic UTF-16 validation and character manipulation.-module Data.Text.Internal.Encoding.Utf16-    (-      chr2-    , validate1-    , validate2-    ) where--import GHC.Exts-import GHC.Word (Word16(..))--chr2 :: Word16 -> Word16 -> Char-chr2 (W16# a#) (W16# b#) = C# (chr# (upper# +# lower# +# 0x10000#))-    where-      !x# = word2Int# a#-      !y# = word2Int# b#-      !upper# = uncheckedIShiftL# (x# -# 0xD800#) 10#-      !lower# = y# -# 0xDC00#-{-# INLINE chr2 #-}--validate1    :: Word16 -> Bool-validate1 x1 = x1 < 0xD800 || x1 > 0xDFFF-{-# INLINE validate1 #-}--validate2       ::  Word16 -> Word16 -> Bool-validate2 x1 x2 = x1 >= 0xD800 && x1 <= 0xDBFF &&-                  x2 >= 0xDC00 && x2 <= 0xDFFF-{-# INLINE validate2 #-}
− Data/Text/Internal/Encoding/Utf32.hs 
@@ -1,26 +0,0 @@--- |--- Module      : Data.Text.Internal.Encoding.Utf32--- Copyright   : (c) 2008, 2009 Tom Harper,---               (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Basic UTF-32 validation.-module Data.Text.Internal.Encoding.Utf32-    (-      validate-    ) where--import Data.Word (Word32)--validate    :: Word32 -> Bool-validate x1 = x1 < 0xD800 || (x1 > 0xDFFF && x1 <= 0x10FFFF)-{-# INLINE validate #-}
− Data/Text/Internal/Encoding/Utf8.hs 
@@ -1,168 +0,0 @@-{-# LANGUAGE CPP, MagicHash, BangPatterns #-}---- |--- Module      : Data.Text.Internal.Encoding.Utf8--- Copyright   : (c) 2008, 2009 Tom Harper,---               (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Basic UTF-8 validation and character manipulation.-module Data.Text.Internal.Encoding.Utf8-    (-    -- Decomposition-      ord2-    , ord3-    , ord4-    -- Construction-    , chr2-    , chr3-    , chr4-    -- * Validation-    , validate1-    , validate2-    , validate3-    , validate4-    ) where--#if defined(TEST_SUITE)-# undef ASSERTS-#endif--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Bits ((.&.))-import Data.Text.Internal.Unsafe.Char (ord)-import Data.Text.Internal.Unsafe.Shift (shiftR)-import GHC.Exts-import GHC.Word (Word8(..))--default(Int)--between :: Word8                -- ^ byte to check-        -> Word8                -- ^ lower bound-        -> Word8                -- ^ upper bound-        -> Bool-between x y z = x >= y && x <= z-{-# INLINE between #-}--ord2 :: Char -> (Word8,Word8)-ord2 c =-#if defined(ASSERTS)-    assert (n >= 0x80 && n <= 0x07ff)-#endif-    (x1,x2)-    where-      n  = ord c-      x1 = fromIntegral $ (n `shiftR` 6) + 0xC0-      x2 = fromIntegral $ (n .&. 0x3F)   + 0x80--ord3 :: Char -> (Word8,Word8,Word8)-ord3 c =-#if defined(ASSERTS)-    assert (n >= 0x0800 && n <= 0xffff)-#endif-    (x1,x2,x3)-    where-      n  = ord c-      x1 = fromIntegral $ (n `shiftR` 12) + 0xE0-      x2 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80-      x3 = fromIntegral $ (n .&. 0x3F) + 0x80--ord4 :: Char -> (Word8,Word8,Word8,Word8)-ord4 c =-#if defined(ASSERTS)-    assert (n >= 0x10000)-#endif-    (x1,x2,x3,x4)-    where-      n  = ord c-      x1 = fromIntegral $ (n `shiftR` 18) + 0xF0-      x2 = fromIntegral $ ((n `shiftR` 12) .&. 0x3F) + 0x80-      x3 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80-      x4 = fromIntegral $ (n .&. 0x3F) + 0x80--chr2 :: Word8 -> Word8 -> Char-chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#))-    where-      !y1# = word2Int# x1#-      !y2# = word2Int# x2#-      !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6#-      !z2# = y2# -# 0x80#-{-# INLINE chr2 #-}--chr3 :: Word8 -> Word8 -> Word8 -> Char-chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#))-    where-      !y1# = word2Int# x1#-      !y2# = word2Int# x2#-      !y3# = word2Int# x3#-      !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12#-      !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6#-      !z3# = y3# -# 0x80#-{-# INLINE chr3 #-}--chr4             :: Word8 -> Word8 -> Word8 -> Word8 -> Char-chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) =-    C# (chr# (z1# +# z2# +# z3# +# z4#))-    where-      !y1# = word2Int# x1#-      !y2# = word2Int# x2#-      !y3# = word2Int# x3#-      !y4# = word2Int# x4#-      !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18#-      !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12#-      !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6#-      !z4# = y4# -# 0x80#-{-# INLINE chr4 #-}--validate1 :: Word8 -> Bool-validate1 x1 = x1 <= 0x7F-{-# INLINE validate1 #-}--validate2 :: Word8 -> Word8 -> Bool-validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF-{-# INLINE validate2 #-}--validate3 :: Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate3 #-}-validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4-  where-    validate3_1 = (x1 == 0xE0) &&-                  between x2 0xA0 0xBF &&-                  between x3 0x80 0xBF-    validate3_2 = between x1 0xE1 0xEC &&-                  between x2 0x80 0xBF &&-                  between x3 0x80 0xBF-    validate3_3 = x1 == 0xED &&-                  between x2 0x80 0x9F &&-                  between x3 0x80 0xBF-    validate3_4 = between x1 0xEE 0xEF &&-                  between x2 0x80 0xBF &&-                  between x3 0x80 0xBF--validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate4 #-}-validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3-  where-    validate4_1 = x1 == 0xF0 &&-                  between x2 0x90 0xBF &&-                  between x3 0x80 0xBF &&-                  between x4 0x80 0xBF-    validate4_2 = between x1 0xF1 0xF3 &&-                  between x2 0x80 0xBF &&-                  between x3 0x80 0xBF &&-                  between x4 0x80 0xBF-    validate4_3 = x1 == 0xF4 &&-                  between x2 0x80 0x8F &&-                  between x3 0x80 0xBF &&-                  between x4 0x80 0xBF
− Data/Text/Internal/Functions.hs 
@@ -1,31 +0,0 @@-{-# LANGUAGE CPP, DeriveDataTypeable #-}---- |--- Module      : Data.Text.Internal.Functions--- Copyright   : 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Useful functions.--module Data.Text.Internal.Functions-    (-      intersperse-    ) where---- | A lazier version of Data.List.intersperse.  The other version--- causes space leaks!-intersperse :: a -> [a] -> [a]-intersperse _   []     = []-intersperse sep (x:xs) = x : go xs-  where-    go []     = []-    go (y:ys) = sep : y: go ys-{-# INLINE intersperse #-}
− Data/Text/Internal/Fusion.hs 
@@ -1,244 +0,0 @@-{-# LANGUAGE BangPatterns, MagicHash #-}---- |--- Module      : Data.Text.Internal.Fusion--- Copyright   : (c) Tom Harper 2008-2009,---               (c) Bryan O'Sullivan 2009-2010,---               (c) Duncan Coutts 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Text manipulation functions represented as fusible operations over--- streams.-module Data.Text.Internal.Fusion-    (-    -- * Types-      Stream(..)-    , Step(..)--    -- * Creation and elimination-    , stream-    , unstream-    , reverseStream--    , length--    -- * Transformations-    , reverse--    -- * Construction-    -- ** Scans-    , reverseScanr--    -- ** Accumulating maps-    , mapAccumL--    -- ** Generation and unfolding-    , unfoldrN--    -- * Indexing-    , index-    , findIndex-    , countChar-    ) where--import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int,-                Num(..), Ord(..), ($), (&&),-                fromIntegral, otherwise)-import Data.Bits ((.&.))-import Data.Text.Internal (Text(..))-import Data.Text.Internal.Private (runText)-import Data.Text.Internal.Unsafe.Char (ord, unsafeChr, unsafeWrite)-import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR)-import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Fusion.Common as S-import Data.Text.Internal.Fusion.Types-import Data.Text.Internal.Fusion.Size-import qualified Data.Text.Internal as I-import qualified Data.Text.Internal.Encoding.Utf16 as U16--default(Int)---- | /O(n)/ Convert a 'Text' into a 'Stream Char'.-stream :: Text -> Stream Char-stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 1) len)-    where-      !end = off+len-      next !i-          | i >= end                   = Done-          | n >= 0xD800 && n <= 0xDBFF = Yield (U16.chr2 n n2) (i + 2)-          | otherwise                  = Yield (unsafeChr n) (i + 1)-          where-            n  = A.unsafeIndex arr i-            n2 = A.unsafeIndex arr (i + 1)-{-# INLINE [0] stream #-}---- | /O(n)/ Convert a 'Text' into a 'Stream Char', but iterate--- backwards.-reverseStream :: Text -> Stream Char-reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 1) len)-    where-      {-# INLINE next #-}-      next !i-          | i < off                    = Done-          | n >= 0xDC00 && n <= 0xDFFF = Yield (U16.chr2 n2 n) (i - 2)-          | otherwise                  = Yield (unsafeChr n) (i - 1)-          where-            n  = A.unsafeIndex arr i-            n2 = A.unsafeIndex arr (i - 1)-{-# INLINE [0] reverseStream #-}---- | /O(n)/ Convert a 'Stream Char' into a 'Text'.-unstream :: Stream Char -> Text-unstream (Stream next0 s0 len) = runText $ \done -> do-  -- Before encoding each char we perform a buffer realloc check assuming-  -- worst case encoding size of two 16-bit units for the char. Just add an-  -- extra space to the buffer so that we do not end up reallocating even when-  -- all the chars are encoded as single unit.-  let mlen = upperBound 4 len + 1-  arr0 <- A.new mlen-  let outer !arr !maxi = encode-       where-        -- keep the common case loop as small as possible-        encode !si !di =-            case next0 si of-                Done        -> done arr di-                Skip si'    -> encode si' di-                Yield c si'-                    -- simply check for the worst case-                    | maxi < di + 1 -> realloc si di-                    | otherwise -> do-                            n <- unsafeWrite arr di c-                            encode si' (di + n)--        -- keep uncommon case separate from the common case code-        {-# NOINLINE realloc #-}-        realloc !si !di = do-            let newlen = (maxi + 1) * 2-            arr' <- A.new newlen-            A.copyM arr' 0 arr 0 di-            outer arr' (newlen - 1) si di--  outer arr0 (mlen - 1) s0 0-{-# INLINE [0] unstream #-}-{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}----- ------------------------------------------------------------------------------- * Basic stream functions--length :: Stream Char -> Int-length = S.lengthI-{-# INLINE[0] length #-}---- | /O(n)/ Reverse the characters of a string.-reverse :: Stream Char -> Text-reverse (Stream next s len0)-    | isEmpty len0 = I.empty-    | otherwise    = I.text arr off' len'-  where-    len0' = upperBound 4 (larger len0 4)-    (arr, (off', len')) = A.run2 (A.new len0' >>= loop s (len0'-1) len0')-    loop !s0 !i !len marr =-        case next s0 of-          Done -> return (marr, (j, len-j))-              where j = i + 1-          Skip s1    -> loop s1 i len marr-          Yield x s1 | i < least -> {-# SCC "reverse/resize" #-} do-                       let newLen = len `shiftL` 1-                       marr' <- A.new newLen-                       A.copyM marr' (newLen-len) marr 0 len-                       write s1 (len+i) newLen marr'-                     | otherwise -> write s1 i len marr-            where n = ord x-                  least | n < 0x10000 = 0-                        | otherwise   = 1-                  m = n - 0x10000-                  lo = fromIntegral $ (m `shiftR` 10) + 0xD800-                  hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00-                  write t j l mar-                      | n < 0x10000 = do-                          A.unsafeWrite mar j (fromIntegral n)-                          loop t (j-1) l mar-                      | otherwise = do-                          A.unsafeWrite mar (j-1) lo-                          A.unsafeWrite mar j hi-                          loop t (j-2) l mar-{-# INLINE [0] reverse #-}---- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with--- the input and result reversed.-reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char-reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low-  where-    {-# INLINE next #-}-    next (Scan1 z s) = Yield z (Scan2 z s)-    next (Scan2 z s) = case next0 s of-                         Yield x s' -> let !x' = f x z-                                       in Yield x' (Scan2 x' s')-                         Skip s'    -> Skip (Scan2 z s')-                         Done       -> Done-{-# INLINE reverseScanr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed--- value. However, the length of the result is limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the length of the result is known.-unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Stream Char-unfoldrN n = S.unfoldrNI n-{-# INLINE [0] unfoldrN #-}------------------------------------------------------------------------------------ ** Indexing streams---- | /O(n)/ stream index (subscript) operator, starting from 0.-index :: Stream Char -> Int -> Char-index = S.indexI-{-# INLINE [0] index #-}---- | The 'findIndex' function takes a predicate and a stream and--- returns the index of the first element in the stream--- satisfying the predicate.-findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int-findIndex = S.findIndexI-{-# INLINE [0] findIndex #-}---- | /O(n)/ The 'count' function returns the number of times the query--- element appears in the given stream.-countChar :: Char -> Stream Char -> Int-countChar = S.countCharI-{-# INLINE [0] countChar #-}---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'.-mapAccumL :: (a -> Char -> (a,Char)) -> a -> Stream Char -> (a, Text)-mapAccumL f z0 (Stream next0 s0 len) = (nz, I.text na 0 nl)-  where-    (na,(nz,nl)) = A.run2 (A.new mlen >>= \arr -> outer arr mlen z0 s0 0)-      where mlen = upperBound 4 len-    outer arr top = loop-      where-        loop !z !s !i =-            case next0 s of-              Done          -> return (arr, (z,i))-              Skip s'       -> loop z s' i-              Yield x s'-                | j >= top  -> {-# SCC "mapAccumL/resize" #-} do-                               let top' = (top + 1) `shiftL` 1-                               arr' <- A.new top'-                               A.copyM arr' 0 arr 0 top-                               outer arr' top' z s i-                | otherwise -> do d <- unsafeWrite arr i c-                                  loop z' s' (i+d)-                where (z',c) = f z x-                      j | ord c < 0x10000 = i-                        | otherwise       = i + 1-{-# INLINE [0] mapAccumL #-}
− Data/Text/Internal/Fusion/CaseMapping.hs 
@@ -1,1002 +0,0 @@-{-# LANGUAGE Rank2Types #-}--- AUTOMATICALLY GENERATED - DO NOT EDIT--- Generated by scripts/CaseMapping.hs--- CaseFolding-9.0.0.txt--- Date: 2016-03-02, 18:54:54 GMT--- SpecialCasing-9.0.0.txt--- Date: 2016-03-02, 18:55:13 GMT--module Data.Text.Internal.Fusion.CaseMapping where-import Data.Char-import Data.Text.Internal.Fusion.Types--upperMapping :: forall s. Char -> s -> Step (CC s) Char-{-# NOINLINE upperMapping #-}--- LATIN SMALL LETTER SHARP S-upperMapping '\x00df' s = Yield '\x0053' (CC s '\x0053' '\x0000')--- LATIN SMALL LIGATURE FF-upperMapping '\xfb00' s = Yield '\x0046' (CC s '\x0046' '\x0000')--- LATIN SMALL LIGATURE FI-upperMapping '\xfb01' s = Yield '\x0046' (CC s '\x0049' '\x0000')--- LATIN SMALL LIGATURE FL-upperMapping '\xfb02' s = Yield '\x0046' (CC s '\x004c' '\x0000')--- LATIN SMALL LIGATURE FFI-upperMapping '\xfb03' s = Yield '\x0046' (CC s '\x0046' '\x0049')--- LATIN SMALL LIGATURE FFL-upperMapping '\xfb04' s = Yield '\x0046' (CC s '\x0046' '\x004c')--- LATIN SMALL LIGATURE LONG S T-upperMapping '\xfb05' s = Yield '\x0053' (CC s '\x0054' '\x0000')--- LATIN SMALL LIGATURE ST-upperMapping '\xfb06' s = Yield '\x0053' (CC s '\x0054' '\x0000')--- ARMENIAN SMALL LIGATURE ECH YIWN-upperMapping '\x0587' s = Yield '\x0535' (CC s '\x0552' '\x0000')--- ARMENIAN SMALL LIGATURE MEN NOW-upperMapping '\xfb13' s = Yield '\x0544' (CC s '\x0546' '\x0000')--- ARMENIAN SMALL LIGATURE MEN ECH-upperMapping '\xfb14' s = Yield '\x0544' (CC s '\x0535' '\x0000')--- ARMENIAN SMALL LIGATURE MEN INI-upperMapping '\xfb15' s = Yield '\x0544' (CC s '\x053b' '\x0000')--- ARMENIAN SMALL LIGATURE VEW NOW-upperMapping '\xfb16' s = Yield '\x054e' (CC s '\x0546' '\x0000')--- ARMENIAN SMALL LIGATURE MEN XEH-upperMapping '\xfb17' s = Yield '\x0544' (CC s '\x053d' '\x0000')--- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE-upperMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS-upperMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS-upperMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301')--- LATIN SMALL LETTER J WITH CARON-upperMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000')--- LATIN SMALL LETTER H WITH LINE BELOW-upperMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000')--- LATIN SMALL LETTER T WITH DIAERESIS-upperMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000')--- LATIN SMALL LETTER W WITH RING ABOVE-upperMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER Y WITH RING ABOVE-upperMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER A WITH RIGHT HALF RING-upperMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI-upperMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA-upperMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300')--- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA-upperMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301')--- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI-upperMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI-upperMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ETA WITH PERISPOMENI-upperMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA-upperMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA-upperMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER IOTA WITH PERISPOMENI-upperMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI-upperMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA-upperMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA-upperMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER RHO WITH PSILI-upperMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PERISPOMENI-upperMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI-upperMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI-upperMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI-upperMapping '\x1f80' s = Yield '\x1f08' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI-upperMapping '\x1f81' s = Yield '\x1f09' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1f82' s = Yield '\x1f0a' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1f83' s = Yield '\x1f0b' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1f84' s = Yield '\x1f0c' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1f85' s = Yield '\x1f0d' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1f86' s = Yield '\x1f0e' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1f87' s = Yield '\x1f0f' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI-upperMapping '\x1f88' s = Yield '\x1f08' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI-upperMapping '\x1f89' s = Yield '\x1f09' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1f8a' s = Yield '\x1f0a' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1f8b' s = Yield '\x1f0b' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1f8c' s = Yield '\x1f0c' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1f8d' s = Yield '\x1f0d' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1f8e' s = Yield '\x1f0e' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1f8f' s = Yield '\x1f0f' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI-upperMapping '\x1f90' s = Yield '\x1f28' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI-upperMapping '\x1f91' s = Yield '\x1f29' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1f92' s = Yield '\x1f2a' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1f93' s = Yield '\x1f2b' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1f94' s = Yield '\x1f2c' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1f95' s = Yield '\x1f2d' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1f96' s = Yield '\x1f2e' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1f97' s = Yield '\x1f2f' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI-upperMapping '\x1f98' s = Yield '\x1f28' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI-upperMapping '\x1f99' s = Yield '\x1f29' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1f9a' s = Yield '\x1f2a' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1f9b' s = Yield '\x1f2b' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1f9c' s = Yield '\x1f2c' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1f9d' s = Yield '\x1f2d' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1f9e' s = Yield '\x1f2e' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1f9f' s = Yield '\x1f2f' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI-upperMapping '\x1fa0' s = Yield '\x1f68' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI-upperMapping '\x1fa1' s = Yield '\x1f69' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1fa2' s = Yield '\x1f6a' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI-upperMapping '\x1fa3' s = Yield '\x1f6b' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1fa4' s = Yield '\x1f6c' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI-upperMapping '\x1fa5' s = Yield '\x1f6d' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1fa6' s = Yield '\x1f6e' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1fa7' s = Yield '\x1f6f' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI-upperMapping '\x1fa8' s = Yield '\x1f68' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI-upperMapping '\x1fa9' s = Yield '\x1f69' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1faa' s = Yield '\x1f6a' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI-upperMapping '\x1fab' s = Yield '\x1f6b' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1fac' s = Yield '\x1f6c' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI-upperMapping '\x1fad' s = Yield '\x1f6d' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1fae' s = Yield '\x1f6e' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-upperMapping '\x1faf' s = Yield '\x1f6f' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI-upperMapping '\x1fb3' s = Yield '\x0391' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI-upperMapping '\x1fbc' s = Yield '\x0391' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI-upperMapping '\x1fc3' s = Yield '\x0397' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI-upperMapping '\x1fcc' s = Yield '\x0397' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI-upperMapping '\x1ff3' s = Yield '\x03a9' (CC s '\x0399' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI-upperMapping '\x1ffc' s = Yield '\x03a9' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI-upperMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI-upperMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI-upperMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI-upperMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI-upperMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI-upperMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0399' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0399')--- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0399')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI-upperMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0399')-upperMapping c s = Yield (toUpper c) (CC s '\0' '\0')-lowerMapping :: forall s. Char -> s -> Step (CC s) Char-{-# NOINLINE lowerMapping #-}--- LATIN CAPITAL LETTER I WITH DOT ABOVE-lowerMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000')-lowerMapping c s = Yield (toLower c) (CC s '\0' '\0')-titleMapping :: forall s. Char -> s -> Step (CC s) Char-{-# NOINLINE titleMapping #-}--- LATIN SMALL LETTER SHARP S-titleMapping '\x00df' s = Yield '\x0053' (CC s '\x0073' '\x0000')--- LATIN SMALL LIGATURE FF-titleMapping '\xfb00' s = Yield '\x0046' (CC s '\x0066' '\x0000')--- LATIN SMALL LIGATURE FI-titleMapping '\xfb01' s = Yield '\x0046' (CC s '\x0069' '\x0000')--- LATIN SMALL LIGATURE FL-titleMapping '\xfb02' s = Yield '\x0046' (CC s '\x006c' '\x0000')--- LATIN SMALL LIGATURE FFI-titleMapping '\xfb03' s = Yield '\x0046' (CC s '\x0066' '\x0069')--- LATIN SMALL LIGATURE FFL-titleMapping '\xfb04' s = Yield '\x0046' (CC s '\x0066' '\x006c')--- LATIN SMALL LIGATURE LONG S T-titleMapping '\xfb05' s = Yield '\x0053' (CC s '\x0074' '\x0000')--- LATIN SMALL LIGATURE ST-titleMapping '\xfb06' s = Yield '\x0053' (CC s '\x0074' '\x0000')--- ARMENIAN SMALL LIGATURE ECH YIWN-titleMapping '\x0587' s = Yield '\x0535' (CC s '\x0582' '\x0000')--- ARMENIAN SMALL LIGATURE MEN NOW-titleMapping '\xfb13' s = Yield '\x0544' (CC s '\x0576' '\x0000')--- ARMENIAN SMALL LIGATURE MEN ECH-titleMapping '\xfb14' s = Yield '\x0544' (CC s '\x0565' '\x0000')--- ARMENIAN SMALL LIGATURE MEN INI-titleMapping '\xfb15' s = Yield '\x0544' (CC s '\x056b' '\x0000')--- ARMENIAN SMALL LIGATURE VEW NOW-titleMapping '\xfb16' s = Yield '\x054e' (CC s '\x0576' '\x0000')--- ARMENIAN SMALL LIGATURE MEN XEH-titleMapping '\xfb17' s = Yield '\x0544' (CC s '\x056d' '\x0000')--- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE-titleMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS-titleMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS-titleMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301')--- LATIN SMALL LETTER J WITH CARON-titleMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000')--- LATIN SMALL LETTER H WITH LINE BELOW-titleMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000')--- LATIN SMALL LETTER T WITH DIAERESIS-titleMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000')--- LATIN SMALL LETTER W WITH RING ABOVE-titleMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER Y WITH RING ABOVE-titleMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER A WITH RIGHT HALF RING-titleMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI-titleMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA-titleMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300')--- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA-titleMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301')--- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI-titleMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI-titleMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ETA WITH PERISPOMENI-titleMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA-titleMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA-titleMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER IOTA WITH PERISPOMENI-titleMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI-titleMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA-titleMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA-titleMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER RHO WITH PSILI-titleMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PERISPOMENI-titleMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI-titleMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI-titleMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI-titleMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI-titleMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI-titleMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI-titleMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI-titleMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI-titleMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0345' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI-titleMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0345')--- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI-titleMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0345')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI-titleMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0345')-titleMapping c s = Yield (toTitle c) (CC s '\0' '\0')-foldMapping :: forall s. Char -> s -> Step (CC s) Char-{-# NOINLINE foldMapping #-}--- MICRO SIGN-foldMapping '\x00b5' s = Yield '\x03bc' (CC s '\x0000' '\x0000')--- LATIN SMALL LETTER SHARP S-foldMapping '\x00df' s = Yield '\x0073' (CC s '\x0073' '\x0000')--- LATIN CAPITAL LETTER I WITH DOT ABOVE-foldMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000')--- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE-foldMapping '\x0149' s = Yield '\x02bc' (CC s '\x006e' '\x0000')--- LATIN SMALL LETTER LONG S-foldMapping '\x017f' s = Yield '\x0073' (CC s '\x0000' '\x0000')--- LATIN SMALL LETTER J WITH CARON-foldMapping '\x01f0' s = Yield '\x006a' (CC s '\x030c' '\x0000')--- COMBINING GREEK YPOGEGRAMMENI-foldMapping '\x0345' s = Yield '\x03b9' (CC s '\x0000' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS-foldMapping '\x0390' s = Yield '\x03b9' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS-foldMapping '\x03b0' s = Yield '\x03c5' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER FINAL SIGMA-foldMapping '\x03c2' s = Yield '\x03c3' (CC s '\x0000' '\x0000')--- GREEK BETA SYMBOL-foldMapping '\x03d0' s = Yield '\x03b2' (CC s '\x0000' '\x0000')--- GREEK THETA SYMBOL-foldMapping '\x03d1' s = Yield '\x03b8' (CC s '\x0000' '\x0000')--- GREEK PHI SYMBOL-foldMapping '\x03d5' s = Yield '\x03c6' (CC s '\x0000' '\x0000')--- GREEK PI SYMBOL-foldMapping '\x03d6' s = Yield '\x03c0' (CC s '\x0000' '\x0000')--- GREEK KAPPA SYMBOL-foldMapping '\x03f0' s = Yield '\x03ba' (CC s '\x0000' '\x0000')--- GREEK RHO SYMBOL-foldMapping '\x03f1' s = Yield '\x03c1' (CC s '\x0000' '\x0000')--- GREEK LUNATE EPSILON SYMBOL-foldMapping '\x03f5' s = Yield '\x03b5' (CC s '\x0000' '\x0000')--- ARMENIAN SMALL LIGATURE ECH YIWN-foldMapping '\x0587' s = Yield '\x0565' (CC s '\x0582' '\x0000')--- CHEROKEE SMALL LETTER YE-foldMapping '\x13f8' s = Yield '\x13f0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER YI-foldMapping '\x13f9' s = Yield '\x13f1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER YO-foldMapping '\x13fa' s = Yield '\x13f2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER YU-foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER YV-foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER MV-foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER ROUNDED VE-foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER LONG-LEGGED DE-foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER NARROW O-foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER WIDE ES-foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER TALL TE-foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER THREE-LEGGED TE-foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER TALL HARD SIGN-foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER TALL YAT-foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000')--- CYRILLIC SMALL LETTER UNBLENDED UK-foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000')--- LATIN SMALL LETTER H WITH LINE BELOW-foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')--- LATIN SMALL LETTER T WITH DIAERESIS-foldMapping '\x1e97' s = Yield '\x0074' (CC s '\x0308' '\x0000')--- LATIN SMALL LETTER W WITH RING ABOVE-foldMapping '\x1e98' s = Yield '\x0077' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER Y WITH RING ABOVE-foldMapping '\x1e99' s = Yield '\x0079' (CC s '\x030a' '\x0000')--- LATIN SMALL LETTER A WITH RIGHT HALF RING-foldMapping '\x1e9a' s = Yield '\x0061' (CC s '\x02be' '\x0000')--- LATIN SMALL LETTER LONG S WITH DOT ABOVE-foldMapping '\x1e9b' s = Yield '\x1e61' (CC s '\x0000' '\x0000')--- LATIN CAPITAL LETTER SHARP S-foldMapping '\x1e9e' s = Yield '\x0073' (CC s '\x0073' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI-foldMapping '\x1f50' s = Yield '\x03c5' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA-foldMapping '\x1f52' s = Yield '\x03c5' (CC s '\x0313' '\x0300')--- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA-foldMapping '\x1f54' s = Yield '\x03c5' (CC s '\x0313' '\x0301')--- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI-foldMapping '\x1f56' s = Yield '\x03c5' (CC s '\x0313' '\x0342')--- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI-foldMapping '\x1f80' s = Yield '\x1f00' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI-foldMapping '\x1f81' s = Yield '\x1f01' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1f82' s = Yield '\x1f02' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1f83' s = Yield '\x1f03' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1f84' s = Yield '\x1f04' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1f85' s = Yield '\x1f05' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1f86' s = Yield '\x1f06' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1f87' s = Yield '\x1f07' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI-foldMapping '\x1f88' s = Yield '\x1f00' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI-foldMapping '\x1f89' s = Yield '\x1f01' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1f8a' s = Yield '\x1f02' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1f8b' s = Yield '\x1f03' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1f8c' s = Yield '\x1f04' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1f8d' s = Yield '\x1f05' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1f8e' s = Yield '\x1f06' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1f8f' s = Yield '\x1f07' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI-foldMapping '\x1f90' s = Yield '\x1f20' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI-foldMapping '\x1f91' s = Yield '\x1f21' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1f92' s = Yield '\x1f22' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1f93' s = Yield '\x1f23' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1f94' s = Yield '\x1f24' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1f95' s = Yield '\x1f25' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1f96' s = Yield '\x1f26' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1f97' s = Yield '\x1f27' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI-foldMapping '\x1f98' s = Yield '\x1f20' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI-foldMapping '\x1f99' s = Yield '\x1f21' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1f9a' s = Yield '\x1f22' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1f9b' s = Yield '\x1f23' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1f9c' s = Yield '\x1f24' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1f9d' s = Yield '\x1f25' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1f9e' s = Yield '\x1f26' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1f9f' s = Yield '\x1f27' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI-foldMapping '\x1fa0' s = Yield '\x1f60' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI-foldMapping '\x1fa1' s = Yield '\x1f61' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1fa2' s = Yield '\x1f62' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI-foldMapping '\x1fa3' s = Yield '\x1f63' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1fa4' s = Yield '\x1f64' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI-foldMapping '\x1fa5' s = Yield '\x1f65' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1fa6' s = Yield '\x1f66' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1fa7' s = Yield '\x1f67' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI-foldMapping '\x1fa8' s = Yield '\x1f60' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI-foldMapping '\x1fa9' s = Yield '\x1f61' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1faa' s = Yield '\x1f62' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI-foldMapping '\x1fab' s = Yield '\x1f63' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1fac' s = Yield '\x1f64' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI-foldMapping '\x1fad' s = Yield '\x1f65' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1fae' s = Yield '\x1f66' (CC s '\x03b9' '\x0000')--- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI-foldMapping '\x1faf' s = Yield '\x1f67' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI-foldMapping '\x1fb2' s = Yield '\x1f70' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI-foldMapping '\x1fb3' s = Yield '\x03b1' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI-foldMapping '\x1fb4' s = Yield '\x03ac' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI-foldMapping '\x1fb6' s = Yield '\x03b1' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1fb7' s = Yield '\x03b1' (CC s '\x0342' '\x03b9')--- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI-foldMapping '\x1fbc' s = Yield '\x03b1' (CC s '\x03b9' '\x0000')--- GREEK PROSGEGRAMMENI-foldMapping '\x1fbe' s = Yield '\x03b9' (CC s '\x0000' '\x0000')--- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI-foldMapping '\x1fc2' s = Yield '\x1f74' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI-foldMapping '\x1fc3' s = Yield '\x03b7' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI-foldMapping '\x1fc4' s = Yield '\x03ae' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER ETA WITH PERISPOMENI-foldMapping '\x1fc6' s = Yield '\x03b7' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1fc7' s = Yield '\x03b7' (CC s '\x0342' '\x03b9')--- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI-foldMapping '\x1fcc' s = Yield '\x03b7' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA-foldMapping '\x1fd2' s = Yield '\x03b9' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA-foldMapping '\x1fd3' s = Yield '\x03b9' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER IOTA WITH PERISPOMENI-foldMapping '\x1fd6' s = Yield '\x03b9' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI-foldMapping '\x1fd7' s = Yield '\x03b9' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA-foldMapping '\x1fe2' s = Yield '\x03c5' (CC s '\x0308' '\x0300')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA-foldMapping '\x1fe3' s = Yield '\x03c5' (CC s '\x0308' '\x0301')--- GREEK SMALL LETTER RHO WITH PSILI-foldMapping '\x1fe4' s = Yield '\x03c1' (CC s '\x0313' '\x0000')--- GREEK SMALL LETTER UPSILON WITH PERISPOMENI-foldMapping '\x1fe6' s = Yield '\x03c5' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI-foldMapping '\x1fe7' s = Yield '\x03c5' (CC s '\x0308' '\x0342')--- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI-foldMapping '\x1ff2' s = Yield '\x1f7c' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI-foldMapping '\x1ff3' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI-foldMapping '\x1ff4' s = Yield '\x03ce' (CC s '\x03b9' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI-foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')--- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI-foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')--- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI-foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')--- LATIN CAPITAL LETTER SMALL CAPITAL I-foldMapping '\xa7ae' s = Yield '\x026a' (CC s '\x0000' '\x0000')--- LATIN CAPITAL LETTER J WITH CROSSED-TAIL-foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')--- LATIN CAPITAL LETTER CHI-foldMapping '\xa7b3' s = Yield '\xab53' (CC s '\x0000' '\x0000')--- LATIN CAPITAL LETTER BETA-foldMapping '\xa7b4' s = Yield '\xa7b5' (CC s '\x0000' '\x0000')--- LATIN CAPITAL LETTER OMEGA-foldMapping '\xa7b6' s = Yield '\xa7b7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER A-foldMapping '\xab70' s = Yield '\x13a0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER E-foldMapping '\xab71' s = Yield '\x13a1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER I-foldMapping '\xab72' s = Yield '\x13a2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER O-foldMapping '\xab73' s = Yield '\x13a3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER U-foldMapping '\xab74' s = Yield '\x13a4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER V-foldMapping '\xab75' s = Yield '\x13a5' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GA-foldMapping '\xab76' s = Yield '\x13a6' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER KA-foldMapping '\xab77' s = Yield '\x13a7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GE-foldMapping '\xab78' s = Yield '\x13a8' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GI-foldMapping '\xab79' s = Yield '\x13a9' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GO-foldMapping '\xab7a' s = Yield '\x13aa' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GU-foldMapping '\xab7b' s = Yield '\x13ab' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER GV-foldMapping '\xab7c' s = Yield '\x13ac' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HA-foldMapping '\xab7d' s = Yield '\x13ad' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HE-foldMapping '\xab7e' s = Yield '\x13ae' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HI-foldMapping '\xab7f' s = Yield '\x13af' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HO-foldMapping '\xab80' s = Yield '\x13b0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HU-foldMapping '\xab81' s = Yield '\x13b1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HV-foldMapping '\xab82' s = Yield '\x13b2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LA-foldMapping '\xab83' s = Yield '\x13b3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LE-foldMapping '\xab84' s = Yield '\x13b4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LI-foldMapping '\xab85' s = Yield '\x13b5' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LO-foldMapping '\xab86' s = Yield '\x13b6' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LU-foldMapping '\xab87' s = Yield '\x13b7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER LV-foldMapping '\xab88' s = Yield '\x13b8' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER MA-foldMapping '\xab89' s = Yield '\x13b9' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER ME-foldMapping '\xab8a' s = Yield '\x13ba' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER MI-foldMapping '\xab8b' s = Yield '\x13bb' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER MO-foldMapping '\xab8c' s = Yield '\x13bc' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER MU-foldMapping '\xab8d' s = Yield '\x13bd' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NA-foldMapping '\xab8e' s = Yield '\x13be' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER HNA-foldMapping '\xab8f' s = Yield '\x13bf' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NAH-foldMapping '\xab90' s = Yield '\x13c0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NE-foldMapping '\xab91' s = Yield '\x13c1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NI-foldMapping '\xab92' s = Yield '\x13c2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NO-foldMapping '\xab93' s = Yield '\x13c3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NU-foldMapping '\xab94' s = Yield '\x13c4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER NV-foldMapping '\xab95' s = Yield '\x13c5' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUA-foldMapping '\xab96' s = Yield '\x13c6' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUE-foldMapping '\xab97' s = Yield '\x13c7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUI-foldMapping '\xab98' s = Yield '\x13c8' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUO-foldMapping '\xab99' s = Yield '\x13c9' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUU-foldMapping '\xab9a' s = Yield '\x13ca' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER QUV-foldMapping '\xab9b' s = Yield '\x13cb' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SA-foldMapping '\xab9c' s = Yield '\x13cc' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER S-foldMapping '\xab9d' s = Yield '\x13cd' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SE-foldMapping '\xab9e' s = Yield '\x13ce' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SI-foldMapping '\xab9f' s = Yield '\x13cf' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SO-foldMapping '\xaba0' s = Yield '\x13d0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SU-foldMapping '\xaba1' s = Yield '\x13d1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER SV-foldMapping '\xaba2' s = Yield '\x13d2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DA-foldMapping '\xaba3' s = Yield '\x13d3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TA-foldMapping '\xaba4' s = Yield '\x13d4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DE-foldMapping '\xaba5' s = Yield '\x13d5' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TE-foldMapping '\xaba6' s = Yield '\x13d6' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DI-foldMapping '\xaba7' s = Yield '\x13d7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TI-foldMapping '\xaba8' s = Yield '\x13d8' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DO-foldMapping '\xaba9' s = Yield '\x13d9' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DU-foldMapping '\xabaa' s = Yield '\x13da' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DV-foldMapping '\xabab' s = Yield '\x13db' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER DLA-foldMapping '\xabac' s = Yield '\x13dc' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLA-foldMapping '\xabad' s = Yield '\x13dd' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLE-foldMapping '\xabae' s = Yield '\x13de' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLI-foldMapping '\xabaf' s = Yield '\x13df' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLO-foldMapping '\xabb0' s = Yield '\x13e0' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLU-foldMapping '\xabb1' s = Yield '\x13e1' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TLV-foldMapping '\xabb2' s = Yield '\x13e2' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSA-foldMapping '\xabb3' s = Yield '\x13e3' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSE-foldMapping '\xabb4' s = Yield '\x13e4' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSI-foldMapping '\xabb5' s = Yield '\x13e5' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSO-foldMapping '\xabb6' s = Yield '\x13e6' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSU-foldMapping '\xabb7' s = Yield '\x13e7' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER TSV-foldMapping '\xabb8' s = Yield '\x13e8' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WA-foldMapping '\xabb9' s = Yield '\x13e9' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WE-foldMapping '\xabba' s = Yield '\x13ea' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WI-foldMapping '\xabbb' s = Yield '\x13eb' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WO-foldMapping '\xabbc' s = Yield '\x13ec' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WU-foldMapping '\xabbd' s = Yield '\x13ed' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER WV-foldMapping '\xabbe' s = Yield '\x13ee' (CC s '\x0000' '\x0000')--- CHEROKEE SMALL LETTER YA-foldMapping '\xabbf' s = Yield '\x13ef' (CC s '\x0000' '\x0000')--- LATIN SMALL LIGATURE FF-foldMapping '\xfb00' s = Yield '\x0066' (CC s '\x0066' '\x0000')--- LATIN SMALL LIGATURE FI-foldMapping '\xfb01' s = Yield '\x0066' (CC s '\x0069' '\x0000')--- LATIN SMALL LIGATURE FL-foldMapping '\xfb02' s = Yield '\x0066' (CC s '\x006c' '\x0000')--- LATIN SMALL LIGATURE FFI-foldMapping '\xfb03' s = Yield '\x0066' (CC s '\x0066' '\x0069')--- LATIN SMALL LIGATURE FFL-foldMapping '\xfb04' s = Yield '\x0066' (CC s '\x0066' '\x006c')--- LATIN SMALL LIGATURE LONG S T-foldMapping '\xfb05' s = Yield '\x0073' (CC s '\x0074' '\x0000')--- LATIN SMALL LIGATURE ST-foldMapping '\xfb06' s = Yield '\x0073' (CC s '\x0074' '\x0000')--- ARMENIAN SMALL LIGATURE MEN NOW-foldMapping '\xfb13' s = Yield '\x0574' (CC s '\x0576' '\x0000')--- ARMENIAN SMALL LIGATURE MEN ECH-foldMapping '\xfb14' s = Yield '\x0574' (CC s '\x0565' '\x0000')--- ARMENIAN SMALL LIGATURE MEN INI-foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')--- ARMENIAN SMALL LIGATURE VEW NOW-foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')--- ARMENIAN SMALL LIGATURE MEN XEH-foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')--- OSAGE CAPITAL LETTER A-foldMapping '\x104b0' s = Yield '\x104d8' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER AI-foldMapping '\x104b1' s = Yield '\x104d9' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER AIN-foldMapping '\x104b2' s = Yield '\x104da' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER AH-foldMapping '\x104b3' s = Yield '\x104db' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER BRA-foldMapping '\x104b4' s = Yield '\x104dc' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER CHA-foldMapping '\x104b5' s = Yield '\x104dd' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EHCHA-foldMapping '\x104b6' s = Yield '\x104de' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER E-foldMapping '\x104b7' s = Yield '\x104df' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EIN-foldMapping '\x104b8' s = Yield '\x104e0' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER HA-foldMapping '\x104b9' s = Yield '\x104e1' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER HYA-foldMapping '\x104ba' s = Yield '\x104e2' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER I-foldMapping '\x104bb' s = Yield '\x104e3' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER KA-foldMapping '\x104bc' s = Yield '\x104e4' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EHKA-foldMapping '\x104bd' s = Yield '\x104e5' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER KYA-foldMapping '\x104be' s = Yield '\x104e6' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER LA-foldMapping '\x104bf' s = Yield '\x104e7' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER MA-foldMapping '\x104c0' s = Yield '\x104e8' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER NA-foldMapping '\x104c1' s = Yield '\x104e9' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER O-foldMapping '\x104c2' s = Yield '\x104ea' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER OIN-foldMapping '\x104c3' s = Yield '\x104eb' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER PA-foldMapping '\x104c4' s = Yield '\x104ec' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EHPA-foldMapping '\x104c5' s = Yield '\x104ed' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER SA-foldMapping '\x104c6' s = Yield '\x104ee' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER SHA-foldMapping '\x104c7' s = Yield '\x104ef' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER TA-foldMapping '\x104c8' s = Yield '\x104f0' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EHTA-foldMapping '\x104c9' s = Yield '\x104f1' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER TSA-foldMapping '\x104ca' s = Yield '\x104f2' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER EHTSA-foldMapping '\x104cb' s = Yield '\x104f3' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER TSHA-foldMapping '\x104cc' s = Yield '\x104f4' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER DHA-foldMapping '\x104cd' s = Yield '\x104f5' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER U-foldMapping '\x104ce' s = Yield '\x104f6' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER WA-foldMapping '\x104cf' s = Yield '\x104f7' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER KHA-foldMapping '\x104d0' s = Yield '\x104f8' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER GHA-foldMapping '\x104d1' s = Yield '\x104f9' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER ZA-foldMapping '\x104d2' s = Yield '\x104fa' (CC s '\x0000' '\x0000')--- OSAGE CAPITAL LETTER ZHA-foldMapping '\x104d3' s = Yield '\x104fb' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER A-foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER AA-foldMapping '\x10c81' s = Yield '\x10cc1' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EB-foldMapping '\x10c82' s = Yield '\x10cc2' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER AMB-foldMapping '\x10c83' s = Yield '\x10cc3' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EC-foldMapping '\x10c84' s = Yield '\x10cc4' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ENC-foldMapping '\x10c85' s = Yield '\x10cc5' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ECS-foldMapping '\x10c86' s = Yield '\x10cc6' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ED-foldMapping '\x10c87' s = Yield '\x10cc7' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER AND-foldMapping '\x10c88' s = Yield '\x10cc8' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER E-foldMapping '\x10c89' s = Yield '\x10cc9' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER CLOSE E-foldMapping '\x10c8a' s = Yield '\x10cca' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EE-foldMapping '\x10c8b' s = Yield '\x10ccb' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EF-foldMapping '\x10c8c' s = Yield '\x10ccc' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EG-foldMapping '\x10c8d' s = Yield '\x10ccd' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EGY-foldMapping '\x10c8e' s = Yield '\x10cce' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EH-foldMapping '\x10c8f' s = Yield '\x10ccf' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER I-foldMapping '\x10c90' s = Yield '\x10cd0' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER II-foldMapping '\x10c91' s = Yield '\x10cd1' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EJ-foldMapping '\x10c92' s = Yield '\x10cd2' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EK-foldMapping '\x10c93' s = Yield '\x10cd3' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER AK-foldMapping '\x10c94' s = Yield '\x10cd4' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER UNK-foldMapping '\x10c95' s = Yield '\x10cd5' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EL-foldMapping '\x10c96' s = Yield '\x10cd6' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ELY-foldMapping '\x10c97' s = Yield '\x10cd7' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EM-foldMapping '\x10c98' s = Yield '\x10cd8' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EN-foldMapping '\x10c99' s = Yield '\x10cd9' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ENY-foldMapping '\x10c9a' s = Yield '\x10cda' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER O-foldMapping '\x10c9b' s = Yield '\x10cdb' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER OO-foldMapping '\x10c9c' s = Yield '\x10cdc' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE-foldMapping '\x10c9d' s = Yield '\x10cdd' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE-foldMapping '\x10c9e' s = Yield '\x10cde' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER OEE-foldMapping '\x10c9f' s = Yield '\x10cdf' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EP-foldMapping '\x10ca0' s = Yield '\x10ce0' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EMP-foldMapping '\x10ca1' s = Yield '\x10ce1' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ER-foldMapping '\x10ca2' s = Yield '\x10ce2' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER SHORT ER-foldMapping '\x10ca3' s = Yield '\x10ce3' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ES-foldMapping '\x10ca4' s = Yield '\x10ce4' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ESZ-foldMapping '\x10ca5' s = Yield '\x10ce5' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ET-foldMapping '\x10ca6' s = Yield '\x10ce6' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ENT-foldMapping '\x10ca7' s = Yield '\x10ce7' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ETY-foldMapping '\x10ca8' s = Yield '\x10ce8' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ECH-foldMapping '\x10ca9' s = Yield '\x10ce9' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER U-foldMapping '\x10caa' s = Yield '\x10cea' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER UU-foldMapping '\x10cab' s = Yield '\x10ceb' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE-foldMapping '\x10cac' s = Yield '\x10cec' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE-foldMapping '\x10cad' s = Yield '\x10ced' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EV-foldMapping '\x10cae' s = Yield '\x10cee' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EZ-foldMapping '\x10caf' s = Yield '\x10cef' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER EZS-foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN-foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')--- OLD HUNGARIAN CAPITAL LETTER US-foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER ALIF-foldMapping '\x1e900' s = Yield '\x1e922' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER DAALI-foldMapping '\x1e901' s = Yield '\x1e923' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER LAAM-foldMapping '\x1e902' s = Yield '\x1e924' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER MIIM-foldMapping '\x1e903' s = Yield '\x1e925' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER BA-foldMapping '\x1e904' s = Yield '\x1e926' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER SINNYIIYHE-foldMapping '\x1e905' s = Yield '\x1e927' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER PE-foldMapping '\x1e906' s = Yield '\x1e928' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER BHE-foldMapping '\x1e907' s = Yield '\x1e929' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER RA-foldMapping '\x1e908' s = Yield '\x1e92a' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER E-foldMapping '\x1e909' s = Yield '\x1e92b' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER FA-foldMapping '\x1e90a' s = Yield '\x1e92c' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER I-foldMapping '\x1e90b' s = Yield '\x1e92d' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER O-foldMapping '\x1e90c' s = Yield '\x1e92e' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER DHA-foldMapping '\x1e90d' s = Yield '\x1e92f' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER YHE-foldMapping '\x1e90e' s = Yield '\x1e930' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER WAW-foldMapping '\x1e90f' s = Yield '\x1e931' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER NUN-foldMapping '\x1e910' s = Yield '\x1e932' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER KAF-foldMapping '\x1e911' s = Yield '\x1e933' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER YA-foldMapping '\x1e912' s = Yield '\x1e934' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER U-foldMapping '\x1e913' s = Yield '\x1e935' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER JIIM-foldMapping '\x1e914' s = Yield '\x1e936' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER CHI-foldMapping '\x1e915' s = Yield '\x1e937' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER HA-foldMapping '\x1e916' s = Yield '\x1e938' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER QAAF-foldMapping '\x1e917' s = Yield '\x1e939' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER GA-foldMapping '\x1e918' s = Yield '\x1e93a' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER NYA-foldMapping '\x1e919' s = Yield '\x1e93b' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER TU-foldMapping '\x1e91a' s = Yield '\x1e93c' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER NHA-foldMapping '\x1e91b' s = Yield '\x1e93d' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER VA-foldMapping '\x1e91c' s = Yield '\x1e93e' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER KHA-foldMapping '\x1e91d' s = Yield '\x1e93f' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER GBE-foldMapping '\x1e91e' s = Yield '\x1e940' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER ZAL-foldMapping '\x1e91f' s = Yield '\x1e941' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER KPO-foldMapping '\x1e920' s = Yield '\x1e942' (CC s '\x0000' '\x0000')--- ADLAM CAPITAL LETTER SHA-foldMapping '\x1e921' s = Yield '\x1e943' (CC s '\x0000' '\x0000')-foldMapping c s = Yield (toLower c) (CC s '\0' '\0')
− Data/Text/Internal/Fusion/Common.hs 
@@ -1,949 +0,0 @@-{-# LANGUAGE PatternGuards, BangPatterns, MagicHash, Rank2Types #-}--- |--- Module      : Data.Text.Internal.Fusion.Common--- Copyright   : (c) Bryan O'Sullivan 2009, 2012------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Common stream fusion functionality for text.--module Data.Text.Internal.Fusion.Common-    (-    -- * Creation and elimination-      singleton-    , streamList-    , unstreamList-    , streamCString#--    -- * Basic interface-    , cons-    , snoc-    , append-    , head-    , uncons-    , last-    , tail-    , init-    , null-    , lengthI-    , compareLengthI-    , isSingleton--    -- * Transformations-    , map-    , intercalate-    , intersperse--    -- ** Case conversion-    -- $case-    , toCaseFold-    , toLower-    , toTitle-    , toUpper--    -- ** Justification-    , justifyLeftI--    -- * Folds-    , foldl-    , foldl'-    , foldl1-    , foldl1'-    , foldr-    , foldr1--    -- ** Special folds-    , concat-    , concatMap-    , any-    , all-    , maximum-    , minimum--    -- * Construction-    -- ** Scans-    , scanl--    -- ** Generation and unfolding-    , replicateCharI-    , replicateI-    , unfoldr-    , unfoldrNI--    -- * Substrings-    -- ** Breaking strings-    , take-    , drop-    , takeWhile-    , dropWhile--    -- * Predicates-    , isPrefixOf--    -- * Searching-    , elem-    , filter--    -- * Indexing-    , findBy-    , indexI-    , findIndexI-    , countCharI--    -- * Zipping and unzipping-    , zipWith-    ) where--import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..),-                Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++),-                (&&), fromIntegral, otherwise)-import qualified Data.List as L-import qualified Prelude as P-import Data.Bits (shiftL)-import Data.Char (isLetter, isSpace)-import Data.Int (Int64)-import Data.Text.Internal.Fusion.Types-import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,-                                     upperMapping)-import Data.Text.Internal.Fusion.Size-import GHC.Prim (Addr#, chr#, indexCharOffAddr#, ord#)-import GHC.Types (Char(..), Int(..))--singleton :: Char -> Stream Char-singleton c = Stream next False (codePointsSize 1)-    where next False = Yield c True-          next True  = Done-{-# INLINE [0] singleton #-}--streamList :: [a] -> Stream a-{-# INLINE [0] streamList #-}-streamList s  = Stream next s unknownSize-    where next []       = Done-          next (x:xs)   = Yield x xs--unstreamList :: Stream a -> [a]-unstreamList (Stream next s0 _len) = unfold s0-    where unfold !s = case next s of-                        Done       -> []-                        Skip s'    -> unfold s'-                        Yield x s' -> x : unfold s'-{-# INLINE [0] unstreamList #-}--{-# RULES "STREAM streamList/unstreamList fusion" forall s. streamList (unstreamList s) = s #-}---- | Stream the UTF-8-like packed encoding used by GHC to represent--- constant strings in generated code.------ This encoding uses the byte sequence "\xc0\x80" to represent NUL,--- and the string is NUL-terminated.-streamCString# :: Addr# -> Stream Char-streamCString# addr = Stream step 0 unknownSize-  where-    step !i-        | b == 0    = Done-        | b <= 0x7f = Yield (C# b#) (i+1)-        | b <= 0xdf = let !c = chr $ ((b-0xc0) `shiftL` 6) + next 1-                      in Yield c (i+2)-        | b <= 0xef = let !c = chr $ ((b-0xe0) `shiftL` 12) +-                                      (next 1  `shiftL` 6) +-                                       next 2-                      in Yield c (i+3)-        | otherwise = let !c = chr $ ((b-0xf0) `shiftL` 18) +-                                      (next 1  `shiftL` 12) +-                                      (next 2  `shiftL` 6) +-                                       next 3-                      in Yield c (i+4)-      where b      = I# (ord# b#)-            next n = I# (ord# (at# (i+n))) - 0x80-            !b#    = at# i-    at# (I# i#) = indexCharOffAddr# addr i#-    chr (I# i#) = C# (chr# i#)-{-# INLINE [0] streamCString# #-}---- ------------------------------------------------------------------------------- * Basic stream functions--data C s = C0 !s-         | C1 !s---- | /O(n)/ Adds a character to the front of a Stream Char.-cons :: Char -> Stream Char -> Stream Char-cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1)-    where-      next (C1 s) = Yield w (C0 s)-      next (C0 s) = case next0 s of-                          Done -> Done-                          Skip s' -> Skip (C0 s')-                          Yield x s' -> Yield x (C0 s')-{-# INLINE [0] cons #-}--data Snoc a = N-            | J !a---- | /O(n)/ Adds a character to the end of a stream.-snoc :: Stream Char -> Char -> Stream Char-snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1)-  where-    next (J xs) = case next0 xs of-      Done        -> Yield w N-      Skip xs'    -> Skip    (J xs')-      Yield x xs' -> Yield x (J xs')-    next N = Done-{-# INLINE [0] snoc #-}--data E l r = L !l-           | R !r---- | /O(n)/ Appends one Stream to the other.-append :: Stream Char -> Stream Char -> Stream Char-append (Stream next0 s01 len1) (Stream next1 s02 len2) =-    Stream next (L s01) (len1 + len2)-    where-      next (L s1) = case next0 s1 of-                         Done        -> Skip    (R s02)-                         Skip s1'    -> Skip    (L s1')-                         Yield x s1' -> Yield x (L s1')-      next (R s2) = case next1 s2 of-                          Done        -> Done-                          Skip s2'    -> Skip    (R s2')-                          Yield x s2' -> Yield x (R s2')-{-# INLINE [0] append #-}---- | /O(1)/ Returns the first character of a Text, which must be non-empty.--- Subject to array fusion.-head :: Stream Char -> Char-head (Stream next s0 _len) = loop_head s0-    where-      loop_head !s = case next s of-                      Yield x _ -> x-                      Skip s'   -> loop_head s'-                      Done      -> head_empty-{-# INLINE [0] head #-}--head_empty :: a-head_empty = streamError "head" "Empty stream"-{-# NOINLINE head_empty #-}---- | /O(1)/ Returns the first character and remainder of a 'Stream--- Char', or 'Nothing' if empty.  Subject to array fusion.-uncons :: Stream Char -> Maybe (Char, Stream Char)-uncons (Stream next s0 len) = loop_uncons s0-    where-      loop_uncons !s = case next s of-                         Yield x s1 -> Just (x, Stream next s1 (len - codePointsSize 1))-                         Skip s'    -> loop_uncons s'-                         Done       -> Nothing-{-# INLINE [0] uncons #-}---- | /O(n)/ Returns the last character of a 'Stream Char', which must--- be non-empty.-last :: Stream Char -> Char-last (Stream next s0 _len) = loop0_last s0-    where-      loop0_last !s = case next s of-                        Done       -> emptyError "last"-                        Skip s'    -> loop0_last  s'-                        Yield x s' -> loop_last x s'-      loop_last !x !s = case next s of-                         Done        -> x-                         Skip s'     -> loop_last x  s'-                         Yield x' s' -> loop_last x' s'-{-# INLINE[0] last #-}---- | /O(1)/ Returns all characters after the head of a Stream Char, which must--- be non-empty.-tail :: Stream Char -> Stream Char-tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1)-    where-      next (C0 s) = case next0 s of-                      Done       -> emptyError "tail"-                      Skip s'    -> Skip (C0 s')-                      Yield _ s' -> Skip (C1 s')-      next (C1 s) = case next0 s of-                      Done       -> Done-                      Skip s'    -> Skip    (C1 s')-                      Yield x s' -> Yield x (C1 s')-{-# INLINE [0] tail #-}--data Init s = Init0 !s-            | Init1 {-# UNPACK #-} !Char !s---- | /O(1)/ Returns all but the last character of a Stream Char, which--- must be non-empty.-init :: Stream Char -> Stream Char-init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1)-    where-      next (Init0 s) = case next0 s of-                         Done       -> emptyError "init"-                         Skip s'    -> Skip (Init0 s')-                         Yield x s' -> Skip (Init1 x s')-      next (Init1 x s)  = case next0 s of-                            Done        -> Done-                            Skip s'     -> Skip    (Init1 x s')-                            Yield x' s' -> Yield x (Init1 x' s')-{-# INLINE [0] init #-}---- | /O(1)/ Tests whether a Stream Char is empty or not.-null :: Stream Char -> Bool-null (Stream next s0 _len) = loop_null s0-    where-      loop_null !s = case next s of-                       Done      -> True-                       Yield _ _ -> False-                       Skip s'   -> loop_null s'-{-# INLINE[0] null #-}---- | /O(n)/ Returns the number of characters in a string.-lengthI :: Integral a => Stream Char -> a-lengthI (Stream next s0 _len) = loop_length 0 s0-    where-      loop_length !z s  = case next s of-                           Done       -> z-                           Skip    s' -> loop_length z s'-                           Yield _ s' -> loop_length (z + 1) s'-{-# INLINE[0] lengthI #-}---- | /O(n)/ Compares the count of characters in a string to a number.--- Subject to fusion.------ This function gives the same answer as comparing against the result--- of 'lengthI', but can short circuit if the count of characters is--- greater than the number or if the stream can't possibly be as long--- as the number supplied, and hence be more efficient.-compareLengthI :: Integral a => Stream Char -> a -> Ordering-compareLengthI (Stream next s0 len) n-    -- Note that @len@ tracks code units whereas we want to compare the length-    -- in code points. Specifically, a stream with hint @len@ may consist of-    -- anywhere from @len/2@ to @len@ code points.-  | Just r <- compareSize len n' = r-  | otherwise = loop_cmp 0 s0-    where-      n' = codePointsSize $ fromIntegral n-      loop_cmp !z s  = case next s of-                         Done       -> compare z n-                         Skip    s' -> loop_cmp z s'-                         Yield _ s' | z > n     -> GT-                                    | otherwise -> loop_cmp (z + 1) s'-{-# INLINE[0] compareLengthI #-}---- | /O(n)/ Indicate whether a string contains exactly one element.-isSingleton :: Stream Char -> Bool-isSingleton (Stream next s0 _len) = loop 0 s0-    where-      loop !z s  = case next s of-                     Done            -> z == (1::Int)-                     Skip    s'      -> loop z s'-                     Yield _ s'-                         | z >= 1    -> False-                         | otherwise -> loop (z+1) s'-{-# INLINE[0] isSingleton #-}---- ------------------------------------------------------------------------------- * Stream transformations---- | /O(n)/ 'map' @f @xs is the Stream Char obtained by applying @f@--- to each element of @xs@.-map :: (Char -> Char) -> Stream Char -> Stream Char-map f (Stream next0 s0 len) = Stream next s0 len-    where-      next !s = case next0 s of-                  Done       -> Done-                  Skip s'    -> Skip s'-                  Yield x s' -> Yield (f x) s'-{-# INLINE [0] map #-}--{-#-  RULES "STREAM map/map fusion" forall f g s.-     map f (map g s) = map (\x -> f (g x)) s- #-}--data I s = I1 !s-         | I2 !s {-# UNPACK #-} !Char-         | I3 !s---- | /O(n)/ Take a character and place it between each of the--- characters of a 'Stream Char'.-intersperse :: Char -> Stream Char -> Stream Char-intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize)-    where-      next (I1 s) = case next0 s of-        Done       -> Done-        Skip s'    -> Skip (I1 s')-        Yield x s' -> Skip (I2 s' x)-      next (I2 s x)  = Yield x (I3 s)-      next (I3 s) = case next0 s of-        Done       -> Done-        Skip s'    -> Skip    (I3 s')-        Yield x s' -> Yield c (I2 s' x)-{-# INLINE [0] intersperse #-}---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ With Unicode text, it is incorrect to use combinators like @map--- toUpper@ to case convert each character of a string individually.--- Instead, use the whole-string case conversion functions from this--- module.  For correctness in different writing systems, these--- functions may map one input character to two or three output--- characters.---- | Map a 'Stream' through the given case-mapping function.-caseConvert :: (forall s. Char -> s -> Step (CC s) Char)-            -> Stream Char -> Stream Char-caseConvert remap (Stream next0 s0 len) =-    Stream next (CC s0 '\0' '\0') (len `unionSize` (3*len))-  where-    next (CC s '\0' _) =-        case next0 s of-          Done       -> Done-          Skip s'    -> Skip (CC s' '\0' '\0')-          Yield c s' -> remap c s'-    next (CC s a b)  =  Yield a (CC s b '\0')---- | /O(n)/ Convert a string to folded case.  This function is mainly--- useful for performing caseless (or case insensitive) string--- comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string.  For instance,--- the Armenian small ligature men now (U+FB13) is case folded to the--- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is--- case folded to the Greek small letter letter mu (U+03BC) instead of--- itself.-toCaseFold :: Stream Char -> Stream Char-toCaseFold = caseConvert foldMapping-{-# INLINE [0] toCaseFold #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.  The result string may be longer than the input string.--- For instance, the German eszett (U+00DF) maps to the two-letter--- sequence SS.-toUpper :: Stream Char -> Stream Char-toUpper = caseConvert upperMapping-{-# INLINE [0] toUpper #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.  The result string may be longer than the input string.--- For instance, the Latin capital letter I with dot above (U+0130)--- maps to the sequence Latin small letter i (U+0069) followed by--- combining dot above (U+0307).-toLower :: Stream Char -> Stream Char-toLower = caseConvert lowerMapping-{-# INLINE [0] toLower #-}---- | /O(n)/ Convert a string to title case, using simple case--- conversion.------ The first letter of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature &#xfb02; (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased&#x2014;but this function will--- capitalize /every/ word.-toTitle :: Stream Char -> Stream Char-toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') (len + unknownSize)-  where-    next (CC (letter :*: s) '\0' _) =-      case next0 s of-        Done            -> Done-        Skip s'         -> Skip (CC (letter :*: s') '\0' '\0')-        Yield c s'-          | nonSpace    -> if letter-                           then lowerMapping c (nonSpace :*: s')-                           else titleMapping c (letter' :*: s')-          | otherwise   -> Yield c (CC (letter' :*: s') '\0' '\0')-          where nonSpace = P.not (isSpace c)-                letter'  = isLetter c-    next (CC s a b)      = Yield a (CC s b '\0')-{-# INLINE [0] toTitle #-}--data Justify i s = Just1 !i !s-                 | Just2 !i !s--justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char-justifyLeftI k c (Stream next0 s0 len) =-    Stream next (Just1 0 s0) (larger (fromIntegral k * charSize c + len) len)-  where-    next (Just1 n s) =-        case next0 s of-          Done       -> next (Just2 n s)-          Skip s'    -> Skip (Just1 n s')-          Yield x s' -> Yield x (Just1 (n+1) s')-    next (Just2 n s)-        | n < k       = Yield c (Just2 (n+1) s)-        | otherwise   = Done-    {-# INLINE next #-}-{-# INLINE [0] justifyLeftI #-}---- ------------------------------------------------------------------------------- * Reducing Streams (folds)---- | foldl, applied to a binary operator, a starting value (typically the--- left-identity of the operator), and a Stream, reduces the Stream using the--- binary operator, from left to right.-foldl :: (b -> Char -> b) -> b -> Stream Char -> b-foldl f z0 (Stream next s0 _len) = loop_foldl z0 s0-    where-      loop_foldl z !s = case next s of-                          Done -> z-                          Skip s' -> loop_foldl z s'-                          Yield x s' -> loop_foldl (f z x) s'-{-# INLINE [0] foldl #-}---- | A strict version of foldl.-foldl' :: (b -> Char -> b) -> b -> Stream Char -> b-foldl' f z0 (Stream next s0 _len) = loop_foldl' z0 s0-    where-      loop_foldl' !z !s = case next s of-                            Done -> z-                            Skip s' -> loop_foldl' z s'-                            Yield x s' -> loop_foldl' (f z x) s'-{-# INLINE [0] foldl' #-}---- | foldl1 is a variant of foldl that has no starting value argument,--- and thus must be applied to non-empty Streams.-foldl1 :: (Char -> Char -> Char) -> Stream Char -> Char-foldl1 f (Stream next s0 _len) = loop0_foldl1 s0-    where-      loop0_foldl1 !s = case next s of-                          Skip s' -> loop0_foldl1 s'-                          Yield x s' -> loop_foldl1 x s'-                          Done -> emptyError "foldl1"-      loop_foldl1 z !s = case next s of-                           Done -> z-                           Skip s' -> loop_foldl1 z s'-                           Yield x s' -> loop_foldl1 (f z x) s'-{-# INLINE [0] foldl1 #-}---- | A strict version of foldl1.-foldl1' :: (Char -> Char -> Char) -> Stream Char -> Char-foldl1' f (Stream next s0 _len) = loop0_foldl1' s0-    where-      loop0_foldl1' !s = case next s of-                           Skip s' -> loop0_foldl1' s'-                           Yield x s' -> loop_foldl1' x s'-                           Done -> emptyError "foldl1"-      loop_foldl1' !z !s = case next s of-                             Done -> z-                             Skip s' -> loop_foldl1' z s'-                             Yield x s' -> loop_foldl1' (f z x) s'-{-# INLINE [0] foldl1' #-}---- | 'foldr', applied to a binary operator, a starting value (typically the--- right-identity of the operator), and a stream, reduces the stream using the--- binary operator, from right to left.-foldr :: (Char -> b -> b) -> b -> Stream Char -> b-foldr f z (Stream next s0 _len) = loop_foldr s0-    where-      loop_foldr !s = case next s of-                        Done -> z-                        Skip s' -> loop_foldr s'-                        Yield x s' -> f x (loop_foldr s')-{-# INLINE [0] foldr #-}---- | foldr1 is a variant of 'foldr' that has no starting value argument,--- and thus must be applied to non-empty streams.--- Subject to array fusion.-foldr1 :: (Char -> Char -> Char) -> Stream Char -> Char-foldr1 f (Stream next s0 _len) = loop0_foldr1 s0-  where-    loop0_foldr1 !s = case next s of-      Done       -> emptyError "foldr1"-      Skip    s' -> loop0_foldr1  s'-      Yield x s' -> loop_foldr1 x s'--    loop_foldr1 x !s = case next s of-      Done        -> x-      Skip     s' -> loop_foldr1 x s'-      Yield x' s' -> f x (loop_foldr1 x' s')-{-# INLINE [0] foldr1 #-}--intercalate :: Stream Char -> [Stream Char] -> Stream Char-intercalate s = concat . (L.intersperse s)-{-# INLINE [0] intercalate #-}---- ------------------------------------------------------------------------------- ** Special folds---- | /O(n)/ Concatenate a list of streams. Subject to array fusion.-concat :: [Stream Char] -> Stream Char-concat = L.foldr append empty-{-# INLINE [0] concat #-}---- | Map a function over a stream that results in a stream and concatenate the--- results.-concatMap :: (Char -> Stream Char) -> Stream Char -> Stream Char-concatMap f = foldr (append . f) empty-{-# INLINE [0] concatMap #-}---- | /O(n)/ any @p @xs determines if any character in the stream--- @xs@ satisfies the predicate @p@.-any :: (Char -> Bool) -> Stream Char -> Bool-any p (Stream next0 s0 _len) = loop_any s0-    where-      loop_any !s = case next0 s of-                      Done                   -> False-                      Skip s'                -> loop_any s'-                      Yield x s' | p x       -> True-                                 | otherwise -> loop_any s'-{-# INLINE [0] any #-}---- | /O(n)/ all @p @xs determines if all characters in the 'Text'--- @xs@ satisfy the predicate @p@.-all :: (Char -> Bool) -> Stream Char -> Bool-all p (Stream next0 s0 _len) = loop_all s0-    where-      loop_all !s = case next0 s of-                      Done                   -> True-                      Skip s'                -> loop_all s'-                      Yield x s' | p x       -> loop_all s'-                                 | otherwise -> False-{-# INLINE [0] all #-}---- | /O(n)/ maximum returns the maximum value from a stream, which must be--- non-empty.-maximum :: Stream Char -> Char-maximum (Stream next0 s0 _len) = loop0_maximum s0-    where-      loop0_maximum !s   = case next0 s of-                             Done       -> emptyError "maximum"-                             Skip s'    -> loop0_maximum s'-                             Yield x s' -> loop_maximum x s'-      loop_maximum !z !s = case next0 s of-                             Done            -> z-                             Skip s'         -> loop_maximum z s'-                             Yield x s'-                                 | x > z     -> loop_maximum x s'-                                 | otherwise -> loop_maximum z s'-{-# INLINE [0] maximum #-}---- | /O(n)/ minimum returns the minimum value from a 'Text', which must be--- non-empty.-minimum :: Stream Char -> Char-minimum (Stream next0 s0 _len) = loop0_minimum s0-    where-      loop0_minimum !s   = case next0 s of-                             Done       -> emptyError "minimum"-                             Skip s'    -> loop0_minimum s'-                             Yield x s' -> loop_minimum x s'-      loop_minimum !z !s = case next0 s of-                             Done            -> z-                             Skip s'         -> loop_minimum z s'-                             Yield x s'-                                 | x < z     -> loop_minimum x s'-                                 | otherwise -> loop_minimum z s'-{-# INLINE [0] minimum #-}---- -------------------------------------------------------------------------------- * Building streams--scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char-scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low-  where-    {-# INLINE next #-}-    next (Scan1 z s) = Yield z (Scan2 z s)-    next (Scan2 z s) = case next0 s of-                         Yield x s' -> let !x' = f z x-                                       in Yield x' (Scan2 x' s')-                         Skip s'    -> Skip (Scan2 z s')-                         Done       -> Done-{-# INLINE [0] scanl #-}---- -------------------------------------------------------------------------------- ** Generating and unfolding streams--replicateCharI :: Integral a => a -> Char -> Stream Char-replicateCharI !n !c-    | n < 0     = empty-    | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low-  where-    next !i | i >= n    = Done-            | otherwise = Yield c (i + 1)-{-# INLINE [0] replicateCharI #-}--data RI s = RI !s {-# UNPACK #-} !Int64--replicateI :: Int64 -> Stream Char -> Stream Char-replicateI n (Stream next0 s0 len) =-    Stream next (RI s0 0) (fromIntegral (max 0 n) * len)-  where-    next (RI s k)-        | k >= n = Done-        | otherwise = case next0 s of-                        Done       -> Skip    (RI s0 (k+1))-                        Skip s'    -> Skip    (RI s' k)-                        Yield x s' -> Yield x (RI s' k)-{-# INLINE [0] replicateI #-}---- | /O(n)/, where @n@ is the length of the result. The unfoldr function--- is analogous to the List 'unfoldr'. unfoldr builds a stream--- from a seed value. The function takes the element and returns--- Nothing if it is done producing the stream or returns Just--- (a,b), in which case, a is the next Char in the string, and b is--- the seed value for further production.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char-unfoldr f s0 = Stream next s0 unknownSize-    where-      {-# INLINE next #-}-      next !s = case f s of-                 Nothing      -> Done-                 Just (w, s') -> Yield w s'-{-# INLINE [0] unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrNI' builds a stream from a seed--- value. However, the length of the result is limited by the--- first argument to 'unfoldrNI'. This function is more efficient than--- 'unfoldr' when the length of the result is known.-unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char-unfoldrNI n f s0 | n <  0    = empty-                 | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2))-    where-      {-# INLINE next #-}-      next (z :*: s) = case f s of-          Nothing                  -> Done-          Just (w, s') | z >= n    -> Done-                       | otherwise -> Yield w ((z + 1) :*: s')-{-# INLINE unfoldrNI #-}------------------------------------------------------------------------------------  * Substreams---- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the--- stream of length @n@, or the stream itself if @n@ is greater than the--- length of the stream.-take :: Integral a => a -> Stream Char -> Stream Char-take n0 (Stream next0 s0 len) =-    Stream next (n0' :*: s0) (smaller len (codePointsSize $ fromIntegral n0'))-    where-      n0' = max n0 0--      {-# INLINE next #-}-      next (n :*: s) | n <= 0    = Done-                     | otherwise = case next0 s of-                                     Done -> Done-                                     Skip s' -> Skip (n :*: s')-                                     Yield x s' -> Yield x ((n-1) :*: s')-{-# INLINE [0] take #-}--data Drop a s = NS !s-              | JS !a !s---- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the--- stream after the first @n@ characters, or the empty stream if @n@--- is greater than the length of the stream.-drop :: Integral a => a -> Stream Char -> Stream Char-drop n0 (Stream next0 s0 len) =-    Stream next (JS n0' s0) (len - codePointsSize (fromIntegral n0'))-  where-    n0' = max n0 0--    {-# INLINE next #-}-    next (JS n s)-      | n <= 0    = Skip (NS s)-      | otherwise = case next0 s of-          Done       -> Done-          Skip    s' -> Skip (JS n    s')-          Yield _ s' -> Skip (JS (n-1) s')-    next (NS s) = case next0 s of-      Done       -> Done-      Skip    s' -> Skip    (NS s')-      Yield x s' -> Yield x (NS s')-{-# INLINE [0] drop #-}---- | 'takeWhile', applied to a predicate @p@ and a stream, returns the--- longest prefix (possibly empty) of elements that satisfy @p@.-takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char-takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize)-    where-      {-# INLINE next #-}-      next !s = case next0 s of-                  Done    -> Done-                  Skip s' -> Skip s'-                  Yield x s' | p x       -> Yield x s'-                             | otherwise -> Done-{-# INLINE [0] takeWhile #-}---- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@.-dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char-dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize)-    where-    {-# INLINE next #-}-    next (L s)  = case next0 s of-      Done                   -> Done-      Skip    s'             -> Skip    (L s')-      Yield x s' | p x       -> Skip    (L s')-                 | otherwise -> Yield x (R s')-    next (R s) = case next0 s of-      Done       -> Done-      Skip    s' -> Skip    (R s')-      Yield x s' -> Yield x (R s')-{-# INLINE [0] dropWhile #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns--- 'True' iff the first is a prefix of the second.-isPrefixOf :: (Eq a) => Stream a -> Stream a -> Bool-isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)-    where-      loop Done      _ = True-      loop _    Done = False-      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')-      loop (Skip s1')     x2             = loop (next1 s1') x2-      loop x1             (Skip s2')     = loop x1          (next2 s2')-      loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&-                                           loop (next1 s1') (next2 s2')-{-# INLINE [0] isPrefixOf #-}---- ------------------------------------------------------------------------------- * Searching------------------------------------------------------------------------------------ ** Searching by equality---- | /O(n)/ 'elem' is the stream membership predicate.-elem :: Char -> Stream Char -> Bool-elem w (Stream next s0 _len) = loop_elem s0-    where-      loop_elem !s = case next s of-                       Done -> False-                       Skip s' -> loop_elem s'-                       Yield x s' | x == w -> True-                                  | otherwise -> loop_elem s'-{-# INLINE [0] elem #-}------------------------------------------------------------------------------------ ** Searching with a predicate---- | /O(n)/ The 'findBy' function takes a predicate and a stream,--- and returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.--findBy :: (Char -> Bool) -> Stream Char -> Maybe Char-findBy p (Stream next s0 _len) = loop_find s0-    where-      loop_find !s = case next s of-                       Done -> Nothing-                       Skip s' -> loop_find s'-                       Yield x s' | p x -> Just x-                                  | otherwise -> loop_find s'-{-# INLINE [0] findBy #-}---- | /O(n)/ Stream index (subscript) operator, starting from 0.-indexI :: Integral a => Stream Char -> a -> Char-indexI (Stream next s0 _len) n0-  | n0 < 0    = streamError "index" "Negative index"-  | otherwise = loop_index n0 s0-  where-    loop_index !n !s = case next s of-      Done                   -> streamError "index" "Index too large"-      Skip    s'             -> loop_index  n    s'-      Yield x s' | n == 0    -> x-                 | otherwise -> loop_index (n-1) s'-{-# INLINE [0] indexI #-}---- | /O(n)/ 'filter', applied to a predicate and a stream,--- returns a stream containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Stream Char -> Stream Char-filter p (Stream next0 s0 len) =-    Stream next s0 (len - unknownSize) -- HINT maybe too high-  where-    next !s = case next0 s of-                Done                   -> Done-                Skip    s'             -> Skip    s'-                Yield x s' | p x       -> Yield x s'-                           | otherwise -> Skip    s'-{-# INLINE [0] filter #-}--{-# RULES-  "STREAM filter/filter fusion" forall p q s.-  filter p (filter q s) = filter (\x -> q x && p x) s-  #-}---- | The 'findIndexI' function takes a predicate and a stream and--- returns the index of the first element in the stream satisfying the--- predicate.-findIndexI :: Integral a => (Char -> Bool) -> Stream Char -> Maybe a-findIndexI p s = case findIndicesI p s of-                  (i:_) -> Just i-                  _     -> Nothing-{-# INLINE [0] findIndexI #-}---- | The 'findIndicesI' function takes a predicate and a stream and--- returns all indices of the elements in the stream satisfying the--- predicate.-findIndicesI :: Integral a => (Char -> Bool) -> Stream Char -> [a]-findIndicesI p (Stream next s0 _len) = loop_findIndex 0 s0-  where-    loop_findIndex !i !s = case next s of-      Done                   -> []-      Skip    s'             -> loop_findIndex i     s' -- hmm. not caught by QC-      Yield x s' | p x       -> i : loop_findIndex (i+1) s'-                 | otherwise -> loop_findIndex (i+1) s'-{-# INLINE [0] findIndicesI #-}------------------------------------------------------------------------------------ * Zipping---- | Strict triple.-data Zip a b m = Z1 !a !b-               | Z2 !a !b !m---- | zipWith generalises 'zip' by zipping with the function given as--- the first argument, instead of a tupling function.-zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b-zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) =-    Stream next (Z1 sa0 sb0) (smaller len1 len2)-    where-      next (Z1 sa sb) = case next0 sa of-                          Done -> Done-                          Skip sa' -> Skip (Z1 sa' sb)-                          Yield a sa' -> Skip (Z2 sa' sb a)--      next (Z2 sa' sb a) = case next1 sb of-                             Done -> Done-                             Skip sb' -> Skip (Z2 sa' sb' a)-                             Yield b sb' -> Yield (f a b) (Z1 sa' sb')-{-# INLINE [0] zipWith #-}---- | /O(n)/ The 'countCharI' function returns the number of times the--- query element appears in the given stream.-countCharI :: Integral a => Char -> Stream Char -> a-countCharI a (Stream next s0 _len) = loop 0 s0-  where-    loop !i !s = case next s of-      Done                   -> i-      Skip    s'             -> loop i s'-      Yield x s' | a == x    -> loop (i+1) s'-                 | otherwise -> loop i s'-{-# INLINE [0] countCharI #-}--streamError :: String -> String -> a-streamError func msg = P.error $ "Data.Text.Internal.Fusion.Common." ++ func ++ ": " ++ msg--emptyError :: String -> a-emptyError func = internalError func "Empty input"--internalError :: String -> a-internalError func = streamError func "Internal error"
− Data/Text/Internal/Fusion/Size.hs 
@@ -1,187 +0,0 @@-{-# LANGUAGE CPP, PatternGuards #-}-{-# OPTIONS_GHC -fno-warn-missing-methods #-}--- |--- Module      : Data.Text.Internal.Fusion.Internal--- Copyright   : (c) Roman Leshchinskiy 2008,---               (c) Bryan O'Sullivan 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Size hints.--module Data.Text.Internal.Fusion.Size-    (-      Size-      -- * Sizes-    , exactSize-    , maxSize-    , betweenSize-    , unknownSize-    , unionSize-    , charSize-    , codePointsSize-      -- * Querying sizes-    , exactly-    , smaller-    , larger-    , upperBound-    , lowerBound-    , compareSize-    , isEmpty-    ) where--import Data.Char (ord)-import Data.Text.Internal (mul)-#if defined(ASSERTS)-import Control.Exception (assert)-#endif---- | A size in UTF-16 code units.-data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.-          | Unknown                                         -- ^ Unknown size.-            deriving (Eq, Show)--exactly :: Size -> Maybe Int-exactly (Between na nb) | na == nb = Just na-exactly _ = Nothing-{-# INLINE exactly #-}---- | The 'Size' of the given code point.-charSize :: Char -> Size-charSize c-  | ord c < 0x10000 = exactSize 1-  | otherwise       = exactSize 2---- | The 'Size' of @n@ code points.-codePointsSize :: Int -> Size-codePointsSize n =-#if defined(ASSERTS)-    assert (n >= 0)-#endif-    Between n (2*n)-{-# INLINE codePointsSize #-}--exactSize :: Int -> Size-exactSize n =-#if defined(ASSERTS)-    assert (n >= 0)-#endif-    Between n n-{-# INLINE exactSize #-}--maxSize :: Int -> Size-maxSize n =-#if defined(ASSERTS)-    assert (n >= 0)-#endif-    Between 0 n-{-# INLINE maxSize #-}--betweenSize :: Int -> Int -> Size-betweenSize m n =-#if defined(ASSERTS)-    assert (m >= 0)-    assert (n >= m)-#endif-    Between m n-{-# INLINE betweenSize #-}--unionSize :: Size -> Size -> Size-unionSize (Between a b) (Between c d) = Between (min a c) (max b d)-unionSize _ _ = Unknown--unknownSize :: Size-unknownSize = Unknown-{-# INLINE unknownSize #-}--instance Num Size where-    (+) = addSize-    (-) = subtractSize-    (*) = mulSize--    fromInteger = f where f = exactSize . fromInteger-                          {-# INLINE f #-}--add :: Int -> Int -> Int-add m n | mn >=   0 = mn-        | otherwise = overflowError-  where mn = m + n-{-# INLINE add #-}--addSize :: Size -> Size -> Size-addSize (Between ma mb) (Between na nb) = Between (add ma na) (add mb nb)-addSize _               _               = Unknown-{-# INLINE addSize #-}--subtractSize :: Size -> Size -> Size-subtractSize (Between ma mb) (Between na nb) = Between (max (ma-nb) 0) (max (mb-na) 0)-subtractSize a@(Between 0 _) Unknown         = a-subtractSize (Between _ mb)  Unknown         = Between 0 mb-subtractSize _               _               = Unknown-{-# INLINE subtractSize #-}--mulSize :: Size -> Size -> Size-mulSize (Between ma mb) (Between na nb) = Between (mul ma na) (mul mb nb)-mulSize _               _               = Unknown-{-# INLINE mulSize #-}---- | Minimum of two size hints.-smaller :: Size -> Size -> Size-smaller a@(Between ma mb) b@(Between na nb)-    | mb <= na  = a-    | nb <= ma  = b-    | otherwise = Between (ma `min` na) (mb `min` nb)-smaller a@(Between 0 _) Unknown         = a-smaller (Between _ mb)  Unknown         = Between 0 mb-smaller Unknown         b@(Between 0 _) = b-smaller Unknown         (Between _ nb)  = Between 0 nb-smaller Unknown         Unknown         = Unknown-{-# INLINE smaller #-}---- | Maximum of two size hints.-larger :: Size -> Size -> Size-larger a@(Between ma mb) b@(Between na nb)-    | ma >= nb  = a-    | na >= mb  = b-    | otherwise = Between (ma `max` na) (mb `max` nb)-larger _ _ = Unknown-{-# INLINE larger #-}---- | Compute the maximum size from a size hint, if possible.-upperBound :: Int -> Size -> Int-upperBound _ (Between _ n) = n-upperBound k _             = k-{-# INLINE upperBound #-}---- | Compute the maximum size from a size hint, if possible.-lowerBound :: Int -> Size -> Int-lowerBound _ (Between n _) = n-lowerBound k _             = k-{-# INLINE lowerBound #-}---- | Determine the ordering relationship between two 'Size's, or 'Nothing' in--- the indeterminate case.-compareSize :: Size -> Size -> Maybe Ordering-compareSize (Between ma mb) (Between na nb)-  | mb < na            = Just LT-  | ma > nb            = Just GT-  | ma == mb-  , ma == na-  , ma == nb           = Just EQ-compareSize _ _        = Nothing---isEmpty :: Size -> Bool-isEmpty (Between _ n) = n <= 0-isEmpty _             = False-{-# INLINE isEmpty #-}--overflowError :: Int-overflowError = error "Data.Text.Internal.Fusion.Size: size overflow"
− Data/Text/Internal/Fusion/Types.hs 
@@ -1,122 +0,0 @@-{-# LANGUAGE BangPatterns, ExistentialQuantification #-}--- |--- Module      : Data.Text.Internal.Fusion.Types--- Copyright   : (c) Tom Harper 2008-2009,---               (c) Bryan O'Sullivan 2009,---               (c) Duncan Coutts 2009,---               (c) Jasper Van der Jeugt 2011------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Core stream fusion functionality for text.--module Data.Text.Internal.Fusion.Types-    (-      CC(..)-    , PairS(..)-    , Scan(..)-    , RS(..)-    , Step(..)-    , Stream(..)-    , empty-    ) where--import Data.Text.Internal.Fusion.Size-import Data.Word (Word8)---- | Specialised tuple for case conversion.-data CC s = CC !s {-# UNPACK #-} !Char {-# UNPACK #-} !Char---- | Restreaming state.-data RS s-    = RS0 !s-    | RS1 !s {-# UNPACK #-} !Word8-    | RS2 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8-    | RS3 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8---- | Strict pair.-data PairS a b = !a :*: !b-                 -- deriving (Eq, Ord, Show)-infixl 2 :*:---- | An intermediate result in a scan.-data Scan s = Scan1 {-# UNPACK #-} !Char !s-            | Scan2 {-# UNPACK #-} !Char !s---- | Intermediate result in a processing pipeline.-data Step s a = Done-              | Skip !s-              | Yield !a !s--{--instance (Show a) => Show (Step s a)-    where show Done        = "Done"-          show (Skip _)    = "Skip"-          show (Yield x _) = "Yield " ++ show x--}--instance (Eq a) => Eq (Stream a) where-    (==) = eq--instance (Ord a) => Ord (Stream a) where-    compare = cmp---- The length hint in a Stream has two roles.  If its value is zero,--- we trust it, and treat the stream as empty.  Otherwise, we treat it--- as a hint: it should usually be accurate, so we use it when--- unstreaming to decide what size array to allocate.  However, the--- unstreaming functions must be able to cope with the hint being too--- small or too large.------ The size hint tries to track the UTF-16 code units in a stream,--- but often counts the number of code points instead.  It can easily--- undercount if, for instance, a transformed stream contains astral--- plane code points (those above 0x10000).--data Stream a =-    forall s. Stream-    (s -> Step s a)             -- stepper function-    !s                          -- current state-    !Size                       -- size hint in code units---- | /O(n)/ Determines if two streams are equal.-eq :: (Eq a) => Stream a -> Stream a -> Bool-eq (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)-    where-      loop Done Done                     = True-      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')-      loop (Skip s1')     x2             = loop (next1 s1') x2-      loop x1             (Skip s2')     = loop x1          (next2 s2')-      loop Done _                        = False-      loop _    Done                     = False-      loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&-                                           loop (next1 s1') (next2 s2')-{-# INLINE [0] eq #-}--cmp :: (Ord a) => Stream a -> Stream a -> Ordering-cmp (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)-    where-      loop Done Done                     = EQ-      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')-      loop (Skip s1')     x2             = loop (next1 s1') x2-      loop x1             (Skip s2')     = loop x1          (next2 s2')-      loop Done _                        = LT-      loop _    Done                     = GT-      loop (Yield x1 s1') (Yield x2 s2') =-          case compare x1 x2 of-            EQ    -> loop (next1 s1') (next2 s2')-            other -> other-{-# INLINE [0] cmp #-}---- | The empty stream.-empty :: Stream a-empty = Stream next () 0-    where next _ = Done-{-# INLINE [0] empty #-}
− Data/Text/Internal/IO.hs 
@@ -1,166 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-}--- |--- Module      : Data.Text.Internal.IO--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Simon Marlow--- License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Low-level support for text I\/O.--module Data.Text.Internal.IO-    (-      hGetLineWith-    , readChunk-    ) where--import qualified Control.Exception as E-import Data.IORef (readIORef, writeIORef)-import Data.Text (Text)-import Data.Text.Internal.Fusion (unstream)-import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Size (exactSize, maxSize)-import Data.Text.Unsafe (inlinePerformIO)-import Foreign.Storable (peekElemOff)-import GHC.IO.Buffer (Buffer(..), CharBuffer, RawCharBuffer, bufferAdjustL,-                      bufferElems, charSize, isEmptyBuffer, readCharBuf,-                      withRawBuffer, writeCharBuf)-import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_)-import GHC.IO.Handle.Types (Handle__(..), Newline(..))-import System.IO (Handle)-import System.IO.Error (isEOFError)-import qualified Data.Text as T---- | Read a single line of input from a handle, constructing a list of--- decoded chunks as we go.  When we're done, transform them into the--- destination type.-hGetLineWith :: ([Text] -> t) -> Handle -> IO t-hGetLineWith f h = wantReadableHandle_ "hGetLine" h go-  where-    go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh []--hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text]-hGetLineLoop hh@Handle__{..} = go where- go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do-  let findEOL raw r | r == w    = return (False, w)-                    | otherwise = do-        (c,r') <- readCharBuf raw r-        if c == '\n'-          then return (True, r)-          else findEOL raw r'-  (eol, off) <- findEOL raw0 r0-  (t,r') <- if haInputNL == CRLF-            then unpack_nl raw0 r0 off-            else do t <- unpack raw0 r0 off-                    return (t,off)-  if eol-    then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf)-            return $ reverse (t:ts)-    else do-      let buf1 = bufferAdjustL r' buf-      maybe_buf <- maybeFillReadBuffer hh buf1-      case maybe_buf of-         -- Nothing indicates we caught an EOF, and we may have a-         -- partial line to return.-         Nothing -> do-              -- we reached EOF.  There might be a lone \r left-              -- in the buffer, so check for that and-              -- append it to the line if necessary.-              let pre | isEmptyBuffer buf1 = T.empty-                      | otherwise          = T.singleton '\r'-              writeIORef haCharBuffer buf1{ bufL=0, bufR=0 }-              let str = reverse . filter (not . T.null) $ pre:t:ts-              if null str-                then ioe_EOF-                else return str-         Just new_buf -> go (t:ts) new_buf---- This function is lifted almost verbatim from GHC.IO.Handle.Text.-maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer)-maybeFillReadBuffer handle_ buf-  = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e ->-      if isEOFError e-      then return Nothing-      else ioError e--unpack :: RawCharBuffer -> Int -> Int -> IO Text-unpack !buf !r !w- | charSize /= 4 = sizeError "unpack"- | r >= w        = return T.empty- | otherwise     = withRawBuffer buf go- where-  go pbuf = return $! unstream (Stream next r (exactSize (w-r)))-   where-    next !i | i >= w    = Done-            | otherwise = Yield (ix i) (i+1)-    ix i = inlinePerformIO $ peekElemOff pbuf i--unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int)-unpack_nl !buf !r !w- | charSize /= 4 = sizeError "unpack_nl"- | r >= w        = return (T.empty, 0)- | otherwise     = withRawBuffer buf $ go- where-  go pbuf = do-    let !t = unstream (Stream next r (maxSize (w-r)))-        w' = w - 1-    return $ if ix w' == '\r'-             then (t,w')-             else (t,w)-   where-    next !i | i >= w = Done-            | c == '\r' = let i' = i + 1-                          in if i' < w-                             then if ix i' == '\n'-                                  then Yield '\n' (i+2)-                                  else Yield '\n' i'-                             else Done-            | otherwise = Yield c (i+1)-            where c = ix i-    ix i = inlinePerformIO $ peekElemOff pbuf i---- This function is completely lifted from GHC.IO.Handle.Text.-getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer-getSomeCharacters handle_@Handle__{..} buf@Buffer{..} =-  case bufferElems buf of-    -- buffer empty: read some more-    0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf--    -- if the buffer has a single '\r' in it and we're doing newline-    -- translation: read some more-    1 | haInputNL == CRLF -> do-      (c,_) <- readCharBuf bufRaw bufL-      if c == '\r'-         then do -- shuffle the '\r' to the beginning.  This is only safe-                 -- if we're about to call readTextDevice, otherwise it-                 -- would mess up flushCharBuffer.-                 -- See [note Buffer Flushing], GHC.IO.Handle.Types-                 _ <- writeCharBuf bufRaw 0 '\r'-                 let buf' = buf{ bufL=0, bufR=1 }-                 readTextDevice handle_ buf'-         else do-                 return buf--    -- buffer has some chars in it already: just return it-    _otherwise -> {-# SCC "otherwise" #-} return buf---- | Read a single chunk of strict text from a buffer. Used by both--- the strict and lazy implementations of hGetContents.-readChunk :: Handle__ -> CharBuffer -> IO Text-readChunk hh@Handle__{..} buf = do-  buf'@Buffer{..} <- getSomeCharacters hh buf-  (t,r) <- if haInputNL == CRLF-           then unpack_nl bufRaw bufL bufR-           else do t <- unpack bufRaw bufL bufR-                   return (t,bufR)-  writeIORef haCharBuffer (bufferAdjustL r buf')-  return t--sizeError :: String -> a-sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"
− Data/Text/Internal/Lazy.hs 
@@ -1,119 +0,0 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module      : Data.Text.Internal.Lazy--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.--module Data.Text.Internal.Lazy-    (-      Text(..)-    , chunk-    , empty-    , foldrChunks-    , foldlChunks-    -- * Data type invariant and abstraction functions--    -- $invariant-    , strictInvariant-    , lazyInvariant-    , showStructure--    -- * Chunk allocation sizes-    , defaultChunkSize-    , smallChunkSize-    , chunkOverhead-    ) where--import Data.Text ()-import Data.Text.Internal.Unsafe.Shift (shiftL)-import Data.Typeable (Typeable)-import Foreign.Storable (sizeOf)-import qualified Data.Text.Internal as T--data Text = Empty-          | Chunk {-# UNPACK #-} !T.Text Text-            deriving (Typeable)---- $invariant------ The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or--- consists of non-null 'T.Text's.  All functions must preserve this,--- and the QC properties must check this.---- | Check the invariant strictly.-strictInvariant :: Text -> Bool-strictInvariant Empty = True-strictInvariant x@(Chunk (T.Text _ _ len) cs)-    | len > 0   = strictInvariant cs-    | otherwise = error $ "Data.Text.Lazy: invariant violation: "-                  ++ showStructure x---- | Check the invariant lazily.-lazyInvariant :: Text -> Text-lazyInvariant Empty = Empty-lazyInvariant x@(Chunk c@(T.Text _ _ len) cs)-    | len > 0   = Chunk c (lazyInvariant cs)-    | otherwise = error $ "Data.Text.Lazy: invariant violation: "-                  ++ showStructure x---- | Display the internal structure of a lazy 'Text'.-showStructure :: Text -> String-showStructure Empty           = "Empty"-showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"-showStructure (Chunk t ts)    =-    "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"---- | Smart constructor for 'Chunk'. Guarantees the data type invariant.-chunk :: T.Text -> Text -> Text-{-# INLINE chunk #-}-chunk t@(T.Text _ _ len) ts | len == 0 = ts-                            | otherwise = Chunk t ts---- | Smart constructor for 'Empty'.-empty :: Text-{-# INLINE [0] empty #-}-empty = Empty---- | Consume the chunks of a lazy 'Text' with a natural right fold.-foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a-foldrChunks f z = go-  where go Empty        = z-        go (Chunk c cs) = f c (go cs)-{-# INLINE foldrChunks #-}---- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,--- accumulating left fold.-foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a-foldlChunks f z = go z-  where go !a Empty        = a-        go !a (Chunk c cs) = go (f a c) cs-{-# INLINE foldlChunks #-}---- | Currently set to 16 KiB, less the memory management overhead.-defaultChunkSize :: Int-defaultChunkSize = 16384 - chunkOverhead-{-# INLINE defaultChunkSize #-}---- | Currently set to 128 bytes, less the memory management overhead.-smallChunkSize :: Int-smallChunkSize = 128 - chunkOverhead-{-# INLINE smallChunkSize #-}---- | The memory management overhead. Currently this is tuned for GHC only.-chunkOverhead :: Int-chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1-{-# INLINE chunkOverhead #-}
− Data/Text/Internal/Lazy/Encoding/Fusion.hs 
@@ -1,324 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}---- |--- Module      : Data.Text.Lazy.Encoding.Fusion--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Fusible 'Stream'-oriented functions for converting between lazy--- 'Text' and several common encodings.--module Data.Text.Internal.Lazy.Encoding.Fusion-    (-    -- * Streaming-    --  streamASCII-      streamUtf8-    , streamUtf16LE-    , streamUtf16BE-    , streamUtf32LE-    , streamUtf32BE--    -- * Unstreaming-    , unstream--    , module Data.Text.Internal.Encoding.Fusion.Common-    ) where--import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize)-import qualified Data.ByteString as B-import qualified Data.ByteString.Unsafe as B-import Data.Text.Internal.Encoding.Fusion.Common-import Data.Text.Encoding.Error-import Data.Text.Internal.Fusion (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Size-import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32)-import Data.Text.Internal.Unsafe.Shift (shiftL)-import Data.Word (Word8, Word16, Word32)-import qualified Data.Text.Internal.Encoding.Utf8 as U8-import qualified Data.Text.Internal.Encoding.Utf16 as U16-import qualified Data.Text.Internal.Encoding.Utf32 as U32-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)-import Foreign.Storable (pokeByteOff)-import Data.ByteString.Internal (mallocByteString, memcpy)-#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import qualified Data.ByteString.Internal as B--data S = S0-       | S1 {-# UNPACK #-} !Word8-       | S2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8-       | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8-       | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8--data T = T !ByteString !S {-# UNPACK #-} !Int---- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using--- UTF-8 encoding.-streamUtf8 :: OnDecodeError -> ByteString -> Stream Char-streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize-  where-    next (T bs@(Chunk ps _) S0 i)-      | i < len && U8.validate1 a =-          Yield (unsafeChr8 a)    (T bs S0 (i+1))-      | i + 1 < len && U8.validate2 a b =-          Yield (U8.chr2 a b)     (T bs S0 (i+2))-      | i + 2 < len && U8.validate3 a b c =-          Yield (U8.chr3 a b c)   (T bs S0 (i+3))-      | i + 3 < len && U8.validate4 a b c d =-          Yield (U8.chr4 a b c d) (T bs S0 (i+4))-      where len = B.length ps-            a = B.unsafeIndex ps i-            b = B.unsafeIndex ps (i+1)-            c = B.unsafeIndex ps (i+2)-            d = B.unsafeIndex ps (i+3)-    next st@(T bs s i) =-      case s of-        S1 a       | U8.validate1 a       -> Yield (unsafeChr8 a)    es-        S2 a b     | U8.validate2 a b     -> Yield (U8.chr2 a b)     es-        S3 a b c   | U8.validate3 a b c   -> Yield (U8.chr3 a b c)   es-        S4 a b c d | U8.validate4 a b c d -> Yield (U8.chr4 a b c d) es-        _ -> consume st-       where es = T bs S0 i-    consume (T bs@(Chunk ps rest) s i)-        | i >= B.length ps = consume (T rest s 0)-        | otherwise =-      case s of-        S0         -> next (T bs (S1 x)       (i+1))-        S1 a       -> next (T bs (S2 a x)     (i+1))-        S2 a b     -> next (T bs (S3 a b x)   (i+1))-        S3 a b c   -> next (T bs (S4 a b c x) (i+1))-        S4 a b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a)-                           (T bs (S3 b c d)   (i+1))-        where x = B.unsafeIndex ps i-    consume (T Empty S0 _) = Done-    consume st             = decodeError "streamUtf8" "UTF-8" onErr Nothing st-{-# INLINE [0] streamUtf8 #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little--- endian UTF-16 encoding.-streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char-streamUtf16LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize-  where-    next (T bs@(Chunk ps _) S0 i)-      | i + 1 < len && U16.validate1 x1 =-          Yield (unsafeChr x1)         (T bs S0 (i+2))-      | i + 3 < len && U16.validate2 x1 x2 =-          Yield (U16.chr2 x1 x2)       (T bs S0 (i+4))-      where len = B.length ps-            x1   = c (idx  i)      (idx (i + 1))-            x2   = c (idx (i + 2)) (idx (i + 3))-            c w1 w2 = w1 + (w2 `shiftL` 8)-            idx = fromIntegral . B.unsafeIndex ps :: Int -> Word16-    next st@(T bs s i) =-      case s of-        S2 w1 w2       | U16.validate1 (c w1 w2)           ->-          Yield (unsafeChr (c w1 w2))   es-        S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) ->-          Yield (U16.chr2 (c w1 w2) (c w3 w4)) es-        _ -> consume st-       where es = T bs S0 i-             c :: Word8 -> Word8 -> Word16-             c w1 w2 = fromIntegral w1 + (fromIntegral w2 `shiftL` 8)-    consume (T bs@(Chunk ps rest) s i)-        | i >= B.length ps = consume (T rest s 0)-        | otherwise =-      case s of-        S0             -> next (T bs (S1 x)          (i+1))-        S1 w1          -> next (T bs (S2 w1 x)       (i+1))-        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))-        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))-        S4 w1 w2 w3 w4 -> decodeError "streamUtf16LE" "UTF-16LE" onErr (Just w1)-                           (T bs (S3 w2 w3 w4)       (i+1))-        where x = B.unsafeIndex ps i-    consume (T Empty S0 _) = Done-    consume st             = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing st-{-# INLINE [0] streamUtf16LE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big--- endian UTF-16 encoding.-streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char-streamUtf16BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize-  where-    next (T bs@(Chunk ps _) S0 i)-      | i + 1 < len && U16.validate1 x1 =-          Yield (unsafeChr x1)         (T bs S0 (i+2))-      | i + 3 < len && U16.validate2 x1 x2 =-          Yield (U16.chr2 x1 x2)       (T bs S0 (i+4))-      where len = B.length ps-            x1   = c (idx  i)      (idx (i + 1))-            x2   = c (idx (i + 2)) (idx (i + 3))-            c w1 w2 = (w1 `shiftL` 8) + w2-            idx = fromIntegral . B.unsafeIndex ps :: Int -> Word16-    next st@(T bs s i) =-      case s of-        S2 w1 w2       | U16.validate1 (c w1 w2)           ->-          Yield (unsafeChr (c w1 w2))   es-        S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) ->-          Yield (U16.chr2 (c w1 w2) (c w3 w4)) es-        _ -> consume st-       where es = T bs S0 i-             c :: Word8 -> Word8 -> Word16-             c w1 w2 = (fromIntegral w1 `shiftL` 8) + fromIntegral w2-    consume (T bs@(Chunk ps rest) s i)-        | i >= B.length ps = consume (T rest s 0)-        | otherwise =-      case s of-        S0             -> next (T bs (S1 x)          (i+1))-        S1 w1          -> next (T bs (S2 w1 x)       (i+1))-        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))-        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))-        S4 w1 w2 w3 w4 -> decodeError "streamUtf16BE" "UTF-16BE" onErr (Just w1)-                           (T bs (S3 w2 w3 w4)       (i+1))-        where x = B.unsafeIndex ps i-    consume (T Empty S0 _) = Done-    consume st             = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing st-{-# INLINE [0] streamUtf16BE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big--- endian UTF-32 encoding.-streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char-streamUtf32BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize-  where-    next (T bs@(Chunk ps _) S0 i)-      | i + 3 < len && U32.validate x =-          Yield (unsafeChr32 x)       (T bs S0 (i+4))-      where len = B.length ps-            x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4-            x1    = idx i-            x2    = idx (i+1)-            x3    = idx (i+2)-            x4    = idx (i+3)-            idx = fromIntegral . B.unsafeIndex ps :: Int -> Word32-    next st@(T bs s i) =-      case s of-        S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) ->-          Yield (unsafeChr32 (c w1 w2 w3 w4)) es-        _ -> consume st-       where es = T bs S0 i-             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32-             c w1 w2 w3 w4 = shifted-              where-               shifted = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4-               x1 = fromIntegral w1-               x2 = fromIntegral w2-               x3 = fromIntegral w3-               x4 = fromIntegral w4-    consume (T bs@(Chunk ps rest) s i)-        | i >= B.length ps = consume (T rest s 0)-        | otherwise =-      case s of-        S0             -> next (T bs (S1 x)          (i+1))-        S1 w1          -> next (T bs (S2 w1 x)       (i+1))-        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))-        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))-        S4 w1 w2 w3 w4 -> decodeError "streamUtf32BE" "UTF-32BE" onErr (Just w1)-                           (T bs (S3 w2 w3 w4)       (i+1))-        where x = B.unsafeIndex ps i-    consume (T Empty S0 _) = Done-    consume st             = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing st-{-# INLINE [0] streamUtf32BE #-}---- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little--- endian UTF-32 encoding.-streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char-streamUtf32LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize-  where-    next (T bs@(Chunk ps _) S0 i)-      | i + 3 < len && U32.validate x =-          Yield (unsafeChr32 x)       (T bs S0 (i+4))-      where len = B.length ps-            x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1-            x1    = idx i-            x2    = idx (i+1)-            x3    = idx (i+2)-            x4    = idx (i+3)-            idx = fromIntegral . B.unsafeIndex ps :: Int -> Word32-    next st@(T bs s i) =-      case s of-        S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) ->-          Yield (unsafeChr32 (c w1 w2 w3 w4)) es-        _ -> consume st-       where es = T bs S0 i-             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32-             c w1 w2 w3 w4 = shifted-              where-               shifted = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1-               x1 = fromIntegral w1-               x2 = fromIntegral w2-               x3 = fromIntegral w3-               x4 = fromIntegral w4-    consume (T bs@(Chunk ps rest) s i)-        | i >= B.length ps = consume (T rest s 0)-        | otherwise =-      case s of-        S0             -> next (T bs (S1 x)          (i+1))-        S1 w1          -> next (T bs (S2 w1 x)       (i+1))-        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))-        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))-        S4 w1 w2 w3 w4 -> decodeError "streamUtf32LE" "UTF-32LE" onErr (Just w1)-                           (T bs (S3 w2 w3 w4)       (i+1))-        where x = B.unsafeIndex ps i-    consume (T Empty S0 _) = Done-    consume st             = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing st-{-# INLINE [0] streamUtf32LE #-}---- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.-unstreamChunks :: Int -> Stream Word8 -> ByteString-unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 (upperBound 4 len0)-  where chunk s1 len1 = unsafeDupablePerformIO $ do-          let len = max 4 (min len1 chunkSize)-          mallocByteString len >>= loop len 0 s1-          where-            loop !n !off !s fp = case next s of-                Done | off == 0 -> return Empty-                     | otherwise -> return $! Chunk (trimUp fp off) Empty-                Skip s' -> loop n off s' fp-                Yield x s'-                    | off == chunkSize -> do-                      let !newLen = n - off-                      return $! Chunk (trimUp fp off) (chunk s newLen)-                    | off == n -> realloc fp n off s' x-                    | otherwise -> do-                      withForeignPtr fp $ \p -> pokeByteOff p off x-                      loop n (off+1) s' fp-            {-# NOINLINE realloc #-}-            realloc fp n off s x = do-              let n' = min (n+n) chunkSize-              fp' <- copy0 fp n n'-              withForeignPtr fp' $ \p -> pokeByteOff p off x-              loop n' (off+1) s fp'-            trimUp fp off = B.PS fp 0 off-            copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)-            copy0 !src !srcLen !destLen =-#if defined(ASSERTS)-              assert (srcLen <= destLen) $-#endif-              do-                dest <- mallocByteString destLen-                withForeignPtr src  $ \src'  ->-                    withForeignPtr dest $ \dest' ->-                        memcpy dest' src' (fromIntegral srcLen)-                return dest---- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.-unstream :: Stream Word8 -> ByteString-unstream = unstreamChunks defaultChunkSize--decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8-            -> s -> Step s Char-decodeError func kind onErr mb i =-    case onErr desc mb of-      Nothing -> Skip i-      Just c  -> Yield c i-    where desc = "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Invalid " ++-                 kind ++ " stream"
− Data/Text/Internal/Lazy/Fusion.hs 
@@ -1,120 +0,0 @@-{-# LANGUAGE BangPatterns #-}--- |--- Module      : Data.Text.Lazy.Fusion--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Core stream fusion functionality for text.--module Data.Text.Internal.Lazy.Fusion-    (-      stream-    , unstream-    , unstreamChunks-    , length-    , unfoldrN-    , index-    , countChar-    ) where--import Prelude hiding (length)-import qualified Data.Text.Internal.Fusion.Common as S-import Control.Monad.ST (runST)-import Data.Text.Internal.Fusion.Types-import Data.Text.Internal.Fusion.Size (isEmpty, unknownSize)-import Data.Text.Internal.Lazy-import qualified Data.Text.Internal as I-import qualified Data.Text.Array as A-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Data.Text.Internal.Unsafe.Shift (shiftL)-import Data.Text.Unsafe (Iter(..), iter)-import Data.Int (Int64)--default(Int64)---- | /O(n)/ Convert a 'Text' into a 'Stream Char'.-stream :: Text -> Stream Char-stream text = Stream next (text :*: 0) unknownSize-  where-    next (Empty :*: _) = Done-    next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i)-        | i >= len  = next (ts :*: 0)-        | otherwise = Yield c (txt :*: i+d)-        where Iter c d = iter t i-{-# INLINE [0] stream #-}---- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given--- chunk size.-unstreamChunks :: Int -> Stream Char -> Text-unstreamChunks !chunkSize (Stream next s0 len0)-  | isEmpty len0 = Empty-  | otherwise    = outer s0-  where-    outer so = {-# SCC "unstreamChunks/outer" #-}-              case next so of-                Done       -> Empty-                Skip s'    -> outer s'-                Yield x s' -> runST $ do-                                a <- A.new unknownLength-                                unsafeWrite a 0 x >>= inner a unknownLength s'-                    where unknownLength = 4-      where-        inner marr !len s !i-            | i + 1 >= chunkSize = finish marr i s-            | i + 1 >= len       = {-# SCC "unstreamChunks/resize" #-} do-                let newLen = min (len `shiftL` 1) chunkSize-                marr' <- A.new newLen-                A.copyM marr' 0 marr 0 len-                inner marr' newLen s i-            | otherwise =-                {-# SCC "unstreamChunks/inner" #-}-                case next s of-                  Done        -> finish marr i s-                  Skip s'     -> inner marr len s' i-                  Yield x s'  -> do d <- unsafeWrite marr i x-                                    inner marr len s' (i+d)-        finish marr len s' = do-          arr <- A.unsafeFreeze marr-          return (I.Text arr 0 len `Chunk` outer s')-{-# INLINE [0] unstreamChunks #-}---- | /O(n)/ Convert a 'Stream Char' into a 'Text', using--- 'defaultChunkSize'.-unstream :: Stream Char -> Text-unstream = unstreamChunks defaultChunkSize-{-# INLINE [0] unstream #-}---- | /O(n)/ Returns the number of characters in a text.-length :: Stream Char -> Int64-length = S.lengthI-{-# INLINE[0] length #-}--{-# RULES "LAZY STREAM stream/unstream fusion" forall s.-    stream (unstream s) = s #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed--- value. However, the length of the result is limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the length of the result is known.-unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Stream Char-unfoldrN n = S.unfoldrNI n-{-# INLINE [0] unfoldrN #-}---- | /O(n)/ stream index (subscript) operator, starting from 0.-index :: Stream Char -> Int64 -> Char-index = S.indexI-{-# INLINE [0] index #-}---- | /O(n)/ The 'count' function returns the number of times the query--- element appears in the given stream.-countChar :: Char -> Stream Char -> Int64-countChar = S.countCharI-{-# INLINE [0] countChar #-}
− Data/Text/Internal/Lazy/Search.hs 
@@ -1,134 +0,0 @@-{-# LANGUAGE BangPatterns, ScopedTypeVariables #-}---- |--- Module      : Data.Text.Lazy.Search--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Fast substring search for lazy 'Text', based on work by Boyer,--- Moore, Horspool, Sunday, and Lundh.  Adapted from the strict--- implementation.--module Data.Text.Internal.Lazy.Search-    (-      indices-    ) where--import qualified Data.Text.Array as A-import Data.Int (Int64)-import Data.Word (Word16, Word64)-import qualified Data.Text.Internal as T-import Data.Text.Internal.Fusion.Types (PairS(..))-import Data.Text.Internal.Lazy (Text(..), foldlChunks)-import Data.Bits ((.|.), (.&.))-import Data.Text.Internal.Unsafe.Shift (shiftL)---- | /O(n+m)/ Find the offsets of all non-overlapping indices of--- @needle@ within @haystack@.------ This function is strict in @needle@, and lazy (as far as possible)--- in the chunks of @haystack@.------ In (unlikely) bad cases, this algorithm's complexity degrades--- towards /O(n*m)/.-indices :: Text              -- ^ Substring to search for (@needle@)-        -> Text              -- ^ Text to search in (@haystack@)-        -> [Int64]-indices needle@(Chunk n ns) _haystack@(Chunk k ks)-    | nlen <= 0  = []-    | nlen == 1  = indicesOne (nindex 0) 0 k ks-    | otherwise  = advance k ks 0 0-  where-    advance x@(T.Text _ _ l) xs = scan-     where-      scan !g !i-         | i >= m = case xs of-                      Empty           -> []-                      Chunk y ys      -> advance y ys g (i-m)-         | lackingHay (i + nlen) x xs  = []-         | c == z && candidateMatch 0  = g : scan (g+nlen) (i+nlen)-         | otherwise                   = scan (g+delta) (i+delta)-       where-         m = fromIntegral l-         c = hindex (i + nlast)-         delta | nextInPattern = nlen + 1-               | c == z        = skip + 1-               | otherwise     = 1-         nextInPattern         = mask .&. swizzle (hindex (i+nlen)) == 0-         candidateMatch !j-             | j >= nlast               = True-             | hindex (i+j) /= nindex j = False-             | otherwise                = candidateMatch (j+1)-         hindex                         = index x xs-    nlen      = wordLength needle-    nlast     = nlen - 1-    nindex    = index n ns-    z         = foldlChunks fin 0 needle-        where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1)-    (mask :: Word64) :*: skip = buildTable n ns 0 0 0 (nlen-2)-    swizzle w = 1 `shiftL` (fromIntegral w .&. 0x3f)-    buildTable (T.Text xarr xoff xlen) xs = go-      where-        go !(g::Int64) !i !msk !skp-            | i >= xlast = case xs of-                             Empty      -> (msk .|. swizzle z) :*: skp-                             Chunk y ys -> buildTable y ys g 0 msk' skp'-            | otherwise = go (g+1) (i+1) msk' skp'-            where c                = A.unsafeIndex xarr (xoff+i)-                  msk'             = msk .|. swizzle c-                  skp' | c == z    = nlen - g - 2-                       | otherwise = skp-                  xlast = xlen - 1-    -- | Check whether an attempt to index into the haystack at the-    -- given offset would fail.-    lackingHay q = go 0-      where-        go p (T.Text _ _ l) ps = p' < q && case ps of-                                             Empty      -> True-                                             Chunk r rs -> go p' r rs-            where p' = p + fromIntegral l-indices _ _ = []---- | Fast index into a partly unpacked 'Text'.  We take into account--- the possibility that the caller might try to access one element--- past the end.-index :: T.Text -> Text -> Int64 -> Word16-index (T.Text arr off len) xs !i-    | j < len   = A.unsafeIndex arr (off+j)-    | otherwise = case xs of-                    Empty-                        -- out of bounds, but legal-                        | j == len  -> 0-                        -- should never happen, due to lackingHay above-                        | otherwise -> emptyError "index"-                    Chunk c cs -> index c cs (i-fromIntegral len)-    where j = fromIntegral i---- | A variant of 'indices' that scans linearly for a single 'Word16'.-indicesOne :: Word16 -> Int64 -> T.Text -> Text -> [Int64]-indicesOne c = chunk-  where-    chunk !i (T.Text oarr ooff olen) os = go 0-      where-        go h | h >= olen = case os of-                             Empty      -> []-                             Chunk y ys -> chunk (i+fromIntegral olen) y ys-             | on == c = i + fromIntegral h : go (h+1)-             | otherwise = go (h+1)-             where on = A.unsafeIndex oarr (ooff+h)---- | The number of 'Word16' values in a 'Text'.-wordLength :: Text -> Int64-wordLength = foldlChunks sumLength 0-    where sumLength i (T.Text _ _ l) = i + fromIntegral l--emptyError :: String -> a-emptyError fun = error ("Data.Text.Lazy.Search." ++ fun ++ ": empty input")
− Data/Text/Internal/Private.hs 
@@ -1,37 +0,0 @@-{-# LANGUAGE BangPatterns, Rank2Types, UnboxedTuples #-}---- |--- Module      : Data.Text.Internal.Private--- Copyright   : (c) 2011 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC--module Data.Text.Internal.Private-    (-      runText-    , span_-    ) where--import Control.Monad.ST (ST, runST)-import Data.Text.Internal (Text(..), text)-import Data.Text.Unsafe (Iter(..), iter)-import qualified Data.Text.Array as A--span_ :: (Char -> Bool) -> Text -> (# Text, Text #)-span_ p t@(Text arr off len) = (# hd,tl #)-  where hd = text arr off k-        tl = text arr (off+k) (len-k)-        !k = loop 0-        loop !i | i < len && p c = loop (i+d)-                | otherwise      = i-            where Iter c d       = iter t i-{-# INLINE span_ #-}--runText :: (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text-runText act = runST (act $ \ !marr !len -> do-                             arr <- A.unsafeFreeze marr-                             return $! text arr 0 len)-{-# INLINE runText #-}
− Data/Text/Internal/Read.hs 
@@ -1,69 +0,0 @@--- |--- Module      : Data.Text.Internal.Read--- Copyright   : (c) 2014 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ Common internal functions for reading textual data.-module Data.Text.Internal.Read-    (-      IReader-    , IParser(..)-    , T(..)-    , digitToInt-    , hexDigitToInt-    , perhaps-    ) where--import Control.Applicative as App (Applicative(..))-import Control.Arrow (first)-import Control.Monad (ap)-import Data.Char (ord)--type IReader t a = t -> Either String (a,t)--newtype IParser t a = P {-      runP :: IReader t a-    }--instance Functor (IParser t) where-    fmap f m = P $ fmap (first f) . runP m--instance Applicative (IParser t) where-    pure a = P $ \t -> Right (a,t)-    {-# INLINE pure #-}-    (<*>) = ap--instance Monad (IParser t) where-    return = App.pure-    m >>= k  = P $ \t -> case runP m t of-                           Left err     -> Left err-                           Right (a,t') -> runP (k a) t'-    {-# INLINE (>>=) #-}---- If we ever need a `MonadFail` instance the definition below can be used------ > instance MonadFail (IParser t) where--- >   fail msg = P $ \_ -> Left msg------ But given the code compiles fine with a post-MFP GHC 8.6+ we don't need--- one just yet.--data T = T !Integer !Int--perhaps :: a -> IParser t a -> IParser t a-perhaps def m = P $ \t -> case runP m t of-                            Left _      -> Right (def,t)-                            r@(Right _) -> r--hexDigitToInt :: Char -> Int-hexDigitToInt c-    | c >= '0' && c <= '9' = ord c - ord '0'-    | c >= 'a' && c <= 'f' = ord c - (ord 'a' - 10)-    | otherwise            = ord c - (ord 'A' - 10)--digitToInt :: Char -> Int-digitToInt c = ord c - ord '0'
− Data/Text/Internal/Search.hs 
@@ -1,89 +0,0 @@-{-# LANGUAGE BangPatterns, ScopedTypeVariables #-}---- |--- Module      : Data.Text.Internal.Search--- Copyright   : (c) Bryan O'Sullivan 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ Fast substring search for 'Text', based on work by Boyer, Moore,--- Horspool, Sunday, and Lundh.------ References:------ * R. S. Boyer, J. S. Moore: A Fast String Searching Algorithm.---   Communications of the ACM, 20, 10, 762-772 (1977)------ * R. N. Horspool: Practical Fast Searching in Strings.  Software ----   Practice and Experience 10, 501-506 (1980)------ * D. M. Sunday: A Very Fast Substring Search Algorithm.---   Communications of the ACM, 33, 8, 132-142 (1990)------ * F. Lundh: The Fast Search Algorithm.---   <http://effbot.org/zone/stringlib.htm> (2006)--module Data.Text.Internal.Search-    (-      indices-    ) where--import qualified Data.Text.Array as A-import Data.Word (Word64)-import Data.Text.Internal (Text(..))-import Data.Bits ((.|.), (.&.))-import Data.Text.Internal.Unsafe.Shift (shiftL)--data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int---- | /O(n+m)/ Find the offsets of all non-overlapping indices of--- @needle@ within @haystack@.  The offsets returned represent--- uncorrected indices in the low-level \"needle\" array, to which its--- offset must be added.------ In (unlikely) bad cases, this algorithm's complexity degrades--- towards /O(n*m)/.-indices :: Text                -- ^ Substring to search for (@needle@)-        -> Text                -- ^ Text to search in (@haystack@)-        -> [Int]-indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen)-    | nlen == 1              = scanOne (nindex 0)-    | nlen <= 0 || ldiff < 0 = []-    | otherwise              = scan 0-  where-    ldiff    = hlen - nlen-    nlast    = nlen - 1-    z        = nindex nlast-    nindex k = A.unsafeIndex narr (noff+k)-    hindex k = A.unsafeIndex harr (hoff+k)-    hindex' k | k == hlen  = 0-              | otherwise = A.unsafeIndex harr (hoff+k)-    buildTable !i !msk !skp-        | i >= nlast           = (msk .|. swizzle z) :* skp-        | otherwise            = buildTable (i+1) (msk .|. swizzle c) skp'-        where c                = nindex i-              skp' | c == z    = nlen - i - 2-                   | otherwise = skp-    swizzle k = 1 `shiftL` (fromIntegral k .&. 0x3f)-    scan !i-        | i > ldiff                  = []-        | c == z && candidateMatch 0 = i : scan (i + nlen)-        | otherwise                  = scan (i + delta)-        where c = hindex (i + nlast)-              candidateMatch !j-                    | j >= nlast               = True-                    | hindex (i+j) /= nindex j = False-                    | otherwise                = candidateMatch (j+1)-              delta | nextInPattern = nlen + 1-                    | c == z        = skip + 1-                    | otherwise     = 1-                where nextInPattern = mask .&. swizzle (hindex' (i+nlen)) == 0-              !(mask :* skip)       = buildTable 0 0 (nlen-2)-    scanOne c = loop 0-        where loop !i | i >= hlen     = []-                      | hindex i == c = i : loop (i+1)-                      | otherwise     = loop (i+1)-{-# INLINE indices #-}
− Data/Text/Internal/Unsafe.hs 
@@ -1,56 +0,0 @@-{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module      : Data.Text.Internal.Unsafe--- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan--- License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : portable------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ A module containing /unsafe/ operations, for /very very careful/ use--- in /heavily tested/ code.-module Data.Text.Internal.Unsafe-    (-      inlineInterleaveST-    , inlinePerformIO-    ) where--import GHC.ST (ST(..))-#if defined(__GLASGOW_HASKELL__)-import GHC.IO (IO(IO))-import GHC.Base (realWorld#)-#endif----- | Just like unsafePerformIO, but we inline it. Big performance gains as--- it exposes lots of things to further inlining. /Very unsafe/. In--- particular, you should do no memory allocation inside an--- 'inlinePerformIO' block. On Hugs this is just @unsafePerformIO@.----{-# INLINE inlinePerformIO #-}-inlinePerformIO :: IO a -> a-#if defined(__GLASGOW_HASKELL__)-inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r-#else-inlinePerformIO = unsafePerformIO-#endif---- | Allow an 'ST' computation to be deferred lazily. When passed an--- action of type 'ST' @s@ @a@, the action will only be performed when--- the value of @a@ is demanded.------ This function is identical to the normal unsafeInterleaveST, but is--- inlined and hence faster.------ /Note/: This operation is highly unsafe, as it can introduce--- externally visible non-determinism into an 'ST' action.-inlineInterleaveST :: ST s a -> ST s a-inlineInterleaveST (ST m) = ST $ \ s ->-    let r = case m s of (# _, res #) -> res in (# s, r #)-{-# INLINE inlineInterleaveST #-}
− Data/Text/Internal/Unsafe/Char.hs 
@@ -1,95 +0,0 @@-{-# LANGUAGE CPP, MagicHash #-}---- |--- Module      : Data.Text.Internal.Unsafe.Char--- Copyright   : (c) 2008, 2009 Tom Harper,---               (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Duncan Coutts------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Fast character manipulation functions.-module Data.Text.Internal.Unsafe.Char-    (-      ord-    , unsafeChr-    , unsafeChr8-    , unsafeChr32-    , unsafeWrite-    -- , unsafeWriteRev-    ) where--#ifdef ASSERTS-import Control.Exception (assert)-#endif-import Control.Monad.ST (ST)-import Data.Bits ((.&.))-import Data.Text.Internal.Unsafe.Shift (shiftR)-import GHC.Exts (Char(..), Int(..), chr#, ord#, word2Int#)-import GHC.Word (Word8(..), Word16(..), Word32(..))-import qualified Data.Text.Array as A--ord :: Char -> Int-ord (C# c#) = I# (ord# c#)-{-# INLINE ord #-}--unsafeChr :: Word16 -> Char-unsafeChr (W16# w#) = C# (chr# (word2Int# w#))-{-# INLINE unsafeChr #-}--unsafeChr8 :: Word8 -> Char-unsafeChr8 (W8# w#) = C# (chr# (word2Int# w#))-{-# INLINE unsafeChr8 #-}--unsafeChr32 :: Word32 -> Char-unsafeChr32 (W32# w#) = C# (chr# (word2Int# w#))-{-# INLINE unsafeChr32 #-}---- | Write a character into the array at the given offset.  Returns--- the number of 'Word16's written.-unsafeWrite :: A.MArray s -> Int -> Char -> ST s Int-unsafeWrite marr i c-    | n < 0x10000 = do-#if defined(ASSERTS)-        assert (i >= 0) . assert (i < A.length marr) $ return ()-#endif-        A.unsafeWrite marr i (fromIntegral n)-        return 1-    | otherwise = do-#if defined(ASSERTS)-        assert (i >= 0) . assert (i < A.length marr - 1) $ return ()-#endif-        A.unsafeWrite marr i lo-        A.unsafeWrite marr (i+1) hi-        return 2-    where n = ord c-          m = n - 0x10000-          lo = fromIntegral $ (m `shiftR` 10) + 0xD800-          hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00-{-# INLINE unsafeWrite #-}--{--unsafeWriteRev :: A.MArray s Word16 -> Int -> Char -> ST s Int-unsafeWriteRev marr i c-    | n < 0x10000 = do-        assert (i >= 0) . assert (i < A.length marr) $-          A.unsafeWrite marr i (fromIntegral n)-        return (i-1)-    | otherwise = do-        assert (i >= 1) . assert (i < A.length marr) $-          A.unsafeWrite marr (i-1) lo-        A.unsafeWrite marr i hi-        return (i-2)-    where n = ord c-          m = n - 0x10000-          lo = fromIntegral $ (m `shiftR` 10) + 0xD800-          hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00-{-# INLINE unsafeWriteRev #-}--}
− Data/Text/Internal/Unsafe/Shift.hs 
@@ -1,72 +0,0 @@-{-# LANGUAGE MagicHash #-}---- |--- Module      : Data.Text.Internal.Unsafe.Shift--- Copyright   : (c) Bryan O'Sullivan 2009------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Fast, unchecked bit shifting functions.--module Data.Text.Internal.Unsafe.Shift-    (-      UnsafeShift(..)-    ) where---- import qualified Data.Bits as Bits-import GHC.Base-import GHC.Word---- | This is a workaround for poor optimisation in GHC 6.8.2.  It--- fails to notice constant-width shifts, and adds a test and branch--- to every shift.  This imposes about a 10% performance hit.------ These functions are undefined when the amount being shifted by is--- greater than the size in bits of a machine Int#.-class UnsafeShift a where-    shiftL :: a -> Int -> a-    shiftR :: a -> Int -> a--instance UnsafeShift Word16 where-    {-# INLINE shiftL #-}-    shiftL (W16# x#) (I# i#) = W16# (narrow16Word# (x# `uncheckedShiftL#` i#))--    {-# INLINE shiftR #-}-    shiftR (W16# x#) (I# i#) = W16# (x# `uncheckedShiftRL#` i#)--instance UnsafeShift Word32 where-    {-# INLINE shiftL #-}-    shiftL (W32# x#) (I# i#) = W32# (narrow32Word# (x# `uncheckedShiftL#` i#))--    {-# INLINE shiftR #-}-    shiftR (W32# x#) (I# i#) = W32# (x# `uncheckedShiftRL#` i#)--instance UnsafeShift Word64 where-    {-# INLINE shiftL #-}-    shiftL (W64# x#) (I# i#) = W64# (x# `uncheckedShiftL64#` i#)--    {-# INLINE shiftR #-}-    shiftR (W64# x#) (I# i#) = W64# (x# `uncheckedShiftRL64#` i#)--instance UnsafeShift Int where-    {-# INLINE shiftL #-}-    shiftL (I# x#) (I# i#) = I# (x# `iShiftL#` i#)--    {-# INLINE shiftR #-}-    shiftR (I# x#) (I# i#) = I# (x# `iShiftRA#` i#)--{--instance UnsafeShift Integer where-    {-# INLINE shiftL #-}-    shiftL = Bits.shiftL--    {-# INLINE shiftR #-}-    shiftR = Bits.shiftR--}
− Data/Text/Lazy.hs 
@@ -1,1725 +0,0 @@-{-# OPTIONS_GHC -fno-warn-orphans #-}-{-# LANGUAGE BangPatterns, MagicHash, CPP #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif-#if __GLASGOW_HASKELL__ >= 708-{-# LANGUAGE TypeFamilies #-}-#endif---- |--- Module      : Data.Text.Lazy--- Copyright   : (c) 2009, 2010, 2012 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text using--- lists of packed arrays.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ The representation used by this module is suitable for high--- performance use and for streaming large quantities of data.  It--- provides a means to manipulate a large body of text without--- requiring that the entire content be resident in memory.------ Some operations, such as 'concat', 'append', 'reverse' and 'cons',--- have better time complexity than their "Data.Text" equivalents, due--- to the underlying representation being a list of chunks. For other--- operations, lazy 'Text's are usually within a few percent of strict--- ones, but often with better heap usage if used in a streaming--- fashion. For data larger than available memory, or if you have--- tight memory constraints, this module will be the only option.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions.  eg.------ > import qualified Data.Text.Lazy as L--module Data.Text.Lazy-    (-    -- * Fusion-    -- $fusion--    -- * Acceptable data-    -- $replacement--    -- * Types-      Text--    -- * Creation and elimination-    , pack-    , unpack-    , singleton-    , empty-    , fromChunks-    , toChunks-    , toStrict-    , fromStrict-    , foldrChunks-    , foldlChunks--    -- * Basic interface-    , cons-    , snoc-    , append-    , uncons-    , unsnoc-    , head-    , last-    , tail-    , init-    , null-    , length-    , compareLength--    -- * Transformations-    , map-    , intercalate-    , intersperse-    , transpose-    , reverse-    , replace--    -- ** Case conversion-    -- $case-    , toCaseFold-    , toLower-    , toUpper-    , toTitle--    -- ** Justification-    , justifyLeft-    , justifyRight-    , center--    -- * Folds-    , foldl-    , foldl'-    , foldl1-    , foldl1'-    , foldr-    , foldr1--    -- ** Special folds-    , concat-    , concatMap-    , any-    , all-    , maximum-    , minimum--    -- * Construction--    -- ** Scans-    , scanl-    , scanl1-    , scanr-    , scanr1--    -- ** Accumulating maps-    , mapAccumL-    , mapAccumR--    -- ** Generation and unfolding-    , repeat-    , replicate-    , cycle-    , iterate-    , unfoldr-    , unfoldrN--    -- * Substrings--    -- ** Breaking strings-    , take-    , takeEnd-    , drop-    , dropEnd-    , takeWhile-    , takeWhileEnd-    , dropWhile-    , dropWhileEnd-    , dropAround-    , strip-    , stripStart-    , stripEnd-    , splitAt-    , span-    , breakOn-    , breakOnEnd-    , break-    , group-    , groupBy-    , inits-    , tails--    -- ** Breaking into many substrings-    -- $split-    , splitOn-    , split-    , chunksOf-    -- , breakSubstring--    -- ** Breaking into lines and words-    , lines-    , words-    , unlines-    , unwords--    -- * Predicates-    , isPrefixOf-    , isSuffixOf-    , isInfixOf--    -- ** View patterns-    , stripPrefix-    , stripSuffix-    , commonPrefixes--    -- * Searching-    , filter-    , find-    , breakOnAll-    , partition--    -- , findSubstring--    -- * Indexing-    , index-    , count--    -- * Zipping and unzipping-    , zip-    , zipWith--    -- -* Ordered text-    -- , sort-    ) where--import Prelude (Char, Bool(..), Maybe(..), String,-                Eq(..), Ord(..), Ordering(..), Read(..), Show(..),-                (&&), (||), (+), (-), (.), ($), (++),-                error, flip, fmap, fromIntegral, not, otherwise, quot)-import qualified Prelude as P-import Control.DeepSeq (NFData(..))-import Data.Int (Int64)-import qualified Data.List as L-import Data.Char (isSpace)-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,-                  Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Data.Binary (Binary(get, put))-import Data.Monoid (Monoid(..))-#if MIN_VERSION_base(4,9,0)-import Data.Semigroup (Semigroup(..))-#endif-import Data.String (IsString(..))-import qualified Data.Text as T-import qualified Data.Text.Internal as T-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Unsafe as T-import qualified Data.Text.Internal.Lazy.Fusion as S-import Data.Text.Internal.Fusion.Types (PairS(..))-import Data.Text.Internal.Lazy.Fusion (stream, unstream)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,-                                foldrChunks, smallChunkSize)-import Data.Text.Internal (firstf, safe, text)-import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8)-import qualified Data.Text.Internal.Functions as F-import Data.Text.Internal.Lazy.Search (indices)-#if __GLASGOW_HASKELL__ >= 702-import qualified GHC.CString as GHC-#else-import qualified GHC.Base as GHC-#endif-#if __GLASGOW_HASKELL__ >= 708-import qualified GHC.Exts as Exts-#endif-import GHC.Prim (Addr#)-#if MIN_VERSION_base(4,7,0)-import Text.Printf (PrintfArg, formatArg, formatString)-#endif---- $fusion------ Most of the functions in this module are subject to /fusion/,--- meaning that a pipeline of such functions will usually allocate at--- most one 'Text' value.------ As an example, consider the following pipeline:------ > import Data.Text.Lazy as T--- > import Data.Text.Lazy.Encoding as E--- > import Data.ByteString.Lazy (ByteString)--- >--- > countChars :: ByteString -> Int--- > countChars = T.length . T.toUpper . E.decodeUtf8------ From the type signatures involved, this looks like it should--- allocate one 'ByteString' value, and two 'Text' values. However,--- when a module is compiled with optimisation enabled under GHC, the--- two intermediate 'Text' values will be optimised away, and the--- function will be compiled down to a single loop over the source--- 'ByteString'.------ Functions that can be fused by the compiler are documented with the--- phrase \"Subject to fusion\".---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"&#xfffd;\" (U+FFFD).  Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\".------ (One reason for this policy of replacement is that internally, a--- 'Text' value is represented as packed UTF-16 data. Values in the--- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate--- code points, and so cannot be represented. The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)--equal :: Text -> Text -> Bool-equal Empty Empty = True-equal Empty _     = False-equal _ Empty     = False-equal (Chunk a as) (Chunk b bs) =-    case compare lenA lenB of-      LT -> a == (T.takeWord16 lenA b) &&-            as `equal` Chunk (T.dropWord16 lenA b) bs-      EQ -> a == b && as `equal` bs-      GT -> T.takeWord16 lenB a == b &&-            Chunk (T.dropWord16 lenB a) as `equal` bs-  where lenA = T.lengthWord16 a-        lenB = T.lengthWord16 b--instance Eq Text where-    (==) = equal-    {-# INLINE (==) #-}--instance Ord Text where-    compare = compareText--compareText :: Text -> Text -> Ordering-compareText Empty Empty = EQ-compareText Empty _     = LT-compareText _     Empty = GT-compareText (Chunk a0 as) (Chunk b0 bs) = outer a0 b0- where-  outer ta@(T.Text arrA offA lenA) tb@(T.Text arrB offB lenB) = go 0 0-   where-    go !i !j-      | i >= lenA = compareText as (chunk (T.Text arrB (offB+j) (lenB-j)) bs)-      | j >= lenB = compareText (chunk (T.Text arrA (offA+i) (lenA-i)) as) bs-      | a < b     = LT-      | a > b     = GT-      | otherwise = go (i+di) (j+dj)-      where T.Iter a di = T.iter ta i-            T.Iter b dj = T.iter tb j--instance Show Text where-    showsPrec p ps r = showsPrec p (unpack ps) r--instance Read Text where-    readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]--#if MIN_VERSION_base(4,9,0)--- | Non-orphan 'Semigroup' instance only defined for--- @base-4.9.0.0@ and later; orphan instances for older GHCs are--- provided by--- the [semigroups](http://hackage.haskell.org/package/semigroups)--- package------ @since 1.2.2.0-instance Semigroup Text where-    (<>) = append-#endif--instance Monoid Text where-    mempty  = empty-#if MIN_VERSION_base(4,9,0)-    mappend = (<>) -- future-proof definition-#else-    mappend = append-#endif-    mconcat = concat--instance IsString Text where-    fromString = pack--#if __GLASGOW_HASKELL__ >= 708--- | @since 1.2.0.0-instance Exts.IsList Text where-    type Item Text = Char-    fromList       = pack-    toList         = unpack-#endif--instance NFData Text where-    rnf Empty        = ()-    rnf (Chunk _ ts) = rnf ts---- | @since 1.2.1.0-instance Binary Text where-    put t = put (encodeUtf8 t)-    get   = do-      bs <- get-      case decodeUtf8' bs of-        P.Left exn -> P.fail (P.show exn)-        P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Text".@'Data.Text.Text'-instance Data Text where-  gfoldl f z txt = z pack `f` (unpack txt)-  toConstr _     = packConstr-  gunfold k z c  = case constrIndex c of-    1 -> k (z pack)-    _ -> error "Data.Text.Lazy.Text.gunfold"-  dataTypeOf _   = textDataType--#if MIN_VERSION_base(4,7,0)--- | Only defined for @base-4.7.0.0@ and later------ @since 1.2.2.0-instance PrintfArg Text where-  formatArg txt = formatString $ unpack txt-#endif--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr]---- | /O(n)/ Convert a 'String' into a 'Text'.------ Subject to fusion.  Performs replacement on invalid scalar values.-pack :: String -> Text-pack = unstream . S.streamList . L.map safe-{-# INLINE [1] pack #-}---- | /O(n)/ Convert a 'Text' into a 'String'.--- Subject to fusion.-unpack :: Text -> String-unpack t = S.unstreamList (stream t)-{-# INLINE [1] unpack #-}---- | /O(n)/ Convert a literal string into a Text.-unpackCString# :: Addr# -> Text-unpackCString# addr# = unstream (S.streamCString# addr#)-{-# NOINLINE unpackCString# #-}--{-# RULES "TEXT literal" forall a.-    unstream (S.streamList (L.map safe (GHC.unpackCString# a)))-      = unpackCString# a #-}--{-# RULES "TEXT literal UTF8" forall a.-    unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a)))-      = unpackCString# a #-}--{-# RULES "LAZY TEXT empty literal"-    unstream (S.streamList (L.map safe []))-      = Empty #-}--{-# RULES "LAZY TEXT empty literal" forall a.-    unstream (S.streamList (L.map safe [a]))-      = Chunk (T.singleton a) Empty #-}---- | /O(1)/ Convert a character into a Text.  Subject to fusion.--- Performs replacement on invalid scalar values.-singleton :: Char -> Text-singleton c = Chunk (T.singleton c) Empty-{-# INLINE [1] singleton #-}--{-# RULES-"LAZY TEXT singleton -> fused" [~1] forall c.-    singleton c = unstream (S.singleton c)-"LAZY TEXT singleton -> unfused" [1] forall c.-    unstream (S.singleton c) = singleton c-  #-}---- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'.-fromChunks :: [T.Text] -> Text-fromChunks cs = L.foldr chunk Empty cs---- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's.-toChunks :: Text -> [T.Text]-toChunks cs = foldrChunks (:) [] cs---- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'.-toStrict :: Text -> T.Text-toStrict t = T.concat (toChunks t)-{-# INLINE [1] toStrict #-}---- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'.-fromStrict :: T.Text -> Text-fromStrict t = chunk t Empty-{-# INLINE [1] fromStrict #-}---- -------------------------------------------------------------------------------- * Basic functions---- | /O(1)/ Adds a character to the front of a 'Text'.  Subject to fusion.-cons :: Char -> Text -> Text-cons c t = Chunk (T.singleton c) t-{-# INLINE [1] cons #-}--infixr 5 `cons`--{-# RULES-"LAZY TEXT cons -> fused" [~1] forall c t.-    cons c t = unstream (S.cons c (stream t))-"LAZY TEXT cons -> unfused" [1] forall c t.-    unstream (S.cons c (stream t)) = cons c t- #-}---- | /O(n)/ Adds a character to the end of a 'Text'.  This copies the--- entire array in the process, unless fused.  Subject to fusion.-snoc :: Text -> Char -> Text-snoc t c = foldrChunks Chunk (singleton c) t-{-# INLINE [1] snoc #-}--{-# RULES-"LAZY TEXT snoc -> fused" [~1] forall t c.-    snoc t c = unstream (S.snoc (stream t) c)-"LAZY TEXT snoc -> unfused" [1] forall t c.-    unstream (S.snoc (stream t) c) = snoc t c- #-}---- | /O(n\/c)/ Appends one 'Text' to another.  Subject to fusion.-append :: Text -> Text -> Text-append xs ys = foldrChunks Chunk ys xs-{-# INLINE [1] append #-}--{-# RULES-"LAZY TEXT append -> fused" [~1] forall t1 t2.-    append t1 t2 = unstream (S.append (stream t1) (stream t2))-"LAZY TEXT append -> unfused" [1] forall t1 t2.-    unstream (S.append (stream t1) (stream t2)) = append t1 t2- #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty. Subject to fusion.-uncons :: Text -> Maybe (Char, Text)-uncons Empty        = Nothing-uncons (Chunk t ts) = Just (T.unsafeHead t, ts')-  where ts' | T.compareLength t 1 == EQ = ts-            | otherwise                 = Chunk (T.unsafeTail t) ts-{-# INLINE uncons #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty.  Subject to fusion.-head :: Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty.  Subject to fusion.-tail :: Text -> Text-tail (Chunk t ts) = chunk (T.tail t) ts-tail Empty        = emptyError "tail"-{-# INLINE [1] tail #-}--{-# RULES-"LAZY TEXT tail -> fused" [~1] forall t.-    tail t = unstream (S.tail (stream t))-"LAZY TEXT tail -> unfused" [1] forall t.-    unstream (S.tail (stream t)) = tail t- #-}---- | /O(n\/c)/ Returns all but the last character of a 'Text', which must--- be non-empty.  Subject to fusion.-init :: Text -> Text-init (Chunk t0 ts0) = go t0 ts0-    where go t (Chunk t' ts) = Chunk t (go t' ts)-          go t Empty         = chunk (T.init t) Empty-init Empty = emptyError "init"-{-# INLINE [1] init #-}--{-# RULES-"LAZY TEXT init -> fused" [~1] forall t.-    init t = unstream (S.init (stream t))-"LAZY TEXT init -> unfused" [1] forall t.-    unstream (S.init (stream t)) = init t- #-}---- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if--- empty.------ * It is no faster than using 'init' and 'last'.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc Empty          = Nothing-unsnoc ts@(Chunk _ _) = Just (init ts, last ts)-{-# INLINE unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.  Subject to--- fusion.-null :: Text -> Bool-null Empty = True-null _     = False-{-# INLINE [1] null #-}--{-# RULES-"LAZY TEXT null -> fused" [~1] forall t.-    null t = S.null (stream t)-"LAZY TEXT null -> unfused" [1] forall t.-    S.null (stream t) = null t- #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.--- Subject to fusion.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n\/c)/ Returns the last character of a 'Text', which must be--- non-empty.  Subject to fusion.-last :: Text -> Char-last Empty        = emptyError "last"-last (Chunk t ts) = go t ts-    where go _ (Chunk t' ts') = go t' ts'-          go t' Empty         = T.last t'-{-# INLINE [1] last #-}--{-# RULES-"LAZY TEXT last -> fused" [~1] forall t.-    last t = S.last (stream t)-"LAZY TEXT last -> unfused" [1] forall t.-    S.last (stream t) = last t-  #-}---- | /O(n)/ Returns the number of characters in a 'Text'.--- Subject to fusion.-length :: Text -> Int64-length = foldlChunks go 0-    where go l t = l + fromIntegral (T.length t)-{-# INLINE [1] length #-}--{-# RULES-"LAZY TEXT length -> fused" [~1] forall t.-    length t = S.length (stream t)-"LAZY TEXT length -> unfused" [1] forall t.-    S.length (stream t) = length t- #-}---- | /O(n)/ Compare the count of characters in a 'Text' to a number.--- Subject to fusion.------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int64 -> Ordering-compareLength t n = S.compareLengthI (stream t) n-{-# INLINE [1] compareLength #-}---- We don't apply those otherwise appealing length-to-compareLength--- rewrite rules here, because they can change the strictness--- properties of code.---- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.  Subject to fusion.  Performs replacement on--- invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f t = unstream (S.map (safe . f) (stream t))-{-# INLINE [1] map #-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.-intercalate :: Text -> [Text] -> Text-intercalate t = concat . (F.intersperse t)-{-# INLINE intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.  Subject to fusion.  Performs--- replacement on invalid scalar values.-intersperse :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse (safe c) (stream t))-{-# INLINE intersperse #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Subject to fusion.  Performs--- replacement on invalid scalar values.------ Examples:------ > justifyLeft 7 'x' "foo"    == "fooxxxx"--- > justifyLeft 3 'x' "foobar" == "foobar"-justifyLeft :: Int64 -> Char -> Text -> Text-justifyLeft k c t-    | len >= k  = t-    | otherwise = t `append` replicateChar (k-len) c-  where len = length t-{-# INLINE [1] justifyLeft #-}--{-# RULES-"LAZY TEXT justifyLeft -> fused" [~1] forall k c t.-    justifyLeft k c t = unstream (S.justifyLeftI k c (stream t))-"LAZY TEXT justifyLeft -> unfused" [1] forall k c t.-    unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t-  #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left.  Performs replacement on--- invalid scalar values.------ Examples:------ > justifyRight 7 'x' "bar"    == "xxxxbar"--- > justifyRight 3 'x' "foobar" == "foobar"-justifyRight :: Int64 -> Char -> Text -> Text-justifyRight k c t-    | len >= k  = t-    | otherwise = replicateChar (k-len) c `append` t-  where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side.  Performs replacement on invalid--- scalar values.------ Examples:------ > center 8 'x' "HS" = "xxxHSxxx"-center :: Int64 -> Char -> Text -> Text-center k c t-    | len >= k  = t-    | otherwise = replicateChar l c `append` t `append` replicateChar r c-  where len = length t-        d   = k - len-        r   = d `quot` 2-        l   = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument.  Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.-transpose :: [Text] -> [Text]-transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty)-                     (L.transpose (L.map unpack ts))--- TODO: make this fast---- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order.-reverse :: Text -> Text-reverse = rev Empty-  where rev a Empty        = a-        rev a (Chunk t ts) = rev (Chunk (T.reverse t) a) ts---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =---   'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once.  So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ > replace "oo" "foo" "oo" == "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ > replace "ofo" "bar" "ofofo" == "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: Text-        -- ^ @needle@ to search for.  If this string is empty, an-        -- error will occur.-        -> Text-        -- ^ @replacement@ to replace @needle@ with.-        -> Text-        -- ^ @haystack@ in which to search.-        -> Text-replace s d = intercalate d . splitOn s-{-# INLINE replace #-}---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ With Unicode text, it is incorrect to use combinators like @map--- toUpper@ to case convert each character of a string individually.--- Instead, use the whole-string case conversion functions from this--- module.  For correctness in different writing systems, these--- functions may map one input character to two or three output--- characters.---- | /O(n)/ Convert a string to folded case.  Subject to fusion.------ This function is mainly useful for performing caseless (or case--- insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string.  For instance,--- the Armenian small ligature men now (U+FB13) is case folded to the--- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is--- case folded to the Greek small letter letter mu (U+03BC) instead of--- itself.-toCaseFold :: Text -> Text-toCaseFold t = unstream (S.toCaseFold (stream t))-{-# INLINE [0] toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.  Subject to fusion.------ The result string may be longer than the input string.  For--- instance, the Latin capital letter I with dot above (U+0130) maps--- to the sequence Latin small letter i (U+0069) followed by combining--- dot above (U+0307).-toLower :: Text -> Text-toLower t = unstream (S.toLower (stream t))-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.  Subject to fusion.------ The result string may be longer than the input string.  For--- instance, the German eszett (U+00DF) maps to the two-letter--- sequence SS.-toUpper :: Text -> Text-toUpper t = unstream (S.toUpper (stream t))-{-# INLINE toUpper #-}----- | /O(n)/ Convert a string to title case, using simple case--- conversion.  Subject to fusion.------ The first letter of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature &#xfb02; (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased&#x2014;but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle t = unstream (S.toTitle (stream t))-{-# INLINE toTitle #-}---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.--- Subject to fusion.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.--- Subject to fusion.-foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.  Subject to fusion.-foldl1 :: (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.  Subject to fusion.-foldl1' :: (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.--- Subject to fusion.-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.  Subject to--- fusion.-foldr1 :: (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat = to-  where-    go Empty        css = to css-    go (Chunk c cs) css = Chunk c (go cs css)-    to []               = Empty-    to (cs:css)         = go cs css-{-# INLINE concat #-}---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty. Subject to fusion.-maximum :: Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty. Subject to fusion.-minimum :: Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left. Subject to fusion.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ Note that------ > last (scanl f z xs) == foldl f z xs.-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))-    where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument.  Subject to fusion.  Performs replacement on--- invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t0 = case uncons t0 of-                Nothing -> empty-                Just (t,ts) -> scanl f t ts-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f v = reverse . scanl g v . reverse-    where g a b = safe (f b a)---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument.  Performs replacement on invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t    = empty-           | otherwise = scanr f (last t) (init t)---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'.  Performs--- replacement on invalid scalar values.-mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumL f = go-  where-    go z (Chunk c cs)    = (z'', Chunk c' cs')-        where (z',  c')  = T.mapAccumL f z c-              (z'', cs') = go z' cs-    go z Empty           = (z, Empty)-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'.  Performs replacement on invalid scalar values.-mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumR f = go-  where-    go z (Chunk c cs)   = (z'', Chunk c' cs')-        where (z'', c') = T.mapAccumR f z' c-              (z', cs') = go z cs-    go z Empty          = (z, Empty)-{-# INLINE mapAccumR #-}---- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every--- element.------ @since 1.2.0.5-repeat :: Char -> Text-repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t-            in t---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int64 -> Text -> Text-replicate n t-    | null t || n <= 0 = empty-    | isSingleton t    = replicateChar n (head t)-    | otherwise        = concat (rep 0)-    where rep !i | i >= n    = []-                 | otherwise = t : rep (i+1)-{-# INLINE [1] replicate #-}---- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or--- equivalently, the infinite repetition of the original 'Text'.------ @since 1.2.0.5-cycle :: Text -> Text-cycle Empty = emptyError "cycle"-cycle t     = let t' = foldrChunks Chunk t' t-               in t'---- | @'iterate' f x@ returns an infinite 'Text' of repeated applications--- of @f@ to @x@:------ > iterate f x == [x, f x, f (f x), ...]------ @since 1.2.0.5-iterate :: (Char -> Char) -> Char -> Text-iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))-               in t c---- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the--- value of every element. Subject to fusion.-replicateChar :: Int64 -> Char -> Text-replicateChar n c = unstream (S.replicateCharI n (safe c))-{-# INLINE replicateChar #-}--{-# RULES-"LAZY TEXT replicate/singleton -> replicateChar" [~1] forall n c.-    replicate n (singleton c) = replicateChar n c-  #-}---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.  Performs--- replacement on invalid scalar values.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'.--- Performs replacement on invalid scalar values.-unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text. Subject to fusion.-take :: Int64 -> Text -> Text-take i _ | i <= 0 = Empty-take i t0         = take' i t0-  where take' 0 _            = Empty-        take' _ Empty        = Empty-        take' n (Chunk t ts)-            | n < len   = Chunk (T.take (fromIntegral n) t) Empty-            | otherwise = Chunk t (take' (n - len) ts)-            where len = fromIntegral (T.length t)-{-# INLINE [1] take #-}--{-# RULES-"LAZY TEXT take -> fused" [~1] forall n t.-    take n t = unstream (S.take n (stream t))-"LAZY TEXT take -> unfused" [1] forall n t.-    unstream (S.take n (stream t)) = take n t-  #-}---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ > takeEnd 3 "foobar" == "bar"------ @since 1.1.1.0-takeEnd :: Int64 -> Text -> Text-takeEnd n t0-    | n <= 0    = empty-    | otherwise = takeChunk n empty . L.reverse . toChunks $ t0-  where takeChunk _ acc [] = acc-        takeChunk i acc (t:ts)-          | i <= l    = chunk (T.takeEnd (fromIntegral i) t) acc-          | otherwise = takeChunk (i-l) (Chunk t acc) ts-          where l = fromIntegral (T.length t)---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'. Subject to fusion.-drop :: Int64 -> Text -> Text-drop i t0-    | i <= 0    = t0-    | otherwise = drop' i t0-  where drop' 0 ts           = ts-        drop' _ Empty        = Empty-        drop' n (Chunk t ts)-            | n < len   = Chunk (T.drop (fromIntegral n) t) ts-            | otherwise = drop' (n - len) ts-            where len   = fromIntegral (T.length t)-{-# INLINE [1] drop #-}--{-# RULES-"LAZY TEXT drop -> fused" [~1] forall n t.-    drop n t = unstream (S.drop n (stream t))-"LAZY TEXT drop -> unfused" [1] forall n t.-    unstream (S.drop n (stream t)) = drop n t-  #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ > dropEnd 3 "foobar" == "foo"------ @since 1.1.1.0-dropEnd :: Int64 -> Text -> Text-dropEnd n t0-    | n <= 0    = t0-    | otherwise = dropChunk n . L.reverse . toChunks $ t0-  where dropChunk _ [] = empty-        dropChunk m (t:ts)-          | m >= l    = dropChunk (m-l) ts-          | otherwise = fromChunks . L.reverse $-                        T.dropEnd (fromIntegral m) t : ts-          where l = fromIntegral (T.length t)---- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word16'--- values dropped, or the empty 'Text' if @n@ is greater than the--- number of 'Word16' values present.-dropWords :: Int64 -> Text -> Text-dropWords i t0-    | i <= 0    = t0-    | otherwise = drop' i t0-  where drop' 0 ts           = ts-        drop' _ Empty        = Empty-        drop' n (Chunk (T.Text arr off len) ts)-            | n < len'  = chunk (text arr (off+n') (len-n')) ts-            | otherwise = drop' (n - len') ts-            where len'  = fromIntegral len-                  n'    = fromIntegral n---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.  Subject to fusion.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t0 = takeWhile' t0-  where takeWhile' Empty        = Empty-        takeWhile' (Chunk t ts) =-          case T.findIndex (not . p) t of-            Just n | n > 0     -> Chunk (T.take n t) Empty-                   | otherwise -> Empty-            Nothing            -> Chunk t (takeWhile' ts)-{-# INLINE [1] takeWhile #-}--{-# RULES-"LAZY TEXT takeWhile -> fused" [~1] forall p t.-    takeWhile p t = unstream (S.takeWhile p (stream t))-"LAZY TEXT takeWhile -> unfused" [1] forall p t.-    unstream (S.takeWhile p (stream t)) = takeWhile p t-  #-}--- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.--- Examples:------ > takeWhileEnd (=='o') "foo" == "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p = takeChunk empty . L.reverse . toChunks-  where takeChunk acc []     = acc-        takeChunk acc (t:ts)-          | T.lengthWord16 t' < T.lengthWord16 t-                             = chunk t' acc-          | otherwise        = takeChunk (Chunk t' acc) ts-          where t' = T.takeWhileEnd p t-{-# INLINE takeWhileEnd #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@.  Subject to fusion.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t0 = dropWhile' t0-  where dropWhile' Empty        = Empty-        dropWhile' (Chunk t ts) =-          case T.findIndex (not . p) t of-            Just n  -> Chunk (T.drop n t) ts-            Nothing -> dropWhile' ts-{-# INLINE [1] dropWhile #-}--{-# RULES-"LAZY TEXT dropWhile -> fused" [~1] forall p t.-    dropWhile p t = unstream (S.dropWhile p (stream t))-"LAZY TEXT dropWhile -> unfused" [1] forall p t.-    unstream (S.dropWhile p (stream t)) = dropWhile p t-  #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.------ Examples:------ > dropWhileEnd (=='.') "foo..." == "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p = go-  where go Empty = Empty-        go (Chunk t Empty) = if T.null t'-                             then Empty-                             else Chunk t' Empty-            where t' = T.dropWhileEnd p t-        go (Chunk t ts) = case go ts of-                            Empty -> go (Chunk t Empty)-                            ts' -> Chunk t ts'-{-# INLINE dropWhileEnd #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.  Subject to fusion.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string.  Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile isSpace-{-# INLINE [1] stripStart #-}---- | /O(n)/ Remove trailing white space from a string.  Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int64 -> Text -> (Text, Text)-splitAt = loop-  where loop _ Empty      = (empty, empty)-        loop n t | n <= 0 = (empty, t)-        loop n (Chunk t ts)-             | n < len   = let (t',t'') = T.splitAt (fromIntegral n) t-                           in (Chunk t' Empty, Chunk t'' ts)-             | otherwise = let (ts',ts'') = loop (n - len) ts-                           in (Chunk t ts', ts'')-             where len = fromIntegral (T.length t)---- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first--- element is a prefix of @t@ whose chunks contain @n@ 'Word16'--- values, and whose second is the remainder of the string.-splitAtWord :: Int64 -> Text -> PairS Text Text-splitAtWord _ Empty = empty :*: empty-splitAtWord x (Chunk c@(T.Text arr off len) cs)-    | y >= len  = let h :*: t = splitAtWord (x-fromIntegral len) cs-                  in  Chunk c h :*: t-    | otherwise = chunk (text arr off y) empty :*:-                  chunk (text arr (off+y) (len-y)) cs-    where y = fromIntegral x---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@.  The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched.  The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ > breakOn "::" "a::b::c" ==> ("a", "::b::c")--- > breakOn "/" "foobar"   ==> ("foobar", "")------ Laws:------ > append prefix match == haystack--- >   where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: Text -> Text -> (Text, Text)-breakOn pat src-    | null pat  = emptyError "breakOn"-    | otherwise = case indices pat src of-                    []    -> (src, empty)-                    (x:_) -> let h :*: t = splitAtWord x src-                             in  (h, t)---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@.  The second is the--- remainder of @haystack@, following the match.------ > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")-breakOnEnd :: Text -> Text -> (Text, Text)-breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)-                   in  (reverse b, reverse a)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@.  Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ > breakOnAll "::" ""--- > ==> []--- > breakOnAll "/" "a/b/c/"--- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: Text              -- ^ @needle@ to search for-           -> Text              -- ^ @haystack@ in which to search-           -> [(Text, Text)]-breakOnAll pat src-    | null pat  = emptyError "breakOnAll"-    | otherwise = go 0 empty src (indices pat src)-  where-    go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s-                           h'      = append p h-                       in (h',t) : go x h' t xs-    go _  _ _ _      = []---- | /O(n)/ 'break' is like 'span', but the prefix returned is over--- elements that fail the predicate @p@.-break :: (Char -> Bool) -> Text -> (Text, Text)-break p t0 = break' t0-  where break' Empty          = (empty, empty)-        break' c@(Chunk t ts) =-          case T.findIndex p t of-            Nothing      -> let (ts', ts'') = break' ts-                            in (Chunk t ts', ts'')-            Just n | n == 0    -> (Empty, c)-                   | otherwise -> let (a,b) = T.splitAt n t-                                  in (Chunk a Empty, Chunk b ts)---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the list.-span :: (Char -> Bool) -> Text -> (Text, Text)-span p = break (not . p)-{-# INLINE span #-}---- | The 'group' function takes a 'Text' and returns a list of 'Text's--- such that the concatenation of the result is equal to the argument.--- Moreover, each sublist in the result contains only equal elements.--- For example,------ > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"]------ It is a special case of 'groupBy', which allows the programmer to--- supply their own equality test.-group :: Text -> [Text]-group =  groupBy (==)-{-# INLINE group #-}---- | The 'groupBy' function is the non-overloaded version of 'group'.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy _  Empty        = []-groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs-                          where (ys,zs) = span (eq x) xs-                                x  = T.unsafeHead t-                                xs = chunk (T.unsafeTail t) ts---- | /O(n)/ Return all initial segments of the given 'Text',--- shortest first.-inits :: Text -> [Text]-inits = (Empty :) . inits'-  where inits' Empty        = []-        inits' (Chunk t ts) = L.map (\t' -> Chunk t' Empty) (L.tail (T.inits t))-                           ++ L.map (Chunk t) (inits' ts)---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails Empty         = Empty : []-tails ts@(Chunk t ts')-  | T.length t == 1 = ts : tails ts'-  | otherwise       = ts : tails (Chunk (T.unsafeTail t) ts')---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be an empty string), consuming the--- delimiter. An empty delimiter is invalid, and will cause an error--- to be raised.------ Examples:------ > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > splitOn "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]--- > splitOn "x"    "x"                == ["",""]------ and------ > intercalate s . splitOn s         == id--- > splitOn (singleton c)             == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: Text-        -- ^ String to split on. If this string is empty, an error-        -- will occur.-        -> Text-        -- ^ Input text.-        -> [Text]-splitOn pat src-    | null pat        = emptyError "splitOn"-    | isSingleton pat = split (== head pat) src-    | otherwise       = go 0 (indices pat src) src-  where-    go  _ []     cs = [cs]-    go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs-                      in  h : go (x+l) xs (dropWords l t)-    l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat-{-# INLINE [1] splitOn #-}--{-# RULES-"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.-    splitOn (singleton c) t = split (==c) t-  #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element.  The--- resulting components do not contain the separators.  Two adjacent--- separators result in an empty component in the output.  eg.------ > split (=='a') "aabbaca" == ["","","bb","c",""]--- > split (=='a') []        == [""]-split :: (Char -> Bool) -> Text -> [Text]-split _ Empty = [Empty]-split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0-  where comb acc (s:[]) Empty        = revChunks (s:acc) : []-        comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts-        comb acc (s:ss) ts           = revChunks (s:acc) : comb [] ss ts-        comb _   []     _            = impossibleError "split"-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@.  The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ > chunksOf 3 "foobarbaz"   == ["foo","bar","baz"]--- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]-chunksOf :: Int64 -> Text -> [Text]-chunksOf k = go-  where-    go t = case splitAt k t of-             (a,b) | null a    -> []-                   | otherwise -> a : go b-{-# INLINE chunksOf #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at--- newline 'Char's. The resulting strings do not contain newlines.-lines :: Text -> [Text]-lines Empty = []-lines t = let (l,t') = break ((==) '\n') t-          in l : if null t' then []-                 else lines (tail t')---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words = L.filter (not . null) . split isSpace-{-# INLINE words #-}---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.map (`snoc` '\n')-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.  Subject to fusion.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf Empty _  = True-isPrefixOf _ Empty  = False-isPrefixOf (Chunk x xs) (Chunk y ys)-    | lx == ly  = x == y  && isPrefixOf xs ys-    | lx <  ly  = x == yh && isPrefixOf xs (Chunk yt ys)-    | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys-  where (xh,xt) = T.splitAt ly x-        (yh,yt) = T.splitAt lx y-        lx = T.length x-        ly = T.length y-{-# INLINE [1] isPrefixOf #-}--{-# RULES-"LAZY TEXT isPrefixOf -> fused" [~1] forall s t.-    isPrefixOf s t = S.isPrefixOf (stream s) (stream t)-"LAZY TEXT isPrefixOf -> unfused" [1] forall s t.-    S.isPrefixOf (stream s) (stream t) = isPrefixOf s t-  #-}---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' iff the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf x y = reverse x `isPrefixOf` reverse y-{-# INLINE isSuffixOf #-}--- TODO: a better implementation---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' iff the first is contained, wholly and intact, anywhere--- within the second.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf :: Text -> Text -> Bool-isInfixOf needle haystack-    | null needle        = True-    | isSingleton needle = S.elem (head needle) . S.stream $ haystack-    | otherwise          = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}--{-# RULES-"LAZY TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h.-    isInfixOf (singleton n) h = S.elem n (S.stream h)-  #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ > stripPrefix "foo" "foobar" == Just "bar"--- > stripPrefix ""    "baz"    == Just "baz"--- > stripPrefix "foo" "quux"   == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _                                 = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p t-    | null p    = Just t-    | otherwise = case commonPrefixes p t of-                    Just (_,c,r) | null c -> Just r-                    _                     -> Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")--- > commonPrefixes "veeble" "fetzer"  == Nothing--- > commonPrefixes "" "baz"           == Nothing-commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)-commonPrefixes Empty _ = Nothing-commonPrefixes _ Empty = Nothing-commonPrefixes a0 b0   = Just (go a0 b0 [])-  where-    go t0@(Chunk x xs) t1@(Chunk y ys) ps-        = case T.commonPrefixes x y of-            Just (p,a,b)-              | T.null a  -> go xs (chunk b ys) (p:ps)-              | T.null b  -> go (chunk a xs) ys (p:ps)-              | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys)-            Nothing       -> (fromChunks (L.reverse ps),t0,t1)-    go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1)---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ > stripSuffix "bar" "foobar" == Just "foo"--- > stripSuffix ""    "baz"    == Just "baz"--- > stripSuffix "foo" "quux"   == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _                                = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t)---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p t = unstream (S.filter p (stream t))-{-# INLINE filter #-}---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: Text -> Int64 -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: Text -> Text -> Int64-count pat src-    | null pat        = emptyError "count"-    | otherwise       = go 0 (indices pat src)-  where go !n []     = n-        go !n (_:xs) = go (n+1) xs-{-# INLINE [1] count #-}--{-# RULES-"LAZY TEXT count/singleton -> countChar" [~1] forall c t.-    count (singleton c) t = countChar c t-  #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'.  Subject to fusion.-countChar :: Char -> Text -> Int64-countChar c t = S.countChar c (stream t)---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE [0] zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))-    where g a b = safe (f a b)-{-# INLINE [0] zipWith #-}--revChunks :: [T.Text] -> Text-revChunks = L.foldl' (flip chunk) Empty--emptyError :: String -> a-emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input")--impossibleError :: String -> a-impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case")
− Data/Text/Lazy/Builder.hs 
@@ -1,57 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---------------------------------------------------------------------------------- |--- Module      : Data.Text.Lazy.Builder--- Copyright   : (c) 2013 Bryan O'Sullivan---               (c) 2010 Johan Tibell--- License     : BSD-style (see LICENSE)------ Maintainer  : Johan Tibell <johan.tibell@gmail.com>--- Portability : portable to Hugs and GHC------ Efficient construction of lazy @Text@ values.  The principal--- operations on a @Builder@ are @singleton@, @fromText@, and--- @fromLazyText@, which construct new builders, and 'mappend', which--- concatenates two builders.------ To get maximum performance when building lazy @Text@ values using a--- builder, associate @mappend@ calls to the right.  For example,--- prefer------ > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')------ to------ > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'------ as the latter associates @mappend@ to the left. Or, equivalently,--- prefer------  > singleton 'a' <> singleton 'b' <> singleton 'c'------ since the '<>' from recent versions of 'Data.Monoid' associates--- to the right.---------------------------------------------------------------------------------module Data.Text.Lazy.Builder-   ( -- * The Builder type-     Builder-   , toLazyText-   , toLazyTextWith--     -- * Constructing Builders-   , singleton-   , fromText-   , fromLazyText-   , fromString--     -- * Flushing the buffer state-   , flush-   ) where--import Data.Text.Internal.Builder
− Data/Text/Lazy/Builder/Int.hs 
@@ -1,264 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, ScopedTypeVariables,-    UnboxedTuples #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---- Module:      Data.Text.Lazy.Builder.Int--- Copyright:   (c) 2013 Bryan O'Sullivan---              (c) 2011 MailRank, Inc.--- License:     BSD-style--- Maintainer:  Bryan O'Sullivan <bos@serpentine.com>--- Portability: portable------ Efficiently write an integral value to a 'Builder'.--module Data.Text.Lazy.Builder.Int-    (-      decimal-    , hexadecimal-    ) where--import Data.Int (Int8, Int16, Int32, Int64)-import Data.Monoid (mempty)-import qualified Data.ByteString.Unsafe as B-import Data.Text.Internal.Builder.Functions ((<>), i2d)-import Data.Text.Internal.Builder-import Data.Text.Internal.Builder.Int.Digits (digits)-import Data.Text.Array-import Data.Word (Word, Word8, Word16, Word32, Word64)-import GHC.Base (quotInt, remInt)-import GHC.Num (quotRemInteger)-import GHC.Types (Int(..))-import Control.Monad.ST-#if MIN_VERSION_base(4,11,0)-import Prelude hiding ((<>))-#endif--#ifdef  __GLASGOW_HASKELL__-# if defined(INTEGER_GMP)-import GHC.Integer.GMP.Internals (Integer(S#))-# elif defined(INTEGER_SIMPLE)-import GHC.Integer-# else-# error "You need to use either GMP or integer-simple."-# endif-#endif--#if defined(INTEGER_GMP) || defined(INTEGER_SIMPLE)-# define PAIR(a,b) (# a,b #)-#else-# define PAIR(a,b) (a,b)-#endif--decimal :: Integral a => a -> Builder-{-# RULES "decimal/Int8" decimal = boundedDecimal :: Int8 -> Builder #-}-{-# RULES "decimal/Int" decimal = boundedDecimal :: Int -> Builder #-}-{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-}-{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-}-{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-}-{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-}-{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-}-{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-}-{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-}-{-# RULES "decimal/Word64" decimal = positive :: Word64 -> Builder #-}-{-# RULES "decimal/Integer" decimal = integer 10 :: Integer -> Builder #-}-decimal i = decimal' (<= -128) i-{-# NOINLINE decimal #-}--boundedDecimal :: (Integral a, Bounded a) => a -> Builder-{-# SPECIALIZE boundedDecimal :: Int -> Builder #-}-{-# SPECIALIZE boundedDecimal :: Int8 -> Builder #-}-{-# SPECIALIZE boundedDecimal :: Int16 -> Builder #-}-{-# SPECIALIZE boundedDecimal :: Int32 -> Builder #-}-{-# SPECIALIZE boundedDecimal :: Int64 -> Builder #-}-boundedDecimal i = decimal' (== minBound) i--decimal' :: (Integral a) => (a -> Bool) -> a -> Builder-{-# INLINE decimal' #-}-decimal' p i-    | i < 0 = if p i-              then let (q, r) = i `quotRem` 10-                       qq = -q-                       !n = countDigits qq-                   in writeN (n + 2) $ \marr off -> do-                       unsafeWrite marr off minus-                       posDecimal marr (off+1) n qq-                       unsafeWrite marr (off+n+1) (i2w (-r))-              else let j = -i-                       !n = countDigits j-                   in writeN (n + 1) $ \marr off ->-                       unsafeWrite marr off minus >> posDecimal marr (off+1) n j-    | otherwise = positive i--positive :: (Integral a) => a -> Builder-{-# SPECIALIZE positive :: Int -> Builder #-}-{-# SPECIALIZE positive :: Int8 -> Builder #-}-{-# SPECIALIZE positive :: Int16 -> Builder #-}-{-# SPECIALIZE positive :: Int32 -> Builder #-}-{-# SPECIALIZE positive :: Int64 -> Builder #-}-{-# SPECIALIZE positive :: Word -> Builder #-}-{-# SPECIALIZE positive :: Word8 -> Builder #-}-{-# SPECIALIZE positive :: Word16 -> Builder #-}-{-# SPECIALIZE positive :: Word32 -> Builder #-}-{-# SPECIALIZE positive :: Word64 -> Builder #-}-positive i-    | i < 10    = writeN 1 $ \marr off -> unsafeWrite marr off (i2w i)-    | otherwise = let !n = countDigits i-                  in writeN n $ \marr off -> posDecimal marr off n i--posDecimal :: (Integral a) =>-              forall s. MArray s -> Int -> Int -> a -> ST s ()-{-# INLINE posDecimal #-}-posDecimal marr off0 ds v0 = go (off0 + ds - 1) v0-  where go off v-           | v >= 100 = do-               let (q, r) = v `quotRem` 100-               write2 off r-               go (off - 2) q-           | v < 10    = unsafeWrite marr off (i2w v)-           | otherwise = write2 off v-        write2 off i0 = do-          let i = fromIntegral i0; j = i + i-          unsafeWrite marr off $ get (j + 1)-          unsafeWrite marr (off - 1) $ get j-        get = fromIntegral . B.unsafeIndex digits--minus, zero :: Word16-{-# INLINE minus #-}-{-# INLINE zero #-}-minus = 45-zero = 48--i2w :: (Integral a) => a -> Word16-{-# INLINE i2w #-}-i2w v = zero + fromIntegral v--countDigits :: (Integral a) => a -> Int-{-# INLINE countDigits #-}-countDigits v0-  | fromIntegral v64 == v0 = go 1 v64-  | otherwise              = goBig 1 (fromIntegral v0)-  where v64 = fromIntegral v0-        goBig !k (v :: Integer)-           | v > big   = goBig (k + 19) (v `quot` big)-           | otherwise = go k (fromIntegral v)-        big = 10000000000000000000-        go !k (v :: Word64)-           | v < 10    = k-           | v < 100   = k + 1-           | v < 1000  = k + 2-           | v < 1000000000000 =-               k + if v < 100000000-                   then if v < 1000000-                        then if v < 10000-                             then 3-                             else 4 + fin v 100000-                        else 6 + fin v 10000000-                   else if v < 10000000000-                        then 8 + fin v 1000000000-                        else 10 + fin v 100000000000-           | otherwise = go (k + 12) (v `quot` 1000000000000)-        fin v n = if v >= n then 1 else 0--hexadecimal :: Integral a => a -> Builder-{-# SPECIALIZE hexadecimal :: Int -> Builder #-}-{-# SPECIALIZE hexadecimal :: Int8 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Int16 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Int32 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Int64 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Word -> Builder #-}-{-# SPECIALIZE hexadecimal :: Word8 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Word16 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Word32 -> Builder #-}-{-# SPECIALIZE hexadecimal :: Word64 -> Builder #-}-{-# RULES "hexadecimal/Integer"-    hexadecimal = hexInteger :: Integer -> Builder #-}-hexadecimal i-    | i < 0     = error hexErrMsg-    | otherwise = go i-  where-    go n | n < 16    = hexDigit n-         | otherwise = go (n `quot` 16) <> hexDigit (n `rem` 16)-{-# NOINLINE[0] hexadecimal #-}--hexInteger :: Integer -> Builder-hexInteger i-    | i < 0     = error hexErrMsg-    | otherwise = integer 16 i--hexErrMsg :: String-hexErrMsg = "Data.Text.Lazy.Builder.Int.hexadecimal: applied to negative number"--hexDigit :: Integral a => a -> Builder-hexDigit n-    | n <= 9    = singleton $! i2d (fromIntegral n)-    | otherwise = singleton $! toEnum (fromIntegral n + 87)-{-# INLINE hexDigit #-}--data T = T !Integer !Int--integer :: Int -> Integer -> Builder-#ifdef INTEGER_GMP-integer 10 (S# i#) = decimal (I# i#)-integer 16 (S# i#) = hexadecimal (I# i#)-#endif-integer base i-    | i < 0     = singleton '-' <> go (-i)-    | otherwise = go i-  where-    go n | n < maxInt = int (fromInteger n)-         | otherwise  = putH (splitf (maxInt * maxInt) n)--    splitf p n-      | p > n       = [n]-      | otherwise   = splith p (splitf (p*p) n)--    splith p (n:ns) = case n `quotRemInteger` p of-                        PAIR(q,r) | q > 0     -> q : r : splitb p ns-                                  | otherwise -> r : splitb p ns-    splith _ _      = error "splith: the impossible happened."--    splitb p (n:ns) = case n `quotRemInteger` p of-                        PAIR(q,r) -> q : r : splitb p ns-    splitb _ _      = []--    T maxInt10 maxDigits10 =-        until ((>mi) . (*10) . fstT) (\(T n d) -> T (n*10) (d+1)) (T 10 1)-      where mi = fromIntegral (maxBound :: Int)-    T maxInt16 maxDigits16 =-        until ((>mi) . (*16) . fstT) (\(T n d) -> T (n*16) (d+1)) (T 16 1)-      where mi = fromIntegral (maxBound :: Int)--    fstT (T a _) = a--    maxInt | base == 10 = maxInt10-           | otherwise  = maxInt16-    maxDigits | base == 10 = maxDigits10-              | otherwise  = maxDigits16--    putH (n:ns) = case n `quotRemInteger` maxInt of-                    PAIR(x,y)-                        | q > 0     -> int q <> pblock r <> putB ns-                        | otherwise -> int r <> putB ns-                        where q = fromInteger x-                              r = fromInteger y-    putH _ = error "putH: the impossible happened"--    putB (n:ns) = case n `quotRemInteger` maxInt of-                    PAIR(x,y) -> pblock q <> pblock r <> putB ns-                        where q = fromInteger x-                              r = fromInteger y-    putB _ = Data.Monoid.mempty--    int :: Int -> Builder-    int x | base == 10 = decimal x-          | otherwise  = hexadecimal x--    pblock = loop maxDigits-      where-        loop !d !n-            | d == 1    = hexDigit n-            | otherwise = loop (d-1) q <> hexDigit r-            where q = n `quotInt` base-                  r = n `remInt` base
− Data/Text/Lazy/Builder/RealFloat.hs 
@@ -1,253 +0,0 @@-{-# LANGUAGE CPP, OverloadedStrings #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---- |--- Module:    Data.Text.Lazy.Builder.RealFloat--- Copyright: (c) The University of Glasgow 1994-2002--- License:   see libraries/base/LICENSE------ Write a floating point value to a 'Builder'.--module Data.Text.Lazy.Builder.RealFloat-    (-      FPFormat(..)-    , realFloat-    , formatRealFloat-    ) where--import Data.Array.Base (unsafeAt)-import Data.Array.IArray-import Data.Text.Internal.Builder.Functions ((<>), i2d)-import Data.Text.Lazy.Builder.Int (decimal)-import Data.Text.Internal.Builder.RealFloat.Functions (roundTo)-import Data.Text.Lazy.Builder-import qualified Data.Text as T-#if MIN_VERSION_base(4,11,0)-import Prelude hiding ((<>))-#endif---- | Control the rendering of floating point numbers.-data FPFormat = Exponent-              -- ^ Scientific notation (e.g. @2.3e123@).-              | Fixed-              -- ^ Standard decimal notation.-              | Generic-              -- ^ Use decimal notation for values between @0.1@ and-              -- @9,999,999@, and scientific notation otherwise.-                deriving (Enum, Read, Show)---- | Show a signed 'RealFloat' value to full precision,--- using standard decimal notation for arguments whose absolute value lies--- between @0.1@ and @9,999,999@, and scientific notation otherwise.-realFloat :: (RealFloat a) => a -> Builder-{-# SPECIALIZE realFloat :: Float -> Builder #-}-{-# SPECIALIZE realFloat :: Double -> Builder #-}-realFloat x = formatRealFloat Generic Nothing x---- | Encode a signed 'RealFloat' according to 'FPFormat' and optionally requested precision.------ This corresponds to the @show{E,F,G}Float@ operations provided by @base@'s "Numeric" module.------ __NOTE__: The functions in @base-4.12@ changed the serialisation in--- case of a @Just 0@ precision; this version of @text@ still provides--- the serialisation as implemented in @base-4.11@. The next major--- version of @text@ will switch to the more correct @base-4.12@ serialisation.-formatRealFloat :: (RealFloat a) =>-                   FPFormat-                -> Maybe Int  -- ^ Number of decimal places to render.-                -> a-                -> Builder-{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Float -> Builder #-}-{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Double -> Builder #-}-formatRealFloat fmt decs x-   | isNaN x                   = "NaN"-   | isInfinite x              = if x < 0 then "-Infinity" else "Infinity"-   | x < 0 || isNegativeZero x = singleton '-' <> doFmt fmt (floatToDigits (-x))-   | otherwise                 = doFmt fmt (floatToDigits x)- where-  doFmt format (is, e) =-    let ds = map i2d is in-    case format of-     Generic ->-      doFmt (if e < 0 || e > 7 then Exponent else Fixed)-            (is,e)-     Exponent ->-      case decs of-       Nothing ->-        let show_e' = decimal (e-1) in-        case ds of-          "0"     -> "0.0e0"-          [d]     -> singleton d <> ".0e" <> show_e'-          (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> show_e'-          []      -> error "formatRealFloat/doFmt/Exponent/Nothing: []"-       Just dec ->-        let dec' = max dec 1 in-        case is of-         [0] -> "0." <> fromText (T.replicate dec' "0") <> "e0"-         _ ->-          let (ei,is') = roundTo (dec'+1) is-              is'' = map i2d (if ei > 0 then init is' else is')-          in case is'' of-               [] -> error "formatRealFloat/doFmt/Exponent/Just: []"-               (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> decimal (e-1+ei)-     Fixed ->-      let-       mk0 ls = case ls of { "" -> "0" ; _ -> fromString ls}-      in-      case decs of-       Nothing-          | e <= 0    -> "0." <> fromText (T.replicate (-e) "0") <> fromString ds-          | otherwise ->-             let-                f 0 s    rs  = mk0 (reverse s) <> singleton '.' <> mk0 rs-                f n s    ""  = f (n-1) ('0':s) ""-                f n s (r:rs) = f (n-1) (r:s) rs-             in-                f e "" ds-       Just dec ->-        let dec' = max dec 0 in-        if e >= 0 then-         let-          (ei,is') = roundTo (dec' + e) is-          (ls,rs)  = splitAt (e+ei) (map i2d is')-         in-         mk0 ls <> (if null rs then "" else singleton '.' <> fromString rs)-        else-         let (ei,is') = roundTo dec' (replicate (-e) 0 ++ is)-             is'' = map i2d (if ei > 0 then is' else 0:is')-         in case is'' of-              [] -> error "formatRealFloat/doFmt/Fixed: []"-              (d:ds') -> singleton d <> (if null ds' then "" else singleton '.' <> fromString ds')----- Based on "Printing Floating-Point Numbers Quickly and Accurately"--- by R.G. Burger and R.K. Dybvig in PLDI 96.--- This version uses a much slower logarithm estimator. It should be improved.---- | 'floatToDigits' takes a base and a non-negative 'RealFloat' number,--- and returns a list of digits and an exponent.--- In particular, if @x>=0@, and------ > floatToDigits base x = ([d1,d2,...,dn], e)------ then------      (1) @n >= 1@------      (2) @x = 0.d1d2...dn * (base**e)@------      (3) @0 <= di <= base-1@--floatToDigits :: (RealFloat a) => a -> ([Int], Int)-{-# SPECIALIZE floatToDigits :: Float -> ([Int], Int) #-}-{-# SPECIALIZE floatToDigits :: Double -> ([Int], Int) #-}-floatToDigits 0 = ([0], 0)-floatToDigits x =- let-  (f0, e0) = decodeFloat x-  (minExp0, _) = floatRange x-  p = floatDigits x-  b = floatRadix x-  minExp = minExp0 - p -- the real minimum exponent-  -- Haskell requires that f be adjusted so denormalized numbers-  -- will have an impossibly low exponent.  Adjust for this.-  (f, e) =-   let n = minExp - e0 in-   if n > 0 then (f0 `quot` (expt b n), e0+n) else (f0, e0)-  (r, s, mUp, mDn) =-   if e >= 0 then-    let be = expt b e in-    if f == expt b (p-1) then-      (f*be*b*2, 2*b, be*b, be)     -- according to Burger and Dybvig-    else-      (f*be*2, 2, be, be)-   else-    if e > minExp && f == expt b (p-1) then-      (f*b*2, expt b (-e+1)*2, b, 1)-    else-      (f*2, expt b (-e)*2, 1, 1)-  k :: Int-  k =-   let-    k0 :: Int-    k0 =-     if b == 2 then-        -- logBase 10 2 is very slightly larger than 8651/28738-        -- (about 5.3558e-10), so if log x >= 0, the approximation-        -- k1 is too small, hence we add one and need one fixup step less.-        -- If log x < 0, the approximation errs rather on the high side.-        -- That is usually more than compensated for by ignoring the-        -- fractional part of logBase 2 x, but when x is a power of 1/2-        -- or slightly larger and the exponent is a multiple of the-        -- denominator of the rational approximation to logBase 10 2,-        -- k1 is larger than logBase 10 x. If k1 > 1 + logBase 10 x,-        -- we get a leading zero-digit we don't want.-        -- With the approximation 3/10, this happened for-        -- 0.5^1030, 0.5^1040, ..., 0.5^1070 and values close above.-        -- The approximation 8651/28738 guarantees k1 < 1 + logBase 10 x-        -- for IEEE-ish floating point types with exponent fields-        -- <= 17 bits and mantissae of several thousand bits, earlier-        -- convergents to logBase 10 2 would fail for long double.-        -- Using quot instead of div is a little faster and requires-        -- fewer fixup steps for negative lx.-        let lx = p - 1 + e0-            k1 = (lx * 8651) `quot` 28738-        in if lx >= 0 then k1 + 1 else k1-     else-        -- f :: Integer, log :: Float -> Float,-        --               ceiling :: Float -> Int-        ceiling ((log (fromInteger (f+1) :: Float) +-                 fromIntegral e * log (fromInteger b)) /-                   log 10)---WAS:            fromInt e * log (fromInteger b))--    fixup n =-      if n >= 0 then-        if r + mUp <= expt 10 n * s then n else fixup (n+1)-      else-        if expt 10 (-n) * (r + mUp) <= s then n else fixup (n+1)-   in-   fixup k0--  gen ds rn sN mUpN mDnN =-   let-    (dn, rn') = (rn * 10) `quotRem` sN-    mUpN' = mUpN * 10-    mDnN' = mDnN * 10-   in-   case (rn' < mDnN', rn' + mUpN' > sN) of-    (True,  False) -> dn : ds-    (False, True)  -> dn+1 : ds-    (True,  True)  -> if rn' * 2 < sN then dn : ds else dn+1 : ds-    (False, False) -> gen (dn:ds) rn' sN mUpN' mDnN'--  rds =-   if k >= 0 then-      gen [] r (s * expt 10 k) mUp mDn-   else-     let bk = expt 10 (-k) in-     gen [] (r * bk) s (mUp * bk) (mDn * bk)- in- (map fromIntegral (reverse rds), k)---- Exponentiation with a cache for the most common numbers.-minExpt, maxExpt :: Int-minExpt = 0-maxExpt = 1100--expt :: Integer -> Int -> Integer-expt base n-    | base == 2 && n >= minExpt && n <= maxExpt = expts `unsafeAt` n-    | base == 10 && n <= maxExpt10              = expts10 `unsafeAt` n-    | otherwise                                 = base^n--expts :: Array Int Integer-expts = array (minExpt,maxExpt) [(n,2^n) | n <- [minExpt .. maxExpt]]--maxExpt10 :: Int-maxExpt10 = 324--expts10 :: Array Int Integer-expts10 = array (minExpt,maxExpt10) [(n,10^n) | n <- [minExpt .. maxExpt10]]
− Data/Text/Lazy/Encoding.hs 
@@ -1,250 +0,0 @@-{-# LANGUAGE BangPatterns,CPP #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif--- |--- Module      : Data.Text.Lazy.Encoding--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : portable------ Functions for converting lazy 'Text' values to and from lazy--- 'ByteString', using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Lazy.Encoding-    (-    -- * Decoding ByteStrings to Text-    -- $strict-      decodeASCII-    , decodeLatin1-    , decodeUtf8-    , decodeUtf16LE-    , decodeUtf16BE-    , decodeUtf32LE-    , decodeUtf32BE--    -- ** Catchable failure-    , decodeUtf8'--    -- ** Controllable error handling-    , decodeUtf8With-    , decodeUtf16LEWith-    , decodeUtf16BEWith-    , decodeUtf32LEWith-    , decodeUtf32BEWith--    -- * Encoding Text to ByteStrings-    , encodeUtf8-    , encodeUtf16LE-    , encodeUtf16BE-    , encodeUtf32LE-    , encodeUtf32BE--    -- * Encoding Text using ByteString Builders-    , encodeUtf8Builder-    , encodeUtf8BuilderEscaped-    ) where--import Control.Exception (evaluate, try)-import Data.Monoid (Monoid(..))-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)-import Data.Word (Word8)-import qualified Data.ByteString as S-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Lazy as B-import qualified Data.ByteString.Lazy.Internal as B-import qualified Data.ByteString.Unsafe as B-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE-import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E-import qualified Data.Text.Internal.Lazy.Fusion as F-import Data.Text.Unsafe (unsafeDupablePerformIO)---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- | /Deprecated/.  Decode a 'ByteString' containing 7-bit ASCII--- encoded text.-decodeASCII :: B.ByteString -> Text-decodeASCII = decodeUtf8-{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.-decodeLatin1 :: B.ByteString -> Text-decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks---- | Decode a 'ByteString' containing UTF-8 encoded text.-decodeUtf8With :: OnDecodeError -> B.ByteString -> Text-decodeUtf8With onErr (B.Chunk b0 bs0) =-    case TE.streamDecodeUtf8With onErr b0 of-      TE.Some t l f -> chunk t (go f l bs0)-  where-    go f0 _ (B.Chunk b bs) =-      case f0 b of-        TE.Some t l f -> chunk t (go f l bs)-    go _ l _-      | S.null l  = empty-      | otherwise = case onErr desc (Just (B.unsafeHead l)) of-                      Nothing -> empty-                      Just c  -> Chunk (T.singleton c) Empty-    desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"-decodeUtf8With _ _ = empty---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code.  For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.-decodeUtf8 :: B.ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}---- This rule seems to cause performance loss.-{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1]-   forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-}---- | Decode a 'ByteString' containing UTF-8 encoded text..------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.------ /Note/: this function is /not/ lazy, as it must decode its entire--- input before it can return a result.  If you need lazy (streaming)--- decoding, use 'decodeUtf8With' in lenient mode.-decodeUtf8' :: B.ByteString -> Either UnicodeException Text-decodeUtf8' bs = unsafeDupablePerformIO $ do-                   let t = decodeUtf8 bs-                   try (evaluate (rnf t `seq` t))-  where-    rnf Empty        = ()-    rnf (Chunk _ ts) = rnf ts-{-# INLINE decodeUtf8' #-}---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> B.ByteString-encodeUtf8    Empty       = B.empty-encodeUtf8 lt@(Chunk t _) =-    B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt-  where-    -- To improve our small string performance, we use a strategy that-    -- allocates a buffer that is guaranteed to be large enough for the-    -- encoding of the first chunk, but not larger than the default-    -- B.smallChunkSize. We clamp the firstChunkSize to ensure that we don't-    -- generate too large buffers which hamper streaming.-    firstChunkSize  = min B.smallChunkSize (4 * (T.length t + 1))-    strategy        = B.safeStrategy firstChunkSize B.defaultChunkSize---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder =-    foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped prim =-    foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: B.ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: B.ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> B.ByteString-encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt)-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> B.ByteString-encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt)-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: B.ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown.  For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: B.ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> B.ByteString-encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt)-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> B.ByteString-encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt)-{-# INLINE encodeUtf32BE #-}
− Data/Text/Lazy/IO.hs 
@@ -1,195 +0,0 @@-{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif--- |--- Module      : Data.Text.Lazy.IO--- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,---               (c) 2009 Simon Marlow--- License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Efficient locale-sensitive support for lazy text I\/O.------ Skip past the synopsis for some important notes on performance and--- portability across different versions of GHC.--module Data.Text.Lazy.IO-    (-    -- * Performance-    -- $performance--    -- * Locale support-    -- $locale-    -- * File-at-a-time operations-      readFile-    , writeFile-    , appendFile-    -- * Operations on handles-    , hGetContents-    , hGetLine-    , hPutStr-    , hPutStrLn-    -- * Special cases for standard input and output-    , interact-    , getContents-    , getLine-    , putStr-    , putStrLn-    ) where--import Data.Text.Lazy (Text)-import Prelude hiding (appendFile, getContents, getLine, interact,-                       putStr, putStrLn, readFile, writeFile)-import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout,-                  withFile)-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as L-import qualified Control.Exception as E-import Control.Monad (when)-import Data.IORef (readIORef)-import Data.Text.Internal.IO (hGetLineWith, readChunk)-import Data.Text.Internal.Lazy (chunk, empty)-import GHC.IO.Buffer (isEmptyBuffer)-import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException)-import GHC.IO.Handle.Internals (augmentIOError, hClose_help,-                                wantReadableHandle, withHandle)-import GHC.IO.Handle.Types (Handle__(..), HandleType(..))-import System.IO (BufferMode(..), hGetBuffering, hSetBuffering)-import System.IO.Error (isEOFError)-import System.IO.Unsafe (unsafeInterleaveIO)---- $performance------ The functions in this module obey the runtime system's locale,--- character set encoding, and line ending conversion settings.------ If you know in advance that you will be working with data that has--- a specific encoding (e.g. UTF-8), and your application is highly--- performance sensitive, you may find that it is faster to perform--- I\/O with bytestrings and to encode and decode yourself than to use--- the functions in this module.------ Whether this will hold depends on the version of GHC you are using,--- the platform you are working on, the data you are working with, and--- the encodings you are using, so be sure to test for yourself.---- | Read a file and return its contents as a string.  The file is--- read lazily, as with 'getContents'.-readFile :: FilePath -> IO Text-readFile name = openFile name ReadMode >>= hGetContents---- | Write a string to a file.  The file is truncated to zero length--- before writing begins.-writeFile :: FilePath -> Text -> IO ()-writeFile p = withFile p WriteMode . flip hPutStr---- | Write a string the end of a file.-appendFile :: FilePath -> Text -> IO ()-appendFile p = withFile p AppendMode . flip hPutStr---- | Lazily read the remaining contents of a 'Handle'.  The 'Handle'--- will be closed after the read completes, or on error.-hGetContents :: Handle -> IO Text-hGetContents h = do-  chooseGoodBuffering h-  wantReadableHandle "hGetContents" h $ \hh -> do-    ts <- lazyRead h-    return (hh{haType=SemiClosedHandle}, ts)---- | Use a more efficient buffer size if we're reading in--- block-buffered mode with the default buffer size.-chooseGoodBuffering :: Handle -> IO ()-chooseGoodBuffering h = do-  bufMode <- hGetBuffering h-  when (bufMode == BlockBuffering Nothing) $-    hSetBuffering h (BlockBuffering (Just 16384))--lazyRead :: Handle -> IO Text-lazyRead h = unsafeInterleaveIO $-  withHandle "hGetContents" h $ \hh -> do-    case haType hh of-      ClosedHandle     -> return (hh, L.empty)-      SemiClosedHandle -> lazyReadBuffered h hh-      _                -> ioException-                          (IOError (Just h) IllegalOperation "hGetContents"-                           "illegal handle type" Nothing Nothing)--lazyReadBuffered :: Handle -> Handle__ -> IO (Handle__, Text)-lazyReadBuffered h hh@Handle__{..} = do-   buf <- readIORef haCharBuffer-   (do t <- readChunk hh buf-       ts <- lazyRead h-       return (hh, chunk t ts)) `E.catch` \e -> do-         (hh', _) <- hClose_help hh-         if isEOFError e-           then return $ if isEmptyBuffer buf-                         then (hh', empty)-                         else (hh', L.singleton '\r')-           else E.throwIO (augmentIOError e "hGetContents" h)---- | Read a single line from a handle.-hGetLine :: Handle -> IO Text-hGetLine = hGetLineWith L.fromChunks---- | Write a string to a handle.-hPutStr :: Handle -> Text -> IO ()-hPutStr h = mapM_ (T.hPutStr h) . L.toChunks---- | Write a string to a handle, followed by a newline.-hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h t = hPutStr h t >> hPutChar h '\n'---- | The 'interact' function takes a function of type @Text -> Text@--- as its argument. The entire input from the standard input device is--- passed (lazily) to this function as its argument, and the resulting--- string is output on the standard output device.-interact :: (Text -> Text) -> IO ()-interact f = putStr . f =<< getContents---- | Lazily read all user input on 'stdin' as a single string.-getContents :: IO Text-getContents = hGetContents stdin---- | Read a single line of user input from 'stdin'.-getLine :: IO Text-getLine = hGetLine stdin---- | Write a string to 'stdout'.-putStr :: Text -> IO ()-putStr = hPutStr stdout---- | Write a string to 'stdout', followed by a newline.-putStrLn :: Text -> IO ()-putStrLn = hPutStrLn stdout---- $locale------ /Note/: The behaviour of functions in this module depends on the--- version of GHC you are using.------ Beginning with GHC 6.12, text I\/O is performed using the system or--- handle's current locale and line ending conventions.------ Under GHC 6.10 and earlier, the system I\/O libraries /do not--- support/ locale-sensitive I\/O or line ending conversion.  On these--- versions of GHC, functions in this library all use UTF-8.  What--- does this mean in practice?------ * All data that is read will be decoded as UTF-8.------ * Before data is written, it is first encoded as UTF-8.------ * On both reading and writing, the platform's native newline---   conversion is performed.------ If you must use a non-UTF-8 locale on an older version of GHC, you--- will have to perform the transcoding yourself, e.g. as follows:------ > import qualified Data.ByteString.Lazy as B--- > import Data.Text.Lazy (Text)--- > import Data.Text.Lazy.Encoding (encodeUtf16)--- >--- > putStr_Utf16LE :: Text -> IO ()--- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t)
− Data/Text/Lazy/Internal.hs 
@@ -1,20 +0,0 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}--- |--- Module      : Data.Text.Lazy.Internal--- Copyright   : (c) 2013 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC------ This module has been renamed to 'Data.Text.Internal.Lazy'. This--- name for the module will be removed in the next major release.--module Data.Text.Lazy.Internal-    {-# DEPRECATED "Use Data.Text.Internal.Lazy instead" #-}-    (-      module Data.Text.Internal.Lazy-    ) where--import Data.Text.Internal.Lazy
− Data/Text/Lazy/Read.hs 
@@ -1,192 +0,0 @@-{-# LANGUAGE OverloadedStrings, CPP #-}-#if __GLASGOW_HASKELL__ >= 704-{-# LANGUAGE Safe #-}-#elif __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---- |--- Module      : Data.Text.Lazy.Read--- Copyright   : (c) 2010, 2011 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Functions used frequently when reading textual data.-module Data.Text.Lazy.Read-    (-      Reader-    , decimal-    , hexadecimal-    , signed-    , rational-    , double-    ) where--import Control.Monad (liftM)-import Data.Char (isDigit, isHexDigit)-import Data.Int (Int8, Int16, Int32, Int64)-import Data.Ratio ((%))-import Data.Text.Internal.Read-import Data.Text.Lazy as T-import Data.Word (Word, Word8, Word16, Word32, Word64)---- | Read some text.  If the read succeeds, return its value and the--- remaining text, otherwise an error message.-type Reader a = IReader Text a-type Parser = IParser Text---- | Read a decimal integer.  The input must begin with at least one--- decimal digit, and is consumed until a non-digit or end of string--- is reached.------ This function does not handle leading sign characters.  If you need--- to handle signed input, use @'signed' 'decimal'@.------ /Note/: For fixed-width integer types, this function does not--- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.  If you are worried about overflow, use--- 'Integer' for your result type.-decimal :: Integral a => Reader a-{-# SPECIALIZE decimal :: Reader Int #-}-{-# SPECIALIZE decimal :: Reader Int8 #-}-{-# SPECIALIZE decimal :: Reader Int16 #-}-{-# SPECIALIZE decimal :: Reader Int32 #-}-{-# SPECIALIZE decimal :: Reader Int64 #-}-{-# SPECIALIZE decimal :: Reader Integer #-}-{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}-{-# SPECIALIZE decimal :: Reader Word8 #-}-{-# SPECIALIZE decimal :: Reader Word16 #-}-{-# SPECIALIZE decimal :: Reader Word32 #-}-{-# SPECIALIZE decimal :: Reader Word64 #-}-decimal txt-    | T.null h  = Left "input does not start with a digit"-    | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.span isDigit txt-        go n d = (n * 10 + fromIntegral (digitToInt d))---- | Read a hexadecimal integer, consisting of an optional leading--- @\"0x\"@ followed by at least one hexadecimal digit. Input is--- consumed until a non-hex-digit or end of string is reached.--- This function is case insensitive.------ This function does not handle leading sign characters.  If you need--- to handle signed input, use @'signed' 'hexadecimal'@.------ /Note/: For fixed-width integer types, this function does not--- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.  If you are worried about overflow, use--- 'Integer' for your result type.-hexadecimal :: Integral a => Reader a-{-# SPECIALIZE hexadecimal :: Reader Int #-}-{-# SPECIALIZE hexadecimal :: Reader Integer #-}-hexadecimal txt-    | h == "0x" || h == "0X" = hex t-    | otherwise              = hex txt- where (h,t) = T.splitAt 2 txt--hex :: Integral a => Reader a-{-# SPECIALIZE hexadecimal :: Reader Int #-}-{-# SPECIALIZE hexadecimal :: Reader Int8 #-}-{-# SPECIALIZE hexadecimal :: Reader Int16 #-}-{-# SPECIALIZE hexadecimal :: Reader Int32 #-}-{-# SPECIALIZE hexadecimal :: Reader Int64 #-}-{-# SPECIALIZE hexadecimal :: Reader Integer #-}-{-# SPECIALIZE hexadecimal :: Reader Word #-}-{-# SPECIALIZE hexadecimal :: Reader Word8 #-}-{-# SPECIALIZE hexadecimal :: Reader Word16 #-}-{-# SPECIALIZE hexadecimal :: Reader Word32 #-}-{-# SPECIALIZE hexadecimal :: Reader Word64 #-}-hex txt-    | T.null h  = Left "input does not start with a hexadecimal digit"-    | otherwise = Right (T.foldl' go 0 h, t)-  where (h,t)  = T.span isHexDigit txt-        go n d = (n * 16 + fromIntegral (hexDigitToInt d))---- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and--- apply it to the result of applying the given reader.-signed :: Num a => Reader a -> Reader a-{-# INLINE signed #-}-signed f = runP (signa (P f))---- | Read a rational number.------ This function accepts an optional leading sign character, followed--- by at least one decimal digit.  The syntax similar to that accepted--- by the 'read' function, with the exception that a trailing @\'.\'@--- or @\'e\'@ /not/ followed by a number is not consumed.------ Examples:------ >rational "3"     == Right (3.0, "")--- >rational "3.1"   == Right (3.1, "")--- >rational "3e4"   == Right (30000.0, "")--- >rational "3.1e4" == Right (31000.0, "")--- >rational ".3"    == Left "input does not start with a digit"--- >rational "e3"    == Left "input does not start with a digit"------ Examples of differences from 'read':------ >rational "3.foo" == Right (3.0, ".foo")--- >rational "3e"    == Right (3.0, "e")-rational :: Fractional a => Reader a-{-# SPECIALIZE rational :: Reader Double #-}-rational = floaty $ \real frac fracDenom -> fromRational $-                     real % 1 + frac % fracDenom---- | Read a rational number.------ The syntax accepted by this function is the same as for 'rational'.------ /Note/: This function is almost ten times faster than 'rational',--- but is slightly less accurate.------ The 'Double' type supports about 16 decimal places of accuracy.--- For 94.2% of numbers, this function and 'rational' give identical--- results, but for the remaining 5.8%, this function loses precision--- around the 15th decimal place.  For 0.001% of numbers, this--- function will lose precision at the 13th or 14th decimal place.-double :: Reader Double-double = floaty $ \real frac fracDenom ->-                   fromIntegral real +-                   fromIntegral frac / fromIntegral fracDenom--signa :: Num a => Parser a -> Parser a-{-# SPECIALIZE signa :: Parser Int -> Parser Int #-}-{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-}-{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-}-{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-}-{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-}-{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-}-signa p = do-  sign <- perhaps '+' $ char (\c -> c == '-' || c == '+')-  if sign == '+' then p else negate `liftM` p--char :: (Char -> Bool) -> Parser Char-char p = P $ \t -> case T.uncons t of-                     Just (c,t') | p c -> Right (c,t')-                     _                 -> Left "character does not match"--floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a-{-# INLINE floaty #-}-floaty f = runP $ do-  sign <- perhaps '+' $ char (\c -> c == '-' || c == '+')-  real <- P decimal-  T fraction fracDigits <- perhaps (T 0 0) $ do-    _ <- char (=='.')-    digits <- P $ \t -> Right (fromIntegral . T.length $ T.takeWhile isDigit t, t)-    n <- P decimal-    return $ T n digits-  let e c = c == 'e' || c == 'E'-  power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int)-  let n = if fracDigits == 0-          then if power == 0-               then fromIntegral real-               else fromIntegral real * (10 ^^ power)-          else if power == 0-               then f real fraction (10 ^ fracDigits)-               else f real fraction (10 ^ fracDigits) * (10 ^^ power)-  return $! if sign == '+'-            then n-            else -n
− Data/Text/Read.hs 
@@ -1,200 +0,0 @@-{-# LANGUAGE OverloadedStrings, UnboxedTuples, CPP #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---- |--- Module      : Data.Text.Read--- Copyright   : (c) 2010, 2011 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : GHC------ Functions used frequently when reading textual data.-module Data.Text.Read-    (-      Reader-    , decimal-    , hexadecimal-    , signed-    , rational-    , double-    ) where--import Control.Monad (liftM)-import Data.Char (isDigit, isHexDigit)-import Data.Int (Int8, Int16, Int32, Int64)-import Data.Ratio ((%))-import Data.Text as T-import Data.Text.Internal.Private (span_)-import Data.Text.Internal.Read-import Data.Word (Word, Word8, Word16, Word32, Word64)---- | Read some text.  If the read succeeds, return its value and the--- remaining text, otherwise an error message.-type Reader a = IReader Text a-type Parser a = IParser Text a---- | Read a decimal integer.  The input must begin with at least one--- decimal digit, and is consumed until a non-digit or end of string--- is reached.------ This function does not handle leading sign characters.  If you need--- to handle signed input, use @'signed' 'decimal'@.------ /Note/: For fixed-width integer types, this function does not--- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.  If you are worried about overflow, use--- 'Integer' for your result type.-decimal :: Integral a => Reader a-{-# SPECIALIZE decimal :: Reader Int #-}-{-# SPECIALIZE decimal :: Reader Int8 #-}-{-# SPECIALIZE decimal :: Reader Int16 #-}-{-# SPECIALIZE decimal :: Reader Int32 #-}-{-# SPECIALIZE decimal :: Reader Int64 #-}-{-# SPECIALIZE decimal :: Reader Integer #-}-{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}-{-# SPECIALIZE decimal :: Reader Word8 #-}-{-# SPECIALIZE decimal :: Reader Word16 #-}-{-# SPECIALIZE decimal :: Reader Word32 #-}-{-# SPECIALIZE decimal :: Reader Word64 #-}-decimal txt-    | T.null h  = Left "input does not start with a digit"-    | otherwise = Right (T.foldl' go 0 h, t)-  where (# h,t #)  = span_ isDigit txt-        go n d = (n * 10 + fromIntegral (digitToInt d))---- | Read a hexadecimal integer, consisting of an optional leading--- @\"0x\"@ followed by at least one hexadecimal digit. Input is--- consumed until a non-hex-digit or end of string is reached.--- This function is case insensitive.------ This function does not handle leading sign characters.  If you need--- to handle signed input, use @'signed' 'hexadecimal'@.------ /Note/: For fixed-width integer types, this function does not--- attempt to detect overflow, so a sufficiently long input may give--- incorrect results.  If you are worried about overflow, use--- 'Integer' for your result type.-hexadecimal :: Integral a => Reader a-{-# SPECIALIZE hexadecimal :: Reader Int #-}-{-# SPECIALIZE hexadecimal :: Reader Int8 #-}-{-# SPECIALIZE hexadecimal :: Reader Int16 #-}-{-# SPECIALIZE hexadecimal :: Reader Int32 #-}-{-# SPECIALIZE hexadecimal :: Reader Int64 #-}-{-# SPECIALIZE hexadecimal :: Reader Integer #-}-{-# SPECIALIZE hexadecimal :: Reader Word #-}-{-# SPECIALIZE hexadecimal :: Reader Word8 #-}-{-# SPECIALIZE hexadecimal :: Reader Word16 #-}-{-# SPECIALIZE hexadecimal :: Reader Word32 #-}-{-# SPECIALIZE hexadecimal :: Reader Word64 #-}-hexadecimal txt-    | h == "0x" || h == "0X" = hex t-    | otherwise              = hex txt- where (h,t) = T.splitAt 2 txt--hex :: Integral a => Reader a-{-# SPECIALIZE hex :: Reader Int #-}-{-# SPECIALIZE hex :: Reader Int8 #-}-{-# SPECIALIZE hex :: Reader Int16 #-}-{-# SPECIALIZE hex :: Reader Int32 #-}-{-# SPECIALIZE hex :: Reader Int64 #-}-{-# SPECIALIZE hex :: Reader Integer #-}-{-# SPECIALIZE hex :: Reader Word #-}-{-# SPECIALIZE hex :: Reader Word8 #-}-{-# SPECIALIZE hex :: Reader Word16 #-}-{-# SPECIALIZE hex :: Reader Word32 #-}-{-# SPECIALIZE hex :: Reader Word64 #-}-hex txt-    | T.null h  = Left "input does not start with a hexadecimal digit"-    | otherwise = Right (T.foldl' go 0 h, t)-  where (# h,t #)  = span_ isHexDigit txt-        go n d = (n * 16 + fromIntegral (hexDigitToInt d))---- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and--- apply it to the result of applying the given reader.-signed :: Num a => Reader a -> Reader a-{-# INLINE signed #-}-signed f = runP (signa (P f))---- | Read a rational number.------ This function accepts an optional leading sign character, followed--- by at least one decimal digit.  The syntax similar to that accepted--- by the 'read' function, with the exception that a trailing @\'.\'@--- or @\'e\'@ /not/ followed by a number is not consumed.------ Examples (with behaviour identical to 'read'):------ >rational "3"     == Right (3.0, "")--- >rational "3.1"   == Right (3.1, "")--- >rational "3e4"   == Right (30000.0, "")--- >rational "3.1e4" == Right (31000.0, "")--- >rational ".3"    == Left "input does not start with a digit"--- >rational "e3"    == Left "input does not start with a digit"------ Examples of differences from 'read':------ >rational "3.foo" == Right (3.0, ".foo")--- >rational "3e"    == Right (3.0, "e")-rational :: Fractional a => Reader a-{-# SPECIALIZE rational :: Reader Double #-}-rational = floaty $ \real frac fracDenom -> fromRational $-                     real % 1 + frac % fracDenom---- | Read a rational number.------ The syntax accepted by this function is the same as for 'rational'.------ /Note/: This function is almost ten times faster than 'rational',--- but is slightly less accurate.------ The 'Double' type supports about 16 decimal places of accuracy.--- For 94.2% of numbers, this function and 'rational' give identical--- results, but for the remaining 5.8%, this function loses precision--- around the 15th decimal place.  For 0.001% of numbers, this--- function will lose precision at the 13th or 14th decimal place.-double :: Reader Double-double = floaty $ \real frac fracDenom ->-                   fromIntegral real +-                   fromIntegral frac / fromIntegral fracDenom--signa :: Num a => Parser a -> Parser a-{-# SPECIALIZE signa :: Parser Int -> Parser Int #-}-{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-}-{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-}-{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-}-{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-}-{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-}-signa p = do-  sign <- perhaps '+' $ char (\c -> c == '-' || c == '+')-  if sign == '+' then p else negate `liftM` p--char :: (Char -> Bool) -> Parser Char-char p = P $ \t -> case T.uncons t of-                     Just (c,t') | p c -> Right (c,t')-                     _                 -> Left "character does not match"--floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a-{-# INLINE floaty #-}-floaty f = runP $ do-  sign <- perhaps '+' $ char (\c -> c == '-' || c == '+')-  real <- P decimal-  T fraction fracDigits <- perhaps (T 0 0) $ do-    _ <- char (=='.')-    digits <- P $ \t -> Right (T.length $ T.takeWhile isDigit t, t)-    n <- P decimal-    return $ T n digits-  let e c = c == 'e' || c == 'E'-  power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int)-  let n = if fracDigits == 0-          then if power == 0-               then fromIntegral real-               else fromIntegral real * (10 ^^ power)-          else if power == 0-               then f real fraction (10 ^ fracDigits)-               else f real fraction (10 ^ fracDigits) * (10 ^^ power)-  return $! if sign == '+'-            then n-            else -n
− Data/Text/Show.hs 
@@ -1,91 +0,0 @@-{-# LANGUAGE CPP, MagicHash #-}-{-# OPTIONS_GHC -fno-warn-orphans #-}-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif---- |--- Module      : Data.Text.Show--- Copyright   : (c) 2009-2015 Bryan O'Sullivan------ License     : BSD-style--- Maintainer  : bos@serpentine.com--- Stability   : experimental--- Portability : GHC--module Data.Text.Show-    (-      singleton-    , unpack-    , unpackCString#-    ) where--import Control.Monad.ST (ST)-import Data.Text.Internal (Text(..), empty_, safe)-import Data.Text.Internal.Fusion (stream, unstream)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import GHC.Prim (Addr#)-import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Fusion.Common as S--#if __GLASGOW_HASKELL__ >= 702-import qualified GHC.CString as GHC-#else-import qualified GHC.Base as GHC-#endif--instance Show Text where-    showsPrec p ps r = showsPrec p (unpack ps) r---- | /O(n)/ Convert a 'Text' into a 'String'.  Subject to fusion.-unpack :: Text -> String-unpack = S.unstreamList . stream-{-# INLINE [1] unpack #-}---- | /O(n)/ Convert a literal string into a 'Text'.  Subject to--- fusion.------ This is exposed solely for people writing GHC rewrite rules.------ @since 1.2.1.1-unpackCString# :: Addr# -> Text-unpackCString# addr# = unstream (S.streamCString# addr#)-{-# NOINLINE unpackCString# #-}--{-# RULES "TEXT literal" [1] forall a.-    unstream (S.map safe (S.streamList (GHC.unpackCString# a)))-      = unpackCString# a #-}--{-# RULES "TEXT literal UTF8" [1] forall a.-    unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a)))-      = unpackCString# a #-}--{-# RULES "TEXT empty literal" [1]-    unstream (S.map safe (S.streamList []))-      = empty_ #-}--{-# RULES "TEXT singleton literal" [1] forall a.-    unstream (S.map safe (S.streamList [a]))-      = singleton_ a #-}---- | /O(1)/ Convert a character into a Text.  Subject to fusion.--- Performs replacement on invalid scalar values.-singleton :: Char -> Text-singleton = unstream . S.singleton . safe-{-# INLINE [1] singleton #-}--{-# RULES "TEXT singleton" forall a.-    unstream (S.singleton (safe a))-      = singleton_ a #-}---- This is intended to reduce inlining bloat.-singleton_ :: Char -> Text-singleton_ c = Text (A.run x) 0 len-  where x :: ST s (A.MArray s)-        x = do arr <- A.new len-               _ <- unsafeWrite arr 0 d-               return arr-        len | d < '\x10000' = 1-            | otherwise     = 2-        d = safe c-{-# NOINLINE singleton_ #-}
− Data/Text/Unsafe.hs 
@@ -1,124 +0,0 @@-{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-}--- |--- Module      : Data.Text.Unsafe--- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan--- License     : BSD-style--- Maintainer  : bos@serpentine.com--- Portability : portable------ A module containing unsafe 'Text' operations, for very very careful--- use in heavily tested code.-module Data.Text.Unsafe-    (-      inlineInterleaveST-    , inlinePerformIO-    , unsafeDupablePerformIO-    , Iter(..)-    , iter-    , iter_-    , reverseIter-    , reverseIter_-    , unsafeHead-    , unsafeTail-    , lengthWord16-    , takeWord16-    , dropWord16-    ) where--#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Text.Internal.Encoding.Utf16 (chr2)-import Data.Text.Internal (Text(..))-import Data.Text.Internal.Unsafe (inlineInterleaveST, inlinePerformIO)-import Data.Text.Internal.Unsafe.Char (unsafeChr)-import qualified Data.Text.Array as A-import GHC.IO (unsafeDupablePerformIO)---- | /O(1)/ A variant of 'head' for non-empty 'Text'. 'unsafeHead'--- omits the check for the empty case, so there is an obligation on--- the programmer to provide a proof that the 'Text' is non-empty.-unsafeHead :: Text -> Char-unsafeHead (Text arr off _len)-    | m < 0xD800 || m > 0xDBFF = unsafeChr m-    | otherwise                = chr2 m n-    where m = A.unsafeIndex arr off-          n = A.unsafeIndex arr (off+1)-{-# INLINE unsafeHead #-}---- | /O(1)/ A variant of 'tail' for non-empty 'Text'. 'unsafeTail'--- omits the check for the empty case, so there is an obligation on--- the programmer to provide a proof that the 'Text' is non-empty.-unsafeTail :: Text -> Text-unsafeTail t@(Text arr off len) =-#if defined(ASSERTS)-    assert (d <= len) $-#endif-    Text arr (off+d) (len-d)-  where d = iter_ t 0-{-# INLINE unsafeTail #-}--data Iter = Iter {-# UNPACK #-} !Char {-# UNPACK #-} !Int---- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-16--- array, returning the current character and the delta to add to give--- the next offset to iterate at.-iter :: Text -> Int -> Iter-iter (Text arr off _len) i-    | m < 0xD800 || m > 0xDBFF = Iter (unsafeChr m) 1-    | otherwise                = Iter (chr2 m n) 2-  where m = A.unsafeIndex arr j-        n = A.unsafeIndex arr k-        j = off + i-        k = j + 1-{-# INLINE iter #-}---- | /O(1)/ Iterate one step through a UTF-16 array, returning the--- delta to add to give the next offset to iterate at.-iter_ :: Text -> Int -> Int-iter_ (Text arr off _len) i | m < 0xD800 || m > 0xDBFF = 1-                            | otherwise                = 2-  where m = A.unsafeIndex arr (off+i)-{-# INLINE iter_ #-}---- | /O(1)/ Iterate one step backwards through a UTF-16 array,--- returning the current character and the delta to add (i.e. a--- negative number) to give the next offset to iterate at.-reverseIter :: Text -> Int -> (Char,Int)-reverseIter (Text arr off _len) i-    | m < 0xDC00 || m > 0xDFFF = (unsafeChr m, -1)-    | otherwise                = (chr2 n m,    -2)-  where m = A.unsafeIndex arr j-        n = A.unsafeIndex arr k-        j = off + i-        k = j - 1-{-# INLINE reverseIter #-}---- | /O(1)/ Iterate one step backwards through a UTF-16 array,--- returning the delta to add (i.e. a negative number) to give the--- next offset to iterate at.------ @since 1.1.1.0-reverseIter_ :: Text -> Int -> Int-reverseIter_ (Text arr off _len) i-    | m < 0xDC00 || m > 0xDFFF = -1-    | otherwise                = -2-  where m = A.unsafeIndex arr (off+i)-{-# INLINE reverseIter_ #-}---- | /O(1)/ Return the length of a 'Text' in units of 'Word16'.  This--- is useful for sizing a target array appropriately before using--- 'unsafeCopyToPtr'.-lengthWord16 :: Text -> Int-lengthWord16 (Text _arr _off len) = len-{-# INLINE lengthWord16 #-}---- | /O(1)/ Unchecked take of 'k' 'Word16's from the front of a 'Text'.-takeWord16 :: Int -> Text -> Text-takeWord16 k (Text arr off _len) = Text arr off k-{-# INLINE takeWord16 #-}---- | /O(1)/ Unchecked drop of 'k' 'Word16's from the front of a 'Text'.-dropWord16 :: Int -> Text -> Text-dropWord16 k (Text arr off len) = Text arr (off+k) (len-k)-{-# INLINE dropWord16 #-}
− README.markdown 
@@ -1,31 +0,0 @@-# `text`: Fast, packed Unicode strings, using stream fusion--This package provides the Data.Text library, a library for the space--and time-efficient manipulation of Unicode text in Haskell.--**Please refer to the [package description on Hackage](https://hackage.haskell.org/package/text#description) for more information.**--# Get involved!--Please report bugs via the-[github issue tracker](https://github.com/haskell/text/issues).--Master [git repository](https://github.com/haskell/text):--* `git clone git://github.com/haskell/text.git`--There's also a [Mercurial mirror](https://bitbucket.org/bos/text):--* `hg clone https://bitbucket.org/bos/text`--(You can create and contribute changes using either Mercurial or git.)---# Authors--The base code for this library was originally written by Tom Harper,-based on the stream fusion framework developed by Roman Leshchinskiy,-Duncan Coutts, and Don Stewart.--The core library was fleshed out, debugged, and tested by Bryan-O'Sullivan <bos@serpentine.com>, and he is the current maintainer.
+ README.md view
@@ -0,0 +1,31 @@+# text [![Hackage](http://img.shields.io/hackage/v/text.svg)](https://hackage.haskell.org/package/text) [![Stackage LTS](http://stackage.org/package/text/badge/lts)](http://stackage.org/lts/package/text) [![Stackage Nightly](http://stackage.org/package/text/badge/nightly)](http://stackage.org/nightly/package/text)++Haskell library for space- and time-efficient operations over Unicode text.++# Get involved!++Please report bugs via the+[github issue tracker](https://github.com/haskell/text/issues).++The main repo:++```bash+git clone https://github.com/haskell/text+```++To run benchmarks please clone and unpack test files:++```bash+cd text+git clone https://github.com/haskell/text-test-data benchmarks/text-test-data+make -Cbenchmarks/text-test-data+```++# Authors++The base code for this library was originally written by Tom Harper,+based on the stream fusion framework developed by Roman Leshchinskiy,+Duncan Coutts, and Don Stewart.++The core library was fleshed out, debugged, and tested by Bryan+O'Sullivan. Transition from UTF-16 to UTF-8 is by Andrew Lelechenko.
− benchmarks/Setup.hs 
@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain
− benchmarks/cbits/time_iconv.c 
@@ -1,35 +0,0 @@-#include <iconv.h>-#include <stdlib.h>-#include <stdio.h>-#include <stdint.h>--int time_iconv(char *srcbuf, size_t srcbufsize)-{-  uint16_t *destbuf = NULL;-  size_t destbufsize;-  static uint16_t *origdestbuf;-  static size_t origdestbufsize;-  iconv_t ic = (iconv_t) -1;-  int ret = 0;--  if (ic == (iconv_t) -1) {-    ic = iconv_open("UTF-16LE", "UTF-8");-    if (ic == (iconv_t) -1) {-      ret = -1;-      goto done;-    }-  }-  -  destbufsize = srcbufsize * sizeof(uint16_t);-  if (destbufsize > origdestbufsize) {-    free(origdestbuf);-    origdestbuf = destbuf = malloc(origdestbufsize = destbufsize);-  } else {-    destbuf = origdestbuf;-  }--  iconv(ic, &srcbuf, &srcbufsize, (char**) &destbuf, &destbufsize);-- done:-  return ret;-}
benchmarks/haskell/Benchmarks.hs view
@@ -1,22 +1,29 @@--- | Main module to run the micro benchmarks---+{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-}+ module Main     ( main     ) where -import Criterion.Main (Benchmark, defaultMain, bgroup)+import Test.Tasty.Bench (defaultMain, bgroup, env) import System.FilePath ((</>))-import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8)+import System.IO +#ifdef mingw32_HOST_OS+import System.IO.Temp (emptySystemTempFile)+import System.Directory (removeFile)+#endif+ import qualified Benchmarks.Builder as Builder import qualified Benchmarks.Concat as Concat import qualified Benchmarks.DecodeUtf8 as DecodeUtf8 import qualified Benchmarks.EncodeUtf8 as EncodeUtf8 import qualified Benchmarks.Equality as Equality import qualified Benchmarks.FileRead as FileRead+import qualified Benchmarks.FileWrite as FileWrite import qualified Benchmarks.FoldLines as FoldLines-import qualified Benchmarks.Mul as Mul+import qualified Benchmarks.Micro as Micro+import qualified Benchmarks.Multilang as Multilang import qualified Benchmarks.Pure as Pure import qualified Benchmarks.ReadNumbers as ReadNumbers import qualified Benchmarks.Replace as Replace@@ -31,51 +38,74 @@ import qualified Benchmarks.Programs.StripTags as Programs.StripTags import qualified Benchmarks.Programs.Throughput as Programs.Throughput -main :: IO ()-main = benchmarks >>= defaultMain--benchmarks :: IO [Benchmark]-benchmarks = do-    sink <- openFile "/dev/null" WriteMode+mkSink :: IO (FilePath, Handle)+mkSink = do+#ifdef mingw32_HOST_OS+    sinkFn <- emptySystemTempFile "dev.null"+#else+    let sinkFn = "/dev/null"+#endif+    sink <- openFile sinkFn WriteMode     hSetEncoding sink utf8+    pure (sinkFn, sink) -    -- Traditional benchmarks-    bs <- sequence+rmSink :: FilePath -> IO ()+#ifdef mingw32_HOST_OS+rmSink = removeFile+#else+rmSink _ = pure ()+#endif++main :: IO ()+main = do+    let tf = ("benchmarks/text-test-data" </>)+    -- Cannot use envWithCleanup, because there is no instance NFData Handle+    (sinkFn, sink) <- mkSink+    (fileWriteBenchmarks, fileWriteCleanup) <- FileWrite.mkFileWriteBenchmarks $ do+      (fp, h) <- mkSink+      return (h, rmSink fp)+    defaultMain         [ Builder.benchmark         , Concat.benchmark-        , DecodeUtf8.benchmark "html" (tf "libya-chinese.html")-        , DecodeUtf8.benchmark "xml" (tf "yiwiki.xml")-        , DecodeUtf8.benchmark "ascii" (tf "ascii.txt")-        , DecodeUtf8.benchmark "russian" (tf "russian.txt")-        , DecodeUtf8.benchmark "japanese" (tf "japanese.txt")-        , EncodeUtf8.benchmark "επανάληψη 竺法蘭共譯"-        , Equality.benchmark (tf "japanese.txt")+        , Micro.benchmark+        , bgroup "DecodeUtf8"+            [ env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html")+            , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml")+            , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmark "ascii")+            , env (DecodeUtf8.initEnv (tf "russian.txt")) (DecodeUtf8.benchmark  "russian")+            , env (DecodeUtf8.initEnv (tf "japanese.txt")) (DecodeUtf8.benchmark "japanese")+            , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmarkASCII)+            ]+        , bgroup "EncodeUtf8"+            [ EncodeUtf8.benchmark "non-ASCII" "επανάληψη 竺法蘭共譯"+            , EncodeUtf8.benchmark "ASCII" "lorem ipsum"+            ]+        , env (Equality.initEnv (tf "japanese.txt")) Equality.benchmark         , FileRead.benchmark (tf "russian.txt")+        , fileWriteBenchmarks         , FoldLines.benchmark (tf "russian.txt")-        , Mul.benchmark-        , Pure.benchmark "tiny" (tf "tiny.txt")-        , Pure.benchmark "ascii" (tf "ascii-small.txt")-        -- , Pure.benchmark "france" (tf "france.html")-        , Pure.benchmark "russian" (tf "russian-small.txt")-        , Pure.benchmark "japanese" (tf "japanese.txt")-        , ReadNumbers.benchmark (tf "numbers.txt")-        , Replace.benchmark (tf "russian.txt") "принимая" "своем"-        , Search.benchmark (tf "russian.txt") "принимая"-        , Stream.benchmark (tf "russian.txt")-        , WordFrequencies.benchmark (tf "russian.txt")-        ]--    -- Program-like benchmarks-    ps <- bgroup "Programs" `fmap` sequence-        [ Programs.BigTable.benchmark sink-        , Programs.Cut.benchmark (tf "russian.txt") sink 20 40-        , Programs.Fold.benchmark (tf "russian.txt") sink-        , Programs.Sort.benchmark (tf "russian.txt") sink-        , Programs.StripTags.benchmark (tf "yiwiki.xml") sink-        , Programs.Throughput.benchmark (tf "russian.txt") sink+        , Multilang.benchmark+        , bgroup "Pure"+            [ env (Pure.initEnv (tf "tiny.txt")) (Pure.benchmark "tiny")+            , env (Pure.initEnv (tf "ascii-small.txt")) (Pure.benchmark "ascii-small")+            , env (Pure.initEnv (tf "ascii.txt")) (Pure.benchmark "ascii")+            , env (Pure.initEnv (tf "english.txt")) (Pure.benchmark "english")+            , env (Pure.initEnv (tf "russian-small.txt")) (Pure.benchmark "russian")+            , env (Pure.initEnv (tf "japanese.txt")) (Pure.benchmark "japanese")+            ]+        , env (ReadNumbers.initEnv (tf "numbers.txt")) ReadNumbers.benchmark+        , env (Replace.initEnv (tf "russian.txt")) (Replace.benchmark "принимая" "своем")+        , env (Search.initEnv (tf "russian.txt")) (Search.benchmark "принимая")+        , env (Stream.initEnv (tf "russian.txt")) Stream.benchmark+        , env (WordFrequencies.initEnv (tf "russian.txt")) WordFrequencies.benchmark+        , bgroup "Programs"+            [ Programs.BigTable.benchmark sink+            , Programs.Cut.benchmark (tf "russian.txt") sink 20 40+            , Programs.Fold.benchmark (tf "russian.txt") sink+            , Programs.Sort.benchmark (tf "russian.txt") sink+            , Programs.StripTags.benchmark (tf "yiwiki.xml") sink+            , Programs.Throughput.benchmark (tf "russian.txt") sink+            ]         ]--    return $ bs ++ [ps]-  where-    -- Location of a test file-    tf = ("../tests/text-test-data" </>)+    rmSink sinkFn+    fileWriteCleanup
benchmarks/haskell/Benchmarks/Builder.hs view
@@ -4,36 +4,24 @@ -- -- * Concatenating many small strings using a builder ---{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Builder     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, nf)-import Data.Binary.Builder as B+import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) import Data.ByteString.Char8 ()-import Data.Monoid (mconcat, mempty)-import qualified Blaze.ByteString.Builder as Blaze-import qualified Blaze.ByteString.Builder.Char.Utf8 as Blaze-import qualified Data.ByteString as SB-import qualified Data.ByteString.Lazy as LB import qualified Data.Text as T import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Builder as LTB import qualified Data.Text.Lazy.Builder.Int as Int import Data.Int (Int64) -benchmark :: IO Benchmark-benchmark = return $ bgroup "Builder"+benchmark :: Benchmark+benchmark = bgroup "Builder"     [ bgroup "Comparison"       [ bench "LazyText" $ nf           (LT.length . LTB.toLazyText . mconcat . map LTB.fromText) texts-      , bench "Binary" $ nf-          (LB.length . B.toLazyByteString . mconcat . map B.fromByteString)-          byteStrings-      , bench "Blaze" $ nf-          (LB.length . Blaze.toLazyByteString . mconcat . map Blaze.fromString)-          strings       ]     , bgroup "Int"       [ bgroup "Decimal"@@ -63,13 +51,3 @@ texts :: [T.Text] texts = take 200000 $ cycle ["foo", "λx", "由の"] {-# NOINLINE texts #-}---- Note that the non-ascii characters will be chopped-byteStrings :: [SB.ByteString]-byteStrings = take 200000 $ cycle ["foo", "λx", "由の"]-{-# NOINLINE byteStrings #-}---- Note that the non-ascii characters will be chopped-strings :: [String]-strings = take 200000 $ cycle ["foo", "λx", "由の"]-{-# NOINLINE strings #-}
benchmarks/haskell/Benchmarks/Concat.hs view
@@ -3,11 +3,11 @@ module Benchmarks.Concat (benchmark) where  import Control.Monad.Trans.Writer-import Criterion (Benchmark, bgroup, bench, whnf)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import Data.Text as T -benchmark :: IO Benchmark-benchmark = return $ bgroup "Concat"+benchmark :: Benchmark+benchmark = bgroup "Concat"   [ bench "append" $ whnf (append4 "Text 1" "Text 2" "Text 3") "Text 4"   , bench "concat" $ whnf (concat4 "Text 1" "Text 2" "Text 3") "Text 4"   , bench "write"  $ whnf (write4  "Text 1" "Text 2" "Text 3") "Text 4"
benchmarks/haskell/Benchmarks/DecodeUtf8.hs view
@@ -15,18 +15,13 @@ -- The latter are used for testing stream fusion. -- module Benchmarks.DecodeUtf8-    ( benchmark+    ( initEnv+    , benchmark+    , benchmarkASCII     ) where -import Foreign.C.Types-import Data.ByteString.Internal (ByteString(..)) import Data.ByteString.Lazy.Internal (ByteString(..))-import Foreign.Ptr (Ptr, plusPtr)-import Foreign.ForeignPtr (withForeignPtr)-import Data.Word (Word8)-import qualified Criterion as C-import Criterion (Benchmark, bgroup, nf, whnfIO)-import qualified Codec.Binary.UTF8.Generic as U8+import Test.Tasty.Bench import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T@@ -34,34 +29,39 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: String -> FilePath -> IO Benchmark-benchmark kind fp = do+type Env = (B.ByteString, BL.ByteString)++initEnv :: FilePath -> IO Env+initEnv fp = do     bs  <- B.readFile fp     lbs <- BL.readFile fp-    let bench name = C.bench (name ++ "+" ++ kind)-        decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of+    return (bs, lbs)++benchmark :: String -> Env -> Benchmark+benchmark kind ~(bs, lbs) =+    let decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of                                         T.Some t0 _ f0 -> t0 : go f0 bs0           where go f (Chunk b bs1) = case f b of                                        T.Some t1 _ f1 -> t1 : go f1 bs1                 go _ _ = []         decodeStream _ = []-    return $ bgroup "DecodeUtf8"+    in bgroup kind         [ bench "Strict" $ nf T.decodeUtf8 bs         , bench "Stream" $ nf decodeStream lbs-        , bench "IConv" $ whnfIO $ iconv bs         , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs         , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs         , bench "Lazy" $ nf TL.decodeUtf8 lbs         , bench "LazyLength" $ nf (TL.length . TL.decodeUtf8) lbs         , bench "LazyInitLength" $ nf (TL.length . TL.init . TL.decodeUtf8) lbs-        , bench "StrictStringUtf8" $ nf U8.toString bs-        , bench "StrictStringUtf8Length" $ nf (length . U8.toString) bs-        , bench "LazyStringUtf8" $ nf U8.toString lbs-        , bench "LazyStringUtf8Length" $ nf (length . U8.toString) lbs         ] -iconv :: B.ByteString -> IO CInt-iconv (PS fp off len) = withForeignPtr fp $ \ptr ->-                        time_iconv (ptr `plusPtr` off) (fromIntegral len)--foreign import ccall unsafe time_iconv :: Ptr Word8 -> CSize -> IO CInt+benchmarkASCII :: Env -> Benchmark+benchmarkASCII ~(bs, lbs) =+    bgroup "ascii"+        [ bench "strict decodeUtf8" $ nf T.decodeUtf8 bs+        , bench "strict decodeLatin1" $ nf T.decodeLatin1 bs+        , bench "strict decodeASCII" $ nf T.decodeASCII bs+        , bench "lazy decodeUtf8" $ nf TL.decodeUtf8 lbs+        , bench "lazy decodeLatin1" $ nf TL.decodeLatin1 lbs+        , bench "lazy decodeASCII" $ nf TL.decodeASCII lbs+        ]
benchmarks/haskell/Benchmarks/EncodeUtf8.hs view
@@ -10,19 +10,23 @@     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnf)+import Test.Tasty.Bench (Benchmark, bgroup, bench, nf, whnf) import qualified Data.ByteString as B+import qualified Data.ByteString.Builder as B+import qualified Data.ByteString.Builder.Prim as BP import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: String -> IO Benchmark-benchmark string = do-    return $ bgroup "EncodeUtf8"+benchmark :: String -> String -> Benchmark+benchmark name string =+    bgroup name         [ bench "Text"     $ whnf (B.length . T.encodeUtf8)   text         , bench "LazyText" $ whnf (BL.length . TL.encodeUtf8) lazyText+        , bench "Text/encodeUtf8Builder" $ nf (B.toLazyByteString . T.encodeUtf8Builder) text+        , bench "Text/encodeUtf8BuilderEscaped" $ nf (B.toLazyByteString . T.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) text         ]   where     -- The string in different formats
benchmarks/haskell/Benchmarks/Equality.hs view
@@ -6,10 +6,11 @@ -- * Comparison of strings (Eq instance) -- module Benchmarks.Equality-    ( benchmark+    ( initEnv+    , benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnf)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as BL import qualified Data.Text as T@@ -17,22 +18,17 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL -benchmark :: FilePath -> IO Benchmark-benchmark fp = do+type Env = (T.Text, TL.Text)++initEnv :: FilePath -> IO Env+initEnv fp = do   b <- B.readFile fp   bl1 <- BL.readFile fp-  -- A lazy bytestring is a list of chunks. When we do not explicitly create two-  -- different lazy bytestrings at a different address, the bytestring library-  -- will compare the chunk addresses instead of the chunk contents. This is why-  -- we read the lazy bytestring twice here.-  bl2 <- BL.readFile fp-  l <- readFile fp-  let t  = T.decodeUtf8 b-      tl = TL.decodeUtf8 bl1-  return $ bgroup "Equality"+  return (T.decodeUtf8 b, TL.decodeUtf8 bl1)++benchmark :: Env -> Benchmark+benchmark ~(t, tl) =+  bgroup "Equality"     [ bench "Text" $ whnf (== T.init t `T.snoc` '\xfffd') t     , bench "LazyText" $ whnf (== TL.init tl `TL.snoc` '\xfffd') tl-    , bench "ByteString" $ whnf (== B.init b `B.snoc` '\xfffd') b-    , bench "LazyByteString" $ whnf (== BL.init bl2 `BL.snoc` '\xfffd') bl1-    , bench "String" $ whnf (== init l ++ "\xfffd") l     ]
benchmarks/haskell/Benchmarks/FileRead.hs view
@@ -4,12 +4,14 @@ -- -- * Reading a file from the disk --++{-# LANGUAGE CPP #-}+ module Benchmarks.FileRead     ( benchmark     ) where -import Control.Applicative ((<$>))-import Criterion (Benchmark, bgroup, bench, whnfIO)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) import qualified Data.ByteString as SB import qualified Data.ByteString.Lazy as LB import qualified Data.Text as T@@ -19,12 +21,9 @@ import qualified Data.Text.Lazy.Encoding as LT import qualified Data.Text.Lazy.IO as LT -benchmark :: FilePath -> IO Benchmark-benchmark p = return $ bgroup "FileRead"-    [ bench "String" $ whnfIO $ length <$> readFile p-    , bench "ByteString" $ whnfIO $ SB.length <$> SB.readFile p-    , bench "LazyByteString" $ whnfIO $ LB.length <$> LB.readFile p-    , bench "Text" $ whnfIO $ T.length <$> T.readFile p+benchmark :: FilePath -> Benchmark+benchmark p = bgroup "FileRead"+    [ bench "Text" $ whnfIO $ T.length <$> T.readFile p     , bench "LazyText" $ whnfIO $ LT.length <$> LT.readFile p     , bench "TextByteString" $ whnfIO $         (T.length . T.decodeUtf8) <$> SB.readFile p
+ benchmarks/haskell/Benchmarks/FileWrite.hs view
@@ -0,0 +1,135 @@+-- | Benchmarks simple file writing+--+-- Tested in this benchmark:+--+-- * Writing a file to the disk+--++{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP #-}++module Benchmarks.FileWrite+    ( mkFileWriteBenchmarks+    ) where++import Control.DeepSeq (NFData, deepseq)+import Data.Bifunctor (first)+import Data.List (intercalate, intersperse)+import Data.String (fromString)+import Data.Text (StrictText)+import Data.Text.Internal.Lazy (LazyText, defaultChunkSize)+import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(..), hSetBuffering, hSetNewlineMode)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfAppIO)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Data.Text.IO.Utf8 as Utf8+import qualified Data.Text.Lazy as L+import qualified Data.Text.Lazy.IO as L++mkFileWriteBenchmarks :: IO (Handle, IO ()) -> IO (Benchmark, IO ())+mkFileWriteBenchmarks mkSinkNRemove = do+  let writeData = L.cycle $ fromString [minBound..maxBound]++#ifdef ExtendedBenchmarks+      lengths = [0..5] <> [10,20..100] <> [1000,3000,10000,100000]+#else+      lengths = [0,1,100,3000,10000,100000]+#endif++      testGroup :: NFData text => (Handle -> text -> IO ()) -> ((String, StrictText -> text)) -> Newline -> BufferMode -> IO (Benchmark, IO ())+      testGroup hPutStr (textCharacteristics, select) nl mode = do+        (h, removeFile) <- mkSinkNRemove+        hSetBuffering h mode+        hSetNewlineMode h $ NewlineMode nl nl+        pure+          ( bgroup (intercalate " " [textCharacteristics, show nl, show mode]) $+            lengths <&> \n -> let+              t = select $ L.toStrict $ L.take n writeData+              in bench ("length " <> show n)+                $ deepseq t+                $ whnfAppIO (hPutStr h) t+          , removeFile+          )++  sequenceGroup "FileWrite hPutStr"+#ifdef ExtendedBenchmarks+    [ testGroup T.hPutStr strict                  LF   NoBuffering+    , testGroup L.hPutStr lazy                    LF   NoBuffering++    , testGroup T.hPutStr strict                  LF   LineBuffering+    , testGroup T.hPutStr strict                  CRLF LineBuffering+    , testGroup T.hPutStr strictNewlines          LF   LineBuffering+    , testGroup T.hPutStr strictNewlines          CRLF LineBuffering++    , testGroup L.hPutStr lazy                    LF   LineBuffering+    , testGroup L.hPutStr lazy                    CRLF LineBuffering+    , testGroup L.hPutStr lazySmallChunks         LF   LineBuffering+    , testGroup L.hPutStr lazySmallChunks         CRLF LineBuffering+    , testGroup L.hPutStr lazyNewlines            LF   LineBuffering+    , testGroup L.hPutStr lazyNewlines            CRLF LineBuffering+    , testGroup L.hPutStr lazySmallChunksNewlines LF   LineBuffering+    , testGroup L.hPutStr lazySmallChunksNewlines CRLF LineBuffering++    , testGroup T.hPutStr strict                  LF   (BlockBuffering Nothing)+    , testGroup T.hPutStr strict                  CRLF (BlockBuffering Nothing)+    , testGroup T.hPutStr strictNewlines          LF   (BlockBuffering Nothing)+    , testGroup T.hPutStr strictNewlines          CRLF (BlockBuffering Nothing)++    , testGroup L.hPutStr lazy                    LF   (BlockBuffering Nothing)+    , testGroup L.hPutStr lazy                    CRLF (BlockBuffering Nothing)+    , testGroup L.hPutStr lazySmallChunks         LF   (BlockBuffering Nothing)+    , testGroup L.hPutStr lazySmallChunks         CRLF (BlockBuffering Nothing)+    , testGroup L.hPutStr lazyNewlines            LF   (BlockBuffering Nothing)+    , testGroup L.hPutStr lazyNewlines            CRLF (BlockBuffering Nothing)+    , testGroup L.hPutStr lazySmallChunksNewlines LF   (BlockBuffering Nothing)+    , testGroup L.hPutStr lazySmallChunksNewlines CRLF (BlockBuffering Nothing)++    , sequenceGroup "UTF-8"+      [ testGroup Utf8.hPutStr strict LF NoBuffering+      , testGroup Utf8.hPutStr strict LF LineBuffering+      , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing)+      ]+    ]+#else+    [ testGroup T.hPutStr strictNewlines LF LineBuffering+    , testGroup T.hPutStr strictNewlines CRLF LineBuffering++    , testGroup T.hPutStr strict LF (BlockBuffering Nothing)+    , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing)++    , testGroup L.hPutStr lazyNewlines LF LineBuffering+    , testGroup L.hPutStr lazyNewlines CRLF LineBuffering++    , testGroup L.hPutStr lazy LF (BlockBuffering Nothing)+    , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing)++    , sequenceGroup "UTF-8"+      [ testGroup Utf8.hPutStr strict LF LineBuffering+      , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing)+      ]+    ]+#endif++  where+  lazy, lazyNewlines :: (String, StrictText -> LazyText)+  lazy                    = ("lazy",                            L.fromChunks . T.chunksOf defaultChunkSize)+  lazyNewlines            = ("lazy many newlines",              snd lazy . snd strictNewlines)++#ifdef ExtendedBenchmarks+  lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText)+  lazySmallChunks         = ("lazy small chunks",               L.fromChunks . T.chunksOf 10)+  lazySmallChunksNewlines = ("lazy small chunks many newlines", snd lazySmallChunks . snd strictNewlines)+#endif++  strict, strictNewlines :: (String, StrictText -> StrictText)+  strict                  = ("strict",                          id)+  strictNewlines          = ("strict many newlines",            mconcat . intersperse "\n" . T.chunksOf 5)++  sequenceGroup groupName tgs+    =   first (bgroup groupName)+    .   foldr (\(b,r) (bs,rs) -> (b:bs,r>>rs)) ([], return ())+    <$> sequence tgs++(<&>) :: Functor f => f a -> (a -> b) -> f b+(<&>) = flip fmap+
benchmarks/haskell/Benchmarks/FoldLines.hs view
@@ -10,16 +10,14 @@     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnfIO)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO) import System.IO-import qualified Data.ByteString as B import qualified Data.Text as T import qualified Data.Text.IO as T -benchmark :: FilePath -> IO Benchmark-benchmark fp = return $ bgroup "ReadLines"+benchmark :: FilePath -> Benchmark+benchmark fp = bgroup "ReadLines"     [ bench "Text"       $ withHandle $ foldLinesT (\n _ -> n + 1) (0 :: Int)-    , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int)     ]   where     withHandle f = whnfIO $ do@@ -42,17 +40,3 @@                 l <- T.hGetLine h                 let z' = f z l in go z' {-# INLINE foldLinesT #-}---- | ByteString line fold----foldLinesB :: (a -> B.ByteString -> a) -> a -> Handle -> IO a-foldLinesB f z0 h = go z0-  where-    go !z = do-        eof <- hIsEOF h-        if eof-            then return z-            else do-                l <- B.hGetLine h-                let z' = f z l in go z'-{-# INLINE foldLinesB #-}
+ benchmarks/haskell/Benchmarks/Micro.hs view
@@ -0,0 +1,33 @@+-- | Benchmarks on artificial data. ++module Benchmarks.Micro (benchmark) where++import qualified Data.List.NonEmpty as NE+import qualified Data.Text.Lazy as TL+import qualified Data.Text as T+import Test.Tasty.Bench (Benchmark, Benchmarkable, bgroup, bcompareWithin, bench, nf)++benchmark :: Benchmark+benchmark = bgroup "Micro"+  [ blinear "lazy-inits--last" 500000 2 0.1 $ \len ->+      nf (NE.last . TL.initsNE) (chunks len)+  , blinear "lazy-inits--map-take1" 500000 2 0.1 $ \len ->+      nf (map (TL.take 1) . TL.inits) (chunks len)+  ]++chunks :: Int -> TL.Text+chunks n = TL.fromChunks (replicate n (T.pack "a"))++-- Check that running an action with input length (m * baseLen)+-- runs m times slower than the same action with input length baseLen.+blinear :: String  -- ^ Name (must be globally unique!)+        -> Int     -- ^ Base length+        -> Int     -- ^ Multiplier m+        -> Double  -- ^ Slack s+        -> (Int -> Benchmarkable)  -- ^ Action to measure, parameterized by input length+        -> Benchmark+blinear name baseLen m s run = bgroup name+  [ bench "baseline" $ run baseLen+  , bcompareWithin (fromIntegral m * (1 - s)) (fromIntegral m * (1 + s)) (name ++ ".baseline") $+      bench ("x" ++ show m) $ run (m * baseLen)+  ]
− benchmarks/haskell/Benchmarks/Mul.hs 
@@ -1,138 +0,0 @@-module Benchmarks.Mul (benchmark) where--import Control.Exception (evaluate)-import Criterion.Main-import Data.Int (Int32, Int64)-import Data.Text.Internal (mul32, mul64)-import qualified Data.Vector.Unboxed as U--oldMul :: Int64 -> Int64 -> Int64-oldMul m n-    | n == 0                 = 0-    | m <= maxBound `quot` n = m * n-    | otherwise              = error "overflow"--benchmark :: IO Benchmark-benchmark = do-  _ <- evaluate testVector32-  _ <- evaluate testVector64-  return $ bgroup "Mul" [-      bench "oldMul" $ whnf (U.map (uncurry oldMul)) testVector64-    , bench "mul64" $ whnf (U.map (uncurry mul64)) testVector64-    , bench "*64" $ whnf (U.map (uncurry (*))) testVector64-    , bench "mul32" $ whnf (U.map (uncurry mul32)) testVector32-    , bench "*32" $ whnf (U.map (uncurry (*))) testVector32-    ]--testVector64 :: U.Vector (Int64,Int64)-testVector64 = U.fromList [-  (0,1248868987182846646),(169004623633872,24458),(482549039517835,7614),-  (372,8157063115504364),(27,107095594861148252),(3,63249878517962420),-  (4363,255694473572912),(86678474,1732634806),(1572453024,1800489338),-  (9384523143,77053781),(49024709555,75095046),(7,43457620410239131),-  (8,8201563008844571),(387719037,1520696708),(189869238220197,1423),-  (46788016849611,23063),(503077742109974359,0),(104,1502010908706487),-  (30478140346,207525518),(80961140129236192,14),(4283,368012829143675),-  (1028719181728108146,6),(318904,5874863049591),(56724427166898,110794),-  (234539368,31369110449),(2,251729663598178612),(103291548194451219,5),-  (76013,5345328755566),(1769631,2980846129318),(40898,60598477385754),-  (0,98931348893227155),(573555872156917492,3),(318821187115,4476566),-  (11152874213584,243582),(40274276,16636653248),(127,4249988676030597),-  (103543712111871836,5),(71,16954462148248238),(3963027173504,216570),-  (13000,503523808916753),(17038308,20018685905),(0,510350226577891549),-  (175898,3875698895405),(425299191292676,5651),(17223451323664536,50),-  (61755131,14247665326),(0,1018195131697569303),(36433751497238985,20),-  (3473607861601050,1837),(1392342328,1733971838),(225770297367,3249655),-  (14,127545244155254102),(1751488975299136,2634),(3949208,504190668767),-  (153329,831454434345),(1066212122928663658,2),(351224,2663633539556),-  (344565,53388869217),(35825609350446863,54),(276011553660081475,10),-  (1969754174790470349,3),(35,68088438338633),(506710,3247689556438),-  (11099382291,327739909),(105787303549,32824363),(210366111,14759049409),-  (688893241579,3102676),(8490,70047474429581),(152085,29923000251880),-  (5046974599257095,400),(4183167795,263434071),(10089728,502781960687),-  (44831977765,4725378),(91,8978094664238578),(30990165721,44053350),-  (1772377,149651820860),(243420621763408572,4),(32,5790357453815138),-  (27980806337993771,5),(47696295759774,20848),(1745874142313778,1098),-  (46869334770121,1203),(886995283,1564424789),(40679396544,76002479),-  (1,672849481568486995),(337656187205,3157069),(816980552858963,6003),-  (2271434085804831543,1),(0,1934521023868747186),(6266220038281,15825),-  (4160,107115946987394),(524,246808621791561),(0,1952519482439636339),-  (128,2865935904539691),(1044,3211982069426297),(16000511542473,88922),-  (1253596745404082,2226),(27041,56836278958002),(23201,49247489754471),-  (175906590497,21252392),(185163584757182295,24),(34742225226802197,150),-  (2363228,250824838408),(216327527109550,45),(24,81574076994520675),-  (28559899906542,15356),(10890139774837133,511),(2293,707179303654492),-  (2749366833,40703233),(0,4498229704622845986),(439,4962056468281937),-  (662,1453820621089921),(16336770612459631,220),(24282989393,74239137),-  (2724564648490195,3),(743672760,124992589),(4528103,704330948891),-  (6050483122491561,250),(13322953,13594265152),(181794,22268101450214),-  (25957941712,75384092),(43352,7322262295009),(32838,52609059549923),-  (33003585202001564,2),(103019,68430142267402),(129918230800,8742978),-  (0,2114347379589080688),(2548,905723041545274),(222745067962838382,0),-  (1671683850790425181,1),(455,4836932776795684),(794227702827214,6620),-  (212534135175874,1365),(96432431858,29784975),(466626763743380,3484),-  (29793949,53041519613),(8359,309952753409844),(3908960585331901,26),-  (45185288970365760,114),(10131829775,68110174),(58039242399640479,83),-  (628092278238719399,6),(1,196469106875361889),(302336625,16347502444),-  (148,3748088684181047),(1,1649096568849015456),(1019866864,2349753026),-  (8211344830,569363306),(65647579546873,34753),(2340190,1692053129069),-  (64263301,30758930355),(48681618072372209,110),(7074794736,47640197),-  (249634721521,7991792),(1162917363807215,232),(7446433349,420634045),-  (63398619383,60709817),(51359004508011,14200),(131788797028647,7072),-  (52079887791430043,7),(7,136277667582599838),(28582879735696,50327),-  (1404582800566278,833),(469164435,15017166943),(99567079957578263,49),-  (1015285971,3625801566),(321504843,4104079293),(5196954,464515406632),-  (114246832260876,7468),(8149664437,487119673),(12265299,378168974869),-  (37711995764,30766513),(3971137243,710996152),(483120070302,603162),-  (103009942,61645547145),(8476344625340,6987),(547948761229739,1446),-  (42234,18624767306301),(13486714173011,58948),(4,198309153268019840),-  (9913176974,325539248),(28246225540203,116822),(2882463945582154,18),-  (959,25504987505398),(3,1504372236378217710),(13505229956793,374987),-  (751661959,457611342),(27375926,36219151769),(482168869,5301952074),-  (1,1577425863241520640),(714116235611821,1164),(904492524250310488,0),-  (5983514941763398,68),(10759472423,23540686),(72539568471529,34919),-  (4,176090672310337473),(938702842110356453,1),(673652445,3335287382),-  (3111998893666122,917),(1568013,3168419765469)]--testVector32 :: U.Vector (Int32,Int32)-testVector32 = U.fromList [-  (39242,410),(0,100077553),(2206,9538),(509400240,1),(38048,6368),-  (1789,651480),(2399,157032),(701,170017),(5241456,14),(11212,70449),-  (1,227804876),(749687254,1),(74559,2954),(1158,147957),(410604456,1),-  (170851,1561),(92643422,1),(6192,180509),(7,24202210),(3440,241481),-  (5753677,5),(294327,1622),(252,4454673),(127684121,11),(28315800,30),-  (340370905,0),(1,667887987),(592782090,1),(49023,27641),(750,290387),-  (72886,3847),(0,301047933),(3050276,473),(1,788366142),(59457,15813),-  (637726933,1),(1135,344317),(853616,264),(696816,493),(7038,12046),-  (125219574,4),(803694088,1),(107081726,1),(39294,21699),(16361,38191),-  (132561123,12),(1760,23499),(847543,484),(175687349,1),(2963,252678),-  (6248,224553),(27596,4606),(5422922,121),(1542,485890),(131,583035),-  (59096,4925),(3637115,132),(0,947225435),(86854,6794),(2984745,339),-  (760129569,1),(1,68260595),(380835652,2),(430575,2579),(54514,7211),-  (15550606,3),(9,27367402),(3007053,207),(7060988,60),(28560,27130),-  (1355,21087),(10880,53059),(14563646,4),(461886361,1),(2,169260724),-  (241454126,2),(406797,1),(61631630,16),(44473,5943),(63869104,12),-  (950300,1528),(2113,62333),(120817,9358),(100261456,1),(426764723,1),-  (119,12723684),(3,53358711),(4448071,18),(1,230278091),(238,232102),-  (8,57316440),(42437979,10),(6769,19555),(48590,22006),(11500585,79),-  (2808,97638),(42,26952545),(11,32104194),(23954638,1),(785427272,0),-  (513,81379),(31333960,37),(897772,1009),(4,25679692),(103027993,12),-  (104972702,11),(546,443401),(7,65137092),(88574269,3),(872139069,0),-  (2,97417121),(378802603,0),(141071401,4),(22613,10575),(2191743,118),-  (470,116119),(7062,38166),(231056,1847),(43901963,9),(2400,70640),-  (63553,1555),(34,11249573),(815174,1820),(997894011,0),(98881794,2),-  (5448,43132),(27956,9),(904926,1357),(112608626,3),(124,613021),-  (282086,1966),(99,10656881),(113799,1501),(433318,2085),(442,948171),-  (165380,1043),(28,14372905),(14880,50462),(2386,219918),(229,1797565),-  (1174961,298),(3925,41833),(3903515,299),(15690452,111),(360860521,3),-  (7440846,81),(2541026,507),(0,492448477),(6869,82469),(245,8322939),-  (3503496,253),(123495298,0),(150963,2299),(33,4408482),(1,200911107),-  (305,252121),(13,123369189),(215846,8181),(2440,65387),(776764401,1),-  (1241172,434),(8,15493155),(81953961,6),(17884993,5),(26,6893822),-  (0,502035190),(1,582451018),(2,514870139),(227,3625619),(49,12720258),-  (1456769,207),(94797661,10),(234407,893),(26843,5783),(15688,24547),-  (4091,86268),(4339448,151),(21360,6294),(397046497,2),(1227,205936),-  (9966,21959),(160046791,1),(0,159992224),(27,24974797),(19177,29334),-  (4136148,42),(21179785,53),(61256583,31),(385,344176),(7,11934915),-  (1,18992566),(3488065,5),(768021,224),(36288474,7),(8624,117561),-  (8,20341439),(5903,261475),(561,1007618),(1738,392327),(633049,1708)]
+ benchmarks/haskell/Benchmarks/Multilang.hs view
@@ -0,0 +1,18 @@+{-# LANGUAGE BangPatterns, OverloadedStrings, RankNTypes #-}++module Benchmarks.Multilang (benchmark) where++import qualified Data.ByteString as B+import qualified Data.Text as Text+import Data.Text.Encoding (decodeUtf8)+import Data.Text (Text)+import Test.Tasty.Bench (Benchmark, bgroup, bench, env, nf)++readYiwiki :: IO Text+readYiwiki = decodeUtf8 `fmap` B.readFile "benchmarks/text-test-data/yiwiki.xml"++benchmark :: Benchmark+benchmark = env readYiwiki $ \content -> bgroup "Multilang"+  [ bench "find_first" $ nf (Text.isInfixOf "en:Benin") content+  , bench "find_index" $ nf (Text.findIndex (=='c')) content+  ]
benchmarks/haskell/Benchmarks/Programs/BigTable.hs view
@@ -6,20 +6,19 @@ -- -- * Writing to a handle ---{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.BigTable     ( benchmark     ) where -import Criterion (Benchmark, bench, whnfIO)-import Data.Monoid (mappend, mconcat)+import Test.Tasty.Bench (Benchmark, bench, whnfIO) import Data.Text.Lazy.Builder (Builder, fromText, toLazyText) import Data.Text.Lazy.IO (hPutStr) import System.IO (Handle) import qualified Data.Text as T -benchmark :: Handle -> IO Benchmark-benchmark sink = return $ bench "BigTable" $ whnfIO $ do+benchmark :: Handle -> Benchmark+benchmark sink = bench "BigTable" $ whnfIO $ do     hPutStr sink "Content-Type: text/html\n\n<table>"     hPutStr sink . toLazyText . makeTable =<< rows     hPutStr sink "</table>"
benchmarks/haskell/Benchmarks/Programs/Cut.hs view
@@ -16,12 +16,10 @@     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnfIO)-import System.IO (Handle, hPutStr)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO)+import System.IO (Handle) import qualified Data.ByteString as B-import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as BL-import qualified Data.ByteString.Lazy.Char8 as BLC import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T@@ -29,41 +27,15 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> Int -> Int -> IO Benchmark-benchmark p sink from to = return $ bgroup "Cut"-    [ bench' "String" string-    , bench' "ByteString" byteString-    , bench' "LazyByteString" lazyByteString-    , bench' "Text" text+benchmark :: FilePath -> Handle -> Int -> Int -> Benchmark+benchmark p sink from to = bgroup "Cut"+    [ bench' "Text" text     , bench' "LazyText" lazyText     , bench' "TextByteString" textByteString     , bench' "LazyTextByteString" lazyTextByteString     ]   where     bench' n s = bench n $ whnfIO (s p sink from to)--string :: FilePath -> Handle -> Int -> Int -> IO ()-string fp sink from to = do-    s <- readFile fp-    hPutStr sink $ cut s-  where-    cut = unlines . map (take (to - from) . drop from) . lines--byteString :: FilePath -> Handle -> Int -> Int -> IO ()-byteString fp sink from to = do-    bs <- B.readFile fp-    B.hPutStr sink $ cut bs-  where-    cut = BC.unlines . map (B.take (to - from) . B.drop from) . BC.lines--lazyByteString :: FilePath -> Handle -> Int -> Int -> IO ()-lazyByteString fp sink from to = do-    bs <- BL.readFile fp-    BL.hPutStr sink $ cut bs-  where-    cut = BLC.unlines . map (BL.take (to' - from') . BL.drop from') . BLC.lines-    from' = fromIntegral from-    to' = fromIntegral to  text :: FilePath -> Handle -> Int -> Int -> IO () text fp sink from to = do
benchmarks/haskell/Benchmarks/Programs/Fold.hs view
@@ -12,25 +12,25 @@ -- -- * Writing back to a handle ---{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.Fold     ( benchmark     ) where -import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.List (intersperse)-import Data.Monoid (mempty, mappend, mconcat)+import Prelude hiding (Foldable(..)) import System.IO (Handle)-import Criterion (Benchmark, bench, whnfIO)+import Test.Tasty.Bench (Benchmark, bench, whnfIO) import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy.Builder as TLB import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark-benchmark i o = return $-    bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80+benchmark :: FilePath -> Handle -> Benchmark+benchmark i o =+    bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . foldText 80  -- | We represent a paragraph by a word list --@@ -38,8 +38,8 @@  -- | Fold a text ---fold :: Int -> T.Text -> TL.Text-fold maxWidth = TLB.toLazyText . mconcat .+foldText :: Int -> T.Text -> TL.Text+foldText maxWidth = TLB.toLazyText . mconcat .     intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs  -- | Fold a paragraph
benchmarks/haskell/Benchmarks/Programs/Sort.hs view
@@ -12,18 +12,15 @@ -- -- * Writing back to a handle ---{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP, OverloadedStrings #-} module Benchmarks.Programs.Sort     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnfIO)-import Data.Monoid (mconcat)-import System.IO (Handle, hPutStr)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO)+import System.IO (Handle) import qualified Data.ByteString as B-import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as BL-import qualified Data.ByteString.Lazy.Char8 as BLC import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Encoding as T@@ -33,13 +30,9 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark-benchmark i o = return $ bgroup "Sort"-    [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string-    , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString-    , bench "LazyByteString" $ whnfIO $-      BL.readFile i >>= BL.hPutStr o . lazyByteString-    , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text+benchmark :: FilePath -> Handle -> Benchmark+benchmark i o = bgroup "Sort"+    [ bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text     , bench "LazyText" $ whnfIO $ TL.readFile i >>= TL.hPutStr o . lazyText     , bench "TextByteString" $ whnfIO $ B.readFile i >>=         B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8@@ -48,15 +41,6 @@     , bench "TextBuilder" $ whnfIO $ B.readFile i >>=         BL.hPutStr o . TL.encodeUtf8 . textBuilder . T.decodeUtf8     ]--string :: String -> String-string = unlines . L.sort . lines--byteString :: B.ByteString -> B.ByteString-byteString = BC.unlines . L.sort . BC.lines--lazyByteString :: BL.ByteString -> BL.ByteString-lazyByteString = BLC.unlines . L.sort . BLC.lines  text :: T.Text -> T.Text text = T.unlines . L.sort . T.lines
benchmarks/haskell/Benchmarks/Programs/StripTags.hs view
@@ -15,32 +15,22 @@     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnfIO)-import Data.List (mapAccumL)-import System.IO (Handle, hPutStr)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO)+import System.IO (Handle) import qualified Data.ByteString as B-import qualified Data.ByteString.Char8 as BC import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T -benchmark :: FilePath -> Handle -> IO Benchmark-benchmark i o = return $ bgroup "StripTags"-    [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string-    , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString-    , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text+benchmark :: FilePath -> Handle -> Benchmark+benchmark i o = bgroup "StripTags"+    [ bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text     , bench "TextByteString" $ whnfIO $         B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8     ] -string :: String -> String-string = snd . mapAccumL step 0- text :: T.Text -> T.Text text = snd . T.mapAccumL step 0--byteString :: B.ByteString -> B.ByteString-byteString = snd . BC.mapAccumL step 0  step :: Int -> Char -> (Int, Char) step d c
benchmarks/haskell/Benchmarks/Programs/Throughput.hs view
@@ -18,8 +18,8 @@     ( benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnfIO)-import System.IO (Handle, hPutStr)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfIO)+import System.IO (Handle) import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL import qualified Data.Text.Encoding as T@@ -27,12 +27,9 @@ import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> Handle -> IO Benchmark-benchmark fp sink = return $ bgroup "Throughput"-    [ bench "String" $ whnfIO $ readFile fp >>= hPutStr sink-    , bench "ByteString" $ whnfIO $ B.readFile fp >>= B.hPutStr sink-    , bench "LazyByteString" $ whnfIO $ BL.readFile fp >>= BL.hPutStr sink-    , bench "Text" $ whnfIO $ T.readFile fp >>= T.hPutStr sink+benchmark :: FilePath -> Handle -> Benchmark+benchmark fp sink = bgroup "Throughput"+    [ bench "Text" $ whnfIO $ T.readFile fp >>= T.hPutStr sink     , bench "LazyText" $ whnfIO $ TL.readFile fp >>= TL.hPutStr sink     , bench "TextByteString" $ whnfIO $         B.readFile fp >>= B.hPutStr sink . T.encodeUtf8 .  T.decodeUtf8
benchmarks/haskell/Benchmarks/Pure.hs view
@@ -5,29 +5,49 @@ -- * Most pure functions defined the string types -- {-# LANGUAGE BangPatterns, CPP, GADTs, MagicHash #-}+{-# LANGUAGE DeriveGeneric, RecordWildCards #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Benchmarks.Pure-    ( benchmark+    ( initEnv+    , benchmark     ) where  import Control.DeepSeq (NFData (..)) import Control.Exception (evaluate)-import Criterion (Benchmark, bgroup, bench, nf)-import Data.Char (toLower, toUpper)-import Data.Monoid (mappend, mempty)-import GHC.Base (Char (..), Int (..), chr#, ord#, (+#))+import Data.Char (chr, ord)+import Test.Tasty.Bench (Benchmark, bgroup, bench, nf)+import GHC.Generics (Generic)+import GHC.Int (Int64) import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.Lazy.Char8 as BL-import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import qualified Data.Text.Lazy.Encoding as TL+import Data.Semigroup+import Data.List.NonEmpty (NonEmpty((:|))) -benchmark :: String -> FilePath -> IO Benchmark-benchmark kind fp = do+data Env = Env+    { bsa :: !BS.ByteString+    , ta :: !T.Text+    , tb :: !T.Text+    , tla :: !TL.Text+    , tlb :: !TL.Text+    , bla :: !BL.ByteString+    , bsa_len :: !Int+    , ta_len :: !Int+    , bla_len :: !Int64+    , tla_len :: !Int64+    , tl :: [T.Text]+    , tll :: [TL.Text]+    } deriving (Generic)++instance NFData Env++initEnv :: FilePath -> IO Env+initEnv fp = do     -- Evaluate stuff before actually running the benchmark, we don't want to     -- count it here. @@ -40,64 +60,51 @@     tla     <- evaluate $ TL.fromChunks (T.chunksOf 16376 ta)     tlb     <- evaluate $ TL.fromChunks (T.chunksOf 16376 tb) -    -- ByteString B, LazyByteString A/B-    bsb     <- evaluate $ T.encodeUtf8 tb     bla     <- evaluate $ BL.fromChunks (chunksOf 16376 bsa)-    blb     <- evaluate $ BL.fromChunks (chunksOf 16376 bsb) -    -- String A/B-    sa      <- evaluate $ UTF8.toString bsa-    sb      <- evaluate $ T.unpack tb-     -- Lengths     bsa_len <- evaluate $ BS.length bsa     ta_len  <- evaluate $ T.length ta     bla_len <- evaluate $ BL.length bla     tla_len <- evaluate $ TL.length tla-    sa_len  <- evaluate $ L.length sa      -- Lines-    bsl     <- evaluate $ BS.lines bsa-    bll     <- evaluate $ BL.lines bla     tl      <- evaluate $ T.lines ta     tll     <- evaluate $ TL.lines tla-    sl      <- evaluate $ L.lines sa -    return $ bgroup "Pure"+    return Env{..}++benchmark :: String -> Env -> Benchmark+benchmark kind ~Env{..} =+    bgroup kind         [ bgroup "append"             [ benchT   $ nf (T.append tb) ta             , benchTL  $ nf (TL.append tlb) tla-            , benchBS  $ nf (BS.append bsb) bsa-            , benchBSL $ nf (BL.append blb) bla-            , benchS   $ nf ((++) sb) sa             ]         , bgroup "concat"             [ benchT   $ nf T.concat tl             , benchTL  $ nf TL.concat tll-            , benchBS  $ nf BS.concat bsl-            , benchBSL $ nf BL.concat bll-            , benchS   $ nf L.concat sl             ]+        , bgroup "sconcat"+            [ benchT   $ nf sconcat (T.empty :| tl)+            , benchTL  $ nf sconcat (TL.empty :| tll)+            ]+        , bgroup "stimes"+            [ benchT   $ nf (stimes (10 :: Int)) ta+            , benchTL  $ nf (stimes (10 :: Int)) tla+            ]         , bgroup "cons"             [ benchT   $ nf (T.cons c) ta             , benchTL  $ nf (TL.cons c) tla-            , benchBS  $ nf (BS.cons c) bsa-            , benchBSL $ nf (BL.cons c) bla-            , benchS   $ nf (c:) sa             ]-        , bgroup "concatMap"-            [ benchT   $ nf (T.concatMap (T.replicate 3 . T.singleton)) ta-            , benchTL  $ nf (TL.concatMap (TL.replicate 3 . TL.singleton)) tla-            , benchBS  $ nf (BS.concatMap (BS.replicate 3)) bsa-            , benchBSL $ nf (BL.concatMap (BL.replicate 3)) bla-            , benchS   $ nf (L.concatMap (L.replicate 3 . (:[]))) sa-            ]+        -- concatMap exceeds 4G heap size on current test data+        -- , bgroup "concatMap"+        --     [ benchT   $ nf (T.concatMap (T.replicate 3 . T.singleton)) ta+        --     , benchTL  $ nf (TL.concatMap (TL.replicate 3 . TL.singleton)) tla+        --     ]         , bgroup "decode"             [ benchT   $ nf T.decodeUtf8 bsa             , benchTL  $ nf TL.decodeUtf8 bla-            , benchBS  $ nf BS.unpack bsa-            , benchBSL $ nf BL.unpack bla-            , benchS   $ nf UTF8.toString bsa             ]         , bgroup "decode'"             [ benchT   $ nf T.decodeUtf8' bsa@@ -106,342 +113,224 @@         , bgroup "drop"             [ benchT   $ nf (T.drop (ta_len `div` 3)) ta             , benchTL  $ nf (TL.drop (tla_len `div` 3)) tla-            , benchBS  $ nf (BS.drop (bsa_len `div` 3)) bsa-            , benchBSL $ nf (BL.drop (bla_len `div` 3)) bla-            , benchS   $ nf (L.drop (sa_len `div` 3)) sa             ]         , bgroup "encode"             [ benchT   $ nf T.encodeUtf8 ta             , benchTL  $ nf TL.encodeUtf8 tla-            , benchBS  $ nf BS.pack sa-            , benchBSL $ nf BL.pack sa-            , benchS   $ nf UTF8.fromString sa             ]         , bgroup "filter"             [ benchT   $ nf (T.filter p0) ta             , benchTL  $ nf (TL.filter p0) tla-            , benchBS  $ nf (BS.filter p0) bsa-            , benchBSL $ nf (BL.filter p0) bla-            , benchS   $ nf (L.filter p0) sa             ]         , bgroup "filter.filter"             [ benchT   $ nf (T.filter p1 . T.filter p0) ta             , benchTL  $ nf (TL.filter p1 . TL.filter p0) tla-            , benchBS  $ nf (BS.filter p1 . BS.filter p0) bsa-            , benchBSL $ nf (BL.filter p1 . BL.filter p0) bla-            , benchS   $ nf (L.filter p1 . L.filter p0) sa             ]         , bgroup "foldl'"             [ benchT   $ nf (T.foldl' len 0) ta             , benchTL  $ nf (TL.foldl' len 0) tla-            , benchBS  $ nf (BS.foldl' len 0) bsa-            , benchBSL $ nf (BL.foldl' len 0) bla-            , benchS   $ nf (L.foldl' len 0) sa             ]         , bgroup "foldr"             [ benchT   $ nf (L.length . T.foldr (:) []) ta             , benchTL  $ nf (L.length . TL.foldr (:) []) tla-            , benchBS  $ nf (L.length . BS.foldr (:) []) bsa-            , benchBSL $ nf (L.length . BL.foldr (:) []) bla-            , benchS   $ nf (L.length . L.foldr (:) []) sa             ]         , bgroup "head"             [ benchT   $ nf T.head ta             , benchTL  $ nf TL.head tla-            , benchBS  $ nf BS.head bsa-            , benchBSL $ nf BL.head bla-            , benchS   $ nf L.head sa             ]         , bgroup "init"             [ benchT   $ nf T.init ta             , benchTL  $ nf TL.init tla-            , benchBS  $ nf BS.init bsa-            , benchBSL $ nf BL.init bla-            , benchS   $ nf L.init sa             ]         , bgroup "intercalate"             [ benchT   $ nf (T.intercalate tsw) tl             , benchTL  $ nf (TL.intercalate tlw) tll-            , benchBS  $ nf (BS.intercalate bsw) bsl-            , benchBSL $ nf (BL.intercalate blw) bll-            , benchS   $ nf (L.intercalate lw) sl             ]         , bgroup "intersperse"             [ benchT   $ nf (T.intersperse c) ta             , benchTL  $ nf (TL.intersperse c) tla-            , benchBS  $ nf (BS.intersperse c) bsa-            , benchBSL $ nf (BL.intersperse c) bla-            , benchS   $ nf (L.intersperse c) sa             ]         , bgroup "isInfixOf"             [ benchT   $ nf (T.isInfixOf tsw) ta             , benchTL  $ nf (TL.isInfixOf tlw) tla-            , benchBS  $ nf (BS.isInfixOf bsw) bsa-              -- no isInfixOf for lazy bytestrings-            , benchS   $ nf (L.isInfixOf lw) sa             ]         , bgroup "last"             [ benchT   $ nf T.last ta             , benchTL  $ nf TL.last tla-            , benchBS  $ nf BS.last bsa-            , benchBSL $ nf BL.last bla-            , benchS   $ nf L.last sa             ]         , bgroup "map"             [ benchT   $ nf (T.map f) ta             , benchTL  $ nf (TL.map f) tla-            , benchBS  $ nf (BS.map f) bsa-            , benchBSL $ nf (BL.map f) bla-            , benchS   $ nf (L.map f) sa             ]         , bgroup "mapAccumL"             [ benchT   $ nf (T.mapAccumL g 0) ta             , benchTL  $ nf (TL.mapAccumL g 0) tla-            , benchBS  $ nf (BS.mapAccumL g 0) bsa-            , benchBSL $ nf (BL.mapAccumL g 0) bla-            , benchS   $ nf (L.mapAccumL g 0) sa             ]         , bgroup "mapAccumR"             [ benchT   $ nf (T.mapAccumR g 0) ta             , benchTL  $ nf (TL.mapAccumR g 0) tla-            , benchBS  $ nf (BS.mapAccumR g 0) bsa-            , benchBSL $ nf (BL.mapAccumR g 0) bla-            , benchS   $ nf (L.mapAccumR g 0) sa             ]         , bgroup "map.map"             [ benchT   $ nf (T.map f . T.map f) ta             , benchTL  $ nf (TL.map f . TL.map f) tla-            , benchBS  $ nf (BS.map f . BS.map f) bsa-            , benchBSL $ nf (BL.map f . BL.map f) bla-            , benchS   $ nf (L.map f . L.map f) sa             ]         , bgroup "replicate char"             [ benchT   $ nf (T.replicate bsa_len) (T.singleton c)             , benchTL  $ nf (TL.replicate (fromIntegral bsa_len)) (TL.singleton c)-            , benchBS  $ nf (BS.replicate bsa_len) c-            , benchBSL $ nf (BL.replicate (fromIntegral bsa_len)) c-            , benchS   $ nf (L.replicate bsa_len) c             ]         , bgroup "replicate string"             [ benchT   $ nf (T.replicate (bsa_len `div` T.length tsw)) tsw             , benchTL  $ nf (TL.replicate (fromIntegral bsa_len `div` TL.length tlw)) tlw-            , benchS   $ nf (replicat (bsa_len `div` T.length tsw)) lw             ]         , bgroup "reverse"             [ benchT   $ nf T.reverse ta             , benchTL  $ nf TL.reverse tla-            , benchBS  $ nf BS.reverse bsa-            , benchBSL $ nf BL.reverse bla-            , benchS   $ nf L.reverse sa             ]         , bgroup "take"             [ benchT   $ nf (T.take (ta_len `div` 3)) ta             , benchTL  $ nf (TL.take (tla_len `div` 3)) tla-            , benchBS  $ nf (BS.take (bsa_len `div` 3)) bsa-            , benchBSL $ nf (BL.take (bla_len `div` 3)) bla-            , benchS   $ nf (L.take (sa_len `div` 3)) sa             ]         , bgroup "tail"             [ benchT   $ nf T.tail ta             , benchTL  $ nf TL.tail tla-            , benchBS  $ nf BS.tail bsa-            , benchBSL $ nf BL.tail bla-            , benchS   $ nf L.tail sa             ]         , bgroup "toLower"             [ benchT   $ nf T.toLower ta             , benchTL  $ nf TL.toLower tla-            , benchBS  $ nf (BS.map toLower) bsa-            , benchBSL $ nf (BL.map toLower) bla-            , benchS   $ nf (L.map toLower) sa             ]         , bgroup "toUpper"             [ benchT   $ nf T.toUpper ta             , benchTL  $ nf TL.toUpper tla-            , benchBS  $ nf (BS.map toUpper) bsa-            , benchBSL $ nf (BL.map toUpper) bla-            , benchS   $ nf (L.map toUpper) sa             ]+        , bgroup "toTitle"+            [ benchT   $ nf T.toTitle ta+            , benchTL  $ nf TL.toTitle tla+            ]         , bgroup "uncons"             [ benchT   $ nf T.uncons ta             , benchTL  $ nf TL.uncons tla-            , benchBS  $ nf BS.uncons bsa-            , benchBSL $ nf BL.uncons bla-            , benchS   $ nf L.uncons sa             ]         , bgroup "words"             [ benchT   $ nf T.words ta             , benchTL  $ nf TL.words tla-            , benchBS  $ nf BS.words bsa-            , benchBSL $ nf BL.words bla-            , benchS   $ nf L.words sa             ]         , bgroup "zipWith"             [ benchT   $ nf (T.zipWith min tb) ta             , benchTL  $ nf (TL.zipWith min tlb) tla-            , benchBS  $ nf (BS.zipWith min bsb) bsa-            , benchBSL $ nf (BL.zipWith min blb) bla-            , benchS   $ nf (L.zipWith min sb) sa             ]+        , bgroup "length . unpack" -- length should fuse with unpack+            [ benchT   $ nf (L.length . T.unpack) ta+            , benchTL  $ nf (L.length . TL.unpack) tla+            ]+        , bgroup "length . drop 1 . unpack" -- no list fusion because of drop 1+            [ benchT   $ nf (L.length . L.drop 1 . T.unpack) ta+            , benchTL  $ nf (L.length . L.drop 1 . TL.unpack) tla+            ]         , bgroup "length"             [ bgroup "cons"                 [ benchT   $ nf (T.length . T.cons c) ta                 , benchTL  $ nf (TL.length . TL.cons c) tla-                , benchBS  $ nf (BS.length . BS.cons c) bsa-                , benchBSL $ nf (BL.length . BL.cons c) bla-                , benchS   $ nf (L.length . (:) c) sa                 ]             , bgroup "decode"                 [ benchT   $ nf (T.length . T.decodeUtf8) bsa                 , benchTL  $ nf (TL.length . TL.decodeUtf8) bla-                , benchBS  $ nf (L.length . BS.unpack) bsa-                , benchBSL $ nf (L.length . BL.unpack) bla-                , bench "StringUTF8" $ nf (L.length . UTF8.toString) bsa                 ]             , bgroup "drop"                 [ benchT   $ nf (T.length . T.drop (ta_len `div` 3)) ta                 , benchTL  $ nf (TL.length . TL.drop (tla_len `div` 3)) tla-                , benchBS  $ nf (BS.length . BS.drop (bsa_len `div` 3)) bsa-                , benchBSL $ nf (BL.length . BL.drop (bla_len `div` 3)) bla-                , benchS   $ nf (L.length . L.drop (sa_len `div` 3)) sa                 ]             , bgroup "filter"                 [ benchT   $ nf (T.length . T.filter p0) ta                 , benchTL  $ nf (TL.length . TL.filter p0) tla-                , benchBS  $ nf (BS.length . BS.filter p0) bsa-                , benchBSL $ nf (BL.length . BL.filter p0) bla-                , benchS   $ nf (L.length . L.filter p0) sa                 ]             , bgroup "filter.filter"                 [ benchT   $ nf (T.length . T.filter p1 . T.filter p0) ta                 , benchTL  $ nf (TL.length . TL.filter p1 . TL.filter p0) tla-                , benchBS  $ nf (BS.length . BS.filter p1 . BS.filter p0) bsa-                , benchBSL $ nf (BL.length . BL.filter p1 . BL.filter p0) bla-                , benchS   $ nf (L.length . L.filter p1 . L.filter p0) sa                 ]             , bgroup "init"                 [ benchT   $ nf (T.length . T.init) ta                 , benchTL  $ nf (TL.length . TL.init) tla-                , benchBS  $ nf (BS.length . BS.init) bsa-                , benchBSL $ nf (BL.length . BL.init) bla-                , benchS   $ nf (L.length . L.init) sa                 ]             , bgroup "intercalate"                 [ benchT   $ nf (T.length . T.intercalate tsw) tl                 , benchTL  $ nf (TL.length . TL.intercalate tlw) tll-                , benchBS  $ nf (BS.length . BS.intercalate bsw) bsl-                , benchBSL $ nf (BL.length . BL.intercalate blw) bll-                , benchS   $ nf (L.length . L.intercalate lw) sl                 ]             , bgroup "intersperse"                 [ benchT   $ nf (T.length . T.intersperse c) ta                 , benchTL  $ nf (TL.length . TL.intersperse c) tla-                , benchBS  $ nf (BS.length . BS.intersperse c) bsa-                , benchBSL $ nf (BL.length . BL.intersperse c) bla-                , benchS   $ nf (L.length . L.intersperse c) sa                 ]             , bgroup "map"                 [ benchT   $ nf (T.length . T.map f) ta                 , benchTL  $ nf (TL.length . TL.map f) tla-                , benchBS  $ nf (BS.length . BS.map f) bsa-                , benchBSL $ nf (BL.length . BL.map f) bla-                , benchS   $ nf (L.length . L.map f) sa                 ]             , bgroup "map.map"                 [ benchT   $ nf (T.length . T.map f . T.map f) ta                 , benchTL  $ nf (TL.length . TL.map f . TL.map f) tla-                , benchBS  $ nf (BS.length . BS.map f . BS.map f) bsa-                , benchS   $ nf (L.length . L.map f . L.map f) sa                 ]             , bgroup "replicate char"                 [ benchT   $ nf (T.length . T.replicate bsa_len) (T.singleton c)                 , benchTL  $ nf (TL.length . TL.replicate (fromIntegral bsa_len)) (TL.singleton c)-                , benchBS  $ nf (BS.length . BS.replicate bsa_len) c-                , benchBSL $ nf (BL.length . BL.replicate (fromIntegral bsa_len)) c-                , benchS   $ nf (L.length . L.replicate bsa_len) c                 ]             , bgroup "replicate string"                 [ benchT   $ nf (T.length . T.replicate (bsa_len `div` T.length tsw)) tsw                 , benchTL  $ nf (TL.length . TL.replicate (fromIntegral bsa_len `div` TL.length tlw)) tlw-                , benchS   $ nf (L.length . replicat (bsa_len `div` T.length tsw)) lw                 ]             , bgroup "take"                 [ benchT   $ nf (T.length . T.take (ta_len `div` 3)) ta                 , benchTL  $ nf (TL.length . TL.take (tla_len `div` 3)) tla-                , benchBS  $ nf (BS.length . BS.take (bsa_len `div` 3)) bsa-                , benchBSL $ nf (BL.length . BL.take (bla_len `div` 3)) bla-                , benchS   $ nf (L.length . L.take (sa_len `div` 3)) sa                 ]             , bgroup "tail"                 [ benchT   $ nf (T.length . T.tail) ta                 , benchTL  $ nf (TL.length . TL.tail) tla-                , benchBS  $ nf (BS.length . BS.tail) bsa-                , benchBSL $ nf (BL.length . BL.tail) bla-                , benchS   $ nf (L.length . L.tail) sa                 ]             , bgroup "toLower"                 [ benchT   $ nf (T.length . T.toLower) ta                 , benchTL  $ nf (TL.length . TL.toLower) tla-                , benchBS  $ nf (BS.length . BS.map toLower) bsa-                , benchBSL $ nf (BL.length . BL.map toLower) bla-                , benchS   $ nf (L.length . L.map toLower) sa                 ]             , bgroup "toUpper"                 [ benchT   $ nf (T.length . T.toUpper) ta                 , benchTL  $ nf (TL.length . TL.toUpper) tla-                , benchBS  $ nf (BS.length . BS.map toUpper) bsa-                , benchBSL $ nf (BL.length . BL.map toUpper) bla-                , benchS   $ nf (L.length . L.map toUpper) sa                 ]+            , bgroup "toTitle"+                [ benchT   $ nf (T.length . T.toTitle) ta+                , benchTL  $ nf (TL.length . TL.toTitle) tla+                ]             , bgroup "words"                 [ benchT   $ nf (L.length . T.words) ta                 , benchTL  $ nf (L.length . TL.words) tla-                , benchBS  $ nf (L.length . BS.words) bsa-                , benchBSL $ nf (L.length . BL.words) bla-                , benchS   $ nf (L.length . L.words) sa                 ]             , bgroup "zipWith"                 [ benchT   $ nf (T.length . T.zipWith min tb) ta                 , benchTL  $ nf (TL.length . TL.zipWith min tlb) tla-                , benchBS  $ nf (L.length . BS.zipWith min bsb) bsa-                , benchBSL $ nf (L.length . BL.zipWith min blb) bla-                , benchS   $ nf (L.length . L.zipWith min sb) sa                 ]               ]         , bgroup "Builder"-            [ bench "mappend char" $ nf (TL.length . TB.toLazyText . mappendNChar 'a') 10000-            , bench "mappend 8 char" $ nf (TL.length . TB.toLazyText . mappend8Char) 'a'-            , bench "mappend text" $ nf (TL.length . TB.toLazyText . mappendNText short) 10000+            [ bench "mappend char" $+                nf (TL.length . TB.toLazyText . mappendNChar 'a') 10000+            , bench "mappend 8 char" $+                nf (TL.length . TB.toLazyText . mappend8Char) 'a'+            , bench "mappend text" $+                nf (TL.length . TB.toLazyText . mappendNText short) 10000             ]         ]   where-    benchS   = bench ("String+" ++ kind)-    benchT   = bench ("Text+" ++ kind)-    benchTL  = bench ("LazyText+" ++ kind)-    benchBS  = bench ("ByteString+" ++ kind)-    benchBSL = bench ("LazyByteString+" ++ kind)+    benchT   = bench "Text"+    benchTL  = bench "LazyText"      c  = 'й'     p0 = (== c)     p1 = (/= 'д')     lw  = "право"-    bsw  = UTF8.fromString lw-    blw  = BL.fromChunks [bsw]     tsw  = T.pack lw     tlw  = TL.fromChunks [tsw]-    f (C# c#) = C# (chr# (ord# c# +# 1#))-    g (I# i#) (C# c#) = (I# (i# +# 1#), C# (chr# (ord# c# +# i#)))     len l _ = l + (1::Int)-    replicat n = concat . L.replicate n     short = T.pack "short" -#if !MIN_VERSION_bytestring(0,10,0)-instance NFData BS.ByteString--instance NFData BL.ByteString where-    rnf BL.Empty        = ()-    rnf (BL.Chunk _ ts) = rnf ts-#endif+    -- Valid 'Char' are in range [0..0x10FFFF], otherwise 'chr' throws an 'error'.+    -- 'Data.Text.Internal.safe' does not validate this, it assumes that inputs+    -- has been already sanitized to belong to the range.+    f !ch = chr (min 0x10FFFF (ord ch + 1))+    g !i !ch = (i + 1, chr (min 0x10FFFF (ord ch + i)))  data B where     B :: NFData a => a -> B@@ -467,7 +356,7 @@       | i < n     = TB.singleton c `mappend` go (i+1)       | otherwise = mempty --- | Gives more opportunity for inlining and elimination of unnecesary+-- | Gives more opportunity for inlining and elimination of unnecessary -- bounds checks. -- mappend8Char :: Char -> TB.Builder
benchmarks/haskell/Benchmarks/ReadNumbers.hs view
@@ -17,15 +17,13 @@ -- * Lexing/parsing of different numerical types -- module Benchmarks.ReadNumbers-    ( benchmark+    ( initEnv+    , benchmark     ) where -import Criterion (Benchmark, bgroup, bench, whnf)-import Data.List (foldl')-import Numeric (readDec, readFloat, readHex)-import qualified Data.ByteString.Char8 as B-import qualified Data.ByteString.Lazy.Char8 as BL-import qualified Data.ByteString.Lex.Fractional as B+import Data.Foldable (Foldable(..))+import Prelude hiding (Foldable(..))+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL@@ -33,21 +31,18 @@ import qualified Data.Text.Lazy.Read as TL import qualified Data.Text.Read as T -benchmark :: FilePath -> IO Benchmark-benchmark fp = do-    -- Read all files into lines: string, text, lazy text, bytestring, lazy-    -- bytestring-    s <- lines `fmap` readFile fp+type Env = ([T.Text], [TL.Text])++initEnv :: FilePath -> IO Env+initEnv fp = do     t <- T.lines `fmap` T.readFile fp     tl <- TL.lines `fmap` TL.readFile fp-    b <- B.lines `fmap` B.readFile fp-    bl <- BL.lines `fmap` BL.readFile fp-    return $ bgroup "ReadNumbers"-        [ bench "DecimalString"     $ whnf (int . string readDec) s-        , bench "HexadecimalString" $ whnf (int . string readHex) s-        , bench "DoubleString"      $ whnf (double . string readFloat) s+    return (t, tl) -        , bench "DecimalText"     $ whnf (int . text (T.signed T.decimal)) t+benchmark :: Env -> Benchmark+benchmark ~(t, tl) =+    bgroup "ReadNumbers"+        [ bench "DecimalText"     $ whnf (int . text (T.signed T.decimal)) t         , bench "HexadecimalText" $ whnf (int . text (T.signed T.hexadecimal)) t         , bench "DoubleText"      $ whnf (double . text T.double) t         , bench "RationalText"    $ whnf (double . text T.rational) t@@ -60,12 +55,6 @@             whnf (double . text TL.double) tl         , bench "RationalLazyText" $             whnf (double . text TL.rational) tl--        , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b-        , bench "DoubleByteString"  $ whnf (double . byteString B.readDecimal) b--        , bench "DecimalLazyByteString" $-            whnf (int . byteString BL.readInt) bl         ]   where     -- Used for fixing types@@ -74,20 +63,8 @@     double :: Double -> Double     double = id -string :: (Ord a, Num a) => (t -> [(a, t)]) -> [t] -> a-string reader = foldl' go 1000000-  where-    go z t = case reader t of [(n, _)] -> min n z-                              _        -> z- text :: (Ord a, Num a) => (t -> Either String (a,t)) -> [t] -> a text reader = foldl' go 1000000   where     go z t = case reader t of Left _       -> z                               Right (n, _) -> min n z--byteString :: (Ord a, Num a) => (t -> Maybe (a,t)) -> [t] -> a-byteString reader = foldl' go 1000000-  where-    go z t = case reader t of Nothing     -> z-                              Just (n, _) -> min n z
benchmarks/haskell/Benchmarks/Replace.hs view
@@ -7,37 +7,30 @@ -- module Benchmarks.Replace     ( benchmark+    , initEnv     ) where -import Criterion (Benchmark, bgroup, bench, nf)-import qualified Data.ByteString.Char8 as B-import qualified Data.ByteString.Lazy as BL-import qualified Data.ByteString.Lazy.Search as BL-import qualified Data.ByteString.Search as B+import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) import qualified Data.Text as T-import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> String -> String -> IO Benchmark-benchmark fp pat sub = do+type Env = (T.Text, TL.Text)++initEnv :: FilePath -> IO Env+initEnv fp = do     tl <- TL.readFile fp-    bl <- BL.readFile fp     let !t = TL.toStrict tl-        !b = T.encodeUtf8 t-    return $ bgroup "Replace" [+    return (t, tl)++benchmark :: String -> String -> Env -> Benchmark+benchmark pat sub ~(t, tl) =+    bgroup "Replace" [           bench "Text"           $ nf (T.length . T.replace tpat tsub) t-        , bench "ByteString"     $ nf (BL.length . B.replace bpat bsub) b         , bench "LazyText"       $ nf (TL.length . TL.replace tlpat tlsub) tl-        , bench "LazyByteString" $ nf (BL.length . BL.replace blpat blsub) bl         ]   where     tpat  = T.pack pat     tsub  = T.pack sub     tlpat = TL.pack pat     tlsub = TL.pack sub-    bpat = T.encodeUtf8 tpat-    bsub = T.encodeUtf8 tsub-    blpat = B.concat $ BL.toChunks $ TL.encodeUtf8 tlpat-    blsub = B.concat $ BL.toChunks $ TL.encodeUtf8 tlsub
benchmarks/haskell/Benchmarks/Search.hs view
@@ -1,45 +1,36 @@--- | Search for a pattern in a file, find the number of occurences+-- | Search for a pattern in a file, find the number of occurrences -- -- Tested in this benchmark: ----- * Searching all occurences of a pattern using library routines+-- * Searching all occurrences of a pattern using library routines -- module Benchmarks.Search-    ( benchmark+    ( initEnv+    , benchmark     ) where -import Criterion (Benchmark, bench, bgroup, whnf)-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL-import qualified Data.ByteString.Lazy.Search as BL-import qualified Data.ByteString.Search as B+import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Text as T-import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL -benchmark :: FilePath -> T.Text -> IO Benchmark-benchmark fp needleT = do-    b  <- B.readFile fp-    bl <- BL.readFile fp+type Env = (T.Text, TL.Text)++initEnv :: FilePath -> IO Env+initEnv fp = do     t  <- T.readFile fp     tl <- TL.readFile fp-    return $ bgroup "FileIndices"-        [ bench "ByteString"     $ whnf (byteString needleB)     b-        , bench "LazyByteString" $ whnf (lazyByteString needleB) bl-        , bench "Text"           $ whnf (text needleT)           t+    return (t, tl)++benchmark :: T.Text -> Env -> Benchmark+benchmark needleT ~(t, tl) =+    bgroup "FileIndices"+        [ bench "Text"           $ whnf (text needleT)           t         , bench "LazyText"       $ whnf (lazyText needleTL)      tl         ]   where-    needleB = T.encodeUtf8 needleT     needleTL = TL.fromChunks [needleT]--byteString :: B.ByteString -> B.ByteString -> Int-byteString needle = length . B.indices needle--lazyByteString :: B.ByteString -> BL.ByteString -> Int-lazyByteString needle = length . BL.indices needle  text :: T.Text -> T.Text -> Int text = T.count
benchmarks/haskell/Benchmarks/Stream.hs view
@@ -6,24 +6,31 @@ -- * Most streaming functions -- {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE DeriveGeneric, RecordWildCards #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Benchmarks.Stream-    ( benchmark+    ( initEnv+    , benchmark     ) where  import Control.DeepSeq (NFData (..))-import Criterion (Benchmark, bgroup, bench, nf)+import Test.Tasty.Bench (Benchmark, bgroup, bench, nf)+import qualified Data.Text as T+import qualified Data.ByteString as B+import qualified Data.Text.Lazy as TL+import qualified Data.ByteString.Lazy as BL import Data.Text.Internal.Fusion.Types (Step (..), Stream (..)) import qualified Data.Text.Encoding as T import qualified Data.Text.Encoding.Error as E import qualified Data.Text.Internal.Encoding.Fusion as T import qualified Data.Text.Internal.Encoding.Fusion.Common as F-import qualified Data.Text.Internal.Fusion as F+import qualified Data.Text.Internal.Fusion as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL-import qualified Data.Text.Internal.Lazy.Fusion as FL+import qualified Data.Text.Internal.Lazy.Fusion as TL import qualified Data.Text.Lazy.IO as TL+import GHC.Generics (Generic)  instance NFData a => NFData (Stream a) where     -- Currently, this implementation does not force evaluation of the size hint@@ -34,8 +41,26 @@             Skip s'    -> go s'             Yield x s' -> rnf x `seq` go s' -benchmark :: FilePath -> IO Benchmark-benchmark fp = do+data Env = Env+    { t :: !T.Text+    , utf8 :: !B.ByteString+    , utf16le :: !B.ByteString+    , utf16be :: !B.ByteString+    , utf32le :: !B.ByteString+    , utf32be :: !B.ByteString+    , tl :: !TL.Text+    , utf8L :: !BL.ByteString+    , utf16leL :: !BL.ByteString+    , utf16beL :: !BL.ByteString+    , utf32leL :: !BL.ByteString+    , utf32beL :: !BL.ByteString+    , s :: T.Stream Char+    } deriving (Generic)++instance NFData Env++initEnv :: FilePath -> IO Env+initEnv fp = do     -- Different formats     t  <- T.readFile fp     let !utf8    = T.encodeUtf8 t@@ -53,25 +78,16 @@         !utf32beL = TL.encodeUtf32BE tl      -- For the functions which operate on streams-    let !s = F.stream t--    return $ bgroup "Stream"+    let !s = T.stream t+    return Env{..} +benchmark :: Env -> Benchmark+benchmark ~Env{..} =+    bgroup "Stream"         -- Fusion         [ bgroup "stream" $-            [ bench "Text"     $ nf F.stream t-            , bench "LazyText" $ nf FL.stream tl-            ]-          -- must perform exactly the same as stream above due to-          -- stream/unstream (i.e. stream after unstream) fusion-        , bgroup "stream-fusion" $-            [ bench "Text"     $ nf (F.stream . F.unstream . F.stream) t-            , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl-            ]-          -- measure the overhead of unstream after stream-        , bgroup "stream-unstream" $-            [ bench "Text"     $ nf (F.unstream . F.stream) t-            , bench "LazyText" $ nf (FL.unstream . FL.stream) tl+            [ bench "Text"     $ nf T.stream t+            , bench "LazyText" $ nf TL.stream tl             ]          -- Encoding.Fusion
benchmarks/haskell/Benchmarks/WordFrequencies.hs view
@@ -9,27 +9,29 @@ -- * Comparing: Eq/Ord instances -- module Benchmarks.WordFrequencies-    ( benchmark+    ( initEnv+    , benchmark     ) where -import Criterion (Benchmark, bench, bgroup, whnf)-import Data.Char (toLower)-import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.Map (Map)-import qualified Data.ByteString.Char8 as B+import Prelude hiding (Foldable(..))+import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T -benchmark :: FilePath -> IO Benchmark-benchmark fp = do-    s <- readFile fp-    b <- B.readFile fp+type Env = T.Text++initEnv :: FilePath -> IO Env+initEnv fp = do     t <- T.readFile fp-    return $ bgroup "WordFrequencies"-        [ bench "String"     $ whnf (frequencies . words . map toLower)     s-        , bench "ByteString" $ whnf (frequencies . B.words . B.map toLower) b-        , bench "Text"       $ whnf (frequencies . T.words . T.toLower)     t+    return t++benchmark :: Env -> Benchmark+benchmark ~t =+    bgroup "WordFrequencies"+        [ bench "Text"       $ whnf (frequencies . T.words . T.toLower)     t         ]  frequencies :: Ord a => [a] -> Map a Int
− benchmarks/haskell/Multilang.hs 
@@ -1,32 +0,0 @@-{-# LANGUAGE BangPatterns, OverloadedStrings, RankNTypes #-}--module Main (-  main-  ) where--import Control.Monad (forM_)-import qualified Data.ByteString as B-import qualified Data.Text as Text-import Data.Text.Encoding (decodeUtf8)-import Data.Text (Text)-import System.IO (hFlush, stdout)-import Timer (timer)--type BM = Text -> ()--bm :: forall a. (Text -> a) -> BM-bm f t = f t `seq` ()--benchmarks :: [(String, Text.Text -> ())]-benchmarks = [-    ("find_first", bm $ Text.isInfixOf "en:Benin")-  , ("find_index", bm $ Text.findIndex (=='c'))-  ]--main :: IO ()-main = do-  !contents <- decodeUtf8 `fmap` B.readFile "../tests/text-test-data/yiwiki.xml"-  forM_ benchmarks $ \(name, bmark) -> do-    putStr $ name ++ " "-    hFlush stdout-    putStrLn =<< (timer 100 contents bmark)
− benchmarks/haskell/Timer.hs 
@@ -1,30 +0,0 @@-{-# LANGUAGE BangPatterns #-}--module Timer (timer) where--import Control.Exception (evaluate)-import Data.Time.Clock.POSIX (getPOSIXTime)-import GHC.Float (FFFormat(..), formatRealFloat)--ickyRound :: Int -> Double -> String-ickyRound k = formatRealFloat FFFixed (Just k)--timer :: Int -> a -> (a -> b) -> IO String-timer count a0 f = do-  let loop !k !fastest-        | k <= 0 = return fastest-        | otherwise = do-        start <- getPOSIXTime-        let inner a i-              | i <= 0    = return ()-              | otherwise = evaluate (f a) >> inner a (i-1)-        inner a0 count-        end <- getPOSIXTime-        let elapsed = end - start-        loop (k-1) (min fastest (elapsed / fromIntegral count))-  t <- loop (3::Int) 1e300-  let log10 x = log x / log 10-      ft = realToFrac t-      prec = round (log10 (fromIntegral count) - log10 ft)-  return $! ickyRound prec ft-{-# NOINLINE timer #-}
− benchmarks/python/cut.py 
@@ -1,12 +0,0 @@-#!/usr/bin/env python--import utils, sys, codecs--def cut(filename, l, r):-    content = open(filename, encoding='utf-8')-    for line in content:-        print(line[l:r])--for f in sys.argv[1:]:-    t = utils.benchmark(lambda: cut(f, 20, 40))-    sys.stderr.write('{0}: {1}\n'.format(f, t))
− benchmarks/python/multilang.py 
@@ -1,50 +0,0 @@-#!/usr/bin/env python--import math-import sys-import time--def find_first():-    cf = contents.find-    return timer(lambda: cf("en:Benin"))--def timer(f, count=100):-    a = 1e300-    def g():-        return-    for i in xrange(3):-        start = time.time()-        for j in xrange(count):-            g()-        a = min(a, (time.time() - start) / count)--    b = 1e300-    for i in xrange(3):-        start = time.time()-        for j in xrange(count):-            f()-        b = min(b, (time.time() - start) / count)--    return round(b - a, int(round(math.log(count, 10) - math.log(b - a, 10))))--contents = open('../../tests/text-test-data/yiwiki.xml', 'r').read()-contents = contents.decode('utf-8')--benchmarks = (-    find_first,-    )--to_run = sys.argv[1:]-bms = []-if to_run:-    for r in to_run:-        for b in benchmarks:-            if b.__name__.startswith(r):-                bms.append(b)-else:-    bms = benchmarks--for b in bms:-    sys.stdout.write(b.__name__ + ' ')-    sys.stdout.flush()-    print b()
− benchmarks/python/sort.py 
@@ -1,13 +0,0 @@-#!/usr/bin/env python--import utils, sys, codecs--def sort(filename):-    content = open(filename, encoding='utf-8').read()-    lines = content.splitlines()-    lines.sort()-    print('\n'.join(lines))--for f in sys.argv[1:]:-    t = utils.benchmark(lambda: sort(f))-    sys.stderr.write('{0}: {1}\n'.format(f, t))
− benchmarks/python/strip_tags.py 
@@ -1,25 +0,0 @@-#!/usr/bin/env python--import utils, sys--def strip_tags(filename):-    string = open(filename, encoding='utf-8').read()--    d = 0-    out = []--    for c in string:-        if c == '<': d += 1--        if d > 0:-            out += ' '-        else:-            out += c--        if c == '>': d -= 1--    print(''.join(out))--for f in sys.argv[1:]:-    t = utils.benchmark(lambda: strip_tags(f))-    sys.stderr.write('{0}: {1}\n'.format(f, t))
− benchmarks/python/utils.py 
@@ -1,18 +0,0 @@-#!/usr/bin/env python--import sys, time--def benchmark_once(f):-    start = time.time()-    f()-    end = time.time()-    return end - start--def benchmark(f):-    runs = 100-    total = 0.0-    for i in range(runs):-        result = benchmark_once(f)-        sys.stderr.write('Run {0}: {1}\n'.format(i, result))-        total += result-    return total / runs
− benchmarks/ruby/cut.rb 
@@ -1,16 +0,0 @@-#!/usr/bin/env ruby--require './utils.rb'--def cut(filename, l, r)-  File.open(filename, 'r:utf-8') do |file|-    file.each_line do |line|-      puts line[l, r - l]-    end-  end-end--ARGV.each do |f|-  t = benchmark { cut(f, 20, 40) }-  STDERR.puts "#{f}: #{t}"-end
− benchmarks/ruby/fold.rb 
@@ -1,50 +0,0 @@-#!/usr/bin/env ruby--require './utils.rb'--def fold(filename, max_width)-  File.open(filename, 'r:utf-8') do |file|-    # Words in this paragraph-    paragraph = []--    file.each_line do |line|-      # If we encounter an empty line, we reformat and dump the current-      # paragraph-      if line.strip.empty?-        puts fold_paragraph(paragraph, max_width)-        puts-        paragraph = []-      # Otherwise, we append the words found in the line to the paragraph-      else-        paragraph.concat line.split-      end-    end--    # Last paragraph-    puts fold_paragraph(paragraph, max_width) unless paragraph.empty?-  end-end--# Fold a single paragraph to the desired width-def fold_paragraph(paragraph, max_width)-  # Gradually build our output-  str, *rest = paragraph-  width = str.length--  rest.each do |word|-    if width + word.length + 1 <= max_width-      str << ' ' << word-      width += word.length + 1-    else-      str << "\n" << word-      width = word.length-    end-  end--  str-end--ARGV.each do |f|-  t = benchmark { fold(f, 80) }-  STDERR.puts "#{f}: #{t}"-end
− benchmarks/ruby/sort.rb 
@@ -1,15 +0,0 @@-#!/usr/bin/env ruby--require './utils.rb'--def sort(filename)-  File.open(filename, 'r:utf-8') do |file|-    content = file.read-    puts content.lines.sort.join-  end-end--ARGV.each do |f|-  t = benchmark { sort(f) }-  STDERR.puts "#{f}: #{t}"-end
− benchmarks/ruby/strip_tags.rb 
@@ -1,22 +0,0 @@-#!/usr/bin/env ruby--require './utils.rb'--def strip_tags(filename)-  File.open(filename, 'r:utf-8') do |file|-    str = file.read--    d = 0--    str.each_char do |c|-      d += 1 if c == '<'-      putc(if d > 0 then ' ' else c end)-      d -= 1 if c == '>'-    end-  end-end--ARGV.each do |f|-  t = benchmark { strip_tags(f) }-  STDERR.puts "#{f}: #{t}"-end
− benchmarks/ruby/utils.rb 
@@ -1,14 +0,0 @@-require 'benchmark'--def benchmark(&block)-  runs = 100-  total = 0--  runs.times do |i|-    result = Benchmark.measure(&block).total-    $stderr.puts "Run #{i}: #{result}"-    total += result-  end--  total / runs -end
− benchmarks/text-benchmarks.cabal 
@@ -1,140 +0,0 @@-name:                text-benchmarks-version:             0.0.0.0-synopsis:            Benchmarks for the text package-description:         Benchmarks for the text package-homepage:            https://bitbucket.org/bos/text-license:             BSD2-license-file:        ../LICENSE-author:              Jasper Van der Jeugt <jaspervdj@gmail.com>,-                     Bryan O'Sullivan <bos@serpentine.com>,-                     Tom Harper <rtomharper@googlemail.com>,-                     Duncan Coutts <duncan@haskell.org>-maintainer:          jaspervdj@gmail.com-category:            Text-build-type:          Simple-cabal-version:       >=1.8--flag bytestring-builder-  description: Depend on the bytestring-builder package for backwards compatibility.-  default: False-  manual: False--flag llvm-  description: use LLVM-  default: False-  manual: True--executable text-benchmarks-  ghc-options:    -Wall -O2 -rtsopts-  if flag(llvm)-    ghc-options:  -fllvm-  cpp-options:    -DINTEGER_GMP-  build-depends:  array,-                  base == 4.*,-                  binary,-                  blaze-builder,-                  bytestring-lexing >= 0.5.0,-                  containers,-                  criterion >= 0.10.0.0,-                  deepseq,-                  directory,-                  filepath,-                  ghc-prim,-                  integer-gmp,-                  stringsearch,-                  transformers,-                  utf8-string,-                  vector--  if flag(bytestring-builder)-    build-depends: bytestring         >= 0.9    && < 0.10.4,-                   bytestring-builder >= 0.10.4-  else-    build-depends: bytestring         >= 0.10.4--  -- modules for benchmark proper-  c-sources:      cbits/time_iconv.c-  hs-source-dirs: haskell-  main-is:        Benchmarks.hs-  other-modules:-    Benchmarks.Builder-    Benchmarks.Concat-    Benchmarks.DecodeUtf8-    Benchmarks.EncodeUtf8-    Benchmarks.Equality-    Benchmarks.FileRead-    Benchmarks.FoldLines-    Benchmarks.Mul-    Benchmarks.Programs.BigTable-    Benchmarks.Programs.Cut-    Benchmarks.Programs.Fold-    Benchmarks.Programs.Sort-    Benchmarks.Programs.StripTags-    Benchmarks.Programs.Throughput-    Benchmarks.Pure-    Benchmarks.ReadNumbers-    Benchmarks.Replace-    Benchmarks.Search-    Benchmarks.Stream-    Benchmarks.WordFrequencies--  -- Source code for IUT (implementation under test)-  -- "borrowed" from parent folder-  include-dirs:   ../include-  c-sources:      ../cbits/cbits.c-  hs-source-dirs: ..-  other-modules:-    Data.Text-    Data.Text.Array-    Data.Text.Encoding-    Data.Text.Encoding.Error-    Data.Text.Foreign-    Data.Text.IO-    Data.Text.Internal-    Data.Text.Internal.Builder-    Data.Text.Internal.Builder.Functions-    Data.Text.Internal.Builder.Int.Digits-    Data.Text.Internal.Builder.RealFloat.Functions-    Data.Text.Internal.Encoding.Fusion-    Data.Text.Internal.Encoding.Fusion.Common-    Data.Text.Internal.Encoding.Utf16-    Data.Text.Internal.Encoding.Utf32-    Data.Text.Internal.Encoding.Utf8-    Data.Text.Internal.Functions-    Data.Text.Internal.Fusion-    Data.Text.Internal.Fusion.CaseMapping-    Data.Text.Internal.Fusion.Common-    Data.Text.Internal.Fusion.Size-    Data.Text.Internal.Fusion.Types-    Data.Text.Internal.IO-    Data.Text.Internal.Lazy-    Data.Text.Internal.Lazy.Encoding.Fusion-    Data.Text.Internal.Lazy.Fusion-    Data.Text.Internal.Lazy.Search-    Data.Text.Internal.Private-    Data.Text.Internal.Read-    Data.Text.Internal.Search-    Data.Text.Internal.Unsafe-    Data.Text.Internal.Unsafe.Char-    Data.Text.Internal.Unsafe.Shift-    Data.Text.Lazy-    Data.Text.Lazy.Builder-    Data.Text.Lazy.Builder.Int-    Data.Text.Lazy.Builder.RealFloat-    Data.Text.Lazy.Encoding-    Data.Text.Lazy.IO-    Data.Text.Lazy.Internal-    Data.Text.Lazy.Read-    Data.Text.Read-    Data.Text.Unsafe-    Data.Text.Show--executable text-multilang-  hs-source-dirs: haskell-  main-is:        Multilang.hs-  other-modules:  Timer-  ghc-options:    -Wall -O2-  build-depends:  base == 4.*,-                  bytestring,-                  text,-                  time
+ cbits/aarch64/measure_off.c view
@@ -0,0 +1,109 @@+/*+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>+ */++#include <string.h>+#include <sys/types.h>+#include <arm_neon.h>++/*+  measure_off_naive / measure_off_neon+  take a UTF-8 sequence between src and srcend, and a number of characters cnt.+  If the sequence is long enough to contain cnt characters, then return how many bytes+  remained unconsumed. Otherwise, if the sequence is shorter, return+  negated count of lacking characters. Cf. _hs_text_measure_off below.+*/++static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt)+{+  // Count leading bytes in 8 byte sequence+  while (src < srcend - 7){+    uint64_t w64;+    memcpy(&w64, src, sizeof(uint64_t));+    // find leading bytes by finding every byte that is not a continuation+    // byte. The bit twiddle only results in a 0 if the original byte starts+    // with 0b11...+    w64 =  ((w64 << 1) | ~w64) & 0x8080808080808080ULL;+    // compute the popcount of w64 with two bit shifts and a multiplication+    size_t leads = (  (w64 >> 7)              // w64 >> 7           = Sum{0<= i <= 7} x_i * 256^i    (x_i \in {0,1})+                    * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i+                                              //              (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) +                                              // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) +                                              // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k+                                              // and each i satisfies this equation for exactly one such j+                                              // So each byte of the result contains the sum we want.+                   ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j+    if (cnt < leads) break;+    cnt-= leads;+    src+= 8;+  }++  // Skip until next leading byte+  while (src < srcend){+    uint8_t w8 = *src;+    if ((int8_t)w8 >= -0x40) break;+    src++;+  }++  // Finish up with tail+  while (src < srcend && cnt > 0){+    uint8_t leadByte = *src++;+    cnt--;+    src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0);+  }++  return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt);+}++static inline const ssize_t measure_off_neon(const uint8_t *src, const uint8_t *srcend, size_t cnt)+{+  while (src < srcend - 63){+    uint8x16_t w128[4] = {vld1q_u8(src), vld1q_u8(src + 16), vld1q_u8(src + 32), vld1q_u8(src + 48)};+    // Which bytes are either < 128 or >= 192?+    uint8x16_t mask0 = vcgtq_s8((int8x16_t)w128[0], vdupq_n_s8(0xBF));+    uint8x16_t mask1 = vcgtq_s8((int8x16_t)w128[1], vdupq_n_s8(0xBF));+    uint8x16_t mask2 = vcgtq_s8((int8x16_t)w128[2], vdupq_n_s8(0xBF));+    uint8x16_t mask3 = vcgtq_s8((int8x16_t)w128[3], vdupq_n_s8(0xBF));++    uint8x16_t mask01 = vaddq_u8(mask0, mask1);+    uint8x16_t mask23 = vaddq_u8(mask2, mask3);+    uint8x16_t mask = vaddq_u8(mask01, mask23);++    size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask));++    if (cnt < leads) break;+    cnt-= leads;+    src+= 64;+  }++  while (src < srcend - 15){+    uint8x16_t w128 = vld1q_u8(src);+    // Which bytes are either < 128 or >= 192?+    uint8x16_t mask = vcgtq_s8((int8x16_t)w128, vdupq_n_s8(0xBF));+    size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask));+    if (cnt < leads) break;+    cnt-= leads;+    src+= 16;+  }++  return measure_off_naive(src, srcend, cnt);+}++/*+  _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len),+  and a number of code points (aka characters) cnt. If the buffer is long enough+  to contain cnt characters, then _hs_text_measure_off returns a non-negative number,+  measuring their size in code units (aka bytes). If the buffer is shorter,+  _hs_text_measure_off returns a non-positive number, which is a negated total count+  of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0+  as well.++  This scheme allows us to implement both take/drop and length with the same C function.++  The input buffer (src, off, len) must be a valid UTF-8 sequence,+  this condition is not checked.+*/+ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) {+  ssize_t ret = measure_off_neon(src + off, src + off + len, cnt);+  return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret));+}
− cbits/cbits.c 
@@ -1,306 +0,0 @@-/*- * Copyright (c) 2011 Bryan O'Sullivan <bos@serpentine.com>.- *- * Portions copyright (c) 2008-2010 Björn Höhrmann <bjoern@hoehrmann.de>.- *- * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.- */--#include <string.h>-#include <stdint.h>-#include <stdio.h>-#include "text_cbits.h"--void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff,-		     size_t n)-{-  memcpy(dest + (doff<<1), src + (soff<<1), n<<1);-}--int _hs_text_memcmp(const void *a, size_t aoff, const void *b, size_t boff,-		    size_t n)-{-  return memcmp(a + (aoff<<1), b + (boff<<1), n<<1);-}--#define UTF8_ACCEPT 0-#define UTF8_REJECT 12--static const uint8_t utf8d[] = {-  /*-   * The first part of the table maps bytes to character classes that-   * to reduce the size of the transition table and create bitmasks.-   */-   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,-   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,-   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,--  /*-   * The second part is a transition table that maps a combination of-   * a state of the automaton and a character class to a state.-   */-   0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,-  12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,-  12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,-  12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,-  12,36,12,12,12,12,12,12,12,12,12,12,-};--static inline uint32_t-decode(uint32_t *state, uint32_t* codep, uint32_t byte) {-  uint32_t type = utf8d[byte];--  *codep = (*state != UTF8_ACCEPT) ?-    (byte & 0x3fu) | (*codep << 6) :-    (0xff >> type) & (byte);--  return *state = utf8d[256 + *state + type];-}--/*- * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode- * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to- * an UTF16 array- */-void-_hs_text_decode_latin1(uint16_t *dest, const uint8_t *src,-                       const uint8_t *srcend)-{-  const uint8_t *p = src;--#if defined(__i386__) || defined(__x86_64__)-  /* This optimization works on a little-endian systems by using-     (aligned) 32-bit loads instead of 8-bit loads-   */--  /* consume unaligned prefix */-  while (p != srcend && (uintptr_t)p & 0x3)-    *dest++ = *p++;--  /* iterate over 32-bit aligned loads */-  while (p < srcend - 3) {-    const uint32_t w = *((const uint32_t *)p);--    *dest++ =  w        & 0xff;-    *dest++ = (w >> 8)  & 0xff;-    *dest++ = (w >> 16) & 0xff;-    *dest++ = (w >> 24) & 0xff;--    p += 4;-  }-#endif--  /* handle unaligned suffix */-  while (p != srcend)-    *dest++ = *p++;-}--/*- * A best-effort decoder. Runs until it hits either end of input or- * the start of an invalid byte sequence.- *- * At exit, we update *destoff with the next offset to write to, *src- * with the next source location past the last one successfully- * decoded, and return the next source location to read from.- *- * Moreover, we expose the internal decoder state (state0 and- * codepoint0), allowing one to restart the decoder after it- * terminates (say, due to a partial codepoint).- *- * In particular, there are a few possible outcomes,- *- *   1) We decoded the buffer entirely:- *      In this case we return srcend- *      state0 == UTF8_ACCEPT- *- *   2) We met an invalid encoding- *      In this case we return the address of the first invalid byte- *      state0 == UTF8_REJECT- *- *   3) We reached the end of the buffer while decoding a codepoint- *      In this case we return a pointer to the first byte of the partial codepoint- *      state0 != UTF8_ACCEPT, UTF8_REJECT- *- */-#if defined(__GNUC__) || defined(__clang__)-static inline uint8_t const *-_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,-			 const uint8_t **src, const uint8_t *srcend,-			 uint32_t *codepoint0, uint32_t *state0)-  __attribute((always_inline));-#endif--static inline uint8_t const *-_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff,-			 const uint8_t **src, const uint8_t *srcend,-			 uint32_t *codepoint0, uint32_t *state0)-{-  uint16_t *d = dest + *destoff;-  const uint8_t *s = *src, *last = *src;-  uint32_t state = *state0;-  uint32_t codepoint = *codepoint0;--  while (s < srcend) {-#if defined(__i386__) || defined(__x86_64__)-    /*-     * This code will only work on a little-endian system that-     * supports unaligned loads.-     *-     * It gives a substantial speed win on data that is purely or-     * partly ASCII (e.g. HTML), at only a slight cost on purely-     * non-ASCII text.-     */--    if (state == UTF8_ACCEPT) {-      while (s < srcend - 4) {-	codepoint = *((uint32_t *) s);-	if ((codepoint & 0x80808080) != 0)-	  break;-	s += 4;--	/*-	 * Tried 32-bit stores here, but the extra bit-twiddling-	 * slowed the code down.-	 */--	*d++ = (uint16_t) (codepoint & 0xff);-	*d++ = (uint16_t) ((codepoint >> 8) & 0xff);-	*d++ = (uint16_t) ((codepoint >> 16) & 0xff);-	*d++ = (uint16_t) ((codepoint >> 24) & 0xff);-      }-      last = s;-    }-#endif--    if (decode(&state, &codepoint, *s++) != UTF8_ACCEPT) {-      if (state != UTF8_REJECT)-	continue;-      break;-    }--    if (codepoint <= 0xffff)-      *d++ = (uint16_t) codepoint;-    else {-      *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10));-      *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF));-    }-    last = s;-  }--  *destoff = d - dest;-  *codepoint0 = codepoint;-  *state0 = state;-  *src = last;--  return s;-}--uint8_t const *-_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,-                           const uint8_t **src,-                           const uint8_t *srcend,-                           uint32_t *codepoint0, uint32_t *state0)-{-  _hs_text_decode_utf8_int(dest, destoff, src, srcend, codepoint0, state0);--  return *src;-}--/*- * Helper to decode buffer and discard final decoder state- */-const uint8_t *-_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff,-                     const uint8_t *src, const uint8_t *const srcend)-{-  uint32_t codepoint;-  uint32_t state = UTF8_ACCEPT;-  _hs_text_decode_utf8_int(dest, destoff, &src, srcend,-                          &codepoint, &state);-  return src;-}--void-_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff,-		     size_t srclen)-{-  const uint16_t *srcend;-  uint8_t *dest = *destp;--  src += srcoff;-  srcend = src + srclen;-- ascii:-#if defined(__x86_64__)-  while (srcend - src >= 4) {-    uint64_t w = *((uint64_t *) src);--    if (w & 0xFF80FF80FF80FF80ULL) {-      if (!(w & 0x000000000000FF80ULL)) {-	*dest++ = w & 0xFFFF;-	src++;-	if (!(w & 0x00000000FF800000ULL)) {-	  *dest++ = (w >> 16) & 0xFFFF;-	  src++;-	  if (!(w & 0x0000FF8000000000ULL)) {-	    *dest++ = (w >> 32) & 0xFFFF;-	    src++;-	  }-	}-      }-      break;-    }-    *dest++ = w & 0xFFFF;-    *dest++ = (w >> 16) & 0xFFFF;-    *dest++ = (w >> 32) & 0xFFFF;-    *dest++ = w >> 48;-    src += 4;-  }-#endif--#if defined(__i386__)-  while (srcend - src >= 2) {-    uint32_t w = *((uint32_t *) src);--    if (w & 0xFF80FF80)-      break;-    *dest++ = w & 0xFFFF;-    *dest++ = w >> 16;-    src += 2;-  }-#endif--  while (src < srcend) {-    uint16_t w = *src++;--    if (w <= 0x7F) {-      *dest++ = w;-      /* An ASCII byte is likely to begin a run of ASCII bytes.-	 Falling back into the fast path really helps performance. */-      goto ascii;-    }-    else if (w <= 0x7FF) {-      *dest++ = (w >> 6) | 0xC0;-      *dest++ = (w & 0x3f) | 0x80;-    }-    else if (w < 0xD800 || w > 0xDBFF) {-      *dest++ = (w >> 12) | 0xE0;-      *dest++ = ((w >> 6) & 0x3F) | 0x80;-      *dest++ = (w & 0x3F) | 0x80;-    } else {-      uint32_t c = ((((uint32_t) w) - 0xD800) << 10) +-	(((uint32_t) *src++) - 0xDC00) + 0x10000;-      *dest++ = (c >> 18) | 0xF0;-      *dest++ = ((c >> 12) & 0x3F) | 0x80;-      *dest++ = ((c >> 6) & 0x3F) | 0x80;-      *dest++ = (c & 0x3F) | 0x80;-    }-  }--  *destp = dest;-}
+ cbits/is_ascii.c view
@@ -0,0 +1,54 @@+/*+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>+ */++#include <string.h>+#include <stdint.h>+#include <sys/types.h>+#ifdef __x86_64__+#include <emmintrin.h>+#include <xmmintrin.h>+#endif+#include <stdbool.h>++/*+  _hs_text_is_ascii takes a UTF-8 encoded buffer,+  and returns the length of the ASCII-compatible prefix.+*/+const size_t _hs_text_is_ascii(const uint8_t *src0, const uint8_t *srcend){+  const uint8_t *src = src0;++#ifdef __x86_64__+  // I experimented with larger vector registers,+  // but did not notice any measurable speed up, so let's keep it simple.+  while (src < srcend - 15){+    __m128i w128 = _mm_loadu_si128((__m128i *)src);+    // Which bytes are < 128?+    uint16_t mask = _mm_movemask_epi8(w128);+    if (mask) break;+    src+= 16;+  }+#endif++  while (src < srcend - 7){+    uint64_t w64;+    memcpy(&w64, src, sizeof(uint64_t));+    if (w64 & 0x8080808080808080ULL) break;+    src+= 8;+  }++  while (src < srcend){+    uint8_t leadByte = *src;+    if(leadByte >= 0x80) break;+    src++;+  }++  return src - src0;+}++/*+  _hs_text_is_ascii_offset is a helper for calling _hs_text_is_ascii on Texts.+*/+const size_t _hs_text_is_ascii_offset(const uint8_t *arr, size_t off, size_t len){+    return _hs_text_is_ascii(arr + off, arr + off + len);+}
+ cbits/measure_off.c view
@@ -0,0 +1,231 @@+/*
+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <sys/types.h>
+#ifdef __x86_64__
+#include <emmintrin.h>
+#include <xmmintrin.h>
+#include <immintrin.h>
+#include <cpuid.h>
+#endif
+#include <stdbool.h>
+
+// stdatomic.h has been introduces in gcc 4.9
+#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__))
+#ifndef __STDC_NO_ATOMICS__
+#define __STDC_NO_ATOMICS__
+#endif
+#endif
+
+#ifndef __STDC_NO_ATOMICS__
+#include <stdatomic.h>
+#endif
+
+/*
+  Clang-6 does not enable proper -march flags for assembly modules
+  which leads to "error: instruction requires: AVX-512 ISA"
+  at the assembler phase.
+
+  Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6
+  https://en.wikipedia.org/wiki/Xcode#Toolchain_versions
+  and it's the latest available version on macOS 10.13.
+
+  Disable AVX-512 instructions as they are most likely not supported
+  on the hardware running clang-6.
+*/
+#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \
+      || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__)
+#define COMPILER_SUPPORTS_AVX512
+#endif
+
+
+#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)
+bool has_avx512_vl_bw() {
+#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__)
+  uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
+  __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
+  // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features
+  const bool has_avx512_bw = ebx & (1 << 30);
+  const bool has_avx512_vl = ebx & (1 << 31);
+  // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl);
+  return has_avx512_bw && has_avx512_vl;
+#else
+  return false;
+#endif
+}
+#endif
+
+/*
+  measure_off_naive / measure_off_avx / measure_off_sse
+  take a UTF-8 sequence between src and srcend, and a number of characters cnt.
+  If the sequence is long enough to contain cnt characters, then return how many bytes
+  remained unconsumed. Otherwise, if the sequence is shorter, return
+  negated count of lacking characters. Cf. _hs_text_measure_off below.
+*/
+
+static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt)
+{
+  // Count leading bytes in 8 byte sequence
+  while (src < srcend - 7){
+    uint64_t w64;
+    memcpy(&w64, src, sizeof(uint64_t));
+    // find leading bytes by finding every byte that is not a continuation
+    // byte. The bit twiddle only results in a 0 if the original byte starts
+    // with 0b11...
+    w64 =  ((w64 << 1) | ~w64) & 0x8080808080808080ULL;
+    // compute the popcount of w64 with two bit shifts and a multiplication
+    size_t leads = (  (w64 >> 7)              // w64 >> 7           = Sum{0<= i <= 7} x_i * 256^i    (x_i \in {0,1})
+                    * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i
+                                              //              (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) 
+                                              // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) 
+                                              // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k
+                                              // and each i satisfies this equation for exactly one such j
+                                              // So each byte of the result contains the sum we want.
+                   ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j
+    if (cnt < leads) break;
+    cnt-= leads;
+    src+= 8;
+  }
+
+  // Skip until next leading byte
+  while (src < srcend){
+    uint8_t w8 = *src;
+    if ((int8_t)w8 >= -0x40) break;
+    src++;
+  }
+
+  // Finish up with tail
+  while (src < srcend && cnt > 0){
+    uint8_t leadByte = *src++;
+    cnt--;
+    src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0);
+  }
+
+  return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt);
+}
+
+#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)
+__attribute__((target("avx512vl,avx512bw")))
+static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt)
+{
+  while (src < srcend - 63){
+    __m512i w512 = _mm512_loadu_si512((__m512i *)src);
+    // Which bytes are either < 128 or >= 192?
+    uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF));
+    size_t leads = __builtin_popcountll(mask);
+    if (cnt < leads) break;
+    cnt-= leads;
+    src+= 64;
+  }
+
+  // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties
+  // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html
+
+  if (src < srcend - 31){
+    __m256i w256 = _mm256_loadu_si256((__m256i *)src);
+    uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF));
+    size_t leads = __builtin_popcountl(mask);
+    if (cnt >= leads){
+      cnt-= leads;
+      src+= 32;
+    }
+  }
+
+  if (src < srcend - 15){
+    __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16
+    uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8
+    size_t leads = __builtin_popcountl(mask);
+    if (cnt >= leads){
+      cnt-= leads;
+      src+= 16;
+    }
+  }
+
+  return measure_off_naive(src, srcend, cnt);
+}
+#endif
+
+/* Count the number of bits set in the argument 
+ * 
+   This is a temporary workaround for the fact that
+   the GHC RTS linker does not recognize the `__builtin_popcountll`
+   symbol.
+ 
+   See https://gitlab.haskell.org/ghc/ghc/-/issues/21787
+       https://gitlab.haskell.org/ghc/ghc/-/issues/19900
+       https://github.com/haskell/text/issues/450
+ 
+   It can be removed and the usages of popcount64 replaced with
+   `__builtin_popcountll` once affected versions of the compiler
+   are no longer in widespread use.
+ 
+   Once GHC learns to recognize the symbol, this can be gated
+   by CPP to call __builtin_popcountll when using the appropriate
+   version of GHC.
+*/
+static inline const size_t popcount16(uint16_t x) {
+
+  // Taken from https://en.wikipedia.org/wiki/Hamming_weight
+  const uint16_t m1  = 0x5555; //binary: 0101...
+  const uint16_t m2  = 0x3333; //binary: 00110011..
+  const uint16_t m4  = 0x0f0f; //binary:  4 zeros,  4 ones ...
+  x -= (x >> 1) & m1;             //put count of each 2 bits into those 2 bits
+  x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits 
+  x = (x + (x >> 4)) & m4;        //put count of each 8 bits into those 8 bits 
+  return (x >> 8) + (x & 0x00FF);
+}
+
+static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt)
+{
+#ifdef __x86_64__
+  while (src < srcend - 15){
+    __m128i w128 = _mm_loadu_si128((__m128i *)src);
+    // Which bytes are either < 128 or >= 192?
+    uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF)));
+    size_t leads = popcount16(mask);
+    if (cnt < leads) break;
+    cnt-= leads;
+    src+= 16;
+  }
+#endif
+
+  return measure_off_naive(src, srcend, cnt);
+}
+
+typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t);
+
+/*
+  _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len),
+  and a number of code points (aka characters) cnt. If the buffer is long enough
+  to contain cnt characters, then _hs_text_measure_off returns a non-negative number,
+  measuring their size in code units (aka bytes). If the buffer is shorter,
+  _hs_text_measure_off returns a non-positive number, which is a negated total count
+  of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0
+  as well.
+
+  This scheme allows us to implement both take/drop and length with the same C function.
+
+  The input buffer (src, off, len) must be a valid UTF-8 sequence,
+  this condition is not checked.
+*/
+ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) {
+#ifndef __STDC_NO_ATOMICS__
+  static _Atomic measure_off_t s_impl = (measure_off_t)NULL;
+  measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed);
+  if (!impl) {
+#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)
+    impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse;
+#else
+    impl = measure_off_sse;
+#endif
+    atomic_store_explicit(&s_impl, impl, memory_order_relaxed);
+  }
+#else
+  measure_off_t impl = measure_off_sse;
+#endif
+  ssize_t ret = (*impl)(src + off, src + off + len, cnt);
+  return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret));
+}
+ cbits/reverse.c view
@@ -0,0 +1,42 @@+/*+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>+ */++#include <string.h>+#include <stdint.h>++/*+  _hs_text_reverse takes a UTF-8 encoded buffer, specified by (src0, off, len),+  and reverses it, writing output starting from dst0.++  The input buffer (src0, off, len) must be a valid UTF-8 sequence,+  this condition is not checked.+*/+void _hs_text_reverse(uint8_t *dst0, const uint8_t *src0, size_t off, size_t len)+{+  const uint8_t *src = src0 + off;+  const uint8_t *srcend = src + len;+  uint8_t *dst = dst0 + len - 1;++  while (src < srcend){+    uint8_t leadByte = *src++;+    if (leadByte < 0x80){+      *dst-- = leadByte;+    } else if (leadByte < 0xe0){+      *(dst-1) = leadByte;+      *dst     = *src++;+      dst-=2;+    } else if (leadByte < 0xf0){+      *(dst-2) = leadByte;+      *(dst-1) = *src++;+      *dst     = *src++;+      dst-=3;+    } else {+      *(dst-3) = leadByte;+      *(dst-2) = *src++;+      *(dst-1) = *src++;+      *dst     = *src++;+      dst-=4;+    }+  }+}
+ cbits/utils.c view
@@ -0,0 +1,21 @@+/*+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>+ */++#include <stdint.h>+#include <stdio.h>+#include <string.h>+#include <sys/types.h>++/* Changed name to disambiguate from _hs_text_memcmp,+   which could be present in system-wide headers from installed ghc package */+int _hs_text_memcmp2(const void *arr1, size_t off1, const void *arr2, size_t off2, size_t len)+{+  return memcmp(arr1 + off1, arr2 + off2, len);+}++ssize_t _hs_text_memchr(const void *arr, size_t off, size_t len, uint8_t byte)+{+  const void *ptr = memchr(arr + off, byte, len);+  return ptr == NULL ? -1 : ptr - (arr + off);+}
changelog.md view
@@ -1,219 +1,440 @@-#### 1.2.3.2--* Special release supporting GHC 8.10.1 / `base-4.14.0.0` only--#### 1.2.3.1--* Make `decodeUtf8With` fail explicitly for unsupported non-BMP-  replacement characters instead silent undefined behaviour (gh-213)--* Fix termination condition for file reads via `Data.Text.IO`-  operations (gh-223)--* A serious correctness issue affecting uses of `take` and `drop` with-  negative counts has been fixed (gh-227)--* A bug in the case-mapping functions resulting in unreasonably large-  allocations with large arguments has been fixed (gh-221)--### 1.2.3.0--* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec-  (updated from 8.0).--* Bug fix: the lazy `takeWhileEnd` function violated the-  [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)-  (gh-184).--* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).--* New function: `unsnoc` (gh-173).--* Reduce memory overhead in `encodeUTF8` (gh-194).--* Improve UTF-8 decoder error-recovery (gh-182).--* Minor documentation improvements (`@since` annotations, more-  examples, clarifications).--#### 1.2.2.2--* The `toTitle` function now correctly handles letters that-  immediately follow punctuation. Before, `"there's"` would turn into-  `"There'S"`. Now, it becomes `"There's"`.--* The implementation of unstreaming is faster, resulting in operations-  such as `map` and `intersperse` speeding up by up to 30%, with-  smaller code generated.--* The optimised length comparison function is now more likely to be-  used after some rewrite rule tweaking.--* Bug fix: an off-by-one bug in `takeEnd` is fixed.--* Bug fix: a logic error in `takeWord16` is fixed.--#### 1.2.2.1--* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.-  The build flag has been renamed accordingly.  Your army of diligent-  maintainers apologizes for the churn.--* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec-  (updated from 7.0)--* An STG lint error has been fixed--### 1.2.2.0--* The `integer-simple` package, upon which this package optionally-  depended, has been replaced with `integer-pure`.  The build flag has-  been renamed accordingly.--* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a-  `get`, the error is propagated via `fail` instead of an uncatchable-  crash.--* New function: `takeWhileEnd`--* New instances for the `Text` types:-    * if `base` >= 4.7: `PrintfArg`-    * if `base` >= 4.9: `Semigroup`--#### 1.2.1.3--* Bug fix: As it turns out, moving the literal rewrite rules to simplifier-  phase 2 does not prevent competition with the `unpack` rule, which is-  also active in this phase. Unfortunately this was hidden due to a silly-  test environment mistake. Moving literal rules back to phase 1 finally-  fixes GHC Trac #10528 correctly.--#### 1.2.1.2--* Bug fix: Run literal rewrite rules in simplifier phase 2.-  The behavior of the simplifier changed in GHC 7.10.2,-  causing these rules to fail to fire, leading to poor code generation-  and long compilation times. See-  [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).--#### 1.2.1.1--* Expose unpackCString#, which you should never use.--### 1.2.1.0--* Added Binary instances for both Text types. (If you have previously-  been using the text-binary package to get a Binary instance, it is-  now obsolete.)--#### 1.2.0.6--* Fixed a space leak in UTF-8 decoding--#### 1.2.0.5--* Feature parity: repeat, cycle, iterate are now implemented for lazy-  Text, and the Data instance is more complete--* Build speed: an inliner space explosion has been fixed with toCaseFold--* Bug fix: encoding Int to a Builder would infinite-loop if the-  integer-simple package was used--* Deprecation: OnEncodeError and EncodeError are deprecated, as they-  are never used--* Internals: some types that are used internally in fusion-related-  functions have moved around, been renamed, or been deleted (we don't-  bump the major version if .Internal modules change)--* Spec compliance: toCaseFold now follows the Unicode 7.0 spec-  (updated from 6.3)--#### 1.2.0.4--* Fixed an incompatibility with base < 4.5--#### 1.2.0.3--* Update formatRealFloat to correspond to the definition in versions-  of base newer than 4.5 (https://github.com/bos/text/issues/105)--#### 1.2.0.2--* Bumped lower bound on deepseq to 1.4 for compatibility with the-  upcoming GHC 7.10--#### 1.2.0.1--* Fixed a buffer overflow in rendering of large Integers-  (https://github.com/bos/text/issues/99)--## 1.2.0.0--* Fixed an integer overflow in the replace function-  (https://github.com/bos/text/issues/81)--* Fixed a hang in lazy decodeUtf8With-  (https://github.com/bos/text/issues/87)--* Reduced codegen bloat caused by use of empty and single-character-  literals--* Added an instance of IsList for GHC 7.8 and above--### 1.1.1.0--* The Data.Data instance now allows gunfold to work, via a virtual-  pack constructor--* dropEnd, takeEnd: new functions--* Comparing the length of a Text against a number can now-  short-circuit in more cases--#### 1.1.0.1--* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes-  in single-byte chunks--## 1.1.0.0--* encodeUtf8: Performance is improved by up to 4x.--* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions,-  available only if bytestring >= 0.10.4.0 is installed, that allow-  very fast and flexible encoding of a Text value to a bytestring-  Builder.--  As an example of the performance gain to be had, the-  encodeUtf8BuilderEscaped function helps to double the speed of JSON-  encoding in the latest version of aeson! (Note: if all you need is a-  plain ByteString, encodeUtf8 is still the faster way to go.)--* All of the internal module hierarchy is now publicly exposed.  If a-  module is in the .Internal hierarchy, or is documented as internal,-  use at your own risk - there are no API stability guarantees for-  internal modules!--#### 1.0.0.1--* decodeUtf8: Fixed a regression that caused us to incorrectly-  identify truncated UTF-8 as valid (gh-61)--# 1.0.0.0--* Added support for Unicode 6.3.0 to case conversion functions--* New function toTitle converts words in a string to title case--* New functions peekCStringLen and withCStringLen simplify-  interoperability with C functionns--* Added support for decoding UTF-8 in stream-friendly fashion--* Fixed a bug in mapAccumL--* Added trusted Haskell support--* Removed support for GHC 6.10 (released in 2008) and older+### 2.1.4 - 2026-01-27
+
+* [Upgrade to Unicode 17.0](https://github.com/haskell/text/pull/658) + [Fix `CaseMapping` generation script to not depend on GHC's Unicode data](https://github.com/haskell/text/pull/687)
+
+* [simdutf: update to 8.0.0](https://github.com/haskell/text/pull/685)
+
+* [Add `decodeUtf8Lenient` for lazy `Text`](https://github.com/haskell/text/pull/690)
+
+* [`scanl`/`scanr` should replace invalid `Char` in the initial value](https://github.com/haskell/text/pull/669)
+
+* [Shave off redundant field of `Text.Internal.Buffer`](https://github.com/haskell/text/pull/659)
+
+* [Switch from template-haskell to template-haskell-lift](https://github.com/haskell/text/pull/661)
+
+#### Minor changes
+
+* [Avoid calling `length` on chunks in lazy `splitAt`](https://github.com/haskell/text/pull/676)
+
+* [Check for zero length in internal `isSingleton`](https://github.com/haskell/text/pull/675)
+
+* [Implement folds directly, without resorting to streaming framework](https://github.com/haskell/text/pull/667)
+
+* [Implement `cons`, `snoc`, `head`, `isSingleton`, `isPrefixOf` directly, without resorting to streaming framework](https://github.com/haskell/text/pull/666)
+
+* [Mark `caseConvert` (the underlying implementation of `toUpper` / `toLower` / `toTitle`) as `INLINABLE`, not `INLINE`](https://github.com/haskell/text/pull/664)
+
+* [Express `index` via `measureOff` instead of going through fusion framework](https://github.com/haskell/text/pull/663)
+
+* [Guard `#define __STDC_NO_ATOMICS__` by `#ifndef`](https://github.com/haskell/text/pull/657)
+
+* [Support QuickCheck-2.17](https://github.com/haskell/text/pull/662)
+
+* [Bump lower bound of binary to >= 0.8.3](https://github.com/haskell/text/pull/673)
+
+#### Documentation
+
+* [A bit more documentation for `Data.Text.Internal.Encoding.Utf8`](https://github.com/haskell/text/pull/691)
+
+* [Clarify documentation of `Data.Text.Foreign`](https://github.com/haskell/text/pull/681)
+
+* [Haddocks: Hyperlink some identifiers and modules](https://github.com/haskell/text/pull/677)
+
+* [`since` pragmas for type synonyms](https://github.com/haskell/text/pull/671)
+
+* [Improve documentation for `streamDecodeUtf8With`](https://github.com/haskell/text/pull/665)
+
+* [Add comprehensive documentation for `hGetChunk`](https://github.com/haskell/text/pull/655)
+
+### 2.1.3 - 2025-08-01
+
+* [Fix CRLF handling in IO functions](https://github.com/haskell/text/pull/649)
+
+* [Change `utf8LengthByLeader` to a branching implementation](https://github.com/haskell/text/pull/635)
+
+* [Define `stimes 0` for lazy text](https://github.com/haskell/text/pull/641)
+
+* [Add implementation of `sconcat` and `stimes` for strict `Text`](https://github.com/haskell/text/pull/580) and [Fix `stimes` for strict text when size wraps around `Int`](https://github.com/haskell/text/pull/639)
+
+* [Allow list fusion for `unpack` over both strict and lazy `Text`](https://github.com/haskell/text/pull/629)
+
+### 2.1.2
+
+* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618)
+
+* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581)
+
+* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558)
+
+* [Add `foldlM'`](https://github.com/haskell/text/pull/543)
+
+* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599)
+
+* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608)
+
+* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619)
+
+* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565)
+
+* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579)
+
+* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600)
+
+* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589)
+
+* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617)
+
+* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575)
+
+* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572)
+
+* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611)
+
+* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code.
+
+### 2.1.1
+
+* Add pure Haskell implementations as an alternative to C-based ones,
+  suitable for JavaScript backend.
+
+* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547)
+
+* [Share empty `Text` values](https://github.com/haskell/text/pull/493)
+
+* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553)
+
+* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551)
+
+* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560)
+
+### 2.1
+
+* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474)
+
+* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503)
+
+* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483)
+
+* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527)
+
+* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528)
+
+* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534)
+
+### 2.0.2
+
+* [Add decoding functions in `Data.Text.Encoding` that allow
+  more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge!
+  * `decodeASCIIPrefix`
+  * `decodeUtf8Chunk`
+  * `decodeUtf8More`
+  * `Utf8ValidState`
+  * `startUtf8ValidState`
+  * `StrictBuilder`
+  * `strictBuilderToText`
+  * `textToStrictBuilder`
+  * `validateUtf8Chunk`
+  * `validateUtf8More`
+
+* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495)
+
+* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497)
+
+* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499)
+
+* Add internal module `Data.Text.Internal.StrictBuilder`
+
+* Add internal module `Data.Text.Internal.Encoding`
+
+* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module.
+
+* [Speed up case conversions](https://github.com/haskell/text/pull/460)
+
+* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468)
+
+* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485)
+
+### 2.0.1
+
+* Improve portability of C and C++ code.
+* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413)
+* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402)
+* [Add `fromPtr0`](https://github.com/haskell/text/pull/423)
+* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436)
+* [Add `withCString`](https://github.com/haskell/text/pull/431)
+* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437)
+
+### 2.0
+
+* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365):
+  * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`.
+  * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`.
+  * Rename functions and types in `Data.Text.Foreign` to reflect switch
+    from `Word16` to `Word8`.
+  * Rename slicing functions in `Data.Text.Unsafe` to reflect switch
+    from `Word16` to `Word8`.
+  * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`.
+  * Change semantics and order of arguments of `Data.Text.Array.copyI`:
+    pass length, not end offset.
+  * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines.
+  * Extend interface of `Data.Text.Array` with more utility functions.
+  * Add `instance Show Data.Text.Unsafe.Iter`.
+  * Add `Data.Text.measureOff`.
+  * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`.
+  * Export `Data.Text.Internal.Lazy.equal`.
+  * Export `Data.Text.Internal.append`.
+  * Add `Data.Text.Internal.Private.spanAscii_`.
+  * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane.
+* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348)
+* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342)
+* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343)
+* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354)
+* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355)
+* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355)
+* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388)
+
+### 1.2.5.0
+
+* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305)
+* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320)
+* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274)
+* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281)
+* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325)
+* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302)
+* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334)
+* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333)
+
+### 1.2.4.1
+
+* Support `template-haskell-2.17.0.0`
+* Support `bytestring-0.11`
+* Add `take . drop` related RULE
+
+### 1.2.4.0
+
+* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232)
+
+* Update Haddock documentation to better reflect fusion eligibility; improve fusion
+  rules for `takeWhileEnd` and `length` (gh-241, ghc-202)
+
+* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times,
+  call it only `O(log n)` times on chunks of increasing size. The total
+  asymptotic complexity remains `O(nm)`. (gh-209)
+
+* Support `base-4.13.0.0`
+
+### 1.2.3.1
+
+* Make `decodeUtf8With` fail explicitly for unsupported non-BMP
+  replacement characters instead silent undefined behaviour (gh-213)
+
+* Fix termination condition for file reads via `Data.Text.IO`
+  operations (gh-223)
+
+* A serious correctness issue affecting uses of `take` and `drop` with
+  negative counts has been fixed (gh-227)
+
+* A bug in the case-mapping functions resulting in unreasonably large
+  allocations with large arguments has been fixed (gh-221)
+
+### 1.2.3.0
+
+* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec
+  (updated from 8.0).
+
+* Bug fix: the lazy `takeWhileEnd` function violated the
+  [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)
+  (gh-184).
+
+* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).
+
+* New function: `unsnoc` (gh-173).
+
+* Reduce memory overhead in `encodeUTF8` (gh-194).
+
+* Improve UTF-8 decoder error-recovery (gh-182).
+
+* Minor documentation improvements (`@since` annotations, more
+  examples, clarifications).
+
+#### 1.2.2.2
+
+* The `toTitle` function now correctly handles letters that
+  immediately follow punctuation. Before, `"there's"` would turn into
+  `"There'S"`. Now, it becomes `"There's"`.
+
+* The implementation of unstreaming is faster, resulting in operations
+  such as `map` and `intersperse` speeding up by up to 30%, with
+  smaller code generated.
+
+* The optimised length comparison function is now more likely to be
+  used after some rewrite rule tweaking.
+
+* Bug fix: an off-by-one bug in `takeEnd` is fixed.
+
+* Bug fix: a logic error in `takeWord16` is fixed.
+
+#### 1.2.2.1
+
+* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.
+  The build flag has been renamed accordingly.  Your army of diligent
+  maintainers apologizes for the churn.
+
+* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec
+  (updated from 7.0)
+
+* An STG lint error has been fixed
+
+### 1.2.2.0
+
+* The `integer-simple` package, upon which this package optionally
+  depended, has been replaced with `integer-pure`.  The build flag has
+  been renamed accordingly.
+
+* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a
+  `get`, the error is propagated via `fail` instead of an uncatchable
+  crash.
+
+* New function: `takeWhileEnd`
+
+* New instances for the `Text` types:
+    * if `base` >= 4.7: `PrintfArg`
+    * if `base` >= 4.9: `Semigroup`
+
+#### 1.2.1.3
+
+* Bug fix: As it turns out, moving the literal rewrite rules to simplifier
+  phase 2 does not prevent competition with the `unpack` rule, which is
+  also active in this phase. Unfortunately this was hidden due to a silly
+  test environment mistake. Moving literal rules back to phase 1 finally
+  fixes GHC Trac #10528 correctly.
+
+#### 1.2.1.2
+
+* Bug fix: Run literal rewrite rules in simplifier phase 2.
+  The behavior of the simplifier changed in GHC 7.10.2,
+  causing these rules to fail to fire, leading to poor code generation
+  and long compilation times. See
+  [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).
+
+#### 1.2.1.1
+
+* Expose unpackCString#, which you should never use.
+
+### 1.2.1.0
+
+* Added Binary instances for both Text types. (If you have previously
+  been using the text-binary package to get a Binary instance, it is
+  now obsolete.)
+
+#### 1.2.0.6
+
+* Fixed a space leak in UTF-8 decoding
+
+#### 1.2.0.5
+
+* Feature parity: repeat, cycle, iterate are now implemented for lazy
+  Text, and the Data instance is more complete
+
+* Build speed: an inliner space explosion has been fixed with toCaseFold
+
+* Bug fix: encoding Int to a Builder would infinite-loop if the
+  integer-simple package was used
+
+* Deprecation: OnEncodeError and EncodeError are deprecated, as they
+  are never used
+
+* Internals: some types that are used internally in fusion-related
+  functions have moved around, been renamed, or been deleted (we don't
+  bump the major version if .Internal modules change)
+
+* Spec compliance: toCaseFold now follows the Unicode 7.0 spec
+  (updated from 6.3)
+
+#### 1.2.0.4
+
+* Fixed an incompatibility with base < 4.5
+
+#### 1.2.0.3
+
+* Update formatRealFloat to correspond to the definition in versions
+  of base newer than 4.5 (https://github.com/bos/text/issues/105)
+
+#### 1.2.0.2
+
+* Bumped lower bound on deepseq to 1.4 for compatibility with the
+  upcoming GHC 7.10
+
+#### 1.2.0.1
+
+* Fixed a buffer overflow in rendering of large Integers
+  (https://github.com/bos/text/issues/99)
+
+## 1.2.0.0
+
+* Fixed an integer overflow in the replace function
+  (https://github.com/bos/text/issues/81)
+
+* Fixed a hang in lazy decodeUtf8With
+  (https://github.com/bos/text/issues/87)
+
+* Reduced codegen bloat caused by use of empty and single-character
+  literals
+
+* Added an instance of IsList for GHC 7.8 and above
+
+### 1.1.1.0
+
+* The Data.Data instance now allows gunfold to work, via a virtual
+  pack constructor
+
+* dropEnd, takeEnd: new functions
+
+* Comparing the length of a Text against a number can now
+  short-circuit in more cases
+
+#### 1.1.0.1
+
+* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes
+  in single-byte chunks
+
+## 1.1.0.0
+
+* encodeUtf8: Performance is improved by up to 4x.
+
+* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions,
+  available only if bytestring >= 0.10.4.0 is installed, that allow
+  very fast and flexible encoding of a Text value to a bytestring
+  Builder.
+
+  As an example of the performance gain to be had, the
+  encodeUtf8BuilderEscaped function helps to double the speed of JSON
+  encoding in the latest version of aeson! (Note: if all you need is a
+  plain ByteString, encodeUtf8 is still the faster way to go.)
+
+* All of the internal module hierarchy is now publicly exposed.  If a
+  module is in the .Internal hierarchy, or is documented as internal,
+  use at your own risk - there are no API stability guarantees for
+  internal modules!
+
+#### 1.0.0.1
+
+* decodeUtf8: Fixed a regression that caused us to incorrectly
+  identify truncated UTF-8 as valid (gh-61)
+
+# 1.0.0.0
+
+* Added support for Unicode 6.3.0 to case conversion functions
+
+* New function toTitle converts words in a string to title case
+
+* New functions peekCStringLen and withCStringLen simplify
+  interoperability with C functions
+
+* Added support for decoding UTF-8 in stream-friendly fashion
+
+* Fixed a bug in mapAccumL
+
+* Added trusted Haskell support
+
+* Removed support for GHC 6.10 (released in 2008) and older
− include/text_cbits.h 
@@ -1,11 +0,0 @@-/*- * Copyright (c) 2013 Bryan O'Sullivan <bos@serpentine.com>.- */--#ifndef _text_cbits_h-#define _text_cbits_h--#define UTF8_ACCEPT 0-#define UTF8_REJECT 12--#endif
scripts/Arsec.hs view
@@ -1,44 +1,48 @@-module Arsec-    (-      Comment-    , comment-    , semi-    , showC-    , unichar-    , unichars-    , module Control.Applicative-    , module Control.Monad-    , module Data.Char-    , module Text.ParserCombinators.Parsec.Char-    , module Text.ParserCombinators.Parsec.Combinator-    , module Text.ParserCombinators.Parsec.Error-    , module Text.ParserCombinators.Parsec.Prim-    ) where--import Control.Monad-import Control.Applicative-import Data.Char-import Numeric-import Text.ParserCombinators.Parsec.Char hiding (lower, upper)-import Text.ParserCombinators.Parsec.Combinator hiding (optional)-import Text.ParserCombinators.Parsec.Error-import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many)--type Comment = String--unichar :: Parser Char-unichar = chr . fst . head . readHex <$> many1 hexDigit--unichars :: Parser [Char]-unichars = manyTill (unichar <* spaces) semi--semi :: Parser ()-semi = char ';' *> spaces *> pure ()--comment :: Parser Comment-comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n"--showC :: Char -> String-showC c = "'\\x" ++ d ++ "'"-    where h = showHex (ord c) ""-          d = replicate (4 - length h) '0' ++ h+module Arsec
+    (
+      Comment
+    , comment
+    , semi
+    , showC
+    , unichar
+    , unichars
+    , module Control.Applicative
+    , module Control.Monad
+    , module Text.ParserCombinators.Parsec.Char
+    , module Text.ParserCombinators.Parsec.Combinator
+    , module Text.ParserCombinators.Parsec.Error
+    , module Text.ParserCombinators.Parsec.Prim
+    ) where
+
+import Prelude hiding (head, tail)
+import Control.Monad
+import Control.Applicative
+import Data.Char
+import Numeric (readHex, showHex)
+import Text.ParserCombinators.Parsec.Char hiding (lower, upper)
+import Text.ParserCombinators.Parsec.Combinator hiding (optional)
+import Text.ParserCombinators.Parsec.Error
+import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many)
+
+type Comment = String
+
+unichar :: Parser Char
+unichar = do
+  digits <- many1 hexDigit
+  case readHex digits of
+    [] -> error "unichar: cannot parse hex digits"
+    (hd, _) : _ -> pure $ chr hd
+
+unichars :: Parser [Char]
+unichars = manyTill (unichar <* spaces) semi
+
+semi :: Parser ()
+semi = char ';' *> spaces *> pure ()
+
+comment :: Parser Comment
+comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n"
+
+showC :: Char -> String
+showC c = "'\\x" ++ d ++ "'"
+    where h = showHex (ord c) ""
+          d = replicate (4 - length h) '0' ++ h
scripts/CaseFolding.hs view
@@ -1,46 +1,47 @@--- This script processes the following source file:------   http://unicode.org/Public/UNIDATA/CaseFolding.txt--module CaseFolding-    (-      CaseFolding(..)-    , Fold(..)-    , parseCF-    , mapCF-    ) where--import Arsec--data Fold = Fold {-      code :: Char-    , status :: Char-    , mapping :: [Char]-    , name :: String-    } deriving (Eq, Ord, Show)--data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] }-                 deriving (Show)--entries :: Parser CaseFolding-entries = CF <$> many comment <*> many (entry <* many comment)-  where-    entry = Fold <$> unichar <* semi-                 <*> oneOf "CFST" <* semi-                 <*> unichars-                 <*> (string "# " *> manyTill anyToken (char '\n'))--parseCF :: FilePath -> IO (Either ParseError CaseFolding)-parseCF name = parse entries name <$> readFile name--mapCF :: CaseFolding -> [String]-mapCF (CF _ ms) = typ ++ (map nice . filter p $ ms) ++ [last]-  where-    typ = ["foldMapping :: forall s. Char -> s -> Step (CC s) Char"-           ,"{-# NOINLINE foldMapping #-}"]-    last = "foldMapping c s = Yield (toLower c) (CC s '\\0' '\\0')"-    nice c = "-- " ++ name c ++ "\n" ++-             "foldMapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")"-       where [x,y,z] = (map showC . take 3) (mapping c ++ repeat '\0')-    p f = status f `elem` "CF" &&-          mapping f /= [toLower (code f)]+-- This script processes the following source file:
+--
+--   http://unicode.org/Public/UNIDATA/CaseFolding.txt
+
+module CaseFolding
+    (
+      CaseFolding(..)
+    , Fold(..)
+    , parseCF
+    , mapCF
+    ) where
+
+import Arsec
+import Data.Bits
+import Data.Char (ord)
+
+data Fold = Fold {
+      code :: Char
+    , status :: Char
+    , mapping :: [Char]
+    , name :: String
+    } deriving (Eq, Ord, Show)
+
+data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] }
+                 deriving (Show)
+
+entries :: Parser CaseFolding
+entries = CF <$> many comment <*> many (entry <* many comment)
+  where
+    entry = Fold <$> unichar <* semi
+                 <*> oneOf "CFST" <* semi
+                 <*> unichars
+                 <*> (string "# " *> manyTill anyToken (char '\n'))
+
+parseCF :: FilePath -> IO (Either ParseError CaseFolding)
+parseCF name = parse entries name <$> readFile name
+
+mapCF :: CaseFolding -> [String]
+mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last]
+  where
+    typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}"
+           ,"{-# NOINLINE foldMapping #-}"
+           ,"foldMapping = \\case"]
+    last = "  _ -> unI64 0"
+    printUnusual c = "  -- " ++ name c ++ "\n" ++
+             "  " ++ showC (code c) ++ "# -> unI64 "  ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))
+       where x:y:z:_ = mapping c ++ repeat '\0'
scripts/CaseMapping.hs view
@@ -1,38 +1,66 @@-import System.Environment-import System.IO--import Arsec-import CaseFolding-import SpecialCasing--main = do-  args <- getArgs-  let oname = case args of-                [] -> "../Data/Text/Internal/Fusion/CaseMapping.hs"-                [o] -> o-  psc <- parseSC "SpecialCasing.txt"-  pcf <- parseCF "CaseFolding.txt"-  scs <- case psc of-           Left err -> print err >> return undefined-           Right ms -> return ms-  cfs <- case pcf of-           Left err -> print err >> return undefined-           Right ms -> return ms-  h <- openFile oname WriteMode-  let comments = map ("--" ++) $-                 take 2 (cfComments cfs) ++ take 2 (scComments scs)-  mapM_ (hPutStrLn h) $-                      ["{-# LANGUAGE Rank2Types #-}"-                      ,"-- AUTOMATICALLY GENERATED - DO NOT EDIT"-                      ,"-- Generated by scripts/CaseMapping.hs"] ++-                      comments ++-                      [""-                      ,"module Data.Text.Internal.Fusion.CaseMapping where"-                      ,"import Data.Char"-                      ,"import Data.Text.Internal.Fusion.Types"-                      ,""]-  mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs)-  mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs)-  mapM_ (hPutStrLn h) (mapSC "title" title toTitle scs)-  mapM_ (hPutStrLn h) (mapCF cfs)-  hClose h+import Data.Char (isDigit)
+import Data.Foldable (toList)
+import Data.List (stripPrefix)
+import Data.Maybe (fromJust)
+import System.Environment
+import System.IO
+
+import Arsec
+import CaseFolding
+import SpecialCasing
+import UnicodeData
+
+main = do
+  args <- getArgs
+  let oname = case args of
+                [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs"
+                [o] -> o
+  psc <- parseSC "SpecialCasing.txt"
+  pcf <- parseCF "CaseFolding.txt"
+  ud <- parseUD "UnicodeData.txt"
+  scs <- case psc of
+           Left err -> print err >> return undefined
+           Right ms -> return ms
+  cfs <- case pcf of
+           Left err -> print err >> return undefined
+           Right ms -> return ms
+  ud <- case ud of
+           Left err -> print err >> return undefined
+           Right ms -> return ms
+  h <- openFile oname WriteMode
+  let comments = map ("--" ++) $
+                 take 2 (cfComments cfs) ++ take 2 (scComments scs)
+      version = parseVersion (cfComments cfs)
+  mapM_ (hPutStrLn h) $
+                      ["-- AUTOMATICALLY GENERATED - DO NOT EDIT"
+                      ,"-- Generated by scripts/CaseMapping.hs"] ++
+                      comments ++
+                      [""
+                      ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}"
+                      ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}"
+                      ,"module Data.Text.Internal.Fusion.CaseMapping where"
+                      ,"import GHC.Int"
+                      ,"import GHC.Exts"
+                      ,"import Data.Version (Version, makeVersion)"
+                      ,"unicodeVersion :: Version"
+                      ,"unicodeVersion = makeVersion " ++ version
+                      ,"unI64 :: Int64 -> _ {- unboxed Int64 -}"
+                      ,"unI64 (I64# n) = n"
+                      ,""]
+  let get f = [(k, d) | c <- toList ud, Just d <- [f c], let k = charUD c, k /= d]
+  mapM_ (hPutStrLn h) (mapSC "upper" upper (get toUpperUD) scs)
+  mapM_ (hPutStrLn h) (mapSC "lower" lower (get toLowerUD) scs)
+  mapM_ (hPutStrLn h) (mapSC "title" title (get toTitleUD) scs)
+  mapM_ (hPutStrLn h) (mapCF cfs)
+  hClose h
+
+-- Parse version from CaseFolding comments
+-- and render it as a list (an argument of makeVersion)
+parseVersion :: [String] -> String
+parseVersion comments = fromJust $ do
+  line' : _ <- pure comments
+  line'' <- stripPrefix " CaseFolding-" line'
+  let (v1, line1) = span isDigit line''
+      (v2, line2) = span isDigit (drop 1 line1)
+      (v3, _) = span isDigit (drop 1 line2)
+  pure $ "[" ++ v1 ++ ", " ++ v2 ++ ", " ++ v3 ++ "]"
scripts/SpecialCasing.hs view
@@ -1,56 +1,61 @@--- This script processes the following source file:------   http://unicode.org/Public/UNIDATA/SpecialCasing.txt--module SpecialCasing-    (-      SpecialCasing(..)-    , Case(..)-    , parseSC-    , mapSC-    ) where--import Arsec--data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] }-                   deriving (Show)--data Case = Case {-      code :: Char-    , lower :: [Char]-    , title :: [Char]-    , upper :: [Char]-    , conditions :: String-    , name :: String-    } deriving (Eq, Ord, Show)--entries :: Parser SpecialCasing-entries = SC <$> many comment <*> many (entry <* many comment)-  where-    entry = Case <$> unichar <* semi-                 <*> unichars-                 <*> unichars-                 <*> unichars-                 <*> manyTill anyToken (string "# ")-                 <*> manyTill anyToken (char '\n')--parseSC :: FilePath -> IO (Either ParseError SpecialCasing)-parseSC name = parse entries name <$> readFile name--mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing-         -> [String]-mapSC which access twiddle (SC _ ms) =-    typ ++ (map nice . filter p $ ms) ++ [last]-  where-    typ = [which ++ "Mapping :: forall s. Char -> s -> Step (CC s) Char"-           ,"{-# NOINLINE " ++ which ++ "Mapping #-}"]-    last = which ++ "Mapping c s = Yield (to" ++ ucFirst which ++ " c) (CC s '\\0' '\\0')"-    nice c = "-- " ++ name c ++ "\n" ++-             which ++ "Mapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")"-       where [x,y,z] = (map showC . take 3) (access c ++ repeat '\0')-    p c = [k] /= a && a /= [twiddle k] && null (conditions c)-        where a = access c-              k = code c--ucFirst (c:cs) = toUpper c : cs-ucFirst [] = []+-- This script processes the following source file:
+--
+--   http://unicode.org/Public/UNIDATA/SpecialCasing.txt
+
+module SpecialCasing
+    (
+      SpecialCasing(..)
+    , Case(..)
+    , parseSC
+    , mapSC
+    ) where
+
+import Arsec
+import Data.Bits
+import Data.Char (ord)
+
+data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] }
+                   deriving (Show)
+
+data Case = Case {
+      code :: Char
+    , lower :: [Char]
+    , title :: [Char]
+    , upper :: [Char]
+    , conditions :: String
+    , name :: String
+    } deriving (Eq, Ord, Show)
+
+entries :: Parser SpecialCasing
+entries = SC <$> many comment <*> many (entry <* many comment)
+  where
+    entry = Case <$> unichar <* semi
+                 <*> unichars
+                 <*> unichars
+                 <*> unichars
+                 <*> manyTill anyToken (string "# ")
+                 <*> manyTill anyToken (char '\n')
+
+parseSC :: FilePath -> IO (Either ParseError SpecialCasing)
+parseSC name = parse entries name <$> readFile name
+
+mapSC :: String -> (Case -> String) -> [(Char, Char)] -> SpecialCasing
+         -> [String]
+mapSC which access twiddle (SC _ ms) =
+    typ ++ map printUnusual ms' ++ map printUsual usual ++ [last]
+  where
+    ms' = filter p ms
+    p c = [k] /= a && null (conditions c)
+        where a = access c
+              k = code c
+    unusual = map code ms'
+    usual = filter (\(c, _) -> c `notElem` unusual) twiddle
+
+    typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}"
+           ,"{-# NOINLINE " ++ which ++ "Mapping #-}"
+           ,which ++ "Mapping = \\case"]
+    last = "  _ -> unI64 0"
+    printUnusual c = "  -- " ++ name c ++ "\n" ++
+             "  " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))
+       where x:y:z:_ = access c ++ repeat '\0'
+    printUsual (c, c') = "  " ++ showC c ++ "# -> unI64 " ++ show (ord c')
+ scripts/UnicodeData.hs view
@@ -0,0 +1,51 @@+-- This script processes the following source file:
+--
+--   http://unicode.org/Public/UNIDATA/UnicodeData.txt
+--
+-- Format description: https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt
+
+module UnicodeData
+    ( UnicodeData
+    , Data(..)
+    , toTitleUD
+    , parseUD
+    ) where
+
+import Debug.Trace
+import Arsec hiding (semi)
+import Data.Array
+import Data.Functor (void)
+import Data.List (sort)
+import Data.Maybe (fromMaybe)
+
+type UnicodeData = Array Int Data
+
+-- "Simple_Titlecase_Mapping: If this field is null, then the Simple_Titlecase_Mapping
+-- is the same as the Simple_Uppercase_Mapping for this character."
+-- -- https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt
+toTitleUD :: Data -> Maybe Char
+toTitleUD d = toTitleUD_ d <|> toUpperUD d
+
+data Data = Data {
+      charUD :: {-# UNPACK #-} !Char
+    , toUpperUD :: {-# UNPACK #-} !(Maybe Char)
+    , toLowerUD :: {-# UNPACK #-} !(Maybe Char)
+    , toTitleUD_ :: {-# UNPACK #-} !(Maybe Char)
+    } deriving (Eq, Ord, Show)
+
+-- I'm pretty sure UnicodeData.txt is sorted but still sort it to be 100% certain.
+entries :: Parser UnicodeData
+entries = (\xs -> listArray (0, length xs - 1) xs) <$> many entry <* eof
+  where
+    entry = Data <$> unichar <* semi
+                <* replicateM_ 11 (ignoreField <* semi)
+                <*> optional unichar <* semi
+                <*> optional unichar <* semi
+                <*> optional unichar <* char '\n'
+    semi = char ';'
+
+ignoreField :: Parser ()
+ignoreField = void (many (satisfy (\c -> c /= ';')))
+
+parseUD :: FilePath -> IO (Either ParseError UnicodeData)
+parseUD name = parse entries name <$> readFile name
+ simdutf/LICENSE-APACHE view
@@ -0,0 +1,201 @@+              Apache License+                           Version 2.0, January 2004+                        http://www.apache.org/licenses/++   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++   1. Definitions.++      "License" shall mean the terms and conditions for use, reproduction,+      and distribution as defined by Sections 1 through 9 of this document.++      "Licensor" shall mean the copyright owner or entity authorized by+      the copyright owner that is granting the License.++      "Legal Entity" shall mean the union of the acting entity and all+      other entities that control, are controlled by, or are under common+      control with that entity. For the purposes of this definition,+      "control" means (i) the power, direct or indirect, to cause the+      direction or management of such entity, whether by contract or+      otherwise, or (ii) ownership of fifty percent (50%) or more of the+      outstanding shares, or (iii) beneficial ownership of such entity.++      "You" (or "Your") shall mean an individual or Legal Entity+      exercising permissions granted by this License.++      "Source" form shall mean the preferred form for making modifications,+      including but not limited to software source code, documentation+      source, and configuration files.++      "Object" form shall mean any form resulting from mechanical+      transformation or translation of a Source form, including but+      not limited to compiled object code, generated documentation,+      and conversions to other media types.++      "Work" shall mean the work of authorship, whether in Source or+      Object form, made available under the License, as indicated by a+      copyright notice that is included in or attached to the work+      (an example is provided in the Appendix below).++      "Derivative Works" shall mean any work, whether in Source or Object+      form, that is based on (or derived from) the Work and for which the+      editorial revisions, annotations, elaborations, or other modifications+      represent, as a whole, an original work of authorship. For the purposes+      of this License, Derivative Works shall not include works that remain+      separable from, or merely link (or bind by name) to the interfaces of,+      the Work and Derivative Works thereof.++      "Contribution" shall mean any work of authorship, including+      the original version of the Work and any modifications or additions+      to that Work or Derivative Works thereof, that is intentionally+      submitted to Licensor for inclusion in the Work by the copyright owner+      or by an individual or Legal Entity authorized to submit on behalf of+      the copyright owner. For the purposes of this definition, "submitted"+      means any form of electronic, verbal, or written communication sent+      to the Licensor or its representatives, including but not limited to+      communication on electronic mailing lists, source code control systems,+      and issue tracking systems that are managed by, or on behalf of, the+      Licensor for the purpose of discussing and improving the Work, but+      excluding communication that is conspicuously marked or otherwise+      designated in writing by the copyright owner as "Not a Contribution."++      "Contributor" shall mean Licensor and any individual or Legal Entity+      on behalf of whom a Contribution has been received by Licensor and+      subsequently incorporated within the Work.++   2. Grant of Copyright License. Subject to the terms and conditions of+      this License, each Contributor hereby grants to You a perpetual,+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable+      copyright license to reproduce, prepare Derivative Works of,+      publicly display, publicly perform, sublicense, and distribute the+      Work and such Derivative Works in Source or Object form.++   3. Grant of Patent License. Subject to the terms and conditions of+      this License, each Contributor hereby grants to You a perpetual,+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable+      (except as stated in this section) patent license to make, have made,+      use, offer to sell, sell, import, and otherwise transfer the Work,+      where such license applies only to those patent claims licensable+      by such Contributor that are necessarily infringed by their+      Contribution(s) alone or by combination of their Contribution(s)+      with the Work to which such Contribution(s) was submitted. If You+      institute patent litigation against any entity (including a+      cross-claim or counterclaim in a lawsuit) alleging that the Work+      or a Contribution incorporated within the Work constitutes direct+      or contributory patent infringement, then any patent licenses+      granted to You under this License for that Work shall terminate+      as of the date such litigation is filed.++   4. Redistribution. You may reproduce and distribute copies of the+      Work or Derivative Works thereof in any medium, with or without+      modifications, and in Source or Object form, provided that You+      meet the following conditions:++      (a) You must give any other recipients of the Work or+          Derivative Works a copy of this License; and++      (b) You must cause any modified files to carry prominent notices+          stating that You changed the files; and++      (c) You must retain, in the Source form of any Derivative Works+          that You distribute, all copyright, patent, trademark, and+          attribution notices from the Source form of the Work,+          excluding those notices that do not pertain to any part of+          the Derivative Works; and++      (d) If the Work includes a "NOTICE" text file as part of its+          distribution, then any Derivative Works that You distribute must+          include a readable copy of the attribution notices contained+          within such NOTICE file, excluding those notices that do not+          pertain to any part of the Derivative Works, in at least one+          of the following places: within a NOTICE text file distributed+          as part of the Derivative Works; within the Source form or+          documentation, if provided along with the Derivative Works; or,+          within a display generated by the Derivative Works, if and+          wherever such third-party notices normally appear. The contents+          of the NOTICE file are for informational purposes only and+          do not modify the License. You may add Your own attribution+          notices within Derivative Works that You distribute, alongside+          or as an addendum to the NOTICE text from the Work, provided+          that such additional attribution notices cannot be construed+          as modifying the License.++      You may add Your own copyright statement to Your modifications and+      may provide additional or different license terms and conditions+      for use, reproduction, or distribution of Your modifications, or+      for any such Derivative Works as a whole, provided Your use,+      reproduction, and distribution of the Work otherwise complies with+      the conditions stated in this License.++   5. Submission of Contributions. Unless You explicitly state otherwise,+      any Contribution intentionally submitted for inclusion in the Work+      by You to the Licensor shall be under the terms and conditions of+      this License, without any additional terms or conditions.+      Notwithstanding the above, nothing herein shall supersede or modify+      the terms of any separate license agreement you may have executed+      with Licensor regarding such Contributions.++   6. Trademarks. This License does not grant permission to use the trade+      names, trademarks, service marks, or product names of the Licensor,+      except as required for reasonable and customary use in describing the+      origin of the Work and reproducing the content of the NOTICE file.++   7. Disclaimer of Warranty. Unless required by applicable law or+      agreed to in writing, Licensor provides the Work (and each+      Contributor provides its Contributions) on an "AS IS" BASIS,+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+      implied, including, without limitation, any warranties or conditions+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+      PARTICULAR PURPOSE. You are solely responsible for determining the+      appropriateness of using or redistributing the Work and assume any+      risks associated with Your exercise of permissions under this License.++   8. Limitation of Liability. In no event and under no legal theory,+      whether in tort (including negligence), contract, or otherwise,+      unless required by applicable law (such as deliberate and grossly+      negligent acts) or agreed to in writing, shall any Contributor be+      liable to You for damages, including any direct, indirect, special,+      incidental, or consequential damages of any character arising as a+      result of this License or out of the use or inability to use the+      Work (including but not limited to damages for loss of goodwill,+      work stoppage, computer failure or malfunction, or any and all+      other commercial damages or losses), even if such Contributor+      has been advised of the possibility of such damages.++   9. Accepting Warranty or Additional Liability. While redistributing+      the Work or Derivative Works thereof, You may choose to offer,+      and charge a fee for, acceptance of support, warranty, indemnity,+      or other liability obligations and/or rights consistent with this+      License. However, in accepting such obligations, You may act only+      on Your own behalf and on Your sole responsibility, not on behalf+      of any other Contributor, and only if You agree to indemnify,+      defend, and hold each Contributor harmless for any liability+      incurred by, or claims asserted against, such Contributor by reason+      of your accepting any such warranty or additional liability.++   END OF TERMS AND CONDITIONS++   APPENDIX: How to apply the Apache License to your work.++      To apply the Apache License to your work, attach the following+      boilerplate notice, with the fields enclosed by brackets "{}"+      replaced with your own identifying information. (Don't include+      the brackets!)  The text should be enclosed in the appropriate+      comment syntax for the file format. We also recommend that a+      file or class name and description of purpose be included on the+      same "printed page" as the copyright notice for easier+      identification within third-party archives.++   Copyright 2020 The simdutf authors ++   Licensed under the Apache License, Version 2.0 (the "License");+   you may not use this file except in compliance with the License.+   You may obtain a copy of the License at++       http://www.apache.org/licenses/LICENSE-2.0++   Unless required by applicable law or agreed to in writing, software+   distributed under the License is distributed on an "AS IS" BASIS,+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+   See the License for the specific language governing permissions and+   limitations under the License.
+ simdutf/LICENSE-MIT view
@@ -0,0 +1,18 @@+Copyright 2021 The simdutf authors++Permission is hereby granted, free of charge, to any person obtaining a copy of+this software and associated documentation files (the "Software"), to deal in+the Software without restriction, including without limitation the rights to+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of+the Software, and to permit persons to whom the Software is furnished to do so,+subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ simdutf/hs_simdutf.c view
@@ -0,0 +1,9 @@+#include "simdutf_c.h"
+
+int _hs_text_is_valid_utf8(const char *buf, size_t len) {
+  return simdutf_validate_utf8(buf, len);
+}
+
+int _hs_text_is_valid_utf8_offset(const char *buf, size_t off, size_t len) {
+  return simdutf_validate_utf8(buf + off, len);
+}
+ simdutf/simdutf.cpp view

file too large to diff

+ simdutf/simdutf.h view
@@ -0,0 +1,13573 @@+/* auto-generated on 2026-01-13 09:03:21 +0100. Do not edit! */
+/* begin file include/simdutf.h */
+#ifndef SIMDUTF_H
+#define SIMDUTF_H
+#include <cstring>
+
+/* begin file include/simdutf/compiler_check.h */
+#ifndef SIMDUTF_COMPILER_CHECK_H
+#define SIMDUTF_COMPILER_CHECK_H
+
+#ifndef __cplusplus
+  #error simdutf requires a C++ compiler
+#endif
+
+#ifndef SIMDUTF_CPLUSPLUS
+  #if defined(_MSVC_LANG) && !defined(__clang__)
+    #define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)
+  #else
+    #define SIMDUTF_CPLUSPLUS __cplusplus
+  #endif
+#endif
+
+// C++ 26
+#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L)
+  #define SIMDUTF_CPLUSPLUS26 1
+#endif
+
+// C++ 23
+#if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L)
+  #define SIMDUTF_CPLUSPLUS23 1
+#endif
+
+// C++ 20
+#if !defined(SIMDUTF_CPLUSPLUS20) && (SIMDUTF_CPLUSPLUS >= 202002L)
+  #define SIMDUTF_CPLUSPLUS20 1
+#endif
+
+// C++ 17
+#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L)
+  #define SIMDUTF_CPLUSPLUS17 1
+#endif
+
+// C++ 14
+#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L)
+  #define SIMDUTF_CPLUSPLUS14 1
+#endif
+
+// C++ 11
+#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L)
+  #define SIMDUTF_CPLUSPLUS11 1
+#endif
+
+#ifndef SIMDUTF_CPLUSPLUS11
+  #error simdutf requires a compiler compliant with the C++11 standard
+#endif
+
+#endif // SIMDUTF_COMPILER_CHECK_H
+/* end file include/simdutf/compiler_check.h */
+/* begin file include/simdutf/common_defs.h */
+#ifndef SIMDUTF_COMMON_DEFS_H
+#define SIMDUTF_COMMON_DEFS_H
+
+/* begin file include/simdutf/portability.h */
+#ifndef SIMDUTF_PORTABILITY_H
+#define SIMDUTF_PORTABILITY_H
+
+
+#include <cfloat>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#ifndef _WIN32
+  // strcasecmp, strncasecmp
+  #include <strings.h>
+#endif
+
+#if defined(__apple_build_version__)
+  #if __apple_build_version__ < 14000000
+    #define SIMDUTF_SPAN_DISABLED                                              \
+      1 // apple-clang/13 doesn't support std::convertible_to
+  #endif
+#endif
+
+#if SIMDUTF_CPLUSPLUS20
+  #include <version>
+  #if __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L &&                \
+      !defined(SIMDUTF_SPAN_DISABLED)
+    #define SIMDUTF_SPAN 1
+  #endif // __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L
+  #if __cpp_lib_atomic_ref >= 201806L
+    #define SIMDUTF_ATOMIC_REF 1
+  #endif // __cpp_lib_atomic_ref
+  #if __has_cpp_attribute(maybe_unused) >= 201603L
+    #define SIMDUTF_MAYBE_UNUSED_AVAILABLE 1
+  #endif // __has_cpp_attribute(maybe_unused) >= 201603L
+#endif
+
+/**
+ * We want to check that it is actually a little endian system at
+ * compile-time.
+ */
+
+#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)
+  #define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#elif defined(_WIN32)
+  #define SIMDUTF_IS_BIG_ENDIAN 0
+#else
+  #if defined(__APPLE__) ||                                                    \
+      defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined
+                           // __ORDER_BIG_ENDIAN__
+    #include <machine/endian.h>
+  #elif defined(sun) ||                                                        \
+      defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__)
+    #include <sys/byteorder.h>
+  #else // defined(__APPLE__) || defined(__FreeBSD__)
+
+    #ifdef __has_include
+      #if __has_include(<endian.h>)
+        #include <endian.h>
+      #endif //__has_include(<endian.h>)
+    #endif   //__has_include
+
+  #endif // defined(__APPLE__) || defined(__FreeBSD__)
+
+  #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__)
+    #define SIMDUTF_IS_BIG_ENDIAN 0
+  #endif
+
+  #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    #define SIMDUTF_IS_BIG_ENDIAN 0
+  #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    #define SIMDUTF_IS_BIG_ENDIAN 1
+  #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+
+#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__
+
+/**
+ * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined.
+ */
+
+#ifdef _MSC_VER
+  #define SIMDUTF_VISUAL_STUDIO 1
+  /**
+   * We want to differentiate carefully between
+   * clang under visual studio and regular visual
+   * studio.
+   *
+   * Under clang for Windows, we enable:
+   *  * target pragmas so that part and only part of the
+   *     code gets compiled for advanced instructions.
+   *
+   */
+  #ifdef __clang__
+    // clang under visual studio
+    #define SIMDUTF_CLANG_VISUAL_STUDIO 1
+  #else
+    // just regular visual studio (best guess)
+    #define SIMDUTF_REGULAR_VISUAL_STUDIO 1
+  #endif // __clang__
+#endif   // _MSC_VER
+
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+  // https://en.wikipedia.org/wiki/C_alternative_tokens
+  // This header should have no effect, except maybe
+  // under Visual Studio.
+  #include <iso646.h>
+#endif
+
+#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC)
+  #define SIMDUTF_IS_X86_64 1
+#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
+  #define SIMDUTF_IS_ARM64 1
+#elif defined(__PPC64__) || defined(_M_PPC64)
+  #if defined(__VEC__) && defined(__ALTIVEC__)
+    #define SIMDUTF_IS_PPC64 1
+  #endif
+#elif defined(__s390__)
+// s390 IBM system. Big endian.
+#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64
+  // RISC-V 64-bit
+  #define SIMDUTF_IS_RISCV64 1
+
+  // #if __riscv_v_intrinsic >= 1000000
+  //   #define SIMDUTF_HAS_RVV_INTRINSICS 1
+  //   #define SIMDUTF_HAS_RVV_TARGET_REGION 1
+  // #elif ...
+  //  Check for special compiler versions that implement pre v1.0 intrinsics
+  #if __riscv_v_intrinsic >= 11000
+    #define SIMDUTF_HAS_RVV_INTRINSICS 1
+  #endif
+
+  #define SIMDUTF_HAS_ZVBB_INTRINSICS                                          \
+    0 // there is currently no way to detect this
+
+  #if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector &&                          \
+      __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64
+    // RISC-V V extension
+    #define SIMDUTF_IS_RVV 1
+    #if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000
+      // RISC-V Vector Basic Bit-manipulation
+      #define SIMDUTF_IS_ZVBB 1
+    #endif
+  #endif
+
+#elif defined(__loongarch_lp64)
+  #if defined(__loongarch_sx) && defined(__loongarch_asx)
+    #define SIMDUTF_IS_LSX 1
+    #define SIMDUTF_IS_LASX 1 // We can always run both
+  #elif defined(__loongarch_sx)
+    #define SIMDUTF_IS_LSX 1
+  #endif
+#else
+  // The simdutf library is designed
+  // for 64-bit processors and it seems that you are not
+  // compiling for a known 64-bit platform. Please
+  // use a 64-bit target such as x64 or 64-bit ARM for best performance.
+  #define SIMDUTF_IS_32BITS 1
+
+  // We do not support 32-bit platforms, but it can be
+  // handy to identify them.
+  #if defined(_M_IX86) || defined(__i386__)
+    #define SIMDUTF_IS_X86_32BITS 1
+  #elif defined(__arm__) || defined(_M_ARM)
+    #define SIMDUTF_IS_ARM_32BITS 1
+  #elif defined(__PPC__) || defined(_M_PPC)
+    #define SIMDUTF_IS_PPC_32BITS 1
+  #endif
+
+#endif // defined(__x86_64__) || defined(_M_AMD64)
+
+#ifdef SIMDUTF_IS_32BITS
+  #ifndef SIMDUTF_NO_PORTABILITY_WARNING
+  // In the future, we may want to warn users of 32-bit systems that
+  // the simdutf does not support accelerated kernels for such systems.
+  #endif // SIMDUTF_NO_PORTABILITY_WARNING
+#endif   // SIMDUTF_IS_32BITS
+
+// this is almost standard?
+#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a
+#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a)
+
+// Our fast kernels require 64-bit systems.
+//
+// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions.
+// Furthermore, the number of SIMD registers is reduced.
+//
+// On 32-bit ARM, we would have smaller registers.
+//
+// The simdutf users should still have the fallback kernel. It is
+// slower, but it should run everywhere.
+
+//
+// Enable valid runtime implementations, and select
+// SIMDUTF_BUILTIN_IMPLEMENTATION
+//
+
+// We are going to use runtime dispatch.
+#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX)
+  #ifdef __clang__
+    // clang does not have GCC push pop
+    // warning: clang attribute push can't be used within a namespace in clang
+    // up til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be
+    // *outside* of a namespace.
+    #define SIMDUTF_TARGET_REGION(T)                                           \
+      _Pragma(SIMDUTF_STRINGIFY(clang attribute push(                          \
+          __attribute__((target(T))), apply_to = function)))
+    #define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop")
+  #elif defined(__GNUC__)
+    // GCC is easier
+    #define SIMDUTF_TARGET_REGION(T)                                           \
+      _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T)))
+    #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options")
+  #endif // clang then gcc
+
+#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX)
+
+// Default target region macros don't do anything.
+#ifndef SIMDUTF_TARGET_REGION
+  #define SIMDUTF_TARGET_REGION(T)
+  #define SIMDUTF_UNTARGET_REGION
+#endif
+
+// Is threading enabled?
+#if defined(_REENTRANT) || defined(_MT)
+  #ifndef SIMDUTF_THREADS_ENABLED
+    #define SIMDUTF_THREADS_ENABLED
+  #endif
+#endif
+
+// workaround for large stack sizes under -O0.
+// https://github.com/simdutf/simdutf/issues/691
+#ifdef __APPLE__
+  #ifndef __OPTIMIZE__
+    // Apple systems have small stack sizes in secondary threads.
+    // Lack of compiler optimization may generate high stack usage.
+    // Users may want to disable threads for safety, but only when
+    // in debug mode which we detect by the fact that the __OPTIMIZE__
+    // macro is not defined.
+    #undef SIMDUTF_THREADS_ENABLED
+  #endif
+#endif
+
+#ifdef SIMDUTF_VISUAL_STUDIO
+  // This is one case where we do not distinguish between
+  // regular visual studio and clang under visual studio.
+  // clang under Windows has _stricmp (like visual studio) but not strcasecmp
+  // (as clang normally has)
+  #define simdutf_strcasecmp _stricmp
+  #define simdutf_strncasecmp _strnicmp
+#else
+  // The strcasecmp, strncasecmp, and strcasestr functions do not work with
+  // multibyte strings (e.g. UTF-8). So they are only useful for ASCII in our
+  // context.
+  // https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings
+  #define simdutf_strcasecmp strcasecmp
+  #define simdutf_strncasecmp strncasecmp
+#endif
+
+#if defined(__GNUC__) && !defined(__clang__)
+  #if __GNUC__ >= 11
+    #define SIMDUTF_GCC11ORMORE 1
+  #endif //  __GNUC__ >= 11
+  #if __GNUC__ == 10
+    #define SIMDUTF_GCC10 1
+  #endif //  __GNUC__ == 10
+  #if __GNUC__ < 10
+    #define SIMDUTF_GCC9OROLDER 1
+  #endif //  __GNUC__ == 10
+#endif   // defined(__GNUC__) && !defined(__clang__)
+
+#endif // SIMDUTF_PORTABILITY_H
+/* end file include/simdutf/portability.h */
+/* begin file include/simdutf/avx512.h */
+#ifndef SIMDUTF_AVX512_H_
+#define SIMDUTF_AVX512_H_
+
+/*
+    It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS.
+
+    All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`,
+    where a feature is a code name for extensions.
+
+    Please see the listing below to find which are supported.
+*/
+
+#ifndef SIMDUTF_HAS_AVX512F
+  #if defined(__AVX512F__) && __AVX512F__ == 1
+    #define SIMDUTF_HAS_AVX512F 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512DQ
+  #if defined(__AVX512DQ__) && __AVX512DQ__ == 1
+    #define SIMDUTF_HAS_AVX512DQ 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512IFMA
+  #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1
+    #define SIMDUTF_HAS_AVX512IFMA 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512CD
+  #if defined(__AVX512CD__) && __AVX512CD__ == 1
+    #define SIMDUTF_HAS_AVX512CD 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512BW
+  #if defined(__AVX512BW__) && __AVX512BW__ == 1
+    #define SIMDUTF_HAS_AVX512BW 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512VL
+  #if defined(__AVX512VL__) && __AVX512VL__ == 1
+    #define SIMDUTF_HAS_AVX512VL 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512VBMI
+  #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1
+    #define SIMDUTF_HAS_AVX512VBMI 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512VBMI2
+  #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1
+    #define SIMDUTF_HAS_AVX512VBMI2 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512VNNI
+  #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1
+    #define SIMDUTF_HAS_AVX512VNNI 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512BITALG
+  #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1
+    #define SIMDUTF_HAS_AVX512BITALG 1
+  #endif
+#endif
+
+#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ
+  #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1
+    #define SIMDUTF_HAS_AVX512VPOPCNTDQ 1
+  #endif
+#endif
+
+#endif // SIMDUTF_AVX512_H_
+/* end file include/simdutf/avx512.h */
+
+// Sometimes logging is useful, but we want it disabled by default
+// and free of any logging code in release builds.
+#ifdef SIMDUTF_LOGGING
+  #include <iostream>
+  #define simdutf_log(msg)                                                     \
+    std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl              \
+              << "\t" << __FILE__ << ":" << __LINE__ << std::endl;
+  #define simdutf_log_assert(cond, msg)                                        \
+    do {                                                                       \
+      if (!(cond)) {                                                           \
+        std::cerr << "[" << __FUNCTION__ << "]: " << msg << std::endl          \
+                  << "\t" << __FILE__ << ":" << __LINE__ << std::endl;         \
+        std::abort();                                                          \
+      }                                                                        \
+    } while (0)
+#else
+  #define simdutf_log(msg)
+  #define simdutf_log_assert(cond, msg)
+#endif
+
+#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO)
+  #define SIMDUTF_DEPRECATED __declspec(deprecated)
+
+  #define simdutf_really_inline __forceinline // really inline in release mode
+  #define simdutf_always_inline __forceinline // always inline, no matter what
+  #define simdutf_never_inline __declspec(noinline)
+
+  #define simdutf_unused
+  #define simdutf_warn_unused
+
+  #ifndef simdutf_likely
+    #define simdutf_likely(x) x
+  #endif
+  #ifndef simdutf_unlikely
+    #define simdutf_unlikely(x) x
+  #endif
+
+  #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning(push))
+  #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0))
+  #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER)                           \
+    __pragma(warning(disable : WARNING_NUMBER))
+  // Get rid of Intellisense-only warnings (Code Analysis)
+  // Though __has_include is C++17, it is supported in Visual Studio 2017 or
+  // better (_MSC_VER>=1910).
+  #ifdef __has_include
+    #if __has_include(<CppCoreCheck\Warnings.h>)
+      #include <CppCoreCheck\Warnings.h>
+      #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS                               \
+        SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS)
+    #endif
+  #endif
+
+  #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS
+    #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS
+  #endif
+
+  #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996)
+  #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING
+  #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning(pop))
+  #define SIMDUTF_DISABLE_UNUSED_WARNING
+#else // SIMDUTF_REGULAR_VISUAL_STUDIO
+  #if defined(__OPTIMIZE__) || defined(NDEBUG)
+    #define simdutf_really_inline inline __attribute__((always_inline))
+  #else
+    #define simdutf_really_inline inline
+  #endif
+  #define simdutf_always_inline                                                \
+    inline __attribute__((always_inline)) // always inline, no matter what
+  #define SIMDUTF_DEPRECATED __attribute__((deprecated))
+  #define simdutf_never_inline inline __attribute__((noinline))
+
+  #define simdutf_unused __attribute__((unused))
+  #define simdutf_warn_unused __attribute__((warn_unused_result))
+
+  #ifndef simdutf_likely
+    #define simdutf_likely(x) __builtin_expect(!!(x), 1)
+  #endif
+  #ifndef simdutf_unlikely
+    #define simdutf_unlikely(x) __builtin_expect(!!(x), 0)
+  #endif
+  // clang-format off
+  #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
+  // gcc doesn't seem to disable all warnings with all and extra, add warnings
+  // here as necessary
+  #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS                                    \
+    SIMDUTF_PUSH_DISABLE_WARNINGS                                              \
+    SIMDUTF_DISABLE_GCC_WARNING(-Weffc++)                                      \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wall)                                         \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wconversion)                                  \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wextra)                                       \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wattributes)                                  \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough)                        \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor)                            \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type)                                 \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wshadow)                                      \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter)                            \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable)
+  #define SIMDUTF_PRAGMA(P) _Pragma(#P)
+  #define SIMDUTF_DISABLE_GCC_WARNING(WARNING)                                 \
+    SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING)
+  #if defined(SIMDUTF_CLANG_VISUAL_STUDIO)
+    #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS                                 \
+      SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include)
+  #else
+    #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS
+  #endif
+  #define SIMDUTF_DISABLE_DEPRECATED_WARNING                                   \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations)
+  #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING                              \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow)
+  #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")
+  #define SIMDUTF_DISABLE_UNUSED_WARNING                                       \
+    SIMDUTF_PUSH_DISABLE_WARNINGS                                              \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wunused-function)                             \
+    SIMDUTF_DISABLE_GCC_WARNING(-Wunused-const-variable)
+  // clang-format on
+
+#endif // MSC_VER
+
+// Conditional constexpr macro: expands to constexpr for C++17+, empty otherwise
+#if SIMDUTF_CPLUSPLUS17
+  #define simdutf_constexpr constexpr
+#else
+  #define simdutf_constexpr
+#endif
+
+// Will evaluate to constexpr in C++23 or later. This makes it possible to mark
+// functions constexpr if the "if consteval" feature is available to use.
+#if SIMDUTF_CPLUSPLUS23
+  #define simdutf_constexpr23 constexpr
+#else
+  #define simdutf_constexpr23
+#endif
+
+#ifndef SIMDUTF_DLLIMPORTEXPORT
+  #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio
+                                     /**
+                                      * Windows users need to do some extra work when building
+                                      * or using a dynamic library (DLL). When building, we need
+                                      * to set SIMDUTF_DLLIMPORTEXPORT to __declspec(dllexport).
+                                      * When *using* the DLL, the user needs to set
+                                      * SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport).
+                                      *
+                                      * Static libraries not need require such work.
+                                      *
+                                      * It does not matter here whether you are using
+                                      * the regular visual studio or clang under visual
+                                      * studio, you still need to handle these issues.
+                                      *
+                                      * Non-Windows systems do not have this complexity.
+                                      */
+    #if SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY
+
+      // We set SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL
+      // under Windows. It should never happen that both
+      // SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY and
+      // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY are set.
+      #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport)
+    #elif SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY
+      // Windows user who call a dynamic library should set
+      // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY to 1.
+
+      #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport)
+    #else
+      // We assume by default static linkage
+      #define SIMDUTF_DLLIMPORTEXPORT
+    #endif
+  #else // defined(SIMDUTF_VISUAL_STUDIO)
+    // Non-Windows systems do not have this complexity.
+    #define SIMDUTF_DLLIMPORTEXPORT
+  #endif // defined(SIMDUTF_VISUAL_STUDIO)
+#endif
+
+#if SIMDUTF_MAYBE_UNUSED_AVAILABLE
+  #define simdutf_maybe_unused [[maybe_unused]]
+#else
+  #define simdutf_maybe_unused
+#endif
+
+#endif // SIMDUTF_COMMON_DEFS_H
+/* end file include/simdutf/common_defs.h */
+/* begin file include/simdutf/encoding_types.h */
+#ifndef SIMDUTF_ENCODING_TYPES_H
+#define SIMDUTF_ENCODING_TYPES_H
+#include <string>
+
+#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) &&                                  \
+    defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L
+  #define SIMDUTF_HAS_STD_TEXT_ENCODING 1
+  #include <text_encoding>
+#endif
+
+namespace simdutf {
+
+enum encoding_type {
+  UTF8 = 1,      // BOM 0xef 0xbb 0xbf
+  UTF16_LE = 2,  // BOM 0xff 0xfe
+  UTF16_BE = 4,  // BOM 0xfe 0xff
+  UTF32_LE = 8,  // BOM 0xff 0xfe 0x00 0x00
+  UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff
+  Latin1 = 32,
+
+  unspecified = 0
+};
+
+#ifndef SIMDUTF_IS_BIG_ENDIAN
+  #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined."
+#endif
+
+enum endianness {
+  LITTLE = 0,
+  BIG = 1,
+  NATIVE =
+#if SIMDUTF_IS_BIG_ENDIAN
+      BIG
+#else
+      LITTLE
+#endif
+};
+
+simdutf_warn_unused simdutf_really_inline constexpr bool
+match_system(endianness e) {
+  return e == endianness::NATIVE;
+}
+
+simdutf_warn_unused std::string to_string(encoding_type bom);
+
+// Note that BOM for UTF8 is discouraged.
+namespace BOM {
+
+/**
+ * Checks for a BOM. If not, returns unspecified
+ * @param input         the string to process
+ * @param length        the length of the string in code units
+ * @return the corresponding encoding
+ */
+
+simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length);
+simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length);
+/**
+ * Returns the size, in bytes, of the BOM for a given encoding type.
+ * Note that UTF8 BOM are discouraged.
+ * @param bom         the encoding type
+ * @return the size in bytes of the corresponding BOM
+ */
+simdutf_warn_unused size_t bom_byte_size(encoding_type bom);
+
+} // namespace BOM
+
+#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING
+/**
+ * Convert a simdutf encoding type to a std::text_encoding.
+ *
+ * @param enc  the simdutf encoding type
+ * @return     the corresponding std::text_encoding, or
+ *             std::text_encoding::id::unknown for unspecified/unsupported
+ */
+simdutf_warn_unused constexpr std::text_encoding
+to_std_encoding(encoding_type enc) noexcept {
+  switch (enc) {
+  case UTF8:
+    return std::text_encoding(std::text_encoding::id::UTF8);
+  case UTF16_LE:
+    return std::text_encoding(std::text_encoding::id::UTF16LE);
+  case UTF16_BE:
+    return std::text_encoding(std::text_encoding::id::UTF16BE);
+  case UTF32_LE:
+    return std::text_encoding(std::text_encoding::id::UTF32LE);
+  case UTF32_BE:
+    return std::text_encoding(std::text_encoding::id::UTF32BE);
+  case Latin1:
+    return std::text_encoding(std::text_encoding::id::ISOLatin1);
+  case unspecified:
+  default:
+    return std::text_encoding(std::text_encoding::id::unknown);
+  }
+}
+
+/**
+ * Convert a std::text_encoding to a simdutf encoding type.
+ *
+ * @param enc  the std::text_encoding
+ * @return     the corresponding simdutf encoding type, or
+ *             encoding_type::unspecified if the encoding is not supported
+ */
+simdutf_warn_unused constexpr encoding_type
+from_std_encoding(const std::text_encoding &enc) noexcept {
+  switch (enc.mib()) {
+  case std::text_encoding::id::UTF8:
+    return UTF8;
+  case std::text_encoding::id::UTF16LE:
+    return UTF16_LE;
+  case std::text_encoding::id::UTF16BE:
+    return UTF16_BE;
+  case std::text_encoding::id::UTF32LE:
+    return UTF32_LE;
+  case std::text_encoding::id::UTF32BE:
+    return UTF32_BE;
+  case std::text_encoding::id::ISOLatin1:
+    return Latin1;
+  default:
+    return unspecified;
+  }
+}
+
+/**
+ * Get the native-endian UTF-16 encoding type for this system.
+ *
+ * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems
+ */
+simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept {
+  #if SIMDUTF_IS_BIG_ENDIAN
+  return UTF16_BE;
+  #else
+  return UTF16_LE;
+  #endif
+}
+
+/**
+ * Get the native-endian UTF-32 encoding type for this system.
+ *
+ * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems
+ */
+simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept {
+  #if SIMDUTF_IS_BIG_ENDIAN
+  return UTF32_BE;
+  #else
+  return UTF32_LE;
+  #endif
+}
+
+/**
+ * Convert a std::text_encoding to a simdutf encoding type,
+ * using native endianness for UTF-16/UTF-32 without explicit endianness.
+ *
+ * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE
+ * suffix), this returns the native-endian simdutf variant.
+ *
+ * @param enc  the std::text_encoding
+ * @return     the corresponding simdutf encoding type, or
+ *             encoding_type::unspecified if the encoding is not supported
+ */
+simdutf_warn_unused constexpr encoding_type
+from_std_encoding_native(const std::text_encoding &enc) noexcept {
+  switch (enc.mib()) {
+  case std::text_encoding::id::UTF8:
+    return UTF8;
+  case std::text_encoding::id::UTF16:
+    return native_utf16_encoding();
+  case std::text_encoding::id::UTF16LE:
+    return UTF16_LE;
+  case std::text_encoding::id::UTF16BE:
+    return UTF16_BE;
+  case std::text_encoding::id::UTF32:
+    return native_utf32_encoding();
+  case std::text_encoding::id::UTF32LE:
+    return UTF32_LE;
+  case std::text_encoding::id::UTF32BE:
+    return UTF32_BE;
+  case std::text_encoding::id::ISOLatin1:
+    return Latin1;
+  default:
+    return unspecified;
+  }
+}
+#endif // SIMDUTF_HAS_STD_TEXT_ENCODING
+
+} // namespace simdutf
+#endif
+/* end file include/simdutf/encoding_types.h */
+/* begin file include/simdutf/error.h */
+#ifndef SIMDUTF_ERROR_H
+#define SIMDUTF_ERROR_H
+namespace simdutf {
+
+enum error_code {
+  SUCCESS = 0,
+  HEADER_BITS, // Any byte must have fewer than 5 header bits.
+  TOO_SHORT,   // The leading byte must be followed by N-1 continuation bytes,
+               // where N is the UTF-8 character length This is also the error
+               // when the input is truncated.
+  TOO_LONG,    // We either have too many consecutive continuation bytes or the
+               // string starts with a continuation byte.
+  OVERLONG, // The decoded character must be above U+7F for two-byte characters,
+            // U+7FF for three-byte characters, and U+FFFF for four-byte
+            // characters.
+  TOO_LARGE, // The decoded character must be less than or equal to
+             // U+10FFFF,less than or equal than U+7F for ASCII OR less than
+             // equal than U+FF for Latin1
+  SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or
+             // UTF-32)
+             // OR
+             // a high surrogate must be followed by a low surrogate
+             // and a low surrogate must be preceded by a high surrogate
+             // (UTF-16)
+             // OR
+             // there must be no surrogate at all and one is
+             // found (Latin1 functions)
+             // OR
+             // *specifically* for the function
+             // utf8_length_from_utf16_with_replacement, a surrogate (whether
+             // in error or not) has been found (I.e., whether we are in the
+             // Basic Multilingual Plane or not).
+  INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid
+                            // base64 string. This may include a misplaced
+                            // padding character ('=').
+  BASE64_INPUT_REMAINDER,   // The base64 input terminates with a single
+                            // character, excluding padding (=). It is also used
+                            // in strict mode when padding is not adequate.
+  BASE64_EXTRA_BITS,        // The base64 input terminates with non-zero
+                            // padding bits.
+  OUTPUT_BUFFER_TOO_SMALL,  // The provided buffer is too small.
+  OTHER                     // Not related to validation/transcoding.
+};
+#if SIMDUTF_CPLUSPLUS17
+inline std::string_view error_to_string(error_code code) noexcept {
+  switch (code) {
+  case SUCCESS:
+    return "SUCCESS";
+  case HEADER_BITS:
+    return "HEADER_BITS";
+  case TOO_SHORT:
+    return "TOO_SHORT";
+  case TOO_LONG:
+    return "TOO_LONG";
+  case OVERLONG:
+    return "OVERLONG";
+  case TOO_LARGE:
+    return "TOO_LARGE";
+  case SURROGATE:
+    return "SURROGATE";
+  case INVALID_BASE64_CHARACTER:
+    return "INVALID_BASE64_CHARACTER";
+  case BASE64_INPUT_REMAINDER:
+    return "BASE64_INPUT_REMAINDER";
+  case BASE64_EXTRA_BITS:
+    return "BASE64_EXTRA_BITS";
+  case OUTPUT_BUFFER_TOO_SMALL:
+    return "OUTPUT_BUFFER_TOO_SMALL";
+  default:
+    return "OTHER";
+  }
+}
+#endif
+
+struct result {
+  error_code error;
+  size_t count; // In case of error, indicates the position of the error. In
+                // case of success, indicates the number of code units
+                // validated/written.
+
+  simdutf_really_inline simdutf_constexpr23 result() noexcept
+      : error{error_code::SUCCESS}, count{0} {}
+
+  simdutf_really_inline simdutf_constexpr23 result(error_code err,
+                                                   size_t pos) noexcept
+      : error{err}, count{pos} {}
+
+  simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept {
+    return error == error_code::SUCCESS;
+  }
+
+  simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept {
+    return error != error_code::SUCCESS;
+  }
+};
+
+struct full_result {
+  error_code error;
+  size_t input_count;
+  size_t output_count;
+  bool padding_error = false; // true if the error is due to padding, only
+                              // meaningful when error is not SUCCESS
+
+  simdutf_really_inline simdutf_constexpr23 full_result() noexcept
+      : error{error_code::SUCCESS}, input_count{0}, output_count{0} {}
+
+  simdutf_really_inline simdutf_constexpr23 full_result(error_code err,
+                                                        size_t pos_in,
+                                                        size_t pos_out) noexcept
+      : error{err}, input_count{pos_in}, output_count{pos_out} {}
+  simdutf_really_inline simdutf_constexpr23 full_result(
+      error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept
+      : error{err}, input_count{pos_in}, output_count{pos_out},
+        padding_error{padding_err} {}
+
+  simdutf_really_inline simdutf_constexpr23 operator result() const noexcept {
+    if (error == error_code::SUCCESS) {
+      return result{error, output_count};
+    } else {
+      return result{error, input_count};
+    }
+  }
+};
+
+} // namespace simdutf
+#endif
+/* end file include/simdutf/error.h */
+
+SIMDUTF_PUSH_DISABLE_WARNINGS
+SIMDUTF_DISABLE_UNDESIRED_WARNINGS
+
+// Public API
+/* begin file include/simdutf/simdutf_version.h */
+// /include/simdutf/simdutf_version.h automatically generated by release.py,
+// do not change by hand
+#ifndef SIMDUTF_SIMDUTF_VERSION_H
+#define SIMDUTF_SIMDUTF_VERSION_H
+
+/** The version of simdutf being used (major.minor.revision) */
+#define SIMDUTF_VERSION "8.0.0"
+
+namespace simdutf {
+enum {
+  /**
+   * The major version (MAJOR.minor.revision) of simdutf being used.
+   */
+  SIMDUTF_VERSION_MAJOR = 8,
+  /**
+   * The minor version (major.MINOR.revision) of simdutf being used.
+   */
+  SIMDUTF_VERSION_MINOR = 0,
+  /**
+   * The revision (major.minor.REVISION) of simdutf being used.
+   */
+  SIMDUTF_VERSION_REVISION = 0
+};
+} // namespace simdutf
+
+#endif // SIMDUTF_SIMDUTF_VERSION_H
+/* end file include/simdutf/simdutf_version.h */
+/* begin file include/simdutf/implementation.h */
+#ifndef SIMDUTF_IMPLEMENTATION_H
+#define SIMDUTF_IMPLEMENTATION_H
+#if !defined(SIMDUTF_NO_THREADS)
+  #include <atomic>
+#endif
+#include <string>
+#ifdef SIMDUTF_INTERNAL_TESTS
+  #include <vector>
+#endif
+/* begin file include/simdutf/internal/isadetection.h */
+/* From
+https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h
+Highly modified.
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,
+Iain Melvin, Jason Weston) Copyright (c) 2006      Idiap Research Institute
+(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,
+Samy Bengio, Johnny Mariethoz)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories
+America and IDIAP Research Institute nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef SIMDutf_INTERNAL_ISADETECTION_H
+#define SIMDutf_INTERNAL_ISADETECTION_H
+
+#include <cstdint>
+#include <cstdlib>
+#if defined(_MSC_VER)
+  #include <intrin.h>
+#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
+  #include <cpuid.h>
+#endif
+
+
+// RISC-V ISA detection utilities
+#if SIMDUTF_IS_RISCV64 && defined(__linux__)
+  #include <unistd.h> // for syscall
+// We define these ourselves, for backwards compatibility
+struct simdutf_riscv_hwprobe {
+  int64_t key;
+  uint64_t value;
+};
+  #define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__)
+  #define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4
+  #define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2)
+  #define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17)
+#endif // SIMDUTF_IS_RISCV64 && defined(__linux__)
+
+#if defined(__loongarch__) && defined(__linux__)
+  #include <sys/auxv.h>
+// bits/hwcap.h
+// #define HWCAP_LOONGARCH_LSX             (1 << 4)
+// #define HWCAP_LOONGARCH_LASX            (1 << 5)
+#endif
+
+namespace simdutf {
+namespace internal {
+
+enum instruction_set {
+  DEFAULT = 0x0,
+  NEON = 0x1,
+  AVX2 = 0x4,
+  SSE42 = 0x8,
+  PCLMULQDQ = 0x10,
+  BMI1 = 0x20,
+  BMI2 = 0x40,
+  ALTIVEC = 0x80,
+  AVX512F = 0x100,
+  AVX512DQ = 0x200,
+  AVX512IFMA = 0x400,
+  AVX512PF = 0x800,
+  AVX512ER = 0x1000,
+  AVX512CD = 0x2000,
+  AVX512BW = 0x4000,
+  AVX512VL = 0x8000,
+  AVX512VBMI2 = 0x10000,
+  AVX512VPOPCNTDQ = 0x2000,
+  RVV = 0x4000,
+  ZVBB = 0x8000,
+  LSX = 0x40000,
+  LASX = 0x80000,
+};
+
+#if defined(__PPC64__)
+
+static inline uint32_t detect_supported_architectures() {
+  return instruction_set::ALTIVEC;
+}
+
+#elif SIMDUTF_IS_RISCV64
+
+static inline uint32_t detect_supported_architectures() {
+  uint32_t host_isa = instruction_set::DEFAULT;
+  #if SIMDUTF_IS_RVV
+  host_isa |= instruction_set::RVV;
+  #endif
+  #if SIMDUTF_IS_ZVBB
+  host_isa |= instruction_set::ZVBB;
+  #endif
+  #if defined(__linux__)
+  simdutf_riscv_hwprobe probes[] = {{SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0}};
+  long ret = simdutf_riscv_hwprobe(&probes, sizeof probes / sizeof *probes, 0,
+                                   nullptr, 0);
+  if (ret == 0) {
+    uint64_t extensions = probes[0].value;
+    if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V)
+      host_isa |= instruction_set::RVV;
+    if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB)
+      host_isa |= instruction_set::ZVBB;
+  }
+  #endif
+  #if defined(RUN_IN_SPIKE_SIMULATOR)
+  // Proxy Kernel does not implement yet hwprobe syscall
+  host_isa |= instruction_set::RVV;
+  #endif
+  return host_isa;
+}
+
+#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
+
+static inline uint32_t detect_supported_architectures() {
+  return instruction_set::NEON;
+}
+
+#elif defined(__x86_64__) || defined(_M_AMD64) // x64
+
+namespace {
+namespace cpuid_bit {
+// Can be found on Intel ISA Reference for CPUID
+
+// EAX = 0x01
+constexpr uint32_t pclmulqdq = uint32_t(1)
+                               << 1; ///< @private bit  1 of ECX for EAX=0x1
+constexpr uint32_t sse42 = uint32_t(1)
+                           << 20; ///< @private bit 20 of ECX for EAX=0x1
+constexpr uint32_t osxsave =
+    (uint32_t(1) << 26) |
+    (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1
+
+// EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf)
+// See: "Table 3-8. Information Returned by CPUID Instruction"
+namespace ebx {
+constexpr uint32_t bmi1 = uint32_t(1) << 3;
+constexpr uint32_t avx2 = uint32_t(1) << 5;
+constexpr uint32_t bmi2 = uint32_t(1) << 8;
+constexpr uint32_t avx512f = uint32_t(1) << 16;
+constexpr uint32_t avx512dq = uint32_t(1) << 17;
+constexpr uint32_t avx512ifma = uint32_t(1) << 21;
+constexpr uint32_t avx512cd = uint32_t(1) << 28;
+constexpr uint32_t avx512bw = uint32_t(1) << 30;
+constexpr uint32_t avx512vl = uint32_t(1) << 31;
+} // namespace ebx
+
+namespace ecx {
+constexpr uint32_t avx512vbmi = uint32_t(1) << 1;
+constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6;
+constexpr uint32_t avx512vnni = uint32_t(1) << 11;
+constexpr uint32_t avx512bitalg = uint32_t(1) << 12;
+constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14;
+} // namespace ecx
+namespace edx {
+constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8;
+}
+namespace xcr0_bit {
+constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX
+constexpr uint64_t avx512_saved =
+    uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM
+} // namespace xcr0_bit
+} // namespace cpuid_bit
+} // namespace
+
+static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
+                         uint32_t *edx) {
+  #if defined(_MSC_VER)
+  int cpu_info[4];
+  __cpuidex(cpu_info, *eax, *ecx);
+  *eax = cpu_info[0];
+  *ebx = cpu_info[1];
+  *ecx = cpu_info[2];
+  *edx = cpu_info[3];
+  #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
+  uint32_t level = *eax;
+  __get_cpuid(level, eax, ebx, ecx, edx);
+  #else
+  uint32_t a = *eax, b, c = *ecx, d;
+  asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));
+  *eax = a;
+  *ebx = b;
+  *ecx = c;
+  *edx = d;
+  #endif
+}
+
+static inline uint64_t xgetbv() {
+  #if defined(_MSC_VER)
+  return _xgetbv(0);
+  #else
+  uint32_t xcr0_lo, xcr0_hi;
+  asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0));
+  return xcr0_lo | ((uint64_t)xcr0_hi << 32);
+  #endif
+}
+
+static inline uint32_t detect_supported_architectures() {
+  uint32_t eax;
+  uint32_t ebx = 0;
+  uint32_t ecx = 0;
+  uint32_t edx = 0;
+  uint32_t host_isa = 0x0;
+
+  // EBX for EAX=0x1
+  eax = 0x1;
+  cpuid(&eax, &ebx, &ecx, &edx);
+
+  if (ecx & cpuid_bit::sse42) {
+    host_isa |= instruction_set::SSE42;
+  }
+
+  if (ecx & cpuid_bit::pclmulqdq) {
+    host_isa |= instruction_set::PCLMULQDQ;
+  }
+
+  if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) {
+    return host_isa;
+  }
+
+  // xgetbv for checking if the OS saves registers
+  uint64_t xcr0 = xgetbv();
+
+  if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) {
+    return host_isa;
+  }
+  // ECX for EAX=0x7
+  eax = 0x7;
+  ecx = 0x0; // Sub-leaf = 0
+  cpuid(&eax, &ebx, &ecx, &edx);
+  if (ebx & cpuid_bit::ebx::avx2) {
+    host_isa |= instruction_set::AVX2;
+  }
+  if (ebx & cpuid_bit::ebx::bmi1) {
+    host_isa |= instruction_set::BMI1;
+  }
+  if (ebx & cpuid_bit::ebx::bmi2) {
+    host_isa |= instruction_set::BMI2;
+  }
+  if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) ==
+        cpuid_bit::xcr0_bit::avx512_saved)) {
+    return host_isa;
+  }
+  if (ebx & cpuid_bit::ebx::avx512f) {
+    host_isa |= instruction_set::AVX512F;
+  }
+  if (ebx & cpuid_bit::ebx::avx512bw) {
+    host_isa |= instruction_set::AVX512BW;
+  }
+  if (ebx & cpuid_bit::ebx::avx512cd) {
+    host_isa |= instruction_set::AVX512CD;
+  }
+  if (ebx & cpuid_bit::ebx::avx512dq) {
+    host_isa |= instruction_set::AVX512DQ;
+  }
+  if (ebx & cpuid_bit::ebx::avx512vl) {
+    host_isa |= instruction_set::AVX512VL;
+  }
+  if (ecx & cpuid_bit::ecx::avx512vbmi2) {
+    host_isa |= instruction_set::AVX512VBMI2;
+  }
+  if (ecx & cpuid_bit::ecx::avx512vpopcnt) {
+    host_isa |= instruction_set::AVX512VPOPCNTDQ;
+  }
+  return host_isa;
+}
+#elif defined(__loongarch__)
+
+static inline uint32_t detect_supported_architectures() {
+  uint32_t host_isa = instruction_set::DEFAULT;
+  #if defined(__linux__)
+  uint64_t hwcap = 0;
+  hwcap = getauxval(AT_HWCAP);
+  if (hwcap & HWCAP_LOONGARCH_LSX) {
+    host_isa |= instruction_set::LSX;
+  }
+  if (hwcap & HWCAP_LOONGARCH_LASX) {
+    host_isa |= instruction_set::LASX;
+  }
+  #endif
+  return host_isa;
+}
+#else // fallback
+
+// includes 32-bit ARM.
+static inline uint32_t detect_supported_architectures() {
+  return instruction_set::DEFAULT;
+}
+
+#endif // end SIMD extension detection code
+
+} // namespace internal
+} // namespace simdutf
+
+#endif // SIMDutf_INTERNAL_ISADETECTION_H
+/* end file include/simdutf/internal/isadetection.h */
+
+#if SIMDUTF_SPAN
+  #include <concepts>
+  #include <type_traits>
+  #include <span>
+  #include <tuple>
+#endif
+#if SIMDUTF_CPLUSPLUS17
+  #include <string_view>
+#endif
+// The following defines are conditionally enabled/disabled during amalgamation.
+// By default all features are enabled, regular code shouldn't check them. Only
+// when user code really relies of a selected subset, it's good to verify these
+// flags, like:
+//
+//      #if !SIMDUTF_FEATURE_UTF16
+//      #   error("Please amalgamate simdutf with UTF-16 support")
+//      #endif
+//
+#define SIMDUTF_FEATURE_DETECT_ENCODING 1
+#define SIMDUTF_FEATURE_ASCII 1
+#define SIMDUTF_FEATURE_LATIN1 1
+#define SIMDUTF_FEATURE_UTF8 1
+#define SIMDUTF_FEATURE_UTF16 1
+#define SIMDUTF_FEATURE_UTF32 1
+#define SIMDUTF_FEATURE_BASE64 1
+
+#if SIMDUTF_CPLUSPLUS23
+/* begin file include/simdutf/constexpr_ptr.h */
+#ifndef SIMDUTF_CONSTEXPR_PTR_H
+#define SIMDUTF_CONSTEXPR_PTR_H
+
+#include <cstddef>
+
+namespace simdutf {
+namespace detail {
+/**
+ * The constexpr_ptr class is a workaround for reinterpret_cast not being
+ * allowed during constant evaluation.
+ */
+template <typename to, typename from>
+  requires(sizeof(to) == sizeof(from))
+struct constexpr_ptr {
+  const from *p;
+
+  constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {}
+
+  constexpr to operator*() const noexcept { return static_cast<to>(*p); }
+
+  constexpr constexpr_ptr &operator++() noexcept {
+    ++p;
+    return *this;
+  }
+
+  constexpr constexpr_ptr operator++(int) noexcept {
+    auto old = *this;
+    ++p;
+    return old;
+  }
+
+  constexpr constexpr_ptr &operator--() noexcept {
+    --p;
+    return *this;
+  }
+
+  constexpr constexpr_ptr operator--(int) noexcept {
+    auto old = *this;
+    --p;
+    return old;
+  }
+
+  constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept {
+    p += n;
+    return *this;
+  }
+
+  constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept {
+    p -= n;
+    return *this;
+  }
+
+  constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept {
+    return constexpr_ptr{p + n};
+  }
+
+  constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept {
+    return constexpr_ptr{p - n};
+  }
+
+  constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept {
+    return p - o.p;
+  }
+
+  constexpr to operator[](std::ptrdiff_t n) const noexcept {
+    return static_cast<to>(*(p + n));
+  }
+
+  // to prevent compilation errors for memcpy, even if it is never
+  // called during constant evaluation
+  constexpr operator const void *() const noexcept { return p; }
+};
+
+template <typename to, typename from>
+constexpr constexpr_ptr<to, from> constexpr_cast_ptr(from *p) noexcept {
+  return constexpr_ptr<to, from>{p};
+}
+
+/**
+ * helper type for constexpr_writeptr, so it is possible to
+ * do "*ptr = val;"
+ */
+template <typename SrcType, typename TargetType>
+struct constexpr_write_ptr_proxy {
+
+  constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {}
+
+  constexpr constexpr_write_ptr_proxy &operator=(SrcType v) {
+    *p = static_cast<TargetType>(v);
+    return *this;
+  }
+
+  TargetType *p;
+};
+
+/**
+ * helper for working around reinterpret_cast not being allowed during constexpr
+ * evaluation. will try to act as a SrcType* but actually write to the pointer
+ * given in the constructor, which is of another type TargetType
+ */
+template <typename SrcType, typename TargetType> struct constexpr_write_ptr {
+  constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {}
+
+  constexpr constexpr_write_ptr_proxy<SrcType, TargetType> operator*() const {
+    return constexpr_write_ptr_proxy<SrcType, TargetType>{p};
+  }
+
+  constexpr constexpr_write_ptr_proxy<SrcType, TargetType>
+  operator[](std::ptrdiff_t n) const {
+    return constexpr_write_ptr_proxy<SrcType, TargetType>{p + n};
+  }
+
+  constexpr constexpr_write_ptr &operator++() {
+    ++p;
+    return *this;
+  }
+
+  constexpr constexpr_write_ptr operator++(int) {
+    constexpr_write_ptr old = *this;
+    ++p;
+    return old;
+  }
+
+  constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const {
+    return p - other.p;
+  }
+
+  TargetType *p;
+};
+
+template <typename SrcType, typename TargetType>
+constexpr auto constexpr_cast_writeptr(TargetType *raw) {
+  return constexpr_write_ptr<SrcType, TargetType>{raw};
+}
+
+} // namespace detail
+} // namespace simdutf
+#endif
+/* end file include/simdutf/constexpr_ptr.h */
+#endif
+
+#if SIMDUTF_SPAN
+/// helpers placed in namespace detail are not a part of the public API
+namespace simdutf {
+namespace detail {
+/**
+ * matches a byte, in the many ways C++ allows. note that these
+ * are all distinct types.
+ */
+template <typename T>
+concept byte_like = std::is_same_v<T, std::byte> ||     //
+                    std::is_same_v<T, char> ||          //
+                    std::is_same_v<T, signed char> ||   //
+                    std::is_same_v<T, unsigned char> || //
+                    std::is_same_v<T, char8_t>;
+
+template <typename T>
+concept is_byte_like = byte_like<std::remove_cvref_t<T>>;
+
+template <typename T>
+concept is_pointer = std::is_pointer_v<T>;
+
+/**
+ * matches anything that behaves like std::span and points to character-like
+ * data such as: std::byte, char, unsigned char, signed char, std::int8_t,
+ * std::uint8_t
+ */
+template <typename T>
+concept input_span_of_byte_like = requires(const T &t) {
+  { t.size() } noexcept -> std::convertible_to<std::size_t>;
+  { t.data() } noexcept -> is_pointer;
+  { *t.data() } noexcept -> is_byte_like;
+};
+
+template <typename T>
+concept is_mutable = !std::is_const_v<std::remove_reference_t<T>>;
+
+/**
+ * like span_of_byte_like, but for an output span (intended to be written to)
+ */
+template <typename T>
+concept output_span_of_byte_like = requires(T &t) {
+  { t.size() } noexcept -> std::convertible_to<std::size_t>;
+  { t.data() } noexcept -> is_pointer;
+  { *t.data() } noexcept -> is_byte_like;
+  { *t.data() } noexcept -> is_mutable;
+};
+
+/**
+ * a pointer like object, when indexed, results in a byte like result.
+ * valid examples: char*, const char*, std::array<char,10>
+ * invalid examples: int*, std::array<int,10>
+ */
+template <class InputPtr>
+concept indexes_into_byte_like = requires(InputPtr p) {
+  { std::decay_t<decltype(p[0])>{} } -> simdutf::detail::byte_like;
+};
+template <class InputPtr>
+concept indexes_into_utf16 = requires(InputPtr p) {
+  { std::decay_t<decltype(p[0])>{} } -> std::same_as<char16_t>;
+};
+template <class InputPtr>
+concept indexes_into_utf32 = requires(InputPtr p) {
+  { std::decay_t<decltype(p[0])>{} } -> std::same_as<char32_t>;
+};
+
+template <class InputPtr>
+concept index_assignable_from_char = requires(InputPtr p, char s) {
+  { p[0] = s };
+};
+
+/**
+ * a pointer like object that results in a uint32_t when indexed.
+ * valid examples: uint32_t*
+ */
+template <class InputPtr>
+concept indexes_into_uint32 = requires(InputPtr p) {
+  { std::decay_t<decltype(p[0])>{} } -> std::same_as<std::uint32_t>;
+};
+} // namespace detail
+} // namespace simdutf
+#endif // SIMDUTF_SPAN
+
+// these includes are needed for constexpr support. they are
+// not part of the public api.
+/* begin file include/simdutf/scalar/swap_bytes.h */
+#ifndef SIMDUTF_SWAP_BYTES_H
+#define SIMDUTF_SWAP_BYTES_H
+
+namespace simdutf {
+namespace scalar {
+
+constexpr inline simdutf_warn_unused uint16_t
+u16_swap_bytes(const uint16_t word) {
+  return uint16_t((word >> 8) | (word << 8));
+}
+
+constexpr inline simdutf_warn_unused uint32_t
+u32_swap_bytes(const uint32_t word) {
+  return ((word >> 24) & 0xff) |      // move byte 3 to byte 0
+         ((word << 8) & 0xff0000) |   // move byte 1 to byte 2
+         ((word >> 8) & 0xff00) |     // move byte 2 to byte 1
+         ((word << 24) & 0xff000000); // byte 0 to byte 3
+}
+
+namespace utf32 {
+template <endianness big_endian> constexpr uint32_t swap_if_needed(uint32_t c) {
+  return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c;
+}
+} // namespace utf32
+
+namespace utf16 {
+template <endianness big_endian> constexpr uint16_t swap_if_needed(uint16_t c) {
+  return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c;
+}
+} // namespace utf16
+
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/swap_bytes.h */
+/* begin file include/simdutf/scalar/ascii.h */
+#ifndef SIMDUTF_ASCII_H
+#define SIMDUTF_ASCII_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace ascii {
+
+template <class InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data,
+                                                      size_t len) noexcept {
+  uint64_t pos = 0;
+
+#if SIMDUTF_CPLUSPLUS23
+  // avoid memcpy during constant evaluation
+  if !consteval
+#endif
+  // process in blocks of 16 bytes when possible
+  {
+    for (; pos + 16 <= len; pos += 16) {
+      uint64_t v1;
+      std::memcpy(&v1, data + pos, sizeof(uint64_t));
+      uint64_t v2;
+      std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+      uint64_t v{v1 | v2};
+      if ((v & 0x8080808080808080) != 0) {
+        return false;
+      }
+    }
+  }
+
+  // process the tail byte-by-byte
+  for (; pos < len; pos++) {
+    if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) {
+      return false;
+    }
+  }
+  return true;
+}
+template <class InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_warn_unused simdutf_constexpr23 result
+validate_with_errors(InputPtr data, size_t len) noexcept {
+  size_t pos = 0;
+#if SIMDUTF_CPLUSPLUS23
+  // avoid memcpy during constant evaluation
+  if !consteval
+#endif
+  {
+    // process in blocks of 16 bytes when possible
+    for (; pos + 16 <= len; pos += 16) {
+      uint64_t v1;
+      std::memcpy(&v1, data + pos, sizeof(uint64_t));
+      uint64_t v2;
+      std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+      uint64_t v{v1 | v2};
+      if ((v & 0x8080808080808080) != 0) {
+        for (; pos < len; pos++) {
+          if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) {
+            return result(error_code::TOO_LARGE, pos);
+          }
+        }
+      }
+    }
+  }
+
+  // process the tail byte-by-byte
+  for (; pos < len; pos++) {
+    if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) {
+      return result(error_code::TOO_LARGE, pos);
+    }
+  }
+  return result(error_code::SUCCESS, pos);
+}
+
+} // namespace ascii
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/ascii.h */
+/* begin file include/simdutf/scalar/atomic_util.h */
+#ifndef SIMDUTF_ATOMIC_UTIL_H
+#define SIMDUTF_ATOMIC_UTIL_H
+#if SIMDUTF_ATOMIC_REF
+  #include <atomic>
+namespace simdutf {
+namespace scalar {
+
+// This function is a memcpy that uses atomic operations to read from the
+// source.
+inline void memcpy_atomic_read(char *dst, const char *src, size_t len) {
+  static_assert(std::atomic_ref<char>::required_alignment == sizeof(char),
+                "std::atomic_ref requires the same alignment as char_type");
+  // We expect all 64-bit systems to be able to read 64-bit words from an
+  // aligned memory region atomically. You might be able to do better on
+  // specific systems, e.g., x64 systems can read 128-bit words atomically.
+  constexpr size_t alignment = sizeof(uint64_t);
+
+  // Lambda for atomic byte-by-byte copy
+  auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc,
+                                   size_t bytelen) noexcept {
+    char *mutable_src = const_cast<char *>(bytesrc);
+    for (size_t j = 0; j < bytelen; ++j) {
+      bytedst[j] =
+          std::atomic_ref<char>(mutable_src[j]).load(std::memory_order_relaxed);
+    }
+  };
+
+  // Handle unaligned start
+  size_t offset = reinterpret_cast<std::uintptr_t>(src) % alignment;
+  if (offset) {
+    size_t to_align = std::min(len, alignment - offset);
+    bbb_memcpy_atomic_read(dst, src, to_align);
+    src += to_align;
+    dst += to_align;
+    len -= to_align;
+  }
+
+  // Process aligned 64-bit chunks
+  while (len >= alignment) {
+    auto *src_aligned = reinterpret_cast<uint64_t *>(const_cast<char *>(src));
+    const auto dst_value =
+        std::atomic_ref<uint64_t>(*src_aligned).load(std::memory_order_relaxed);
+    std::memcpy(dst, &dst_value, sizeof(uint64_t));
+    src += alignment;
+    dst += alignment;
+    len -= alignment;
+  }
+
+  // Handle remaining bytes
+  if (len) {
+    bbb_memcpy_atomic_read(dst, src, len);
+  }
+}
+
+// This function is a memcpy that uses atomic operations to write to the
+// destination.
+inline void memcpy_atomic_write(char *dst, const char *src, size_t len) {
+  static_assert(std::atomic_ref<char>::required_alignment == sizeof(char),
+                "std::atomic_ref requires the same alignment as char");
+  // We expect all 64-bit systems to be able to write 64-bit words to an aligned
+  // memory region atomically.
+  // You might be able to do better on specific systems, e.g., x64 systems can
+  // write 128-bit words atomically.
+  constexpr size_t alignment = sizeof(uint64_t);
+
+  // Lambda for atomic byte-by-byte write
+  auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc,
+                                    size_t bytelen) noexcept {
+    for (size_t j = 0; j < bytelen; ++j) {
+      std::atomic_ref<char>(bytedst[j])
+          .store(bytesrc[j], std::memory_order_relaxed);
+    }
+  };
+
+  // Handle unaligned start
+  size_t offset = reinterpret_cast<std::uintptr_t>(dst) % alignment;
+  if (offset) {
+    size_t to_align = std::min(len, alignment - offset);
+    bbb_memcpy_atomic_write(dst, src, to_align);
+    dst += to_align;
+    src += to_align;
+    len -= to_align;
+  }
+
+  // Process aligned 64-bit chunks
+  while (len >= alignment) {
+    auto *dst_aligned = reinterpret_cast<uint64_t *>(dst);
+    uint64_t src_val;
+    std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src
+    std::atomic_ref<uint64_t>(*dst_aligned)
+        .store(src_val, std::memory_order_relaxed);
+    dst += alignment;
+    src += alignment;
+    len -= alignment;
+  }
+
+  // Handle remaining bytes
+  if (len) {
+    bbb_memcpy_atomic_write(dst, src, len);
+  }
+}
+} // namespace scalar
+} // namespace simdutf
+#endif // SIMDUTF_ATOMIC_REF
+#endif // SIMDUTF_ATOMIC_UTIL_H
+/* end file include/simdutf/scalar/atomic_util.h */
+/* begin file include/simdutf/scalar/latin1.h */
+#ifndef SIMDUTF_LATIN1_H
+#define SIMDUTF_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace latin1 {
+
+simdutf_really_inline size_t utf8_length_from_latin1(const char *buf,
+                                                     size_t len) {
+  const uint8_t *c = reinterpret_cast<const uint8_t *>(buf);
+  size_t answer = 0;
+  for (size_t i = 0; i < len; i++) {
+    if ((c[i] >> 7)) {
+      answer++;
+    }
+  }
+  return answer + len;
+}
+
+} // namespace latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/latin1.h */
+/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */
+#ifndef SIMDUTF_LATIN1_TO_UTF16_H
+#define SIMDUTF_LATIN1_TO_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace latin1_to_utf16 {
+
+template <endianness big_endian, typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+
+  while (pos < len) {
+    uint16_t word =
+        uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point
+    *utf16_output++ =
+        char16_t(match_system(big_endian) ? word : u16_swap_bytes(word));
+    pos++;
+  }
+
+  return utf16_output - start;
+}
+
+template <endianness big_endian>
+inline result convert_with_errors(const char *buf, size_t len,
+                                  char16_t *utf16_output) {
+  const uint8_t *data = reinterpret_cast<const uint8_t *>(buf);
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+
+  while (pos < len) {
+    uint16_t word =
+        uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point
+    *utf16_output++ =
+        char16_t(match_system(big_endian) ? word : u16_swap_bytes(word));
+    pos++;
+  }
+
+  return result(error_code::SUCCESS, utf16_output - start);
+}
+
+} // namespace latin1_to_utf16
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */
+/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */
+#ifndef SIMDUTF_LATIN1_TO_UTF32_H
+#define SIMDUTF_LATIN1_TO_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace latin1_to_utf32 {
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   char32_t *utf32_output) {
+  char32_t *start{utf32_output};
+  for (size_t i = 0; i < len; i++) {
+    *utf32_output++ = uint8_t(data[i]);
+  }
+  return utf32_output - start;
+}
+
+} // namespace latin1_to_utf32
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */
+/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */
+#ifndef SIMDUTF_LATIN1_TO_UTF8_H
+#define SIMDUTF_LATIN1_TO_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace latin1_to_utf8 {
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_byte_like<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   OutputPtr utf8_output) {
+  // const unsigned char *data = reinterpret_cast<const unsigned char *>(buf);
+  size_t pos = 0;
+  size_t utf8_pos = 0;
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 |
+                   v2}; // We are only interested in these bits: 1000 1000 1000
+                        // 1000, so it makes sense to concatenate everything
+        if ((v & 0x8080808080808080) ==
+            0) { // if NONE of these are set, e.g. all of them are zero, then
+                 // everything is ASCII
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            utf8_output[utf8_pos++] = char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      } // if (pos + 16 <= len)
+    } // !consteval scope
+
+    unsigned char byte = data[pos];
+    if ((byte & 0x80) == 0) { // if ASCII
+      // will generate one UTF-8 bytes
+      utf8_output[utf8_pos++] = char(byte);
+      pos++;
+    } else {
+      // will generate two UTF-8 bytes
+      utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000);
+      utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000);
+      pos++;
+    }
+  } // while
+  return utf8_pos;
+}
+
+simdutf_really_inline size_t convert(const char *buf, size_t len,
+                                     char *utf8_output) {
+  return convert(reinterpret_cast<const unsigned char *>(buf), len,
+                 utf8_output);
+}
+
+inline size_t convert_safe(const char *buf, size_t len, char *utf8_output,
+                           size_t utf8_len) {
+  const unsigned char *data = reinterpret_cast<const unsigned char *>(buf);
+  size_t pos = 0;
+  size_t skip_pos = 0;
+  size_t utf8_pos = 0;
+  while (pos < len && utf8_pos < utf8_len) {
+    // try to convert the next block of 16 ASCII bytes
+    if (pos >= skip_pos && pos + 16 <= len &&
+        utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes,
+                                     // check that they are ascii
+      uint64_t v1;
+      ::memcpy(&v1, data + pos, sizeof(uint64_t));
+      uint64_t v2;
+      ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+      uint64_t v{v1 |
+                 v2}; // We are only interested in these bits: 1000 1000 1000
+                      // 1000, so it makes sense to concatenate everything
+      if ((v & 0x8080808080808080) ==
+          0) { // if NONE of these are set, e.g. all of them are zero, then
+               // everything is ASCII
+        ::memcpy(utf8_output + utf8_pos, buf + pos, 16);
+        utf8_pos += 16;
+        pos += 16;
+      } else {
+        // At least one of the next 16 bytes are not ASCII, we will process them
+        // one by one
+        skip_pos = pos + 16;
+      }
+    } else {
+      const auto byte = data[pos];
+      if ((byte & 0x80) == 0) { // if ASCII
+        // will generate one UTF-8 bytes
+        utf8_output[utf8_pos++] = char(byte);
+        pos++;
+      } else if (utf8_pos + 2 <= utf8_len) {
+        // will generate two UTF-8 bytes
+        utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000);
+        utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000);
+        pos++;
+      } else {
+        break;
+      }
+    }
+  }
+  return utf8_pos;
+}
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_byte_like<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len,
+                                                  OutputPtr utf8_output,
+                                                  size_t utf8_len) {
+  size_t pos = 0;
+  size_t utf8_pos = 0;
+  while (pos < len && utf8_pos < utf8_len) {
+    const unsigned char byte = data[pos];
+    if ((byte & 0x80) == 0) { // if ASCII
+      // will generate one UTF-8 bytes
+      utf8_output[utf8_pos++] = char(byte);
+      pos++;
+    } else if (utf8_pos + 2 <= utf8_len) {
+      // will generate two UTF-8 bytes
+      utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000);
+      utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      break;
+    }
+  }
+  return utf8_pos;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 simdutf_warn_unused size_t
+utf8_length_from_latin1(InputPtr input, size_t length) noexcept {
+  size_t answer = length;
+  size_t i = 0;
+
+#if SIMDUTF_CPLUSPLUS23
+  if !consteval
+#endif
+  {
+    auto pop = [](uint64_t v) {
+      return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) *
+                          UINT64_C(0x0101010101010101) >>
+                      56);
+    };
+    for (; i + 32 <= length; i += 32) {
+      uint64_t v;
+      memcpy(&v, input + i, 8);
+      answer += pop(v);
+      memcpy(&v, input + i + 8, sizeof(v));
+      answer += pop(v);
+      memcpy(&v, input + i + 16, sizeof(v));
+      answer += pop(v);
+      memcpy(&v, input + i + 24, sizeof(v));
+      answer += pop(v);
+    }
+    for (; i + 8 <= length; i += 8) {
+      uint64_t v;
+      memcpy(&v, input + i, sizeof(v));
+      answer += pop(v);
+    }
+  } // !consteval scope
+  for (; i + 1 <= length; i += 1) {
+    answer += static_cast<uint8_t>(input[i]) >> 7;
+  }
+  return answer;
+}
+
+} // namespace latin1_to_utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */
+/* begin file include/simdutf/scalar/utf16.h */
+#ifndef SIMDUTF_UTF16_H
+#define SIMDUTF_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace utf16 {
+
+template <endianness big_endian>
+simdutf_warn_unused simdutf_constexpr23 bool
+validate_as_ascii(const char16_t *data, size_t len) noexcept {
+  for (size_t pos = 0; pos < len; pos++) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
+    if (word >= 0x80) {
+      return false;
+    }
+  }
+  return true;
+}
+
+template <endianness big_endian>
+inline simdutf_warn_unused simdutf_constexpr23 bool
+validate(const char16_t *data, size_t len) noexcept {
+  uint64_t pos = 0;
+  while (pos < len) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
+    if ((word & 0xF800) == 0xD800) {
+      if (pos + 1 >= len) {
+        return false;
+      }
+      char16_t diff = char16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return false;
+      }
+      char16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      char16_t diff2 = char16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return false;
+      }
+      pos += 2;
+    } else {
+      pos++;
+    }
+  }
+  return true;
+}
+
+template <endianness big_endian>
+inline simdutf_warn_unused simdutf_constexpr23 result
+validate_with_errors(const char16_t *data, size_t len) noexcept {
+  size_t pos = 0;
+  while (pos < len) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
+    if ((word & 0xF800) == 0xD800) {
+      if (pos + 1 >= len) {
+        return result(error_code::SURROGATE, pos);
+      }
+      char16_t diff = char16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      char16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      char16_t diff2 = uint16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      pos += 2;
+    } else {
+      pos++;
+    }
+  }
+  return result(error_code::SUCCESS, pos);
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) {
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
+    counter += ((word & 0xFC00) != 0xDC00);
+  }
+  return counter;
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p,
+                                                  size_t len) {
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
+    counter++; // ASCII
+    counter += static_cast<size_t>(
+        word >
+        0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes
+    counter += static_cast<size_t>((word > 0x7FF && word <= 0xD7FF) ||
+                                   (word >= 0xE000)); // three-byte
+  }
+  return counter;
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p,
+                                                   size_t len) {
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
+    counter += ((word & 0xFC00) != 0xDC00);
+  }
+  return counter;
+}
+
+simdutf_really_inline simdutf_constexpr23 void
+change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) {
+  for (size_t i = 0; i < size; i++) {
+    *output++ = char16_t(input[i] >> 8 | input[i] << 8);
+  }
+}
+
+template <endianness big_endian>
+simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf16(const char16_t *input, size_t length) {
+  if (length == 0) {
+    return 0;
+  }
+  uint16_t last_word = uint16_t(input[length - 1]);
+  last_word = scalar::utf16::swap_if_needed<big_endian>(last_word);
+  length -= ((last_word & 0xFC00) == 0xD800);
+  return length;
+}
+
+template <endianness big_endian>
+simdutf_constexpr bool is_high_surrogate(char16_t c) {
+  c = scalar::utf16::swap_if_needed<big_endian>(c);
+  return (0xd800 <= c && c <= 0xdbff);
+}
+
+template <endianness big_endian>
+simdutf_constexpr bool is_low_surrogate(char16_t c) {
+  c = scalar::utf16::swap_if_needed<big_endian>(c);
+  return (0xdc00 <= c && c <= 0xdfff);
+}
+
+simdutf_really_inline constexpr bool high_surrogate(char16_t c) {
+  return (0xd800 <= c && c <= 0xdbff);
+}
+
+simdutf_really_inline constexpr bool low_surrogate(char16_t c) {
+  return (0xdc00 <= c && c <= 0xdfff);
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 result
+utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) {
+  bool any_surrogates = false;
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    if (is_high_surrogate<big_endian>(p[i])) {
+      any_surrogates = true;
+      // surrogate pair
+      if (i + 1 < len && is_low_surrogate<big_endian>(p[i + 1])) {
+        counter += 4;
+        i++; // skip low surrogate
+      } else {
+        counter += 3; // unpaired high surrogate replaced by U+FFFD
+      }
+      continue;
+    } else if (is_low_surrogate<big_endian>(p[i])) {
+      any_surrogates = true;
+      counter += 3; // unpaired low surrogate replaced by U+FFFD
+      continue;
+    }
+    char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i];
+    counter++; // at least 1 byte
+    counter +=
+        static_cast<size_t>(word > 0x7F); // non-ASCII is at least 2 bytes
+    counter += static_cast<size_t>(word > 0x7FF); // three-byte
+  }
+  return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS,
+          counter};
+}
+
+// variable templates are a C++14 extension
+template <endianness big_endian> constexpr char16_t replacement() {
+  return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd;
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len,
+                                              char16_t *output) {
+  const char16_t replacement = utf16::replacement<big_endian>();
+  bool high_surrogate_prev = false, high_surrogate, low_surrogate;
+  size_t i = 0;
+  for (; i < len; i++) {
+    char16_t c = input[i];
+    high_surrogate = is_high_surrogate<big_endian>(c);
+    low_surrogate = is_low_surrogate<big_endian>(c);
+    if (high_surrogate_prev && !low_surrogate) {
+      output[i - 1] = replacement;
+    }
+
+    if (!high_surrogate_prev && low_surrogate) {
+      output[i] = replacement;
+    } else {
+      output[i] = input[i];
+    }
+    high_surrogate_prev = high_surrogate;
+  }
+
+  /* string may not end with high surrogate */
+  if (high_surrogate_prev) {
+    output[i - 1] = replacement;
+  }
+}
+
+} // namespace utf16
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16.h */
+/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */
+#ifndef SIMDUTF_UTF16_TO_LATIN1_H
+#define SIMDUTF_UTF16_TO_LATIN1_H
+
+#include <cstring> // for std::memcpy
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_latin1 {
+
+template <endianness big_endian, typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf16<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   OutputPtr latin_output) {
+  if (len == 0) {
+    return 0;
+  }
+  size_t pos = 0;
+  const auto latin_output_start = latin_output;
+  uint16_t word = 0;
+  uint16_t too_large = 0;
+
+  while (pos < len) {
+    word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    too_large |= word;
+    *latin_output++ = char(word & 0xFF);
+    pos++;
+  }
+  if ((too_large & 0xFF00) != 0) {
+    return 0;
+  }
+
+  return latin_output - latin_output_start;
+}
+
+template <endianness big_endian, typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf16<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
+                                               OutputPtr latin_output) {
+  if (len == 0) {
+    return result(error_code::SUCCESS, 0);
+  }
+  size_t pos = 0;
+  auto start = latin_output;
+  uint16_t word;
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that
+                             // they are Latin1
+        uint64_t v1, v2, v3, v4;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        ::memcpy(&v2, data + pos + 4, sizeof(uint64_t));
+        ::memcpy(&v3, data + pos + 8, sizeof(uint64_t));
+        ::memcpy(&v4, data + pos + 12, sizeof(uint64_t));
+
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v1 = (v1 >> 8) | (v1 << (64 - 8));
+        }
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v2 = (v2 >> 8) | (v2 << (64 - 8));
+        }
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v3 = (v3 >> 8) | (v3 << (64 - 8));
+        }
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v4 = (v4 >> 8) | (v4 << (64 - 8));
+        }
+
+        if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) {
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *latin_output++ = !match_system(big_endian)
+                                  ? char(u16_swap_bytes(data[pos]))
+                                  : char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xFF00) == 0) {
+      *latin_output++ = char(word & 0xFF);
+      pos++;
+    } else {
+      return result(error_code::TOO_LARGE, pos);
+    }
+  }
+  return result(error_code::SUCCESS, latin_output - start);
+}
+
+} // namespace utf16_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */
+/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */
+#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H
+#define SIMDUTF_VALID_UTF16_TO_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_latin1 {
+
+template <endianness big_endian, class InputIterator, class OutputIterator>
+simdutf_constexpr23 inline size_t
+convert_valid_impl(InputIterator data, size_t len,
+                   OutputIterator latin_output) {
+  static_assert(
+      std::is_same<typename std::decay<decltype(*data)>::type, uint16_t>::value,
+      "must decay to uint16_t");
+  size_t pos = 0;
+  const auto start = latin_output;
+  uint16_t word = 0;
+
+  while (pos < len) {
+    word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    *latin_output++ = char(word);
+    pos++;
+  }
+
+  return latin_output - start;
+}
+
+template <endianness big_endian>
+simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len,
+                                           char *latin_output) {
+  return convert_valid_impl<big_endian>(reinterpret_cast<const uint16_t *>(buf),
+                                        len, latin_output);
+}
+} // namespace utf16_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */
+/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */
+#ifndef SIMDUTF_UTF16_TO_UTF32_H
+#define SIMDUTF_UTF16_TO_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_utf32 {
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t convert(const char16_t *data, size_t len,
+                                   char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xF800) != 0xD800) {
+      // No surrogate pair, extend 16-bit word to 32-bit word
+      *utf32_output++ = char32_t(word);
+      pos++;
+    } else {
+      // must be a surrogate pair
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return 0;
+      }
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return 0;
+      }
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      *utf32_output++ = char32_t(value);
+      pos += 2;
+    }
+  }
+  return utf32_output - start;
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len,
+                                               char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xF800) != 0xD800) {
+      // No surrogate pair, extend 16-bit word to 32-bit word
+      *utf32_output++ = char32_t(word);
+      pos++;
+    } else {
+      // must be a surrogate pair
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      if (pos + 1 >= len) {
+        return result(error_code::SURROGATE, pos);
+      } // minimal bound checking
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      *utf32_output++ = char32_t(value);
+      pos += 2;
+    }
+  }
+  return result(error_code::SUCCESS, utf32_output - start);
+}
+
+} // namespace utf16_to_utf32
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */
+/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */
+#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H
+#define SIMDUTF_VALID_UTF16_TO_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_utf32 {
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len,
+                                         char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xF800) != 0xD800) {
+      // No surrogate pair, extend 16-bit word to 32-bit word
+      *utf32_output++ = char32_t(word);
+      pos++;
+    } else {
+      // must be a surrogate pair
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      *utf32_output++ = char32_t(value);
+      pos += 2;
+    }
+  }
+  return utf32_output - start;
+}
+
+} // namespace utf16_to_utf32
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */
+/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */
+#ifndef SIMDUTF_UTF16_TO_UTF8_H
+#define SIMDUTF_UTF16_TO_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_utf8 {
+
+template <endianness big_endian, typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_utf16<InputPtr>
+// FIXME constrain output as well
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   OutputPtr utf8_output) {
+  size_t pos = 0;
+  const auto start = utf8_output;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 8 bytes
+      if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v = (v >> 8) | (v << (64 - 8));
+        }
+        if ((v & 0xFF80FF80FF80FF80) == 0) {
+          size_t final_pos = pos + 4;
+          while (pos < final_pos) {
+            *utf8_output++ = !match_system(big_endian)
+                                 ? char(u16_swap_bytes(data[pos]))
+                                 : char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xFF80) == 0) {
+      // will generate one UTF-8 bytes
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else if ((word & 0xF800) != 0xD800) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      // must be a surrogate pair
+      if (pos + 1 >= len) {
+        return 0;
+      }
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return 0;
+      }
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return 0;
+      }
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((value >> 18) | 0b11110000);
+      *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((value & 0b111111) | 0b10000000);
+      pos += 2;
+    }
+  }
+  return utf8_output - start;
+}
+
+template <endianness big_endian, bool check_output = false, typename InputPtr,
+          typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf16<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len,
+                                                    OutputPtr utf8_output,
+                                                    size_t utf8_len = 0) {
+  if (check_output && utf8_len == 0) {
+    return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0);
+  }
+
+  size_t pos = 0;
+  auto start = utf8_output;
+  auto end = utf8_output + utf8_len;
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 8 bytes
+      if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if simdutf_constexpr (!match_system(big_endian))
+          v = (v >> 8) | (v << (64 - 8));
+        if ((v & 0xFF80FF80FF80FF80) == 0) {
+          size_t final_pos = pos + 4;
+          while (pos < final_pos) {
+            if (check_output && size_t(end - utf8_output) < 1) {
+              return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
+                                 utf8_output - start);
+            }
+            *utf8_output++ = !match_system(big_endian)
+                                 ? char(u16_swap_bytes(data[pos]))
+                                 : char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xFF80) == 0) {
+      // will generate one UTF-8 bytes
+      if (check_output && size_t(end - utf8_output) < 1) {
+        return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
+                           utf8_output - start);
+      }
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      if (check_output && size_t(end - utf8_output) < 2) {
+        return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
+                           utf8_output - start);
+      }
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+
+    } else if ((word & 0xF800) != 0xD800) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      if (check_output && size_t(end - utf8_output) < 3) {
+        return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
+                           utf8_output - start);
+      }
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+
+      if (check_output && size_t(end - utf8_output) < 4) {
+        return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
+                           utf8_output - start);
+      }
+      // must be a surrogate pair
+      if (pos + 1 >= len) {
+        return full_result(error_code::SURROGATE, pos, utf8_output - start);
+      }
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (diff > 0x3FF) {
+        return full_result(error_code::SURROGATE, pos, utf8_output - start);
+      }
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      if (diff2 > 0x3FF) {
+        return full_result(error_code::SURROGATE, pos, utf8_output - start);
+      }
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((value >> 18) | 0b11110000);
+      *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((value & 0b111111) | 0b10000000);
+      pos += 2;
+    }
+  }
+  return full_result(error_code::SUCCESS, pos, utf8_output - start);
+}
+
+template <endianness big_endian>
+inline result simple_convert_with_errors(const char16_t *buf, size_t len,
+                                         char *utf8_output) {
+  return convert_with_errors<big_endian, false>(buf, len, utf8_output, 0);
+}
+
+} // namespace utf16_to_utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */
+/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */
+#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H
+#define SIMDUTF_VALID_UTF16_TO_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf16_to_utf8 {
+
+template <endianness big_endian, typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf16<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
+                                         OutputPtr utf8_output) {
+  size_t pos = 0;
+  auto start = utf8_output;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 4 ASCII characters
+      if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if simdutf_constexpr (!match_system(big_endian)) {
+          v = (v >> 8) | (v << (64 - 8));
+        }
+        if ((v & 0xFF80FF80FF80FF80) == 0) {
+          size_t final_pos = pos + 4;
+          while (pos < final_pos) {
+            *utf8_output++ = !match_system(big_endian)
+                                 ? char(u16_swap_bytes(data[pos]))
+                                 : char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    uint16_t word =
+        !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
+    if ((word & 0xFF80) == 0) {
+      // will generate one UTF-8 bytes
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else if ((word & 0xF800) != 0xD800) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      // must be a surrogate pair
+      uint16_t diff = uint16_t(word - 0xD800);
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      uint16_t next_word = !match_system(big_endian)
+                               ? u16_swap_bytes(data[pos + 1])
+                               : data[pos + 1];
+      uint16_t diff2 = uint16_t(next_word - 0xDC00);
+      uint32_t value = (diff << 10) + diff2 + 0x10000;
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((value >> 18) | 0b11110000);
+      *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((value & 0b111111) | 0b10000000);
+      pos += 2;
+    }
+  }
+  return utf8_output - start;
+}
+
+} // namespace utf16_to_utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */
+/* begin file include/simdutf/scalar/utf32.h */
+#ifndef SIMDUTF_UTF32_H
+#define SIMDUTF_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace utf32 {
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_uint32<InputPtr>
+#endif
+simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data,
+                                                      size_t len) noexcept {
+  uint64_t pos = 0;
+  for (; pos < len; pos++) {
+    uint32_t word = data[pos];
+    if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf,
+                                                        size_t len) noexcept {
+  return validate(reinterpret_cast<const uint32_t *>(buf), len);
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_uint32<InputPtr>
+#endif
+simdutf_warn_unused simdutf_constexpr23 result
+validate_with_errors(InputPtr data, size_t len) noexcept {
+  size_t pos = 0;
+  for (; pos < len; pos++) {
+    uint32_t word = data[pos];
+    if (word > 0x10FFFF) {
+      return result(error_code::TOO_LARGE, pos);
+    }
+    if (word >= 0xD800 && word <= 0xDFFF) {
+      return result(error_code::SURROGATE, pos);
+    }
+  }
+  return result(error_code::SUCCESS, pos);
+}
+
+simdutf_warn_unused simdutf_really_inline result
+validate_with_errors(const char32_t *buf, size_t len) noexcept {
+  return validate_with_errors(reinterpret_cast<const uint32_t *>(buf), len);
+}
+
+inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p,
+                                                         size_t len) {
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    // credit: @ttsugriy  for the vectorizable approach
+    counter++;                                     // ASCII
+    counter += static_cast<size_t>(p[i] > 0x7F);   // two-byte
+    counter += static_cast<size_t>(p[i] > 0x7FF);  // three-byte
+    counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes
+  }
+  return counter;
+}
+
+inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf16_length_from_utf32(const char32_t *p, size_t len) {
+  // We are not BOM aware.
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    counter++;                                     // non-surrogate word
+    counter += static_cast<size_t>(p[i] > 0xFFFF); // surrogate pair
+  }
+  return counter;
+}
+
+} // namespace utf32
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32.h */
+/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */
+#ifndef SIMDUTF_UTF32_TO_LATIN1_H
+#define SIMDUTF_UTF32_TO_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_latin1 {
+
+inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len,
+                                          char *latin1_output) {
+  char *start = latin1_output;
+  uint32_t utf32_char;
+  size_t pos = 0;
+  uint32_t too_large = 0;
+
+  while (pos < len) {
+    utf32_char = (uint32_t)data[pos];
+    too_large |= utf32_char;
+    *latin1_output++ = (char)(utf32_char & 0xFF);
+    pos++;
+  }
+  if ((too_large & 0xFFFFFF00) != 0) {
+    return 0;
+  }
+  return latin1_output - start;
+}
+
+inline simdutf_constexpr23 result convert_with_errors(const char32_t *data,
+                                                      size_t len,
+                                                      char *latin1_output) {
+  char *start{latin1_output};
+  size_t pos = 0;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are Latin1
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0xFFFFFF00FFFFFF00) == 0) {
+          *latin1_output++ = char(data[pos]);
+          *latin1_output++ = char(data[pos + 1]);
+          pos += 2;
+          continue;
+        }
+      }
+    }
+
+    uint32_t utf32_char = data[pos];
+    if ((utf32_char & 0xFFFFFF00) ==
+        0) { // Check if the character can be represented in Latin-1
+      *latin1_output++ = (char)(utf32_char & 0xFF);
+      pos++;
+    } else {
+      return result(error_code::TOO_LARGE, pos);
+    };
+  }
+  return result(error_code::SUCCESS, latin1_output - start);
+}
+
+} // namespace utf32_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */
+/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */
+#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H
+#define SIMDUTF_VALID_UTF32_TO_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_latin1 {
+
+template <typename ReadPtr, typename WritePtr>
+simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len,
+                                         WritePtr latin1_output) {
+  static_assert(
+      std::is_same<typename std::decay<decltype(*data)>::type, uint32_t>::value,
+      "dereferencing the data pointer must result in a uint32_t");
+  auto start = latin1_output;
+  uint32_t utf32_char;
+  size_t pos = 0;
+
+  while (pos < len) {
+    utf32_char = data[pos];
+
+#if SIMDUTF_CPLUSPLUS23
+    // avoid using the 8 byte at a time optimization in constant evaluation
+    // mode. memcpy can't be used and replacing it with bitwise or gave worse
+    // codegen (when not during constant evaluation).
+    if !consteval {
+#endif
+      if (pos + 2 <= len) {
+        // if it is safe to read 8 more bytes, check that they are Latin1
+        uint64_t v;
+        std::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0xFFFFFF00FFFFFF00) == 0) {
+          *latin1_output++ = char(data[pos]);
+          *latin1_output++ = char(data[pos + 1]);
+          pos += 2;
+          continue;
+        } else {
+          // output can not be represented in latin1
+          return 0;
+        }
+      }
+#if SIMDUTF_CPLUSPLUS23
+    } // if ! consteval
+#endif
+    if ((utf32_char & 0xFFFFFF00) == 0) {
+      *latin1_output++ = char(utf32_char);
+    } else {
+      // output can not be represented in latin1
+      return 0;
+    }
+    pos++;
+  }
+  return latin1_output - start;
+}
+
+simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len,
+                                           char *latin1_output) {
+  return convert_valid(reinterpret_cast<const uint32_t *>(buf), len,
+                       latin1_output);
+}
+
+} // namespace utf32_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */
+/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */
+#ifndef SIMDUTF_UTF32_TO_UTF16_H
+#define SIMDUTF_UTF32_TO_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_utf16 {
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t convert(const char32_t *data, size_t len,
+                                   char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+    uint32_t word = data[pos];
+    if ((word & 0xFFFF0000) == 0) {
+      if (word >= 0xD800 && word <= 0xDFFF) {
+        return 0;
+      }
+      // will not generate a surrogate pair
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(uint16_t(word)))
+                            : char16_t(word);
+    } else {
+      // will generate a surrogate pair
+      if (word > 0x10FFFF) {
+        return 0;
+      }
+      word -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+    }
+    pos++;
+  }
+  return utf16_output - start;
+}
+
+template <endianness big_endian>
+simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len,
+                                               char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+    uint32_t word = data[pos];
+    if ((word & 0xFFFF0000) == 0) {
+      if (word >= 0xD800 && word <= 0xDFFF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      // will not generate a surrogate pair
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(uint16_t(word)))
+                            : char16_t(word);
+    } else {
+      // will generate a surrogate pair
+      if (word > 0x10FFFF) {
+        return result(error_code::TOO_LARGE, pos);
+      }
+      word -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+    }
+    pos++;
+  }
+  return result(error_code::SUCCESS, utf16_output - start);
+}
+
+} // namespace utf32_to_utf16
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */
+/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */
+#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H
+#define SIMDUTF_VALID_UTF32_TO_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_utf16 {
+
+template <endianness big_endian>
+simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len,
+                                         char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+    uint32_t word = data[pos];
+    if ((word & 0xFFFF0000) == 0) {
+      // will not generate a surrogate pair
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(uint16_t(word)))
+                            : char16_t(word);
+      pos++;
+    } else {
+      // will generate a surrogate pair
+      word -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+      pos++;
+    }
+  }
+  return utf16_output - start;
+}
+
+} // namespace utf32_to_utf16
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */
+/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */
+#ifndef SIMDUTF_UTF32_TO_UTF8_H
+#define SIMDUTF_UTF32_TO_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_utf8 {
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf32<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   OutputPtr utf8_output) {
+  size_t pos = 0;
+  auto start = utf8_output;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    { // try to convert the next block of 2 ASCII characters
+      if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0xFFFFFF80FFFFFF80) == 0) {
+          *utf8_output++ = char(data[pos]);
+          *utf8_output++ = char(data[pos + 1]);
+          pos += 2;
+          continue;
+        }
+      }
+    }
+
+    uint32_t word = data[pos];
+    if ((word & 0xFFFFFF80) == 0) {
+      // will generate one UTF-8 bytes
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xFFFFF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else if ((word & 0xFFFF0000) == 0) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      if (word >= 0xD800 && word <= 0xDFFF) {
+        return 0;
+      }
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      if (word > 0x10FFFF) {
+        return 0;
+      }
+      *utf8_output++ = char((word >> 18) | 0b11110000);
+      *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    }
+  }
+  return utf8_output - start;
+}
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf32<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
+                                               OutputPtr utf8_output) {
+  size_t pos = 0;
+  auto start = utf8_output;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    { // try to convert the next block of 2 ASCII characters
+      if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0xFFFFFF80FFFFFF80) == 0) {
+          *utf8_output++ = char(data[pos]);
+          *utf8_output++ = char(data[pos + 1]);
+          pos += 2;
+          continue;
+        }
+      }
+    }
+
+    uint32_t word = data[pos];
+    if ((word & 0xFFFFFF80) == 0) {
+      // will generate one UTF-8 bytes
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xFFFFF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else if ((word & 0xFFFF0000) == 0) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      if (word >= 0xD800 && word <= 0xDFFF) {
+        return result(error_code::SURROGATE, pos);
+      }
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      if (word > 0x10FFFF) {
+        return result(error_code::TOO_LARGE, pos);
+      }
+      *utf8_output++ = char((word >> 18) | 0b11110000);
+      *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    }
+  }
+  return result(error_code::SUCCESS, utf8_output - start);
+}
+
+} // namespace utf32_to_utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */
+/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */
+#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H
+#define SIMDUTF_VALID_UTF32_TO_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf32_to_utf8 {
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_utf32<InputPtr> &&
+           simdutf::detail::index_assignable_from_char<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
+                                         OutputPtr utf8_output) {
+  size_t pos = 0;
+  auto start = utf8_output;
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    { // try to convert the next block of 2 ASCII characters
+      if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0xFFFFFF80FFFFFF80) == 0) {
+          *utf8_output++ = char(data[pos]);
+          *utf8_output++ = char(data[pos + 1]);
+          pos += 2;
+          continue;
+        }
+      }
+    }
+
+    uint32_t word = data[pos];
+    if ((word & 0xFFFFFF80) == 0) {
+      // will generate one UTF-8 bytes
+      *utf8_output++ = char(word);
+      pos++;
+    } else if ((word & 0xFFFFF800) == 0) {
+      // will generate two UTF-8 bytes
+      // we have 0b110XXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 6) | 0b11000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else if ((word & 0xFFFF0000) == 0) {
+      // will generate three UTF-8 bytes
+      // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 12) | 0b11100000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    } else {
+      // will generate four UTF-8 bytes
+      // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
+      *utf8_output++ = char((word >> 18) | 0b11110000);
+      *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000);
+      *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+      *utf8_output++ = char((word & 0b111111) | 0b10000000);
+      pos++;
+    }
+  }
+  return utf8_output - start;
+}
+
+} // namespace utf32_to_utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */
+/* begin file include/simdutf/scalar/utf8.h */
+#ifndef SIMDUTF_UTF8_H
+#define SIMDUTF_UTF8_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8 {
+
+// credit: based on code from Google Fuchsia (Apache Licensed)
+template <class BytePtr>
+simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
+                                                      size_t len) noexcept {
+  static_assert(
+      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
+      "dereferencing the data pointer must result in a uint8_t");
+  uint64_t pos = 0;
+  uint32_t code_point = 0;
+  while (pos < len) {
+    uint64_t next_pos;
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    { // check if the next 16 bytes are ascii.
+      next_pos = pos + 16;
+      if (next_pos <= len) { // if it is safe to read 16 more bytes, check
+                             // that they are ascii
+        uint64_t v1{};
+        std::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2{};
+        std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2};
+        if ((v & 0x8080808080808080) == 0) {
+          pos = next_pos;
+          continue;
+        }
+      }
+    }
+
+    unsigned char byte = data[pos];
+
+    while (byte < 0b10000000) {
+      if (++pos == len) {
+        return true;
+      }
+      byte = data[pos];
+    }
+
+    if ((byte & 0b11100000) == 0b11000000) {
+      next_pos = pos + 2;
+      if (next_pos > len) {
+        return false;
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      // range check
+      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
+      if ((code_point < 0x80) || (0x7ff < code_point)) {
+        return false;
+      }
+    } else if ((byte & 0b11110000) == 0b11100000) {
+      next_pos = pos + 3;
+      if (next_pos > len) {
+        return false;
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      // range check
+      code_point = (byte & 0b00001111) << 12 |
+                   (data[pos + 1] & 0b00111111) << 6 |
+                   (data[pos + 2] & 0b00111111);
+      if ((code_point < 0x800) || (0xffff < code_point) ||
+          (0xd7ff < code_point && code_point < 0xe000)) {
+        return false;
+      }
+    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
+      next_pos = pos + 4;
+      if (next_pos > len) {
+        return false;
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
+        return false;
+      }
+      // range check
+      code_point =
+          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
+          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
+      if (code_point <= 0xffff || 0x10ffff < code_point) {
+        return false;
+      }
+    } else {
+      // we may have a continuation
+      return false;
+    }
+    pos = next_pos;
+  }
+  return true;
+}
+
+simdutf_really_inline simdutf_warn_unused bool validate(const char *buf,
+                                                        size_t len) noexcept {
+  return validate(reinterpret_cast<const uint8_t *>(buf), len);
+}
+
+template <class BytePtr>
+simdutf_constexpr23 simdutf_warn_unused result
+validate_with_errors(BytePtr data, size_t len) noexcept {
+  static_assert(
+      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
+      "dereferencing the data pointer must result in a uint8_t");
+  size_t pos = 0;
+  uint32_t code_point = 0;
+  while (pos < len) {
+    // check of the next 16 bytes are ascii.
+    size_t next_pos = pos + 16;
+    if (next_pos <=
+        len) { // if it is safe to read 16 more bytes, check that they are ascii
+      uint64_t v1;
+      std::memcpy(&v1, data + pos, sizeof(uint64_t));
+      uint64_t v2;
+      std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+      uint64_t v{v1 | v2};
+      if ((v & 0x8080808080808080) == 0) {
+        pos = next_pos;
+        continue;
+      }
+    }
+    unsigned char byte = data[pos];
+
+    while (byte < 0b10000000) {
+      if (++pos == len) {
+        return result(error_code::SUCCESS, len);
+      }
+      byte = data[pos];
+    }
+
+    if ((byte & 0b11100000) == 0b11000000) {
+      next_pos = pos + 2;
+      if (next_pos > len) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
+      if ((code_point < 0x80) || (0x7ff < code_point)) {
+        return result(error_code::OVERLONG, pos);
+      }
+    } else if ((byte & 0b11110000) == 0b11100000) {
+      next_pos = pos + 3;
+      if (next_pos > len) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      code_point = (byte & 0b00001111) << 12 |
+                   (data[pos + 1] & 0b00111111) << 6 |
+                   (data[pos + 2] & 0b00111111);
+      if ((code_point < 0x800) || (0xffff < code_point)) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0xd7ff < code_point && code_point < 0xe000) {
+        return result(error_code::SURROGATE, pos);
+      }
+    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
+      next_pos = pos + 4;
+      if (next_pos > len) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      code_point =
+          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
+          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
+      if (code_point <= 0xffff) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0x10ffff < code_point) {
+        return result(error_code::TOO_LARGE, pos);
+      }
+    } else {
+      // we either have too many continuation bytes or an invalid leading byte
+      if ((byte & 0b11000000) == 0b10000000) {
+        return result(error_code::TOO_LONG, pos);
+      } else {
+        return result(error_code::HEADER_BITS, pos);
+      }
+    }
+    pos = next_pos;
+  }
+  return result(error_code::SUCCESS, len);
+}
+
+simdutf_really_inline simdutf_warn_unused result
+validate_with_errors(const char *buf, size_t len) noexcept {
+  return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len);
+}
+
+// Finds the previous leading byte starting backward from buf and validates with
+// errors from there Used to pinpoint the location of an error when an invalid
+// chunk is detected We assume that the stream starts with a leading byte, and
+// to check that it is the case, we ask that you pass a pointer to the start of
+// the stream (start).
+inline simdutf_warn_unused result rewind_and_validate_with_errors(
+    const char *start, const char *buf, size_t len) noexcept {
+  // First check that we start with a leading byte
+  if ((*start & 0b11000000) == 0b10000000) {
+    return result(error_code::TOO_LONG, 0);
+  }
+  size_t extra_len{0};
+  // A leading byte cannot be further than 4 bytes away
+  for (int i = 0; i < 5; i++) {
+    unsigned char byte = *buf;
+    if ((byte & 0b11000000) != 0b10000000) {
+      break;
+    } else {
+      buf--;
+      extra_len++;
+    }
+  }
+
+  result res = validate_with_errors(buf, len + extra_len);
+  res.count -= extra_len;
+  return res;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) {
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    // -65 is 0b10111111, anything larger in two-complement's should start a new
+    // code point.
+    if (int8_t(data[i]) > -65) {
+      counter++;
+    }
+  }
+  return counter;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) {
+  size_t counter{0};
+  for (size_t i = 0; i < len; i++) {
+    if (int8_t(data[i]) > -65) {
+      counter++;
+    }
+    if (uint8_t(data[i]) >= 240) {
+      counter++;
+    }
+  }
+  return counter;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf8(InputPtr input, size_t length) {
+  if (length < 3) {
+    switch (length) {
+    case 2:
+      if (uint8_t(input[length - 1]) >= 0xc0) {
+        return length - 1;
+      } // 2-, 3- and 4-byte characters with only 1 byte left
+      if (uint8_t(input[length - 2]) >= 0xe0) {
+        return length - 2;
+      } // 3- and 4-byte characters with only 2 bytes left
+      return length;
+    case 1:
+      if (uint8_t(input[length - 1]) >= 0xc0) {
+        return length - 1;
+      } // 2-, 3- and 4-byte characters with only 1 byte left
+      return length;
+    case 0:
+      return length;
+    }
+  }
+  if (uint8_t(input[length - 1]) >= 0xc0) {
+    return length - 1;
+  } // 2-, 3- and 4-byte characters with only 1 byte left
+  if (uint8_t(input[length - 2]) >= 0xe0) {
+    return length - 2;
+  } // 3- and 4-byte characters with only 1 byte left
+  if (uint8_t(input[length - 3]) >= 0xf0) {
+    return length - 3;
+  } // 4-byte characters with only 3 bytes left
+  return length;
+}
+
+} // namespace utf8
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8.h */
+/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */
+#ifndef SIMDUTF_UTF8_TO_LATIN1_H
+#define SIMDUTF_UTF8_TO_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_latin1 {
+
+template <typename InputPtr, typename OutputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires(simdutf::detail::indexes_into_byte_like<InputPtr> &&
+           simdutf::detail::indexes_into_byte_like<OutputPtr>)
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   OutputPtr latin_output) {
+  size_t pos = 0;
+  auto start = latin_output;
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000
+                             // 1000 1000 .... etc
+        if ((v & 0x8080808080808080) ==
+            0) { // if NONE of these are set, e.g. all of them are zero, then
+                 // everything is ASCII
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *latin_output++ = char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    // suppose it is not an all ASCII byte sequence
+    uint8_t leading_byte = data[pos]; // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *latin_output++ = char(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) ==
+               0b11000000) { // the first three bits indicate:
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return 0;
+      } // checks if the next byte is a valid continuation byte in UTF-8. A
+        // valid continuation byte starts with 10.
+      // range check -
+      uint32_t code_point =
+          (leading_byte & 0b00011111) << 6 |
+          (data[pos + 1] &
+           0b00111111); // assembles the Unicode code point from the two bytes.
+                        // It does this by discarding the leading 110 and 10
+                        // bits from the two bytes, shifting the remaining bits
+                        // of the first byte, and then combining the results
+                        // with a bitwise OR operation.
+      if (code_point < 0x80 || 0xFF < code_point) {
+        return 0; // We only care about the range 129-255 which is Non-ASCII
+                  // latin1 characters. A code_point beneath 0x80 is invalid as
+                  // it is already covered by bytes whose leading bit is zero.
+      }
+      *latin_output++ = char(code_point);
+      pos += 2;
+    } else {
+      return 0;
+    }
+  }
+  return latin_output - start;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
+                                               char *latin_output) {
+  size_t pos = 0;
+  char *start{latin_output};
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000
+                             // 1000 1000...etc
+        if ((v & 0x8080808080808080) ==
+            0) { // if NONE of these are set, e.g. all of them are zero, then
+                 // everything is ASCII
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *latin_output++ = char(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+    // suppose it is not an all ASCII byte sequence
+    uint8_t leading_byte = data[pos]; // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *latin_output++ = char(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) ==
+               0b11000000) { // the first three bits indicate:
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      } // checks if the next byte is a valid continuation byte in UTF-8. A
+        // valid continuation byte starts with 10.
+      // range check -
+      uint32_t code_point =
+          (leading_byte & 0b00011111) << 6 |
+          (data[pos + 1] &
+           0b00111111); // assembles the Unicode code point from the two bytes.
+                        // It does this by discarding the leading 110 and 10
+                        // bits from the two bytes, shifting the remaining bits
+                        // of the first byte, and then combining the results
+                        // with a bitwise OR operation.
+      if (code_point < 0x80) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0xFF < code_point) {
+        return result(error_code::TOO_LARGE, pos);
+      } // We only care about the range 129-255 which is Non-ASCII latin1
+        // characters
+      *latin_output++ = char(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8
+      return result(error_code::TOO_LARGE, pos);
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      return result(error_code::TOO_LARGE, pos);
+    } else {
+      // we either have too many continuation bytes or an invalid leading byte
+      if ((leading_byte & 0b11000000) == 0b10000000) {
+        return result(error_code::TOO_LONG, pos);
+      }
+
+      return result(error_code::HEADER_BITS, pos);
+    }
+  }
+  return result(error_code::SUCCESS, latin_output - start);
+}
+
+inline result rewind_and_convert_with_errors(size_t prior_bytes,
+                                             const char *buf, size_t len,
+                                             char *latin1_output) {
+  size_t extra_len{0};
+  // We potentially need to go back in time and find a leading byte.
+  // In theory '3' would be sufficient, but sometimes the error can go back
+  // quite far.
+  size_t how_far_back = prior_bytes;
+  // size_t how_far_back = 3; // 3 bytes in the past + current position
+  // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; }
+  bool found_leading_bytes{false};
+  // important: it is i <= how_far_back and not 'i < how_far_back'.
+  for (size_t i = 0; i <= how_far_back; i++) {
+    unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)];
+    found_leading_bytes = ((byte & 0b11000000) != 0b10000000);
+    if (found_leading_bytes) {
+      if (i > 0 && byte < 128) {
+        // If we had to go back and the leading byte is ascii
+        // then we can stop right away.
+        return result(error_code::TOO_LONG, 0 - i + 1);
+      }
+      buf -= i;
+      extra_len = i;
+      break;
+    }
+  }
+  //
+  // It is possible for this function to return a negative count in its result.
+  // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described
+  // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an
+  // unsigned integral type of the result of the sizeof operator
+  //
+  // An unsigned type will simply wrap round arithmetically (well defined).
+  //
+  if (!found_leading_bytes) {
+    // If how_far_back == 3, we may have four consecutive continuation bytes!!!
+    // [....] [continuation] [continuation] [continuation] | [buf is
+    // continuation] Or we possibly have a stream that does not start with a
+    // leading byte.
+    return result(error_code::TOO_LONG, 0 - how_far_back);
+  }
+  result res = convert_with_errors(buf, len + extra_len, latin1_output);
+  if (res.error) {
+    res.count -= extra_len;
+  }
+  return res;
+}
+
+} // namespace utf8_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */
+/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */
+#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H
+#define SIMDUTF_VALID_UTF8_TO_LATIN1_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_latin1 {
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
+                                         char *latin_output) {
+
+  size_t pos = 0;
+  char *start{latin_output};
+
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 |
+                   v2}; // We are only interested in these bits: 1000 1000 1000
+                        // 1000, so it makes sense to concatenate everything
+        if ((v & 0x8080808080808080) ==
+            0) { // if NONE of these are set, e.g. all of them are zero, then
+                 // everything is ASCII
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *latin_output++ = uint8_t(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    // suppose it is not an all ASCII byte sequence
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *latin_output++ = char(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) ==
+               0b11000000) { // the first three bits indicate:
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        break;
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return 0;
+      } // checks if the next byte is a valid continuation byte in UTF-8. A
+        // valid continuation byte starts with 10.
+      // range check -
+      uint32_t code_point =
+          (leading_byte & 0b00011111) << 6 |
+          (uint8_t(data[pos + 1]) &
+           0b00111111); // assembles the Unicode code point from the two bytes.
+                        // It does this by discarding the leading 110 and 10
+                        // bits from the two bytes, shifting the remaining bits
+                        // of the first byte, and then combining the results
+                        // with a bitwise OR operation.
+      *latin_output++ = char(code_point);
+      pos += 2;
+    } else {
+      // we may have a continuation but we do not do error checking
+      return 0;
+    }
+  }
+  return latin_output - start;
+}
+
+} // namespace utf8_to_latin1
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */
+/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */
+#ifndef SIMDUTF_UTF8_TO_UTF16_H
+#define SIMDUTF_UTF8_TO_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_utf16 {
+
+template <endianness big_endian, typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    // try to convert the next block of 16 ASCII bytes
+    {
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2};
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *utf16_output++ = !match_system(big_endian)
+                                  ? char16_t(u16_swap_bytes(data[pos]))
+                                  : char16_t(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    uint8_t leading_byte = data[pos]; // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(leading_byte))
+                            : char16_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      // range check
+      uint32_t code_point =
+          (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
+      if (code_point < 0x80 || 0x7ff < code_point) {
+        return 0;
+      }
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = uint32_t(u16_swap_bytes(uint16_t(code_point)));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 2 >= len) {
+        return 0;
+      } // minimal bound checking
+
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00001111) << 12 |
+                            (data[pos + 1] & 0b00111111) << 6 |
+                            (data[pos + 2] & 0b00111111);
+      if (code_point < 0x800 || 0xffff < code_point ||
+          (0xd7ff < code_point && code_point < 0xe000)) {
+        return 0;
+      }
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = uint32_t(u16_swap_bytes(uint16_t(code_point)));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        return 0;
+      } // minimal bound checking
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+
+      // range check
+      uint32_t code_point = (leading_byte & 0b00000111) << 18 |
+                            (data[pos + 1] & 0b00111111) << 12 |
+                            (data[pos + 2] & 0b00111111) << 6 |
+                            (data[pos + 3] & 0b00111111);
+      if (code_point <= 0xffff || 0x10ffff < code_point) {
+        return 0;
+      }
+      code_point -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+      pos += 4;
+    } else {
+      return 0;
+    }
+  }
+  return utf16_output - start;
+}
+
+template <endianness big_endian, typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
+                                               char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2};
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            const char16_t byte = uint8_t(data[pos]);
+            *utf16_output++ =
+                !match_system(big_endian) ? u16_swap_bytes(byte) : byte;
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(leading_byte))
+                            : char16_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 1 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00011111) << 6 |
+                            (uint8_t(data[pos + 1]) & 0b00111111);
+      if (code_point < 0x80 || 0x7ff < code_point) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = uint32_t(u16_swap_bytes(uint16_t(code_point)));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 2 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00001111) << 12 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 2]) & 0b00111111);
+      if ((code_point < 0x800) || (0xffff < code_point)) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0xd7ff < code_point && code_point < 0xe000) {
+        return result(error_code::SURROGATE, pos);
+      }
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = uint32_t(u16_swap_bytes(uint16_t(code_point)));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+
+      // range check
+      uint32_t code_point = (leading_byte & 0b00000111) << 18 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 12 |
+                            (uint8_t(data[pos + 2]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 3]) & 0b00111111);
+      if (code_point <= 0xffff) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0x10ffff < code_point) {
+        return result(error_code::TOO_LARGE, pos);
+      }
+      code_point -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+      pos += 4;
+    } else {
+      // we either have too many continuation bytes or an invalid leading byte
+      if ((leading_byte & 0b11000000) == 0b10000000) {
+        return result(error_code::TOO_LONG, pos);
+      } else {
+        return result(error_code::HEADER_BITS, pos);
+      }
+    }
+  }
+  return result(error_code::SUCCESS, utf16_output - start);
+}
+
+/**
+ * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and
+ * we have up to len input bytes left, and we encountered some error. It is
+ * possible that the error is at 'buf' exactly, but it could also be in the
+ * previous bytes  (up to 3 bytes back).
+ *
+ * prior_bytes indicates how many bytes, prior to 'buf' may belong to the
+ * current memory section and can be safely accessed. We prior_bytes to access
+ * safely up to three bytes before 'buf'.
+ *
+ * The caller is responsible to ensure that len > 0.
+ *
+ * If the error is believed to have occurred prior to 'buf', the count value
+ * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.
+ */
+template <endianness endian>
+inline result rewind_and_convert_with_errors(size_t prior_bytes,
+                                             const char *buf, size_t len,
+                                             char16_t *utf16_output) {
+  size_t extra_len{0};
+  // We potentially need to go back in time and find a leading byte.
+  // In theory '3' would be sufficient, but sometimes the error can go back
+  // quite far.
+  size_t how_far_back = prior_bytes;
+  // size_t how_far_back = 3; // 3 bytes in the past + current position
+  // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; }
+  bool found_leading_bytes{false};
+  // important: it is i <= how_far_back and not 'i < how_far_back'.
+  for (size_t i = 0; i <= how_far_back; i++) {
+    unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)];
+    found_leading_bytes = ((byte & 0b11000000) != 0b10000000);
+    if (found_leading_bytes) {
+      if (i > 0 && byte < 128) {
+        // If we had to go back and the leading byte is ascii
+        // then we can stop right away.
+        return result(error_code::TOO_LONG, 0 - i + 1);
+      }
+      buf -= i;
+      extra_len = i;
+      break;
+    }
+  }
+  //
+  // It is possible for this function to return a negative count in its result.
+  // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described
+  // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an
+  // unsigned integral type of the result of the sizeof operator
+  //
+  // An unsigned type will simply wrap round arithmetically (well defined).
+  //
+  if (!found_leading_bytes) {
+    // If how_far_back == 3, we may have four consecutive continuation bytes!!!
+    // [....] [continuation] [continuation] [continuation] | [buf is
+    // continuation] Or we possibly have a stream that does not start with a
+    // leading byte.
+    return result(error_code::TOO_LONG, 0 - how_far_back);
+  }
+  result res = convert_with_errors<endian>(buf, len + extra_len, utf16_output);
+  if (res.error) {
+    res.count -= extra_len;
+  }
+  return res;
+}
+
+} // namespace utf8_to_utf16
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */
+/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */
+#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H
+#define SIMDUTF_VALID_UTF8_TO_UTF16_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_utf16 {
+
+template <endianness big_endian, typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
+                                         char16_t *utf16_output) {
+  size_t pos = 0;
+  char16_t *start{utf16_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {                       // try to convert the next block of 8 ASCII bytes
+      if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 8;
+          while (pos < final_pos) {
+            const char16_t byte = uint8_t(data[pos]);
+            *utf16_output++ =
+                !match_system(big_endian) ? u16_swap_bytes(byte) : byte;
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf16_output++ = !match_system(big_endian)
+                            ? char16_t(u16_swap_bytes(leading_byte))
+                            : char16_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 1 >= len) {
+        break;
+      } // minimal bound checking
+      uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) |
+                                     (uint8_t(data[pos + 1]) & 0b00111111));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = u16_swap_bytes(uint16_t(code_point));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8, it should become
+      // a single UTF-16 word.
+      if (pos + 2 >= len) {
+        break;
+      } // minimal bound checking
+      uint16_t code_point =
+          uint16_t(((leading_byte & 0b00001111) << 12) |
+                   ((uint8_t(data[pos + 1]) & 0b00111111) << 6) |
+                   (uint8_t(data[pos + 2]) & 0b00111111));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        code_point = u16_swap_bytes(uint16_t(code_point));
+      }
+      *utf16_output++ = char16_t(code_point);
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        break;
+      } // minimal bound checking
+      uint32_t code_point = ((leading_byte & 0b00000111) << 18) |
+                            ((uint8_t(data[pos + 1]) & 0b00111111) << 12) |
+                            ((uint8_t(data[pos + 2]) & 0b00111111) << 6) |
+                            (uint8_t(data[pos + 3]) & 0b00111111);
+      code_point -= 0x10000;
+      uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10));
+      uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF));
+      if simdutf_constexpr (!match_system(big_endian)) {
+        high_surrogate = u16_swap_bytes(high_surrogate);
+        low_surrogate = u16_swap_bytes(low_surrogate);
+      }
+      *utf16_output++ = char16_t(high_surrogate);
+      *utf16_output++ = char16_t(low_surrogate);
+      pos += 4;
+    } else {
+      // we may have a continuation but we do not do error checking
+      return 0;
+    }
+  }
+  return utf16_output - start;
+}
+
+} // namespace utf8_to_utf16
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */
+/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */
+#ifndef SIMDUTF_UTF8_TO_UTF32_H
+#define SIMDUTF_UTF8_TO_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_utf32 {
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
+                                   char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2};
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *utf32_output++ = uint8_t(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf32_output++ = char32_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        return 0;
+      } // minimal bound checking
+      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00011111) << 6 |
+                            (uint8_t(data[pos + 1]) & 0b00111111);
+      if (code_point < 0x80 || 0x7ff < code_point) {
+        return 0;
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8
+      if (pos + 2 >= len) {
+        return 0;
+      } // minimal bound checking
+
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00001111) << 12 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 2]) & 0b00111111);
+      if (code_point < 0x800 || 0xffff < code_point ||
+          (0xd7ff < code_point && code_point < 0xe000)) {
+        return 0;
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        return 0;
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+      if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) {
+        return 0;
+      }
+
+      // range check
+      uint32_t code_point = (leading_byte & 0b00000111) << 18 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 12 |
+                            (uint8_t(data[pos + 2]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 3]) & 0b00111111);
+      if (code_point <= 0xffff || 0x10ffff < code_point) {
+        return 0;
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 4;
+    } else {
+      return 0;
+    }
+  }
+  return utf32_output - start;
+}
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
+                                               char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 16 ASCII bytes
+      if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
+                             // they are ascii
+        uint64_t v1;
+        ::memcpy(&v1, data + pos, sizeof(uint64_t));
+        uint64_t v2;
+        ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
+        uint64_t v{v1 | v2};
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 16;
+          while (pos < final_pos) {
+            *utf32_output++ = uint8_t(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf32_output++ = char32_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00011111) << 6 |
+                            (uint8_t(data[pos + 1]) & 0b00111111);
+      if (code_point < 0x80 || 0x7ff < code_point) {
+        return result(error_code::OVERLONG, pos);
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8
+      if (pos + 2 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      // range check
+      uint32_t code_point = (leading_byte & 0b00001111) << 12 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 2]) & 0b00111111);
+      if (code_point < 0x800 || 0xffff < code_point) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0xd7ff < code_point && code_point < 0xe000) {
+        return result(error_code::SURROGATE, pos);
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        return result(error_code::TOO_SHORT, pos);
+      } // minimal bound checking
+      if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+      if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) {
+        return result(error_code::TOO_SHORT, pos);
+      }
+
+      // range check
+      uint32_t code_point = (leading_byte & 0b00000111) << 18 |
+                            (uint8_t(data[pos + 1]) & 0b00111111) << 12 |
+                            (uint8_t(data[pos + 2]) & 0b00111111) << 6 |
+                            (uint8_t(data[pos + 3]) & 0b00111111);
+      if (code_point <= 0xffff) {
+        return result(error_code::OVERLONG, pos);
+      }
+      if (0x10ffff < code_point) {
+        return result(error_code::TOO_LARGE, pos);
+      }
+      *utf32_output++ = char32_t(code_point);
+      pos += 4;
+    } else {
+      // we either have too many continuation bytes or an invalid leading byte
+      if ((leading_byte & 0b11000000) == 0b10000000) {
+        return result(error_code::TOO_LONG, pos);
+      } else {
+        return result(error_code::HEADER_BITS, pos);
+      }
+    }
+  }
+  return result(error_code::SUCCESS, utf32_output - start);
+}
+
+/**
+ * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and
+ * we have up to len input bytes left, and we encountered some error. It is
+ * possible that the error is at 'buf' exactly, but it could also be in the
+ * previous bytes location (up to 3 bytes back).
+ *
+ * prior_bytes indicates how many bytes, prior to 'buf' may belong to the
+ * current memory section and can be safely accessed. We prior_bytes to access
+ * safely up to three bytes before 'buf'.
+ *
+ * The caller is responsible to ensure that len > 0.
+ *
+ * If the error is believed to have occurred prior to 'buf', the count value
+ * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.
+ */
+inline result rewind_and_convert_with_errors(size_t prior_bytes,
+                                             const char *buf, size_t len,
+                                             char32_t *utf32_output) {
+  size_t extra_len{0};
+  // We potentially need to go back in time and find a leading byte.
+  size_t how_far_back = 3; // 3 bytes in the past + current position
+  if (how_far_back > prior_bytes) {
+    how_far_back = prior_bytes;
+  }
+  bool found_leading_bytes{false};
+  // important: it is i <= how_far_back and not 'i < how_far_back'.
+  for (size_t i = 0; i <= how_far_back; i++) {
+    unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)];
+    found_leading_bytes = ((byte & 0b11000000) != 0b10000000);
+    if (found_leading_bytes) {
+      if (i > 0 && byte < 128) {
+        // If we had to go back and the leading byte is ascii
+        // then we can stop right away.
+        return result(error_code::TOO_LONG, 0 - i + 1);
+      }
+      buf -= i;
+      extra_len = i;
+      break;
+    }
+  }
+  //
+  // It is possible for this function to return a negative count in its result.
+  // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described
+  // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an
+  // unsigned integral type of the result of the sizeof operator
+  //
+  // An unsigned type will simply wrap round arithmetically (well defined).
+  //
+  if (!found_leading_bytes) {
+    // If how_far_back == 3, we may have four consecutive continuation bytes!!!
+    // [....] [continuation] [continuation] [continuation] | [buf is
+    // continuation] Or we possibly have a stream that does not start with a
+    // leading byte.
+    return result(error_code::TOO_LONG, 0 - how_far_back);
+  }
+
+  result res = convert_with_errors(buf, len + extra_len, utf32_output);
+  if (res.error) {
+    res.count -= extra_len;
+  }
+  return res;
+}
+
+} // namespace utf8_to_utf32
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */
+/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */
+#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H
+#define SIMDUTF_VALID_UTF8_TO_UTF32_H
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace utf8_to_utf32 {
+
+template <typename InputPtr>
+#if SIMDUTF_CPLUSPLUS20
+  requires simdutf::detail::indexes_into_byte_like<InputPtr>
+#endif
+simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
+                                         char32_t *utf32_output) {
+  size_t pos = 0;
+  char32_t *start{utf32_output};
+  while (pos < len) {
+#if SIMDUTF_CPLUSPLUS23
+    if !consteval
+#endif
+    {
+      // try to convert the next block of 8 ASCII bytes
+      if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that
+                            // they are ascii
+        uint64_t v;
+        ::memcpy(&v, data + pos, sizeof(uint64_t));
+        if ((v & 0x8080808080808080) == 0) {
+          size_t final_pos = pos + 8;
+          while (pos < final_pos) {
+            *utf32_output++ = uint8_t(data[pos]);
+            pos++;
+          }
+          continue;
+        }
+      }
+    }
+    auto leading_byte = uint8_t(data[pos]); // leading byte
+    if (leading_byte < 0b10000000) {
+      // converting one ASCII byte !!!
+      *utf32_output++ = char32_t(leading_byte);
+      pos++;
+    } else if ((leading_byte & 0b11100000) == 0b11000000) {
+      // We have a two-byte UTF-8
+      if (pos + 1 >= len) {
+        break;
+      } // minimal bound checking
+      *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) |
+                                 (uint8_t(data[pos + 1]) & 0b00111111));
+      pos += 2;
+    } else if ((leading_byte & 0b11110000) == 0b11100000) {
+      // We have a three-byte UTF-8
+      if (pos + 2 >= len) {
+        break;
+      } // minimal bound checking
+      *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) |
+                                 ((uint8_t(data[pos + 1]) & 0b00111111) << 6) |
+                                 (uint8_t(data[pos + 2]) & 0b00111111));
+      pos += 3;
+    } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
+      // we have a 4-byte UTF-8 word.
+      if (pos + 3 >= len) {
+        break;
+      } // minimal bound checking
+      uint32_t code_word = ((leading_byte & 0b00000111) << 18) |
+                           ((uint8_t(data[pos + 1]) & 0b00111111) << 12) |
+                           ((uint8_t(data[pos + 2]) & 0b00111111) << 6) |
+                           (uint8_t(data[pos + 3]) & 0b00111111);
+      *utf32_output++ = char32_t(code_word);
+      pos += 4;
+    } else {
+      // we may have a continuation but we do not do error checking
+      return 0;
+    }
+  }
+  return utf32_output - start;
+}
+
+} // namespace utf8_to_utf32
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */
+
+namespace simdutf {
+
+constexpr size_t default_line_length =
+    76; ///< default line length for base64 encoding with lines
+
+#if SIMDUTF_FEATURE_DETECT_ENCODING
+/**
+ * Autodetect the encoding of the input, a single encoding is recommended.
+ * E.g., the function might return simdutf::encoding_type::UTF8,
+ * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or
+ * simdutf::encoding_type::UTF32_LE.
+ *
+ * @param input the string to analyze.
+ * @param length the length of the string in bytes.
+ * @return the detected encoding type
+ */
+simdutf_warn_unused simdutf::encoding_type
+autodetect_encoding(const char *input, size_t length) noexcept;
+simdutf_really_inline simdutf_warn_unused simdutf::encoding_type
+autodetect_encoding(const uint8_t *input, size_t length) noexcept {
+  return autodetect_encoding(reinterpret_cast<const char *>(input), length);
+}
+  #if SIMDUTF_SPAN
+/**
+ * Autodetect the encoding of the input, a single encoding is recommended.
+ * E.g., the function might return simdutf::encoding_type::UTF8,
+ * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or
+ * simdutf::encoding_type::UTF32_LE.
+ *
+ * @param input the string to analyze. can be a anything span-like that has a
+ * data() and size() that points to character data: std::string,
+ * std::string_view, std::vector<char>, std::span<const std::byte> etc.
+ * @return the detected encoding type
+ */
+simdutf_really_inline simdutf_warn_unused simdutf::encoding_type
+autodetect_encoding(
+    const detail::input_span_of_byte_like auto &input) noexcept {
+  return autodetect_encoding(reinterpret_cast<const char *>(input.data()),
+                             input.size());
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Autodetect the possible encodings of the input in one pass.
+ * E.g., if the input might be UTF-16LE or UTF-8, this function returns
+ * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE).
+ *
+ * Overridden by each implementation.
+ *
+ * @param input the string to analyze.
+ * @param length the length of the string in bytes.
+ * @return the detected encoding type
+ */
+simdutf_warn_unused int detect_encodings(const char *input,
+                                         size_t length) noexcept;
+simdutf_really_inline simdutf_warn_unused int
+detect_encodings(const uint8_t *input, size_t length) noexcept {
+  return detect_encodings(reinterpret_cast<const char *>(input), length);
+}
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused int
+detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept {
+  return detect_encodings(reinterpret_cast<const char *>(input.data()),
+                          input.size());
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING
+/**
+ * Validate the UTF-8 string. This function may be best when you expect
+ * the input to be almost always valid. Otherwise, consider using
+ * validate_utf8_with_errors.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the UTF-8 string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if and only if the string is valid UTF-8.
+ */
+simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool
+validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::validate(
+        detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_utf8(reinterpret_cast<const char *>(input.data()),
+                         input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF8
+/**
+ * Validate the UTF-8 string and stop on error.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the UTF-8 string to validate.
+ * @param len the length of the string in bytes.
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_utf8_with_errors(const char *buf,
+                                                     size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result
+validate_utf8_with_errors(
+    const detail::input_span_of_byte_like auto &input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::validate_with_errors(
+        detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_utf8_with_errors(
+        reinterpret_cast<const char *>(input.data()), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8
+
+#if SIMDUTF_FEATURE_ASCII
+/**
+ * Validate the ASCII string.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the ASCII string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if and only if the string is valid ASCII.
+ */
+simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::ascii::validate(
+        detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_ascii(reinterpret_cast<const char *>(input.data()),
+                          input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Validate the ASCII string and stop on error. It might be faster than
+ * validate_utf8 when an error is expected to occur early.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the ASCII string to validate.
+ * @param len the length of the string in bytes.
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_ascii_with_errors(const char *buf,
+                                                      size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+validate_ascii_with_errors(
+    const detail::input_span_of_byte_like auto &input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::ascii::validate_with_errors(
+        detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_ascii_with_errors(
+        reinterpret_cast<const char *>(input.data()), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_ASCII
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII
+/**
+ * Validate the ASCII string as a UTF-16 sequence.
+ * An UTF-16 sequence is considered an ASCII sequence
+ * if it could be converted to an ASCII string losslessly.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the UTF-16 string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if and only if the string is valid ASCII.
+ */
+simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf,
+                                                 size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf16_as_ascii(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_as_ascii<endianness::NATIVE>(input.data(),
+                                                                input.size());
+  } else
+    #endif
+  {
+    return validate_utf16_as_ascii(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Validate the ASCII string as a UTF-16BE sequence.
+ * An UTF-16 sequence is considered an ASCII sequence
+ * if it could be converted to an ASCII string losslessly.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the UTF-16BE string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if and only if the string is valid ASCII.
+ */
+simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf,
+                                                   size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf16be_as_ascii(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_as_ascii<endianness::BIG>(input.data(),
+                                                             input.size());
+  } else
+    #endif
+  {
+    return validate_utf16be_as_ascii(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Validate the ASCII string as a UTF-16LE sequence.
+ * An UTF-16 sequence is considered an ASCII sequence
+ * if it could be converted to an ASCII string losslessly.
+ *
+ * Overridden by each implementation.
+ *
+ * @param buf the UTF-16LE string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if and only if the string is valid ASCII.
+ */
+simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf,
+                                                   size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf16le_as_ascii(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_as_ascii<endianness::LITTLE>(input.data(),
+                                                                input.size());
+  } else
+    #endif
+  {
+    return validate_utf16le_as_ascii(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII
+
+#if SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness; Validate the UTF-16 string.
+ * This function may be best when you expect the input to be almost always
+ * valid. Otherwise, consider using validate_utf16_with_errors.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16 string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return true if and only if the string is valid UTF-16.
+ */
+simdutf_warn_unused bool validate_utf16(const char16_t *buf,
+                                        size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf16(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate<endianness::NATIVE>(input.data(),
+                                                       input.size());
+  } else
+    #endif
+  {
+    return validate_utf16(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING
+/**
+ * Validate the UTF-16LE string. This function may be best when you expect
+ * the input to be almost always valid. Otherwise, consider using
+ * validate_utf16le_with_errors.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16LE string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return true if and only if the string is valid UTF-16LE.
+ */
+simdutf_warn_unused bool validate_utf16le(const char16_t *buf,
+                                          size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool
+validate_utf16le(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate<endianness::LITTLE>(input.data(),
+                                                       input.size());
+  } else
+    #endif
+  {
+    return validate_utf16le(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF16
+/**
+ * Validate the UTF-16BE string. This function may be best when you expect
+ * the input to be almost always valid. Otherwise, consider using
+ * validate_utf16be_with_errors.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16BE string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return true if and only if the string is valid UTF-16BE.
+ */
+simdutf_warn_unused bool validate_utf16be(const char16_t *buf,
+                                          size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf16be(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate<endianness::BIG>(input.data(), input.size());
+  } else
+    #endif
+  {
+    return validate_utf16be(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness; Validate the UTF-16 string and stop on error.
+ * It might be faster than validate_utf16 when an error is expected to occur
+ * early.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16 string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf,
+                                                      size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+validate_utf16_with_errors(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_with_errors<endianness::NATIVE>(
+        input.data(), input.size());
+  } else
+    #endif
+  {
+    return validate_utf16_with_errors(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Validate the UTF-16LE string and stop on error. It might be faster than
+ * validate_utf16le when an error is expected to occur early.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16LE string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf,
+                                                        size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+validate_utf16le_with_errors(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_with_errors<endianness::LITTLE>(
+        input.data(), input.size());
+  } else
+    #endif
+  {
+    return validate_utf16le_with_errors(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Validate the UTF-16BE string and stop on error. It might be faster than
+ * validate_utf16be when an error is expected to occur early.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-16BE string to validate.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf,
+                                                        size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+validate_utf16be_with_errors(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::validate_with_errors<endianness::BIG>(input.data(),
+                                                                input.size());
+  } else
+    #endif
+  {
+    return validate_utf16be_with_errors(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with
+ * the Unicode replacement character U+FFFD. If input and output points to
+ * different memory areas, the procedure copies string, and it's expected that
+ * output memory is at least as big as the input. It's also possible to set
+ * input equal output, that makes replacements an in-place operation.
+ *
+ * @param input the UTF-16LE string to correct.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @param output the output buffer.
+ */
+void to_well_formed_utf16le(const char16_t *input, size_t len,
+                            char16_t *output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 void
+to_well_formed_utf16le(std::span<const char16_t> input,
+                       std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    scalar::utf16::to_well_formed_utf16<endianness::LITTLE>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    to_well_formed_utf16le(input.data(), input.size(), output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with
+ * the Unicode replacement character U+FFFD. If input and output points to
+ * different memory areas, the procedure copies string, and it's expected that
+ * output memory is at least as big as the input. It's also possible to set
+ * input equal output, that makes replacements an in-place operation.
+ *
+ * @param input the UTF-16BE string to correct.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @param output the output buffer.
+ */
+void to_well_formed_utf16be(const char16_t *input, size_t len,
+                            char16_t *output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 void
+to_well_formed_utf16be(std::span<const char16_t> input,
+                       std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    scalar::utf16::to_well_formed_utf16<endianness::BIG>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    to_well_formed_utf16be(input.data(), input.size(), output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the
+ * Unicode replacement character U+FFFD. If input and output points to different
+ * memory areas, the procedure copies string, and it's expected that output
+ * memory is at least as big as the input. It's also possible to set input equal
+ * output, that makes replacements an in-place operation.
+ *
+ * @param input the UTF-16 string to correct.
+ * @param len the length of the string in number of 2-byte code units
+ * (char16_t).
+ * @param output the output buffer.
+ */
+void to_well_formed_utf16(const char16_t *input, size_t len,
+                          char16_t *output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 void
+to_well_formed_utf16(std::span<const char16_t> input,
+                     std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    scalar::utf16::to_well_formed_utf16<endianness::NATIVE>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    to_well_formed_utf16(input.data(), input.size(), output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+#endif // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING
+/**
+ * Validate the UTF-32 string. This function may be best when you expect
+ * the input to be almost always valid. Otherwise, consider using
+ * validate_utf32_with_errors.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-32 string to validate.
+ * @param len the length of the string in number of 4-byte code units
+ * (char32_t).
+ * @return true if and only if the string is valid UTF-32.
+ */
+simdutf_warn_unused bool validate_utf32(const char32_t *buf,
+                                        size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool
+validate_utf32(std::span<const char32_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32::validate(
+        detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_utf32(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF32
+/**
+ * Validate the UTF-32 string and stop on error. It might be faster than
+ * validate_utf32 when an error is expected to occur early.
+ *
+ * Overridden by each implementation.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param buf the UTF-32 string to validate.
+ * @param len the length of the string in number of 4-byte code units
+ * (char32_t).
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf,
+                                                      size_t len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+validate_utf32_with_errors(std::span<const char32_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32::validate_with_errors(
+        detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return validate_utf32_with_errors(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert Latin1 string into UTF-8 string.
+ *
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf8_output   the pointer to buffer that can hold conversion result
+ * @return the number of written char; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input,
+                                                  size_t length,
+                                                  char *utf8_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf8(
+    const detail::input_span_of_byte_like auto &latin1_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf8::convert(
+        detail::constexpr_cast_ptr<char>(latin1_input.data()),
+        latin1_input.size(),
+        detail::constexpr_cast_writeptr<char>(utf8_output.data()));
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf8(
+        reinterpret_cast<const char *>(latin1_input.data()),
+        latin1_input.size(), reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert Latin1 string into UTF-8 string with output limit.
+ *
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * We write as many characters as possible.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf8_output  	the pointer to buffer that can hold conversion result
+ * @param utf8_len      the maximum output length
+ * @return the number of written char; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t
+convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output,
+                            size_t utf8_len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf8_safe(
+    const detail::input_span_of_byte_like auto &input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+      // implementation note: outputspan is a forwarding ref to avoid copying
+      // and allow both lvalues and rvalues. std::span can be copied without
+      // problems, but std::vector should not, and this function should accept
+      // both. it will allow using an owning rvalue ref (example: passing a
+      // temporary std::string) as output, but the user will quickly find out
+      // that he has no way of getting the data out of the object in that case.
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf8::convert_safe_constexpr(
+        input.data(), input.size(), utf8_output.data(), utf8_output.size());
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf8_safe(
+        reinterpret_cast<const char *>(input.data()), input.size(),
+        reinterpret_cast<char *>(utf8_output.data()), utf8_output.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert possibly Latin1 string into UTF-16LE string.
+ *
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_latin1_to_utf16le(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf16le(
+    const detail::input_span_of_byte_like auto &latin1_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf16::convert<endianness::LITTLE>(
+        latin1_input.data(), latin1_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf16le(
+        reinterpret_cast<const char *>(latin1_input.data()),
+        latin1_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert Latin1 string into UTF-16BE string.
+ *
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_latin1_to_utf16be(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input,
+                          std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf16::convert<endianness::BIG>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf16be(
+        reinterpret_cast<const char *>(input.data()), input.size(),
+        output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+/**
+ * Compute the number of bytes that this UTF-16 string would require in Latin1
+ * format.
+ *
+ * @param length        the length of the string in Latin1 code units (char)
+ * @return the length of the string in Latin1 code units (char) required to
+ * encode the UTF-16 string as Latin1
+ */
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+latin1_length_from_utf16(size_t length) noexcept {
+  return length;
+}
+
+/**
+ * Compute the number of code units that this Latin1 string would require in
+ * UTF-16 format.
+ *
+ * @param length        the length of the string in Latin1 code units (char)
+ * @return the length of the string in 2-byte code units (char16_t) required to
+ * encode the Latin1 string as UTF-16
+ */
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf16_length_from_latin1(size_t length) noexcept {
+  return length;
+}
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert Latin1 string into UTF-32 string.
+ *
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf32_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char32_t; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_latin1_to_utf32(
+    const char *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf32(
+    const detail::input_span_of_byte_like auto &latin1_input,
+    std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf32::convert(
+        latin1_input.data(), latin1_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf32(
+        reinterpret_cast<const char *>(latin1_input.data()),
+        latin1_input.size(), utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert possibly broken UTF-8 string into latin1 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param latin1_output  the pointer to buffer that can hold conversion result
+ * @return the number of written char; 0 if the input was not valid UTF-8 string
+ * or if it cannot be represented as Latin1
+ */
+simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input,
+                                                  size_t length,
+                                                  char *latin1_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf8_to_latin1(
+    const detail::input_span_of_byte_like auto &input,
+    detail::output_span_of_byte_like auto &&output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_latin1::convert(input.data(), input.size(),
+                                           output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_latin1(reinterpret_cast<const char *>(input.data()),
+                                  input.size(),
+                                  reinterpret_cast<char *>(output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness, convert possibly broken UTF-8 string into a UTF-16
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t; 0 if the input was not valid UTF-8
+ * string
+ */
+simdutf_warn_unused size_t convert_utf8_to_utf16(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input,
+                      std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert<endianness::NATIVE>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16(reinterpret_cast<const char *>(input.data()),
+                                 input.size(), output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16LE string would require in UTF-8
+ * format even when the UTF-16LE content contains mismatched surrogates
+ * that have to be replaced by the replacement character (0xFFFD).
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) where the count is the number of bytes required to
+ * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or
+ * SURROGATE. The count is correct regardless of the error field.
+ * When SURROGATE is returned, it does not indicate an error in the case of this
+ * function: it indicates that at least one surrogate has been encountered: the
+ * surrogates may be matched or not (thus this function does not validate). If
+ * the returned error code is SUCCESS, then the input contains no surrogate, is
+ * in the Basic Multilingual Plane, and is necessarily valid.
+ */
+simdutf_warn_unused result utf8_length_from_utf16le_with_replacement(
+    const char16_t *input, size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result
+utf8_length_from_utf16le_with_replacement(
+    std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16_with_replacement<
+        endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(),
+                                                     valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16BE string would require in UTF-8
+ * format even when the UTF-16BE content contains mismatched surrogates
+ * that have to be replaced by the replacement character (0xFFFD).
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) where the count is the number of bytes required to
+ * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or
+ * SURROGATE. The count is correct regardless of the error field.
+ * When SURROGATE is returned, it does not indicate an error in the case of this
+ * function: it indicates that at least one surrogate has been encountered: the
+ * surrogates may be matched or not (thus this function does not validate). If
+ * the returned error code is SUCCESS, then the input contains no surrogate, is
+ * in the Basic Multilingual Plane, and is necessarily valid.
+ */
+simdutf_warn_unused result utf8_length_from_utf16be_with_replacement(
+    const char16_t *input, size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+utf8_length_from_utf16be_with_replacement(
+    std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16_with_replacement<
+        endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(),
+                                                     valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Using native endianness, convert a Latin1 string into a UTF-16 string.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t.
+ */
+simdutf_warn_unused size_t convert_latin1_to_utf16(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input,
+                        std::span<char16_t> output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf16::convert<endianness::NATIVE>(
+        input.data(), input.size(), output.data());
+  } else
+    #endif
+  {
+    return convert_latin1_to_utf16(reinterpret_cast<const char *>(input.data()),
+                                   input.size(), output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Convert possibly broken UTF-8 string into UTF-16LE string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t; 0 if the input was not valid UTF-8
+ * string
+ */
+simdutf_warn_unused size_t convert_utf8_to_utf16le(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input,
+                        std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert<endianness::LITTLE>(
+        utf8_input.data(), utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16le(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-8 string into UTF-16BE string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t; 0 if the input was not valid UTF-8
+ * string
+ */
+simdutf_warn_unused size_t convert_utf8_to_utf16be(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input,
+                        std::span<char16_t> utf16_output) noexcept {
+
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert<endianness::BIG>(
+        utf8_input.data(), utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16be(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert possibly broken UTF-8 string into latin1 string with errors.
+ * If the string cannot be represented as Latin1, an error
+ * code is returned.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param latin1_output  the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of code units validated if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf8_to_latin1_with_errors(
+    const char *input, size_t length, char *latin1_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf8_to_latin1_with_errors(
+    const detail::input_span_of_byte_like auto &utf8_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_latin1::convert_with_errors(
+        utf8_input.data(), utf8_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_latin1_with_errors(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness, convert possibly broken UTF-8 string into UTF-16
+ * string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf8_to_utf16_with_errors(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf8_to_utf16_with_errors(
+    const detail::input_span_of_byte_like auto &utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_with_errors<endianness::NATIVE>(
+        utf8_input.data(), utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16_with_errors(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf8_to_utf16le_with_errors(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf8_to_utf16le_with_errors(
+    const detail::input_span_of_byte_like auto &utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_with_errors<endianness::LITTLE>(
+        utf8_input.data(), utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16le_with_errors(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf8_to_utf16be_with_errors(
+    const char *input, size_t length, char16_t *utf16_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf8_to_utf16be_with_errors(
+    const detail::input_span_of_byte_like auto &utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_with_errors<endianness::BIG>(
+        utf8_input.data(), utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf16be_with_errors(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+/**
+ * Convert possibly broken UTF-8 string into UTF-32 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf32_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char32_t; 0 if the input was not valid UTF-8
+ * string
+ */
+simdutf_warn_unused size_t convert_utf8_to_utf32(
+    const char *input, size_t length, char32_t *utf32_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input,
+                      std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(),
+                                          utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf32(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-8 string into UTF-32 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf32_buffer  the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char32_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf8_to_utf32_with_errors(
+    const char *input, size_t length, char32_t *utf32_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf8_to_utf32_with_errors(
+    const detail::input_span_of_byte_like auto &utf8_input,
+    std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf32::convert_with_errors(
+        utf8_input.data(), utf8_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf8_to_utf32_with_errors(
+        reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(),
+        utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert valid UTF-8 string into latin1 string.
+ *
+ * This function assumes that the input string is valid UTF-8 and that it can be
+ * represented as Latin1. If you violate this assumption, the result is
+ * implementation defined and may include system-dependent behavior such as
+ * crashes.
+ *
+ * This function is for expert users only and not part of our public API. Use
+ * convert_utf8_to_latin1 instead. The function may be removed from the library
+ * in the future.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param latin1_output  the pointer to buffer that can hold conversion result
+ * @return the number of written char; 0 if the input was not valid UTF-8 string
+ */
+simdutf_warn_unused size_t convert_valid_utf8_to_latin1(
+    const char *input, size_t length, char *latin1_output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf8_to_latin1(
+    const detail::input_span_of_byte_like auto &valid_utf8_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_latin1::convert_valid(
+        valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf8_to_latin1(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size(), latin1_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness, convert valid UTF-8 string into a UTF-16 string.
+ *
+ * This function assumes that the input string is valid UTF-8.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t
+ */
+simdutf_warn_unused size_t convert_valid_utf8_to_utf16(
+    const char *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf8_to_utf16(
+    const detail::input_span_of_byte_like auto &valid_utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_valid<endianness::NATIVE>(
+        valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf8_to_utf16(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-8 string into UTF-16LE string.
+ *
+ * This function assumes that the input string is valid UTF-8.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t
+ */
+simdutf_warn_unused size_t convert_valid_utf8_to_utf16le(
+    const char *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf8_to_utf16le(
+    const detail::input_span_of_byte_like auto &valid_utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_valid<endianness::LITTLE>(
+        valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf8_to_utf16le(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-8 string into UTF-16BE string.
+ *
+ * This function assumes that the input string is valid UTF-8.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf16_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char16_t
+ */
+simdutf_warn_unused size_t convert_valid_utf8_to_utf16be(
+    const char *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf8_to_utf16be(
+    const detail::input_span_of_byte_like auto &valid_utf8_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf16::convert_valid<endianness::BIG>(
+        valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf8_to_utf16be(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+/**
+ * Convert valid UTF-8 string into UTF-32 string.
+ *
+ * This function assumes that the input string is valid UTF-8.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in bytes
+ * @param utf32_buffer  the pointer to buffer that can hold conversion result
+ * @return the number of written char32_t
+ */
+simdutf_warn_unused size_t convert_valid_utf8_to_utf32(
+    const char *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf8_to_utf32(
+    const detail::input_span_of_byte_like auto &valid_utf8_input,
+    std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8_to_utf32::convert_valid(
+        valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf8_to_utf32(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size(), utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Return the number of bytes that this Latin1 string would require in UTF-8
+ * format.
+ *
+ * @param input         the Latin1 string to convert
+ * @param length        the length of the string bytes
+ * @return the number of bytes required to encode the Latin1 string as UTF-8
+ */
+simdutf_warn_unused size_t utf8_length_from_latin1(const char *input,
+                                                   size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf8_length_from_latin1(
+    const detail::input_span_of_byte_like auto &latin1_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(),
+                                                           latin1_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_latin1(
+        reinterpret_cast<const char *>(latin1_input.data()),
+        latin1_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-8 string would require in Latin1
+ * format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-8 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-8 string to convert
+ * @param length        the length of the string in byte
+ * @return the number of bytes required to encode the UTF-8 string as Latin1
+ */
+simdutf_warn_unused size_t latin1_length_from_utf8(const char *input,
+                                                   size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+latin1_length_from_utf8(
+    const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::count_code_points(valid_utf8_input.data(),
+                                           valid_utf8_input.size());
+  } else
+    #endif
+  {
+    return latin1_length_from_utf8(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Compute the number of 2-byte code units that this UTF-8 string would require
+ * in UTF-16LE format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-8 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-8 string to process
+ * @param length        the length of the string in bytes
+ * @return the number of char16_t code units required to encode the UTF-8 string
+ * as UTF-16LE
+ */
+simdutf_warn_unused size_t utf16_length_from_utf8(const char *input,
+                                                  size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf16_length_from_utf8(
+    const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(),
+                                                valid_utf8_input.size());
+  } else
+    #endif
+  {
+    return utf16_length_from_utf8(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+/**
+ * Compute the number of 4-byte code units that this UTF-8 string would require
+ * in UTF-32 format.
+ *
+ * This function is equivalent to count_utf8
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-8 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-8 string to process
+ * @param length        the length of the string in bytes
+ * @return the number of char32_t code units required to encode the UTF-8 string
+ * as UTF-32
+ */
+simdutf_warn_unused size_t utf32_length_from_utf8(const char *input,
+                                                  size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf32_length_from_utf8(
+    const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept {
+
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::count_code_points(valid_utf8_input.data(),
+                                           valid_utf8_input.size());
+  } else
+    #endif
+  {
+    return utf32_length_from_utf8(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into UTF-8
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input,
+                                                 size_t length,
+                                                 char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16_to_utf8(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(),
+                                 reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into UTF-8
+ * string with output limit.
+ *
+ * We write as many characters as possible into the output buffer,
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 16-bit code units (char16_t)
+ * @param utf8_output  	the pointer to buffer that can hold conversion result
+ * @param utf8_len      the maximum output length
+ * @return the number of written char; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input,
+                                                      size_t length,
+                                                      char *utf8_output,
+                                                      size_t utf8_len) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16_to_utf8_safe(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+      // implementation note: outputspan is a forwarding ref to avoid copying
+      // and allow both lvalues and rvalues. std::span can be copied without
+      // problems, but std::vector should not, and this function should accept
+      // both. it will allow using an owning rvalue ref (example: passing a
+      // temporary std::string) as output, but the user will quickly find out
+      // that he has no way of getting the data out of the object in that case.
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    const full_result r =
+        scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE, true>(
+            utf16_input.data(), utf16_input.size(), utf8_output.data(),
+            utf8_output.size());
+    if (r.error != error_code::SUCCESS &&
+        r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) {
+      return 0;
+    }
+    return r.output_count;
+  } else
+    #endif
+  {
+    return convert_utf16_to_utf8_safe(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()), utf8_output.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into Latin1
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16 string
+ * or if it cannot be represented as Latin1
+ */
+simdutf_warn_unused size_t convert_utf16_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16_to_latin1(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_latin1(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16LE string into Latin1 string.
+ * If the string cannot be represented as Latin1, an error
+ * is returned.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string or if it cannot be represented as Latin1
+ */
+simdutf_warn_unused size_t convert_utf16le_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16le_to_latin1(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_latin1(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into Latin1 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16BE
+ * string or if it cannot be represented as Latin1
+ */
+simdutf_warn_unused size_t convert_utf16be_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16be_to_latin1(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_latin1(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Convert possibly broken UTF-16LE string into UTF-8 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input,
+                                                   size_t length,
+                                                   char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16le_to_utf8(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_utf8(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into UTF-8 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input,
+                                                   size_t length,
+                                                   char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16be_to_utf8(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_utf8(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into Latin1
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16_to_latin1_with_errors(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16_to_latin1_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_with_errors<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_latin1_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16LE string into Latin1 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16le_to_latin1_with_errors(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16le_to_latin1_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_with_errors<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_latin1_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into Latin1 string.
+ * If the string cannot be represented as Latin1, an error
+ * is returned.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16be_to_latin1_with_errors(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16be_to_latin1_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_with_errors<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_latin1_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into UTF-8
+ * string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16_to_utf8_with_errors(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16_to_utf8_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_utf8_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16le_to_utf8_with_errors(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16le_to_utf8_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_with_errors<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_utf8_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16be_to_utf8_with_errors(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16be_to_utf8_with_errors(
+    std::span<const char16_t> utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_with_errors<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_utf8_with_errors(
+        utf16_input.data(), utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert valid UTF-16 string into UTF-8 string.
+ *
+ * This function assumes that the input string is valid UTF-16.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16_to_utf8(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16_to_utf8(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16_to_utf8(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Using native endianness, convert UTF-16 string into Latin1 string.
+ *
+ * This function assumes that the input string is valid UTF-16 and that it can
+ * be represented as Latin1. If you violate this assumption, the result is
+ * implementation defined and may include system-dependent behavior such as
+ * crashes.
+ *
+ * This function is for expert users only and not part of our public API. Use
+ * convert_utf16_to_latin1 instead. The function may be removed from the library
+ * in the future.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16_to_latin1(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_valid_impl<endianness::NATIVE>(
+        detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()),
+        valid_utf16_input.size(),
+        detail::constexpr_cast_writeptr<char>(latin1_output.data()));
+  } else
+    #endif
+  {
+    return convert_valid_utf16_to_latin1(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-16LE string into Latin1 string.
+ *
+ * This function assumes that the input string is valid UTF-16LE and that it can
+ * be represented as Latin1. If you violate this assumption, the result is
+ * implementation defined and may include system-dependent behavior such as
+ * crashes.
+ *
+ * This function is for expert users only and not part of our public API. Use
+ * convert_utf16le_to_latin1 instead. The function may be removed from the
+ * library in the future.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16le_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t
+convert_valid_utf16le_to_latin1(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_valid_impl<endianness::LITTLE>(
+        detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()),
+        valid_utf16_input.size(),
+        detail::constexpr_cast_writeptr<char>(latin1_output.data()));
+  } else
+    #endif
+  {
+    return convert_valid_utf16le_to_latin1(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-16BE string into Latin1 string.
+ *
+ * This function assumes that the input string is valid UTF-16BE and that it can
+ * be represented as Latin1. If you violate this assumption, the result is
+ * implementation defined and may include system-dependent behavior such as
+ * crashes.
+ *
+ * This function is for expert users only and not part of our public API. Use
+ * convert_utf16be_to_latin1 instead. The function may be removed from the
+ * library in the future.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16be_to_latin1(
+    const char16_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t
+convert_valid_utf16be_to_latin1(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_latin1::convert_valid_impl<endianness::BIG>(
+        detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()),
+        valid_utf16_input.size(),
+        detail::constexpr_cast_writeptr<char>(latin1_output.data()));
+  } else
+    #endif
+  {
+    return convert_valid_utf16be_to_latin1(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Convert valid UTF-16LE string into UTF-8 string.
+ *
+ * This function assumes that the input string is valid UTF-16LE
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16le_to_utf8(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16le_to_utf8(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16le_to_utf8(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-16BE string into UTF-8 string.
+ *
+ * This function assumes that the input string is valid UTF-16BE.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16be_to_utf8(
+    const char16_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16be_to_utf8(
+    std::span<const char16_t> valid_utf16_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf8::convert_valid<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16be_to_utf8(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into UTF-32
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16_to_utf32(std::span<const char16_t> utf16_input,
+                       std::span<char32_t> utf32_output) noexcept {
+
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(),
+                                  utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16LE string into UTF-32 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16le_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16le_to_utf32(std::span<const char16_t> utf16_input,
+                         std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(),
+                                    utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into UTF-32 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-16LE
+ * string
+ */
+simdutf_warn_unused size_t convert_utf16be_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf16be_to_utf32(std::span<const char16_t> utf16_input,
+                         std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(),
+                                    utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert possibly broken UTF-16 string into
+ * UTF-32 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char32_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16_to_utf32_with_errors(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16_to_utf32_with_errors(std::span<const char16_t> utf16_input,
+                                   std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_with_errors<endianness::NATIVE>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16_to_utf32_with_errors(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char32_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16le_to_utf32_with_errors(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16le_to_utf32_with_errors(
+    std::span<const char16_t> utf16_input,
+    std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_with_errors<endianness::LITTLE>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16le_to_utf32_with_errors(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char32_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf16be_to_utf32_with_errors(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf16be_to_utf32_with_errors(
+    std::span<const char16_t> utf16_input,
+    std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_with_errors<endianness::BIG>(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_utf16be_to_utf32_with_errors(
+        utf16_input.data(), utf16_input.size(), utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert valid UTF-16 string into UTF-32 string.
+ *
+ * This function assumes that the input string is valid UTF-16 (native
+ * endianness).
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16_to_utf32(std::span<const char16_t> valid_utf16_input,
+                             std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_valid<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16_to_utf32(valid_utf16_input.data(),
+                                        valid_utf16_input.size(),
+                                        utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-16LE string into UTF-32 string.
+ *
+ * This function assumes that the input string is valid UTF-16LE.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16le_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16le_to_utf32(std::span<const char16_t> valid_utf16_input,
+                               std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_valid<endianness::LITTLE>(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16le_to_utf32(valid_utf16_input.data(),
+                                          valid_utf16_input.size(),
+                                          utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-16BE string into UTF-32 string.
+ *
+ * This function assumes that the input string is valid UTF-16LE.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param utf32_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf16be_to_utf32(
+    const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf16be_to_utf32(std::span<const char16_t> valid_utf16_input,
+                               std::span<char32_t> utf32_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16_to_utf32::convert_valid<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size(),
+        utf32_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf16be_to_utf32(valid_utf16_input.data(),
+                                          valid_utf16_input.size(),
+                                          utf32_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+/**
+ * Using native endianness; Compute the number of bytes that this UTF-16
+ * string would require in UTF-8 format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16LE string as UTF-8
+ */
+simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input,
+                                                  size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf8_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16(valid_utf16_input.data(),
+                                  valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness; compute the number of bytes that this UTF-16
+ * string would require in UTF-8 format even when the UTF-16LE content contains
+ * mismatched surrogates that have to be replaced by the replacement character
+ * (0xFFFD).
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) where the count is the number of bytes required to
+ * encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or
+ * SURROGATE. The count is correct regardless of the error field.
+ * When SURROGATE is returned, it does not indicate an error in the case of this
+ * function: it indicates that at least one surrogate has been encountered: the
+ * surrogates may be matched or not (thus this function does not validate). If
+ * the returned error code is SUCCESS, then the input contains no surrogate, is
+ * in the Basic Multilingual Plane, and is necessarily valid.
+ */
+simdutf_warn_unused result utf8_length_from_utf16_with_replacement(
+    const char16_t *input, size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+utf8_length_from_utf16_with_replacement(
+    std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16_with_replacement<
+        endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(),
+                                                   valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16LE string would require in UTF-8
+ * format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16LE string as UTF-8
+ */
+simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input,
+                                                    size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t
+utf8_length_from_utf16le(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16<endianness::LITTLE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16le(valid_utf16_input.data(),
+                                    valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16BE string would require in UTF-8
+ * format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16BE string as UTF-8
+ */
+simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input,
+                                                    size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf8_length_from_utf16be(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf8_length_from_utf16<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf16be(valid_utf16_input.data(),
+                                    valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+/**
+ * Convert possibly broken UTF-32 string into UTF-8 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-32 string
+ */
+simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input,
+                                                 size_t length,
+                                                 char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf32_to_utf8(
+    std::span<const char32_t> utf32_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf8::convert(
+        utf32_input.data(), utf32_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(),
+                                 reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-32 string into UTF-8 string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf8_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf32_to_utf8_with_errors(
+    const char32_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf32_to_utf8_with_errors(
+    std::span<const char32_t> utf32_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf8::convert_with_errors(
+        utf32_input.data(), utf32_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf8_with_errors(
+        utf32_input.data(), utf32_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-32 string into UTF-8 string.
+ *
+ * This function assumes that the input string is valid UTF-32.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf32_to_utf8(
+    const char32_t *input, size_t length, char *utf8_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf32_to_utf8(
+    std::span<const char32_t> valid_utf32_input,
+    detail::output_span_of_byte_like auto &&utf8_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf8::convert_valid(
+        valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf32_to_utf8(
+        valid_utf32_input.data(), valid_utf32_input.size(),
+        reinterpret_cast<char *>(utf8_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+/**
+ * Using native endianness, convert possibly broken UTF-32 string into a UTF-16
+ * string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-32 string
+ */
+simdutf_warn_unused size_t convert_utf32_to_utf16(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf32_to_utf16(std::span<const char32_t> utf32_input,
+                       std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert<endianness::NATIVE>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(),
+                                  utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-32 string into UTF-16LE string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-32 string
+ */
+simdutf_warn_unused size_t convert_utf32_to_utf16le(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf32_to_utf16le(std::span<const char32_t> utf32_input,
+                         std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert<endianness::LITTLE>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(),
+                                    utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+/**
+ * Convert possibly broken UTF-32 string into Latin1 string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-32 string
+ * or if it cannot be represented as Latin1
+ */
+simdutf_warn_unused size_t convert_utf32_to_latin1(
+    const char32_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf32_to_latin1(
+    std::span<const char32_t> utf32_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_latin1::convert(
+        utf32_input.data(), utf32_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_latin1(
+        utf32_input.data(), utf32_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-32 string into Latin1 string and stop on error.
+ * If the string cannot be represented as Latin1, an error is returned.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param latin1_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf32_to_latin1_with_errors(
+    const char32_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf32_to_latin1_with_errors(
+    std::span<const char32_t> utf32_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_latin1::convert_with_errors(
+        utf32_input.data(), utf32_input.size(), latin1_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_latin1_with_errors(
+        utf32_input.data(), utf32_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-32 string into Latin1 string.
+ *
+ * This function assumes that the input string is valid UTF-32 and that it can
+ * be represented as Latin1. If you violate this assumption, the result is
+ * implementation defined and may include system-dependent behavior such as
+ * crashes.
+ *
+ * This function is for expert users only and not part of our public API. Use
+ * convert_utf32_to_latin1 instead. The function may be removed from the library
+ * in the future.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param latin1_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf32_to_latin1(
+    const char32_t *input, size_t length, char *latin1_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t
+convert_valid_utf32_to_latin1(
+    std::span<const char32_t> valid_utf32_input,
+    detail::output_span_of_byte_like auto &&latin1_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_latin1::convert_valid(
+        detail::constexpr_cast_ptr<uint32_t>(valid_utf32_input.data()),
+        valid_utf32_input.size(),
+        detail::constexpr_cast_writeptr<char>(latin1_output.data()));
+  }
+    #endif
+  {
+    return convert_valid_utf32_to_latin1(
+        valid_utf32_input.data(), valid_utf32_input.size(),
+        reinterpret_cast<char *>(latin1_output.data()));
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-32 string would require in Latin1
+ * format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-32 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @return the number of bytes required to encode the UTF-32 string as Latin1
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t
+latin1_length_from_utf32(size_t length) noexcept {
+  return length;
+}
+
+/**
+ * Compute the number of bytes that this Latin1 string would require in UTF-32
+ * format.
+ *
+ * @param length        the length of the string in Latin1 code units (char)
+ * @return the length of the string in 4-byte code units (char32_t) required to
+ * encode the Latin1 string as UTF-32
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t
+utf32_length_from_latin1(size_t length) noexcept {
+  return length;
+}
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+/**
+ * Convert possibly broken UTF-32 string into UTF-16BE string.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return number of written code units; 0 if input is not a valid UTF-32 string
+ */
+simdutf_warn_unused size_t convert_utf32_to_utf16be(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_utf32_to_utf16be(std::span<const char32_t> utf32_input,
+                         std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert<endianness::BIG>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(),
+                                    utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert possibly broken UTF-32 string into UTF-16
+ * string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf32_to_utf16_with_errors(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf32_to_utf16_with_errors(std::span<const char32_t> utf32_input,
+                                   std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_with_errors<endianness::NATIVE>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16_with_errors(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf32_to_utf16le_with_errors(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf32_to_utf16le_with_errors(
+    std::span<const char32_t> utf32_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_with_errors<endianness::LITTLE>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16le_with_errors(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error.
+ *
+ * During the conversion also validation of the input string is done.
+ * This function is suitable to work with inputs from untrusted sources.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to buffer that can hold conversion result
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in code units) if any, or the number of char16_t written if
+ * successful.
+ */
+simdutf_warn_unused result convert_utf32_to_utf16be_with_errors(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+convert_utf32_to_utf16be_with_errors(
+    std::span<const char32_t> utf32_input,
+    std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_with_errors<endianness::BIG>(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_utf32_to_utf16be_with_errors(
+        utf32_input.data(), utf32_input.size(), utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness, convert valid UTF-32 string into a UTF-16 string.
+ *
+ * This function assumes that the input string is valid UTF-32.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf32_to_utf16(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf32_to_utf16(std::span<const char32_t> valid_utf32_input,
+                             std::span<char16_t> utf16_output) noexcept {
+
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_valid<endianness::NATIVE>(
+        valid_utf32_input.data(), valid_utf32_input.size(),
+        utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf32_to_utf16(valid_utf32_input.data(),
+                                        valid_utf32_input.size(),
+                                        utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-32 string into UTF-16LE string.
+ *
+ * This function assumes that the input string is valid UTF-32.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf32_to_utf16le(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf32_to_utf16le(std::span<const char32_t> valid_utf32_input,
+                               std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_valid<endianness::LITTLE>(
+        valid_utf32_input.data(), valid_utf32_input.size(),
+        utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf32_to_utf16le(valid_utf32_input.data(),
+                                          valid_utf32_input.size(),
+                                          utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert valid UTF-32 string into UTF-16BE string.
+ *
+ * This function assumes that the input string is valid UTF-32.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @param utf16_buffer   the pointer to a buffer that can hold the conversion
+ * result
+ * @return number of written code units; 0 if conversion is not possible
+ */
+simdutf_warn_unused size_t convert_valid_utf32_to_utf16be(
+    const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+convert_valid_utf32_to_utf16be(std::span<const char32_t> valid_utf32_input,
+                               std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32_to_utf16::convert_valid<endianness::BIG>(
+        valid_utf32_input.data(), valid_utf32_input.size(),
+        utf16_output.data());
+  } else
+    #endif
+  {
+    return convert_valid_utf32_to_utf16be(valid_utf32_input.data(),
+                                          valid_utf32_input.size(),
+                                          utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16
+/**
+ * Change the endianness of the input. Can be used to go from UTF-16LE to
+ * UTF-16BE or from UTF-16BE to UTF-16LE.
+ *
+ * This function does not validate the input.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to process
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result
+ */
+void change_endianness_utf16(const char16_t *input, size_t length,
+                             char16_t *output) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_constexpr23 void
+change_endianness_utf16(std::span<const char16_t> utf16_input,
+                        std::span<char16_t> utf16_output) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::change_endianness_utf16(
+        utf16_input.data(), utf16_input.size(), utf16_output.data());
+  } else
+    #endif
+  {
+    return change_endianness_utf16(utf16_input.data(), utf16_input.size(),
+                                   utf16_output.data());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+/**
+ * Compute the number of bytes that this UTF-32 string would require in UTF-8
+ * format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-32 strings but in such cases the result is implementation defined.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @return the number of bytes required to encode the UTF-32 string as UTF-8
+ */
+simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input,
+                                                  size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf8_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(),
+                                                 valid_utf32_input.size());
+  } else
+    #endif
+  {
+    return utf8_length_from_utf32(valid_utf32_input.data(),
+                                  valid_utf32_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+/**
+ * Compute the number of two-byte code units that this UTF-32 string would
+ * require in UTF-16 format.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-32 strings but in such cases the result is implementation defined.
+ *
+ * @param input         the UTF-32 string to convert
+ * @param length        the length of the string in 4-byte code units (char32_t)
+ * @return the number of bytes required to encode the UTF-32 string as UTF-16
+ */
+simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input,
+                                                   size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf16_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(),
+                                                  valid_utf32_input.size());
+  } else
+    #endif
+  {
+    return utf16_length_from_utf32(valid_utf32_input.data(),
+                                   valid_utf32_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Using native endianness; Compute the number of bytes that this UTF-16
+ * string would require in UTF-32 format.
+ *
+ * This function is equivalent to count_utf16.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16LE string as UTF-32
+ */
+simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input,
+                                                   size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf32_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf32_length_from_utf16<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf32_length_from_utf16(valid_utf16_input.data(),
+                                   valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16LE string would require in UTF-32
+ * format.
+ *
+ * This function is equivalent to count_utf16le.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16LE string as UTF-32
+ */
+simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input,
+                                                     size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf32_length_from_utf16le(
+    std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf32_length_from_utf16<endianness::LITTLE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf32_length_from_utf16le(valid_utf16_input.data(),
+                                     valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Compute the number of bytes that this UTF-16BE string would require in UTF-32
+ * format.
+ *
+ * This function is equivalent to count_utf16be.
+ *
+ * This function does not validate the input. It is acceptable to pass invalid
+ * UTF-16 strings but in such cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to convert
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return the number of bytes required to encode the UTF-16BE string as UTF-32
+ */
+simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input,
+                                                     size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+utf32_length_from_utf16be(
+    std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::utf32_length_from_utf16<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return utf32_length_from_utf16be(valid_utf16_input.data(),
+                                     valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16
+/**
+ * Count the number of code points (characters) in the string assuming that
+ * it is valid.
+ *
+ * This function assumes that the input string is valid UTF-16 (native
+ * endianness). It is acceptable to pass invalid UTF-16 strings but in such
+ * cases the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16 string to process
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return number of code points
+ */
+simdutf_warn_unused size_t count_utf16(const char16_t *input,
+                                       size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+count_utf16(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::count_code_points<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return count_utf16(valid_utf16_input.data(), valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Count the number of code points (characters) in the string assuming that
+ * it is valid.
+ *
+ * This function assumes that the input string is valid UTF-16LE.
+ * It is acceptable to pass invalid UTF-16 strings but in such cases
+ * the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16LE string to process
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return number of code points
+ */
+simdutf_warn_unused size_t count_utf16le(const char16_t *input,
+                                         size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+count_utf16le(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::count_code_points<endianness::LITTLE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Count the number of code points (characters) in the string assuming that
+ * it is valid.
+ *
+ * This function assumes that the input string is valid UTF-16BE.
+ * It is acceptable to pass invalid UTF-16 strings but in such cases
+ * the result is implementation defined.
+ *
+ * This function is not BOM-aware.
+ *
+ * @param input         the UTF-16BE string to process
+ * @param length        the length of the string in 2-byte code units (char16_t)
+ * @return number of code points
+ */
+simdutf_warn_unused size_t count_utf16be(const char16_t *input,
+                                         size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+count_utf16be(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::count_code_points<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8
+/**
+ * Count the number of code points (characters) in the string assuming that
+ * it is valid.
+ *
+ * This function assumes that the input string is valid UTF-8.
+ * It is acceptable to pass invalid UTF-8 strings but in such cases
+ * the result is implementation defined.
+ *
+ * @param input         the UTF-8 string to process
+ * @param length        the length of the string in bytes
+ * @return number of code points
+ */
+simdutf_warn_unused size_t count_utf8(const char *input,
+                                      size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8(
+    const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::count_code_points(valid_utf8_input.data(),
+                                           valid_utf8_input.size());
+  } else
+    #endif
+  {
+    return count_utf8(reinterpret_cast<const char *>(valid_utf8_input.data()),
+                      valid_utf8_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Given a valid UTF-8 string having a possibly truncated last character,
+ * this function checks the end of string. If the last character is truncated
+ * (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so
+ * that the short UTF-8 strings only contain complete characters. If there is no
+ * truncated character, the original length is returned.
+ *
+ * This function assumes that the input string is valid UTF-8, but possibly
+ * truncated.
+ *
+ * @param input         the UTF-8 string to process
+ * @param length        the length of the string in bytes
+ * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes
+ */
+simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length);
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf8(
+    const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(),
+                                           valid_utf8_input.size());
+  } else
+    #endif
+  {
+    return trim_partial_utf8(
+        reinterpret_cast<const char *>(valid_utf8_input.data()),
+        valid_utf8_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF8
+
+#if SIMDUTF_FEATURE_UTF16
+/**
+ * Given a valid UTF-16BE string having a possibly truncated last character,
+ * this function checks the end of string. If the last character is truncated
+ * (or partial), then it returns a shorter length (shorter by 1 unit) so that
+ * the short UTF-16BE strings only contain complete characters. If there is no
+ * truncated character, the original length is returned.
+ *
+ * This function assumes that the input string is valid UTF-16BE, but possibly
+ * truncated.
+ *
+ * @param input         the UTF-16BE string to process
+ * @param length        the length of the string in bytes
+ * @return the length of the string in bytes, possibly shorter by 1 unit
+ */
+simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input,
+                                                size_t length);
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf16be(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::trim_partial_utf16<endianness::BIG>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return trim_partial_utf16be(valid_utf16_input.data(),
+                                valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Given a valid UTF-16LE string having a possibly truncated last character,
+ * this function checks the end of string. If the last character is truncated
+ * (or partial), then it returns a shorter length (shorter by 1 unit) so that
+ * the short UTF-16LE strings only contain complete characters. If there is no
+ * truncated character, the original length is returned.
+ *
+ * This function assumes that the input string is valid UTF-16LE, but possibly
+ * truncated.
+ *
+ * @param input         the UTF-16LE string to process
+ * @param length        the length of the string in bytes
+ * @return the length of the string in unit, possibly shorter by 1 unit
+ */
+simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input,
+                                                size_t length);
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf16le(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::trim_partial_utf16<endianness::LITTLE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return trim_partial_utf16le(valid_utf16_input.data(),
+                                valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Given a valid UTF-16 string having a possibly truncated last character,
+ * this function checks the end of string. If the last character is truncated
+ * (or partial), then it returns a shorter length (shorter by 1 unit) so that
+ * the short UTF-16 strings only contain complete characters. If there is no
+ * truncated character, the original length is returned.
+ *
+ * This function assumes that the input string is valid UTF-16, but possibly
+ * truncated. We use the native endianness.
+ *
+ * @param input         the UTF-16 string to process
+ * @param length        the length of the string in bytes
+ * @return the length of the string in unit, possibly shorter by 1 unit
+ */
+simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input,
+                                              size_t length);
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+trim_partial_utf16(std::span<const char16_t> valid_utf16_input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::utf16::trim_partial_utf16<endianness::NATIVE>(
+        valid_utf16_input.data(), valid_utf16_input.size());
+  } else
+    #endif
+  {
+    return trim_partial_utf16(valid_utf16_input.data(),
+                              valid_utf16_input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+#endif   // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 ||                         \
+    SIMDUTF_FEATURE_DETECT_ENCODING
+  #ifndef SIMDUTF_NEED_TRAILING_ZEROES
+    #define SIMDUTF_NEED_TRAILING_ZEROES 1
+  #endif
+#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 ||
+       // SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_BASE64
+// base64_options are used to specify the base64 encoding options.
+// ASCII spaces are ' ', '\t', '\n', '\r', '\f'
+// garbage characters are characters that are not part of the base64 alphabet
+// nor ASCII spaces.
+constexpr uint64_t base64_reverse_padding =
+    2; /* modifier for base64_default and base64_url */
+enum base64_options : uint64_t {
+  base64_default = 0, /* standard base64 format (with padding) */
+  base64_url = 1,     /* base64url format (no padding) */
+  base64_default_no_padding =
+      base64_default |
+      base64_reverse_padding, /* standard base64 format without padding */
+  base64_url_with_padding =
+      base64_url | base64_reverse_padding, /* base64url with padding */
+  base64_default_accept_garbage =
+      4, /* standard base64 format accepting garbage characters, the input stops
+            with the first '=' if any */
+  base64_url_accept_garbage =
+      5, /* base64url format accepting garbage characters, the input stops with
+            the first '=' if any */
+  base64_default_or_url =
+      8, /* standard/base64url hybrid format (only meaningful for decoding!) */
+  base64_default_or_url_accept_garbage =
+      12, /* standard/base64url hybrid format accepting garbage characters
+             (only meaningful for decoding!), the input stops with the first '='
+             if any */
+};
+
+// last_chunk_handling_options are used to specify the handling of the last
+// chunk in base64 decoding.
+// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+enum last_chunk_handling_options : uint64_t {
+  loose = 0,  /* standard base64 format, decode partial final chunk */
+  strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and
+                 unpadded, or non-zero bit padding */
+  stop_before_partial =
+      2, /* if the last chunk is partial, ignore it (no error) */
+  only_full_chunks =
+      3 /* only decode full blocks (4 base64 characters, no padding) */
+};
+
+inline simdutf_constexpr23 bool
+is_partial(last_chunk_handling_options options) {
+  return (options == stop_before_partial) || (options == only_full_chunks);
+}
+
+namespace detail {
+simdutf_warn_unused const char *find(const char *start, const char *end,
+                                     char character) noexcept;
+simdutf_warn_unused const char16_t *
+find(const char16_t *start, const char16_t *end, char16_t character) noexcept;
+} // namespace detail
+
+/**
+ * Find the first occurrence of a character in a string. If the character is
+ * not found, return a pointer to the end of the string.
+ * @param start        the start of the string
+ * @param end          the end of the string
+ * @param character    the character to find
+ * @return a pointer to the first occurrence of the character in the string,
+ * or a pointer to the end of the string if the character is not found.
+ *
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char *
+find(const char *start, const char *end, char character) noexcept {
+  #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    for (; start != end; ++start)
+      if (*start == character)
+        return start;
+    return end;
+  } else
+  #endif
+  {
+    return detail::find(start, end, character);
+  }
+}
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t *
+find(const char16_t *start, const char16_t *end, char16_t character) noexcept {
+    // implementation note: this is repeated instead of a template, to ensure
+    // the api is still a function and compiles without concepts
+  #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    for (; start != end; ++start)
+      if (*start == character)
+        return start;
+    return end;
+  } else
+  #endif
+  {
+    return detail::find(start, end, character);
+  }
+}
+}
+  // We include base64_tables once.
+/* begin file include/simdutf/base64_tables.h */
+#ifndef SIMDUTF_BASE64_TABLES_H
+#define SIMDUTF_BASE64_TABLES_H
+#include <cstdint>
+
+namespace simdutf {
+namespace {
+namespace tables {
+namespace base64 {
+namespace base64_default {
+
+constexpr char e0[256] = {
+    'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D',
+    'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H',
+    'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L',
+    'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O',
+    'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S',
+    'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W',
+    'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a',
+    'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd',
+    'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h',
+    'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l',
+    'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p',
+    'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's',
+    't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w',
+    'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0',
+    '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4',
+    '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7',
+    '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/',
+    '/'};
+
+constexpr char e1[256] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+    'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
+    'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+    't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
+    '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+    'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
+    '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+    'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+    'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
+    'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C',
+    'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
+    'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+    'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+    'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+',
+    '/'};
+
+constexpr char e2[256] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+    'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
+    'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+    't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
+    '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+    'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
+    '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+    'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+    'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
+    'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C',
+    'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
+    'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+    'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+    'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+',
+    '/'};
+
+constexpr uint32_t d0[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc,
+    0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
+    0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
+    0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
+    0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
+    0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
+    0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
+    0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
+    0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
+    0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
+    0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+
+constexpr uint32_t d1[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003,
+    0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
+    0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
+    0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
+    0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
+    0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
+    0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
+    0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
+    0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
+    0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
+    0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+
+constexpr uint32_t d2[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00,
+    0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
+    0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
+    0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
+    0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
+    0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
+    0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
+    0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
+    0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
+    0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
+    0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+
+constexpr uint32_t d3[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000,
+    0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
+    0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
+    0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
+    0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
+    0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
+    0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
+    0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
+    0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
+    0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
+    0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+} // namespace base64_default
+
+namespace base64_url {
+
+constexpr char e0[256] = {
+    'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D',
+    'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H',
+    'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L',
+    'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O',
+    'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S',
+    'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W',
+    'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a',
+    'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd',
+    'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h',
+    'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l',
+    'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p',
+    'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's',
+    't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w',
+    'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0',
+    '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4',
+    '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7',
+    '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_',
+    '_'};
+
+constexpr char e1[256] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+    'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
+    'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+    't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
+    '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+    'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
+    '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+    'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+    'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
+    'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C',
+    'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
+    'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+    'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+    'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-',
+    '_'};
+
+constexpr char e2[256] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+    'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
+    'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+    't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
+    '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+    'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
+    '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+    'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+    'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
+    'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C',
+    'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
+    'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+    'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+    'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-',
+    '_'};
+
+constexpr uint32_t d0[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff,
+    0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
+    0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
+    0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
+    0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
+    0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
+    0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc,
+    0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
+    0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
+    0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
+    0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
+    0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d1[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff,
+    0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
+    0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
+    0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
+    0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
+    0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
+    0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003,
+    0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
+    0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
+    0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
+    0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
+    0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d2[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff,
+    0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
+    0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
+    0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
+    0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
+    0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
+    0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00,
+    0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
+    0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
+    0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
+    0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
+    0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d3[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff,
+    0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
+    0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
+    0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
+    0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
+    0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
+    0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000,
+    0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
+    0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
+    0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
+    0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
+    0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+} // namespace base64_url
+
+namespace base64_default_or_url {
+constexpr uint32_t d0[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc,
+    0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
+    0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
+    0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
+    0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
+    0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
+    0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc,
+    0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
+    0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
+    0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
+    0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
+    0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d1[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003,
+    0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
+    0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
+    0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
+    0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
+    0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
+    0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003,
+    0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
+    0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
+    0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
+    0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
+    0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d2[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00,
+    0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
+    0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
+    0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
+    0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
+    0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
+    0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00,
+    0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
+    0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
+    0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
+    0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
+    0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+constexpr uint32_t d3[256] = {
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000,
+    0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
+    0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
+    0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
+    0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
+    0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
+    0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
+    0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000,
+    0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
+    0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
+    0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
+    0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
+    0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
+    0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff};
+} // namespace base64_default_or_url
+constexpr uint64_t thintable_epi8[256] = {
+    0x0706050403020100, 0x0007060504030201, 0x0007060504030200,
+    0x0000070605040302, 0x0007060504030100, 0x0000070605040301,
+    0x0000070605040300, 0x0000000706050403, 0x0007060504020100,
+    0x0000070605040201, 0x0000070605040200, 0x0000000706050402,
+    0x0000070605040100, 0x0000000706050401, 0x0000000706050400,
+    0x0000000007060504, 0x0007060503020100, 0x0000070605030201,
+    0x0000070605030200, 0x0000000706050302, 0x0000070605030100,
+    0x0000000706050301, 0x0000000706050300, 0x0000000007060503,
+    0x0000070605020100, 0x0000000706050201, 0x0000000706050200,
+    0x0000000007060502, 0x0000000706050100, 0x0000000007060501,
+    0x0000000007060500, 0x0000000000070605, 0x0007060403020100,
+    0x0000070604030201, 0x0000070604030200, 0x0000000706040302,
+    0x0000070604030100, 0x0000000706040301, 0x0000000706040300,
+    0x0000000007060403, 0x0000070604020100, 0x0000000706040201,
+    0x0000000706040200, 0x0000000007060402, 0x0000000706040100,
+    0x0000000007060401, 0x0000000007060400, 0x0000000000070604,
+    0x0000070603020100, 0x0000000706030201, 0x0000000706030200,
+    0x0000000007060302, 0x0000000706030100, 0x0000000007060301,
+    0x0000000007060300, 0x0000000000070603, 0x0000000706020100,
+    0x0000000007060201, 0x0000000007060200, 0x0000000000070602,
+    0x0000000007060100, 0x0000000000070601, 0x0000000000070600,
+    0x0000000000000706, 0x0007050403020100, 0x0000070504030201,
+    0x0000070504030200, 0x0000000705040302, 0x0000070504030100,
+    0x0000000705040301, 0x0000000705040300, 0x0000000007050403,
+    0x0000070504020100, 0x0000000705040201, 0x0000000705040200,
+    0x0000000007050402, 0x0000000705040100, 0x0000000007050401,
+    0x0000000007050400, 0x0000000000070504, 0x0000070503020100,
+    0x0000000705030201, 0x0000000705030200, 0x0000000007050302,
+    0x0000000705030100, 0x0000000007050301, 0x0000000007050300,
+    0x0000000000070503, 0x0000000705020100, 0x0000000007050201,
+    0x0000000007050200, 0x0000000000070502, 0x0000000007050100,
+    0x0000000000070501, 0x0000000000070500, 0x0000000000000705,
+    0x0000070403020100, 0x0000000704030201, 0x0000000704030200,
+    0x0000000007040302, 0x0000000704030100, 0x0000000007040301,
+    0x0000000007040300, 0x0000000000070403, 0x0000000704020100,
+    0x0000000007040201, 0x0000000007040200, 0x0000000000070402,
+    0x0000000007040100, 0x0000000000070401, 0x0000000000070400,
+    0x0000000000000704, 0x0000000703020100, 0x0000000007030201,
+    0x0000000007030200, 0x0000000000070302, 0x0000000007030100,
+    0x0000000000070301, 0x0000000000070300, 0x0000000000000703,
+    0x0000000007020100, 0x0000000000070201, 0x0000000000070200,
+    0x0000000000000702, 0x0000000000070100, 0x0000000000000701,
+    0x0000000000000700, 0x0000000000000007, 0x0006050403020100,
+    0x0000060504030201, 0x0000060504030200, 0x0000000605040302,
+    0x0000060504030100, 0x0000000605040301, 0x0000000605040300,
+    0x0000000006050403, 0x0000060504020100, 0x0000000605040201,
+    0x0000000605040200, 0x0000000006050402, 0x0000000605040100,
+    0x0000000006050401, 0x0000000006050400, 0x0000000000060504,
+    0x0000060503020100, 0x0000000605030201, 0x0000000605030200,
+    0x0000000006050302, 0x0000000605030100, 0x0000000006050301,
+    0x0000000006050300, 0x0000000000060503, 0x0000000605020100,
+    0x0000000006050201, 0x0000000006050200, 0x0000000000060502,
+    0x0000000006050100, 0x0000000000060501, 0x0000000000060500,
+    0x0000000000000605, 0x0000060403020100, 0x0000000604030201,
+    0x0000000604030200, 0x0000000006040302, 0x0000000604030100,
+    0x0000000006040301, 0x0000000006040300, 0x0000000000060403,
+    0x0000000604020100, 0x0000000006040201, 0x0000000006040200,
+    0x0000000000060402, 0x0000000006040100, 0x0000000000060401,
+    0x0000000000060400, 0x0000000000000604, 0x0000000603020100,
+    0x0000000006030201, 0x0000000006030200, 0x0000000000060302,
+    0x0000000006030100, 0x0000000000060301, 0x0000000000060300,
+    0x0000000000000603, 0x0000000006020100, 0x0000000000060201,
+    0x0000000000060200, 0x0000000000000602, 0x0000000000060100,
+    0x0000000000000601, 0x0000000000000600, 0x0000000000000006,
+    0x0000050403020100, 0x0000000504030201, 0x0000000504030200,
+    0x0000000005040302, 0x0000000504030100, 0x0000000005040301,
+    0x0000000005040300, 0x0000000000050403, 0x0000000504020100,
+    0x0000000005040201, 0x0000000005040200, 0x0000000000050402,
+    0x0000000005040100, 0x0000000000050401, 0x0000000000050400,
+    0x0000000000000504, 0x0000000503020100, 0x0000000005030201,
+    0x0000000005030200, 0x0000000000050302, 0x0000000005030100,
+    0x0000000000050301, 0x0000000000050300, 0x0000000000000503,
+    0x0000000005020100, 0x0000000000050201, 0x0000000000050200,
+    0x0000000000000502, 0x0000000000050100, 0x0000000000000501,
+    0x0000000000000500, 0x0000000000000005, 0x0000000403020100,
+    0x0000000004030201, 0x0000000004030200, 0x0000000000040302,
+    0x0000000004030100, 0x0000000000040301, 0x0000000000040300,
+    0x0000000000000403, 0x0000000004020100, 0x0000000000040201,
+    0x0000000000040200, 0x0000000000000402, 0x0000000000040100,
+    0x0000000000000401, 0x0000000000000400, 0x0000000000000004,
+    0x0000000003020100, 0x0000000000030201, 0x0000000000030200,
+    0x0000000000000302, 0x0000000000030100, 0x0000000000000301,
+    0x0000000000000300, 0x0000000000000003, 0x0000000000020100,
+    0x0000000000000201, 0x0000000000000200, 0x0000000000000002,
+    0x0000000000000100, 0x0000000000000001, 0x0000000000000000,
+    0x0000000000000000,
+};
+
+constexpr uint8_t pshufb_combine_table[272] = {
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08,
+    0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03,
+    0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff,
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
+    0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08,
+    0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff,
+    0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff,
+    0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
+    0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
+constexpr unsigned char BitsSetTable256mul2[256] = {
+    0,  2,  2,  4,  2,  4,  4,  6,  2,  4,  4,  6,  4,  6,  6,  8,  2,  4,  4,
+    6,  4,  6,  6,  8,  4,  6,  6,  8,  6,  8,  8,  10, 2,  4,  4,  6,  4,  6,
+    6,  8,  4,  6,  6,  8,  6,  8,  8,  10, 4,  6,  6,  8,  6,  8,  8,  10, 6,
+    8,  8,  10, 8,  10, 10, 12, 2,  4,  4,  6,  4,  6,  6,  8,  4,  6,  6,  8,
+    6,  8,  8,  10, 4,  6,  6,  8,  6,  8,  8,  10, 6,  8,  8,  10, 8,  10, 10,
+    12, 4,  6,  6,  8,  6,  8,  8,  10, 6,  8,  8,  10, 8,  10, 10, 12, 6,  8,
+    8,  10, 8,  10, 10, 12, 8,  10, 10, 12, 10, 12, 12, 14, 2,  4,  4,  6,  4,
+    6,  6,  8,  4,  6,  6,  8,  6,  8,  8,  10, 4,  6,  6,  8,  6,  8,  8,  10,
+    6,  8,  8,  10, 8,  10, 10, 12, 4,  6,  6,  8,  6,  8,  8,  10, 6,  8,  8,
+    10, 8,  10, 10, 12, 6,  8,  8,  10, 8,  10, 10, 12, 8,  10, 10, 12, 10, 12,
+    12, 14, 4,  6,  6,  8,  6,  8,  8,  10, 6,  8,  8,  10, 8,  10, 10, 12, 6,
+    8,  8,  10, 8,  10, 10, 12, 8,  10, 10, 12, 10, 12, 12, 14, 6,  8,  8,  10,
+    8,  10, 10, 12, 8,  10, 10, 12, 10, 12, 12, 14, 8,  10, 10, 12, 10, 12, 12,
+    14, 10, 12, 12, 14, 12, 14, 14, 16};
+
+constexpr uint8_t to_base64_value[] = {
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 64,  64,  255, 64,  64,  255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 64,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62,  255,
+    255, 255, 63,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  255, 255,
+    255, 255, 255, 255, 255, 0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+    10,  11,  12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,
+    25,  255, 255, 255, 255, 255, 255, 26,  27,  28,  29,  30,  31,  32,  33,
+    34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47,  48,
+    49,  50,  51,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255};
+
+constexpr uint8_t to_base64_url_value[] = {
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 64,  64,  255, 64,  64,  255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 64,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    62,  255, 255, 52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  255, 255,
+    255, 255, 255, 255, 255, 0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+    10,  11,  12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,
+    25,  255, 255, 255, 255, 63,  255, 26,  27,  28,  29,  30,  31,  32,  33,
+    34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47,  48,
+    49,  50,  51,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255};
+
+constexpr uint8_t to_base64_default_or_url_value[] = {
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 64,  64,  255, 64,  64,  255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 64,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62,  255,
+    62,  255, 63,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  255, 255,
+    255, 255, 255, 255, 255, 0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+    10,  11,  12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,
+    25,  255, 255, 255, 255, 63,  255, 26,  27,  28,  29,  30,  31,  32,  33,
+    34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47,  48,
+    49,  50,  51,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+    255};
+
+static_assert(sizeof(to_base64_value) == 256,
+              "to_base64_value must have 256 elements");
+static_assert(sizeof(to_base64_url_value) == 256,
+              "to_base64_url_value must have 256 elements");
+static_assert(to_base64_value[uint8_t(' ')] == 64,
+              "space must be == 64 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t(' ')] == 64,
+              "space must be == 64 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('\t')] == 64,
+              "tab must be == 64 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('\t')] == 64,
+              "tab must be == 64 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('\r')] == 64,
+              "cr must be == 64 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('\r')] == 64,
+              "cr must be == 64 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('\n')] == 64,
+              "lf must be == 64 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('\n')] == 64,
+              "lf must be == 64 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('\f')] == 64,
+              "ff must be == 64 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('\f')] == 64,
+              "ff must be == 64 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('+')] == 62,
+              "+ must be == 62 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('-')] == 62,
+              "- must be == 62 in to_base64_url_value");
+static_assert(to_base64_value[uint8_t('/')] == 63,
+              "/ must be == 63 in to_base64_value");
+static_assert(to_base64_url_value[uint8_t('_')] == 63,
+              "_ must be == 63 in to_base64_url_value");
+} // namespace base64
+} // namespace tables
+} // unnamed namespace
+} // namespace simdutf
+
+#endif // SIMDUTF_BASE64_TABLES_H
+/* end file include/simdutf/base64_tables.h */
+/* begin file include/simdutf/scalar/base64.h */
+#ifndef SIMDUTF_BASE64_H
+#define SIMDUTF_BASE64_H
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <iostream>
+
+namespace simdutf {
+namespace scalar {
+namespace {
+namespace base64 {
+
+// This function is not expected to be fast. Do not use in long loops.
+// In most instances you should be using is_ignorable.
+template <class char_type> bool is_ascii_white_space(char_type c) {
+  return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f';
+}
+
+template <class char_type> simdutf_constexpr23 bool is_eight_byte(char_type c) {
+  if simdutf_constexpr (sizeof(char_type) == 1) {
+    return true;
+  }
+  return uint8_t(c) == c;
+}
+
+template <class char_type>
+simdutf_constexpr23 bool is_ignorable(char_type c,
+                                      simdutf::base64_options options) {
+  const uint8_t *to_base64 =
+      (options & base64_default_or_url)
+          ? tables::base64::to_base64_default_or_url_value
+          : ((options & base64_url) ? tables::base64::to_base64_url_value
+                                    : tables::base64::to_base64_value);
+  const bool ignore_garbage =
+      (options == base64_options::base64_url_accept_garbage) ||
+      (options == base64_options::base64_default_accept_garbage) ||
+      (options == base64_options::base64_default_or_url_accept_garbage);
+  uint8_t code = to_base64[uint8_t(c)];
+  if (is_eight_byte(c) && code <= 63) {
+    return false;
+  }
+  if (is_eight_byte(c) && code == 64) {
+    return true;
+  }
+  return ignore_garbage;
+}
+template <class char_type>
+simdutf_constexpr23 bool is_base64(char_type c,
+                                   simdutf::base64_options options) {
+  const uint8_t *to_base64 =
+      (options & base64_default_or_url)
+          ? tables::base64::to_base64_default_or_url_value
+          : ((options & base64_url) ? tables::base64::to_base64_url_value
+                                    : tables::base64::to_base64_value);
+  uint8_t code = to_base64[uint8_t(c)];
+  if (is_eight_byte(c) && code <= 63) {
+    return true;
+  }
+  return false;
+}
+
+template <class char_type>
+simdutf_constexpr23 bool is_base64_or_padding(char_type c,
+                                              simdutf::base64_options options) {
+  const uint8_t *to_base64 =
+      (options & base64_default_or_url)
+          ? tables::base64::to_base64_default_or_url_value
+          : ((options & base64_url) ? tables::base64::to_base64_url_value
+                                    : tables::base64::to_base64_value);
+  if (c == '=') {
+    return true;
+  }
+  uint8_t code = to_base64[uint8_t(c)];
+  if (is_eight_byte(c) && code <= 63) {
+    return true;
+  }
+  return false;
+}
+
+template <class char_type>
+bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) {
+  return is_ignorable(c, options) || c == '=';
+}
+
+struct reduced_input {
+  size_t equalsigns;    // number of padding characters '=', typically 0, 1, 2.
+  size_t equallocation; // location of the first padding character if any
+  size_t srclen;        // length of the input buffer before padding
+  size_t full_input_length; // length of the input buffer with padding but
+                            // without ignorable characters
+};
+
+// find the end of the base64 input buffer
+// It returns the number of padding characters, the location of the first
+// padding character if any, the length of the input buffer before padding
+// and the length of the input buffer with padding. The input buffer is not
+// modified. The function assumes that there are at most two padding characters.
+template <class char_type>
+simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen,
+                                           simdutf::base64_options options) {
+  const uint8_t *to_base64 =
+      (options & base64_default_or_url)
+          ? tables::base64::to_base64_default_or_url_value
+          : ((options & base64_url) ? tables::base64::to_base64_url_value
+                                    : tables::base64::to_base64_value);
+  const bool ignore_garbage =
+      (options == base64_options::base64_url_accept_garbage) ||
+      (options == base64_options::base64_default_accept_garbage) ||
+      (options == base64_options::base64_default_or_url_accept_garbage);
+
+  size_t equalsigns = 0;
+  // We intentionally include trailing spaces in the full input length.
+  // See https://github.com/simdutf/simdutf/issues/824
+  size_t full_input_length = srclen;
+  // skip trailing spaces
+  while (!ignore_garbage && srclen > 0 &&
+         scalar::base64::is_eight_byte(src[srclen - 1]) &&
+         to_base64[uint8_t(src[srclen - 1])] == 64) {
+    srclen--;
+  }
+  size_t equallocation =
+      srclen; // location of the first padding character if any
+  if (ignore_garbage) {
+    // Technically, we don't need to find the first padding character, we can
+    // just change our algorithms, but it adds substantial complexity.
+    auto it = simdutf::find(src, src + srclen, '=');
+    if (it != src + srclen) {
+      equallocation = it - src;
+      equalsigns = 1;
+      srclen = equallocation;
+      full_input_length = equallocation + 1;
+    }
+    return {equalsigns, equallocation, srclen, full_input_length};
+  }
+  if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') {
+    // This is the last '=' sign.
+    equallocation = srclen - 1;
+    srclen--;
+    equalsigns = 1;
+    // skip trailing spaces
+    while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) &&
+           to_base64[uint8_t(src[srclen - 1])] == 64) {
+      srclen--;
+    }
+    if (srclen > 0 && src[srclen - 1] == '=') {
+      // This is the second '=' sign.
+      equallocation = srclen - 1;
+      srclen--;
+      equalsigns = 2;
+    }
+  }
+  return {equalsigns, equallocation, srclen, full_input_length};
+}
+
+// Returns true upon success. The destination buffer must be large enough.
+// This functions assumes that the padding (=) has been removed.
+// if check_capacity is true, it will check that the destination buffer is
+// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL.
+template <bool check_capacity, class char_type>
+simdutf_constexpr23 full_result base64_tail_decode_impl(
+    char *dst, size_t outlen, const char_type *src, size_t length,
+    size_t padding_characters, // number of padding characters
+                               // '=', typically 0, 1, 2.
+    base64_options options, last_chunk_handling_options last_chunk_options) {
+  char *dstend = dst + outlen;
+  (void)dstend;
+  // This looks like 10 branches, but we expect the compiler to resolve this to
+  // two branches (easily predicted):
+  const uint8_t *to_base64 =
+      (options & base64_default_or_url)
+          ? tables::base64::to_base64_default_or_url_value
+          : ((options & base64_url) ? tables::base64::to_base64_url_value
+                                    : tables::base64::to_base64_value);
+  const uint32_t *d0 =
+      (options & base64_default_or_url)
+          ? tables::base64::base64_default_or_url::d0
+          : ((options & base64_url) ? tables::base64::base64_url::d0
+                                    : tables::base64::base64_default::d0);
+  const uint32_t *d1 =
+      (options & base64_default_or_url)
+          ? tables::base64::base64_default_or_url::d1
+          : ((options & base64_url) ? tables::base64::base64_url::d1
+                                    : tables::base64::base64_default::d1);
+  const uint32_t *d2 =
+      (options & base64_default_or_url)
+          ? tables::base64::base64_default_or_url::d2
+          : ((options & base64_url) ? tables::base64::base64_url::d2
+                                    : tables::base64::base64_default::d2);
+  const uint32_t *d3 =
+      (options & base64_default_or_url)
+          ? tables::base64::base64_default_or_url::d3
+          : ((options & base64_url) ? tables::base64::base64_url::d3
+                                    : tables::base64::base64_default::d3);
+  const bool ignore_garbage =
+      (options == base64_options::base64_url_accept_garbage) ||
+      (options == base64_options::base64_default_accept_garbage) ||
+      (options == base64_options::base64_default_or_url_accept_garbage);
+
+  const char_type *srcend = src + length;
+  const char_type *srcinit = src;
+  const char *dstinit = dst;
+
+  uint32_t x;
+  size_t idx;
+  uint8_t buffer[4];
+  while (true) {
+    while (srcend - src >= 4 && is_eight_byte(src[0]) &&
+           is_eight_byte(src[1]) && is_eight_byte(src[2]) &&
+           is_eight_byte(src[3]) &&
+           (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] |
+                d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) {
+      if (check_capacity && dstend - dst < 3) {
+        return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit),
+                size_t(dst - dstinit)};
+      }
+      *dst++ = static_cast<char>(x & 0xFF);
+      *dst++ = static_cast<char>((x >> 8) & 0xFF);
+      *dst++ = static_cast<char>((x >> 16) & 0xFF);
+      src += 4;
+    }
+    const char_type *srccur = src;
+    idx = 0;
+    // we need at least four characters.
+#ifdef __clang__
+    // If possible, we read four characters at a time. (It is an optimization.)
+    if (ignore_garbage && src + 4 <= srcend) {
+      char_type c0 = src[0];
+      char_type c1 = src[1];
+      char_type c2 = src[2];
+      char_type c3 = src[3];
+
+      uint8_t code0 = to_base64[uint8_t(c0)];
+      uint8_t code1 = to_base64[uint8_t(c1)];
+      uint8_t code2 = to_base64[uint8_t(c2)];
+      uint8_t code3 = to_base64[uint8_t(c3)];
+
+      buffer[idx] = code0;
+      idx += (is_eight_byte(c0) && code0 <= 63);
+      buffer[idx] = code1;
+      idx += (is_eight_byte(c1) && code1 <= 63);
+      buffer[idx] = code2;
+      idx += (is_eight_byte(c2) && code2 <= 63);
+      buffer[idx] = code3;
+      idx += (is_eight_byte(c3) && code3 <= 63);
+      src += 4;
+    }
+#endif
+    while ((idx < 4) && (src < srcend)) {
+      char_type c = *src;
+
+      uint8_t code = to_base64[uint8_t(c)];
+      buffer[idx] = uint8_t(code);
+      if (is_eight_byte(c) && code <= 63) {
+        idx++;
+      } else if (!ignore_garbage &&
+                 (code > 64 || !scalar::base64::is_eight_byte(c))) {
+        return {INVALID_BASE64_CHARACTER, size_t(src - srcinit),
+                size_t(dst - dstinit)};
+      } else {
+        // We have a space or a newline or garbage. We ignore it.
+      }
+      src++;
+    }
+    if (idx != 4) {
+      simdutf_log_assert(idx < 4, "idx should be less than 4");
+      // We never should have that the number of base64 characters + the
+      // number of padding characters is more than 4.
+      if (!ignore_garbage && (idx + padding_characters > 4)) {
+        return {INVALID_BASE64_CHARACTER, size_t(src - srcinit),
+                size_t(dst - dstinit), true};
+      }
+
+      // The idea here is that in loose mode,
+      // if there is padding at all, it must be used
+      // to form 4-wise chunk. However, in loose mode,
+      // we do accept no padding at all.
+      if (!ignore_garbage &&
+          last_chunk_options == last_chunk_handling_options::loose &&
+          (idx >= 2) && padding_characters > 0 &&
+          ((idx + padding_characters) & 3) != 0) {
+        return {INVALID_BASE64_CHARACTER, size_t(src - srcinit),
+                size_t(dst - dstinit), true};
+      } else
+
+        // The idea here is that in strict mode, we do not want to accept
+        // incomplete base64 chunks. So if the chunk was otherwise valid, we
+        // return BASE64_INPUT_REMAINDER.
+        if (!ignore_garbage &&
+            last_chunk_options == last_chunk_handling_options::strict &&
+            (idx >= 2) && ((idx + padding_characters) & 3) != 0) {
+          // The partial chunk was at src - idx
+          return {BASE64_INPUT_REMAINDER, size_t(src - srcinit),
+                  size_t(dst - dstinit), true};
+        } else
+          // If there is a partial chunk with insufficient padding, with
+          // stop_before_partial, we need to just ignore it. In "only full"
+          // mode, skip the minute there are padding characters.
+          if ((last_chunk_options ==
+                   last_chunk_handling_options::stop_before_partial &&
+               (padding_characters + idx < 4) && (idx != 0) &&
+               (idx >= 2 || padding_characters == 0)) ||
+              (last_chunk_options ==
+                   last_chunk_handling_options::only_full_chunks &&
+               (idx >= 2 || padding_characters == 0))) {
+            // partial means that we are *not* going to consume the read
+            // characters. We need to rewind the src pointer.
+            src = srccur;
+            return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)};
+          } else {
+            if (idx == 2) {
+              uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) +
+                                (uint32_t(buffer[1]) << 2 * 6);
+              if (!ignore_garbage &&
+                  (last_chunk_options == last_chunk_handling_options::strict) &&
+                  (triple & 0xffff)) {
+                return {BASE64_EXTRA_BITS, size_t(src - srcinit),
+                        size_t(dst - dstinit)};
+              }
+              if (check_capacity && dstend - dst < 1) {
+                return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit),
+                        size_t(dst - dstinit)};
+              }
+              *dst++ = static_cast<char>((triple >> 16) & 0xFF);
+            } else if (idx == 3) {
+              uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) +
+                                (uint32_t(buffer[1]) << 2 * 6) +
+                                (uint32_t(buffer[2]) << 1 * 6);
+              if (!ignore_garbage &&
+                  (last_chunk_options == last_chunk_handling_options::strict) &&
+                  (triple & 0xff)) {
+                return {BASE64_EXTRA_BITS, size_t(src - srcinit),
+                        size_t(dst - dstinit)};
+              }
+              if (check_capacity && dstend - dst < 2) {
+                return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit),
+                        size_t(dst - dstinit)};
+              }
+              *dst++ = static_cast<char>((triple >> 16) & 0xFF);
+              *dst++ = static_cast<char>((triple >> 8) & 0xFF);
+            } else if (!ignore_garbage && idx == 1 &&
+                       (!is_partial(last_chunk_options) ||
+                        (is_partial(last_chunk_options) &&
+                         padding_characters > 0))) {
+              return {BASE64_INPUT_REMAINDER, size_t(src - srcinit),
+                      size_t(dst - dstinit)};
+            } else if (!ignore_garbage && idx == 0 && padding_characters > 0) {
+              return {INVALID_BASE64_CHARACTER, size_t(src - srcinit),
+                      size_t(dst - dstinit), true};
+            }
+            return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)};
+          }
+    }
+    if (check_capacity && dstend - dst < 3) {
+      return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit),
+              size_t(dst - dstinit)};
+    }
+    uint32_t triple =
+        (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) +
+        (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6);
+    *dst++ = static_cast<char>((triple >> 16) & 0xFF);
+    *dst++ = static_cast<char>((triple >> 8) & 0xFF);
+    *dst++ = static_cast<char>(triple & 0xFF);
+  }
+}
+
+template <class char_type>
+simdutf_constexpr23 full_result base64_tail_decode(
+    char *dst, const char_type *src, size_t length,
+    size_t padding_characters, // number of padding characters
+                               // '=', typically 0, 1, 2.
+    base64_options options, last_chunk_handling_options last_chunk_options) {
+  return base64_tail_decode_impl<false>(dst, 0, src, length, padding_characters,
+                                        options, last_chunk_options);
+}
+
+// like base64_tail_decode, but it will not write past the end of the output
+// buffer. The outlen parameter is modified to reflect the number of bytes
+// written. This functions assumes that the padding (=) has been removed.
+//
+template <class char_type>
+simdutf_constexpr23 full_result base64_tail_decode_safe(
+    char *dst, size_t outlen, const char_type *src, size_t length,
+    size_t padding_characters, // number of padding characters
+                               // '=', typically 0, 1, 2.
+    base64_options options, last_chunk_handling_options last_chunk_options) {
+  return base64_tail_decode_impl<true>(dst, outlen, src, length,
+                                       padding_characters, options,
+                                       last_chunk_options);
+}
+
+inline simdutf_constexpr23 full_result
+patch_tail_result(full_result r, size_t previous_input, size_t previous_output,
+                  size_t equallocation, size_t full_input_length,
+                  last_chunk_handling_options last_chunk_options) {
+  r.input_count += previous_input;
+  r.output_count += previous_output;
+  if (r.padding_error) {
+    r.input_count = equallocation;
+  }
+
+  if (r.error == error_code::SUCCESS) {
+    if (!is_partial(last_chunk_options)) {
+      // A success when we are not in stop_before_partial mode.
+      // means that we have consumed the whole input buffer.
+      r.input_count = full_input_length;
+    } else if (r.output_count % 3 != 0) {
+      r.input_count = full_input_length;
+    }
+  }
+  return r;
+}
+
+// Returns the number of bytes written. The destination buffer must be large
+// enough. It will add padding (=) if needed.
+template <bool use_lines = false>
+simdutf_constexpr23 size_t tail_encode_base64_impl(
+    char *dst, const char *src, size_t srclen, base64_options options,
+    size_t line_length = simdutf::default_line_length, size_t line_offset = 0) {
+  if simdutf_constexpr (use_lines) {
+    // sanitize line_length and starting_line_offset.
+    // line_length must be greater than 3.
+    if (line_length < 4) {
+      line_length = 4;
+    }
+    simdutf_log_assert(line_offset <= line_length,
+                       "line_offset should be less than line_length");
+  }
+  // By default, we use padding if we are not using the URL variant.
+  // This is check with ((options & base64_url) == 0) which returns true if we
+  // are not using the URL variant. However, we also allow 'inversion' of the
+  // convention with the base64_reverse_padding option. If the
+  // base64_reverse_padding option is set, we use padding if we are using the
+  // URL variant, and we omit it if we are not using the URL variant. This is
+  // checked with
+  // ((options & base64_reverse_padding) == base64_reverse_padding).
+  bool use_padding =
+      ((options & base64_url) == 0) ^
+      ((options & base64_reverse_padding) == base64_reverse_padding);
+  // This looks like 3 branches, but we expect the compiler to resolve this to
+  // a single branch:
+  const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0
+                                          : tables::base64::base64_default::e0;
+  const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1
+                                          : tables::base64::base64_default::e1;
+  const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2
+                                          : tables::base64::base64_default::e2;
+  char *out = dst;
+  size_t i = 0;
+  uint8_t t1, t2, t3;
+  for (; i + 2 < srclen; i += 3) {
+    t1 = uint8_t(src[i]);
+    t2 = uint8_t(src[i + 1]);
+    t3 = uint8_t(src[i + 2]);
+    if simdutf_constexpr (use_lines) {
+      if (line_offset + 3 >= line_length) {
+        if (line_offset == line_length) {
+          *out++ = '\n';
+          *out++ = e0[t1];
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+          *out++ = e2[t3];
+          line_offset = 4;
+        } else if (line_offset + 1 == line_length) {
+          *out++ = e0[t1];
+          *out++ = '\n';
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+          *out++ = e2[t3];
+          line_offset = 3;
+        } else if (line_offset + 2 == line_length) {
+          *out++ = e0[t1];
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = '\n';
+          *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+          *out++ = e2[t3];
+          line_offset = 2;
+        } else if (line_offset + 3 == line_length) {
+          *out++ = e0[t1];
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+          *out++ = '\n';
+          *out++ = e2[t3];
+          line_offset = 1;
+        }
+      } else {
+        *out++ = e0[t1];
+        *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+        *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+        *out++ = e2[t3];
+        line_offset += 4;
+      }
+    } else {
+      *out++ = e0[t1];
+      *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+      *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)];
+      *out++ = e2[t3];
+    }
+  }
+  switch (srclen - i) {
+  case 0:
+    break;
+  case 1:
+    t1 = uint8_t(src[i]);
+    if simdutf_constexpr (use_lines) {
+      if (use_padding) {
+        if (line_offset + 3 >= line_length) {
+          if (line_offset == line_length) {
+            *out++ = '\n';
+            *out++ = e0[t1];
+            *out++ = e1[(t1 & 0x03) << 4];
+            *out++ = '=';
+            *out++ = '=';
+          } else if (line_offset + 1 == line_length) {
+            *out++ = e0[t1];
+            *out++ = '\n';
+            *out++ = e1[(t1 & 0x03) << 4];
+            *out++ = '=';
+            *out++ = '=';
+          } else if (line_offset + 2 == line_length) {
+            *out++ = e0[t1];
+            *out++ = e1[(t1 & 0x03) << 4];
+            *out++ = '\n';
+            *out++ = '=';
+            *out++ = '=';
+          } else if (line_offset + 3 == line_length) {
+            *out++ = e0[t1];
+            *out++ = e1[(t1 & 0x03) << 4];
+            *out++ = '=';
+            *out++ = '\n';
+            *out++ = '=';
+          }
+        } else {
+          *out++ = e0[t1];
+          *out++ = e1[(t1 & 0x03) << 4];
+          *out++ = '=';
+          *out++ = '=';
+        }
+      } else {
+        if (line_offset + 2 >= line_length) {
+          if (line_offset == line_length) {
+            *out++ = '\n';
+            *out++ = e0[uint8_t(src[i])];
+            *out++ = e1[(uint8_t(src[i]) & 0x03) << 4];
+          } else if (line_offset + 1 == line_length) {
+            *out++ = e0[uint8_t(src[i])];
+            *out++ = '\n';
+            *out++ = e1[(uint8_t(src[i]) & 0x03) << 4];
+          } else {
+            *out++ = e0[uint8_t(src[i])];
+            *out++ = e1[(uint8_t(src[i]) & 0x03) << 4];
+            // *out++ = '\n'; ==> no newline at the end of the output
+          }
+        } else {
+          *out++ = e0[uint8_t(src[i])];
+          *out++ = e1[(uint8_t(src[i]) & 0x03) << 4];
+        }
+      }
+    } else {
+      *out++ = e0[t1];
+      *out++ = e1[(t1 & 0x03) << 4];
+      if (use_padding) {
+        *out++ = '=';
+        *out++ = '=';
+      }
+    }
+    break;
+  default: /* case 2 */
+    t1 = uint8_t(src[i]);
+    t2 = uint8_t(src[i + 1]);
+    if simdutf_constexpr (use_lines) {
+      if (use_padding) {
+        if (line_offset + 3 >= line_length) {
+          if (line_offset == line_length) {
+            *out++ = '\n';
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+            *out++ = '=';
+          } else if (line_offset + 1 == line_length) {
+            *out++ = e0[t1];
+            *out++ = '\n';
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+            *out++ = '=';
+          } else if (line_offset + 2 == line_length) {
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = '\n';
+            *out++ = e2[(t2 & 0x0F) << 2];
+            *out++ = '=';
+          } else if (line_offset + 3 == line_length) {
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+            *out++ = '\n';
+            *out++ = '=';
+          }
+        } else {
+          *out++ = e0[t1];
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = e2[(t2 & 0x0F) << 2];
+          *out++ = '=';
+        }
+      } else {
+        if (line_offset + 3 >= line_length) {
+          if (line_offset == line_length) {
+            *out++ = '\n';
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+          } else if (line_offset + 1 == line_length) {
+            *out++ = e0[t1];
+            *out++ = '\n';
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+          } else if (line_offset + 2 == line_length) {
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = '\n';
+            *out++ = e2[(t2 & 0x0F) << 2];
+          } else {
+            *out++ = e0[t1];
+            *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+            *out++ = e2[(t2 & 0x0F) << 2];
+            // *out++ = '\n'; ==> no newline at the end of the output
+          }
+        } else {
+          *out++ = e0[t1];
+          *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+          *out++ = e2[(t2 & 0x0F) << 2];
+        }
+      }
+    } else {
+      *out++ = e0[t1];
+      *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)];
+      *out++ = e2[(t2 & 0x0F) << 2];
+      if (use_padding) {
+        *out++ = '=';
+      }
+    }
+  }
+  return (size_t)(out - dst);
+}
+
+// Returns the number of bytes written. The destination buffer must be large
+// enough. It will add padding (=) if needed.
+inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src,
+                                                     size_t srclen,
+                                                     base64_options options) {
+  return tail_encode_base64_impl(dst, src, srclen, options);
+}
+
+template <class InputPtr>
+simdutf_warn_unused simdutf_constexpr23 size_t
+maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept {
+  // We process the padding characters ('=') at the end to make sure
+  // that we return an exact result when the input has no ignorable characters
+  // (e.g., spaces).
+  size_t padding = 0;
+  if (length > 0) {
+    if (input[length - 1] == '=') {
+      padding++;
+      if (length > 1 && input[length - 2] == '=') {
+        padding++;
+      }
+    }
+  }
+  // The input is not otherwise processed for ignorable characters or
+  // validation, so that the function runs in constant time (very fast). In
+  // practice, base64 inputs without ignorable characters are common and the
+  // common case are line separated inputs with relatively long lines (e.g., 76
+  // characters) which leads this function to a slight (1%) overestimation of
+  // the output size.
+  //
+  // Of course, some inputs might contain an arbitrary number of spaces or
+  // newlines, which would make this function return a very pessimistic output
+  // size but systems that produce base64 outputs typically do not do that and
+  // if they do, they do not care much about minimizing memory usage.
+  //
+  // In specialized applications, users may know that their input is line
+  // separated, which can be checked very quickly by by iterating (e.g., over 76
+  // character chunks, looking for the linefeed characters only). We could
+  // provide a specialized function for that, but it is not clear that the added
+  // complexity is worth it for us.
+  //
+  size_t actual_length = length - padding;
+  if (actual_length % 4 <= 1) {
+    return actual_length / 4 * 3;
+  }
+  // if we have a valid input, then the remainder must be 2 or 3 adding one or
+  // two extra bytes.
+  return actual_length / 4 * 3 + (actual_length % 4) - 1;
+}
+
+template <typename char_type>
+simdutf_warn_unused simdutf_constexpr23 full_result
+base64_to_binary_details_impl(
+    const char_type *input, size_t length, char *output, base64_options options,
+    last_chunk_handling_options last_chunk_options) noexcept {
+  const bool ignore_garbage =
+      (options == base64_options::base64_url_accept_garbage) ||
+      (options == base64_options::base64_default_accept_garbage) ||
+      (options == base64_options::base64_default_or_url_accept_garbage);
+  auto ri = simdutf::scalar::base64::find_end(input, length, options);
+  size_t equallocation = ri.equallocation;
+  size_t equalsigns = ri.equalsigns;
+  length = ri.srclen;
+  size_t full_input_length = ri.full_input_length;
+  if (length == 0) {
+    if (!ignore_garbage && equalsigns > 0) {
+      return {INVALID_BASE64_CHARACTER, equallocation, 0};
+    }
+    return {SUCCESS, full_input_length, 0};
+  }
+  full_result r = scalar::base64::base64_tail_decode(
+      output, input, length, equalsigns, options, last_chunk_options);
+  r = scalar::base64::patch_tail_result(r, 0, 0, equallocation,
+                                        full_input_length, last_chunk_options);
+  if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS &&
+      equalsigns > 0 && !ignore_garbage) {
+    // additional checks
+    if ((r.output_count % 3 == 0) ||
+        ((r.output_count % 3) + 1 + equalsigns != 4)) {
+      return {INVALID_BASE64_CHARACTER, equallocation, r.output_count};
+    }
+  }
+  // When is_partial(last_chunk_options) is true, we must either end with
+  // the end of the stream (beyond whitespace) or right after a non-ignorable
+  // character or at the very beginning of the stream.
+  // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+  if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS &&
+      r.input_count < full_input_length) {
+    // First check if we can extend the input to the end of the stream
+    while (r.input_count < full_input_length &&
+           base64_ignorable(*(input + r.input_count), options)) {
+      r.input_count++;
+    }
+    // If we are still not at the end of the stream, then we must backtrack
+    // to the last non-ignorable character.
+    if (r.input_count < full_input_length) {
+      while (r.input_count > 0 &&
+             base64_ignorable(*(input + r.input_count - 1), options)) {
+        r.input_count--;
+      }
+    }
+  }
+  return r;
+}
+
+template <typename char_type>
+simdutf_constexpr23 simdutf_warn_unused full_result
+base64_to_binary_details_safe_impl(
+    const char_type *input, size_t length, char *output, size_t outlen,
+    base64_options options,
+    last_chunk_handling_options last_chunk_options) noexcept {
+  const bool ignore_garbage =
+      (options == base64_options::base64_url_accept_garbage) ||
+      (options == base64_options::base64_default_accept_garbage) ||
+      (options == base64_options::base64_default_or_url_accept_garbage);
+  auto ri = simdutf::scalar::base64::find_end(input, length, options);
+  size_t equallocation = ri.equallocation;
+  size_t equalsigns = ri.equalsigns;
+  length = ri.srclen;
+  size_t full_input_length = ri.full_input_length;
+  if (length == 0) {
+    if (!ignore_garbage && equalsigns > 0) {
+      return {INVALID_BASE64_CHARACTER, equallocation, 0};
+    }
+    return {SUCCESS, full_input_length, 0};
+  }
+  full_result r = scalar::base64::base64_tail_decode_safe(
+      output, outlen, input, length, equalsigns, options, last_chunk_options);
+  r = scalar::base64::patch_tail_result(r, 0, 0, equallocation,
+                                        full_input_length, last_chunk_options);
+  if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS &&
+      equalsigns > 0 && !ignore_garbage) {
+    // additional checks
+    if ((r.output_count % 3 == 0) ||
+        ((r.output_count % 3) + 1 + equalsigns != 4)) {
+      return {INVALID_BASE64_CHARACTER, equallocation, r.output_count};
+    }
+  }
+
+  // When is_partial(last_chunk_options) is true, we must either end with
+  // the end of the stream (beyond whitespace) or right after a non-ignorable
+  // character or at the very beginning of the stream.
+  // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+  if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS &&
+      r.input_count < full_input_length) {
+    // First check if we can extend the input to the end of the stream
+    while (r.input_count < full_input_length &&
+           base64_ignorable(*(input + r.input_count), options)) {
+      r.input_count++;
+    }
+    // If we are still not at the end of the stream, then we must backtrack
+    // to the last non-ignorable character.
+    if (r.input_count < full_input_length) {
+      while (r.input_count > 0 &&
+             base64_ignorable(*(input + r.input_count - 1), options)) {
+        r.input_count--;
+      }
+    }
+  }
+  return r;
+}
+
+simdutf_warn_unused simdutf_constexpr23 size_t
+base64_length_from_binary(size_t length, base64_options options) noexcept {
+  // By default, we use padding if we are not using the URL variant.
+  // This is check with ((options & base64_url) == 0) which returns true if we
+  // are not using the URL variant. However, we also allow 'inversion' of the
+  // convention with the base64_reverse_padding option. If the
+  // base64_reverse_padding option is set, we use padding if we are using the
+  // URL variant, and we omit it if we are not using the URL variant. This is
+  // checked with
+  // ((options & base64_reverse_padding) == base64_reverse_padding).
+  bool use_padding =
+      ((options & base64_url) == 0) ^
+      ((options & base64_reverse_padding) == base64_reverse_padding);
+  if (!use_padding) {
+    return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0);
+  }
+  return (length + 2) / 3 *
+         4; // We use padding to make the length a multiple of 4.
+}
+
+simdutf_warn_unused simdutf_constexpr23 size_t
+base64_length_from_binary_with_lines(size_t length, base64_options options,
+                                     size_t line_length) noexcept {
+  if (length == 0) {
+    return 0;
+  }
+  size_t base64_length =
+      scalar::base64::base64_length_from_binary(length, options);
+  if (line_length < 4) {
+    line_length = 4;
+  }
+  size_t lines =
+      (base64_length + line_length - 1) / line_length; // number of lines
+  return base64_length + lines - 1;
+}
+
+// Return the length of the prefix that contains count base64 characters.
+// Thus, if count is 3, the function returns the length of the prefix
+// that contains 3 base64 characters.
+// The function returns (size_t)-1 if there is not enough base64 characters in
+// the input.
+template <typename char_type>
+simdutf_warn_unused size_t prefix_length(size_t count,
+                                         simdutf::base64_options options,
+                                         const char_type *input,
+                                         size_t length) noexcept {
+  size_t i = 0;
+  while (i < length && is_ignorable(input[i], options)) {
+    i++;
+  }
+  if (count == 0) {
+    return i; // duh!
+  }
+  for (; i < length; i++) {
+    if (is_ignorable(input[i], options)) {
+      continue;
+    }
+    // We have a base64 character or a padding character.
+    count--;
+    if (count == 0) {
+      return i + 1;
+    }
+  }
+  simdutf_log_assert(false, "You never get here");
+
+  return -1; // should never happen
+}
+
+} // namespace base64
+} // unnamed namespace
+} // namespace scalar
+} // namespace simdutf
+
+#endif
+/* end file include/simdutf/scalar/base64.h */
+
+namespace simdutf {
+
+  #if SIMDUTF_CPLUSPLUS17
+inline std::string_view to_string(base64_options options) {
+  switch (options) {
+  case base64_default:
+    return "base64_default";
+  case base64_url:
+    return "base64_url";
+  case base64_reverse_padding:
+    return "base64_reverse_padding";
+  case base64_url_with_padding:
+    return "base64_url_with_padding";
+  case base64_default_accept_garbage:
+    return "base64_default_accept_garbage";
+  case base64_url_accept_garbage:
+    return "base64_url_accept_garbage";
+  case base64_default_or_url:
+    return "base64_default_or_url";
+  case base64_default_or_url_accept_garbage:
+    return "base64_default_or_url_accept_garbage";
+  }
+  return "<unknown>";
+}
+  #endif // SIMDUTF_CPLUSPLUS17
+
+  #if SIMDUTF_CPLUSPLUS17
+inline std::string_view to_string(last_chunk_handling_options options) {
+  switch (options) {
+  case loose:
+    return "loose";
+  case strict:
+    return "strict";
+  case stop_before_partial:
+    return "stop_before_partial";
+  case only_full_chunks:
+    return "only_full_chunks";
+  }
+  return "<unknown>";
+}
+  #endif
+
+/**
+ * Provide the maximal binary length in bytes given the base64 input.
+ * As long as the input does not contain ignorable characters (e.g., ASCII
+ * spaces or linefeed characters), the result is exact. In particular, the
+ * function checks for padding characters.
+ *
+ * The function is fast (constant time). It checks up to two characters at
+ * the end of the string. The input is not otherwise validated or read.
+ *
+ * @param input         the base64 input to process
+ * @param length        the length of the base64 input in bytes
+ * @return maximum number of binary bytes
+ */
+simdutf_warn_unused size_t
+maximal_binary_length_from_base64(const char *input, size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+maximal_binary_length_from_base64(
+    const detail::input_span_of_byte_like auto &input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::maximal_binary_length_from_base64(
+        detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size());
+  } else
+    #endif
+  {
+    return maximal_binary_length_from_base64(
+        reinterpret_cast<const char *>(input.data()), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Provide the maximal binary length in bytes given the base64 input.
+ * As long as the input does not contain ignorable characters (e.g., ASCII
+ * spaces or linefeed characters), the result is exact. In particular, the
+ * function checks for padding characters.
+ *
+ * The function is fast (constant time). It checks up to two characters at
+ * the end of the string. The input is not otherwise validated or read.
+ *
+ * @param input         the base64 input to process, in ASCII stored as 16-bit
+ * units
+ * @param length        the length of the base64 input in 16-bit units
+ * @return maximal number of binary bytes
+ */
+simdutf_warn_unused size_t maximal_binary_length_from_base64(
+    const char16_t *input, size_t length) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+maximal_binary_length_from_base64(std::span<const char16_t> input) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::maximal_binary_length_from_base64(input.data(),
+                                                             input.size());
+  } else
+    #endif
+  {
+    return maximal_binary_length_from_base64(input.data(), input.size());
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert a base64 input to a binary output.
+ *
+ * This function follows the WHATWG forgiving-base64 format, which means that it
+ * will ignore any ASCII spaces in the input. You may provide a padded input
+ * (with one or two equal signs at the end) or an unpadded input (without any
+ * equal signs at the end).
+ *
+ * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+ *
+ * This function will fail in case of invalid input. When last_chunk_options =
+ * loose, there are two possible reasons for failure: the input contains a
+ * number of base64 characters that when divided by 4, leaves a single remainder
+ * character (BASE64_INPUT_REMAINDER), or the input contains a character that is
+ * not a valid base64 character (INVALID_BASE64_CHARACTER).
+ *
+ * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the
+ * input where the invalid character was found. When the error is
+ * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.
+ *
+ * The default option (simdutf::base64_default) expects the characters `+` and
+ * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the
+ * characters `-` and `_` as part of its alphabet.
+ *
+ * The padding (`=`) is validated if present. There may be at most two padding
+ * characters at the end of the input. If there are any padding characters, the
+ * total number of characters (excluding spaces but including padding
+ * characters) must be divisible by four.
+ *
+ * You should call this function with a buffer that is at least
+ * maximal_binary_length_from_base64(input, length) bytes long. If you fail to
+ * provide that much space, the function may cause a buffer overflow.
+ *
+ * Advanced users may want to tailor how the last chunk is handled. By default,
+ * we use a loose (forgiving) approach but we also support a strict approach
+ * as well as a stop_before_partial approach, as per the following proposal:
+ *
+ * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+ *
+ * @param input         the base64 string to process
+ * @param length        the length of the string in bytes
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result (should be at least maximal_binary_length_from_base64(input, length)
+ * bytes long).
+ * @param options       the base64 options to use, usually base64_default or
+ * base64_url, and base64_default by default.
+ * @param last_chunk_options the last chunk handling options,
+ * last_chunk_handling_options::loose by default
+ * but can also be last_chunk_handling_options::strict or
+ * last_chunk_handling_options::stop_before_partial.
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and either position of the error
+ * (in the input in bytes) if any, or the number of bytes written if successful.
+ */
+simdutf_warn_unused result base64_to_binary(
+    const char *input, size_t length, char *output,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options = loose) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+base64_to_binary(
+    const detail::input_span_of_byte_like auto &input,
+    detail::output_span_of_byte_like auto &&binary_output,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options = loose) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::base64_to_binary_details_impl(
+        input.data(), input.size(), binary_output.data(), options,
+        last_chunk_options);
+  } else
+    #endif
+  {
+    return base64_to_binary(reinterpret_cast<const char *>(input.data()),
+                            input.size(),
+                            reinterpret_cast<char *>(binary_output.data()),
+                            options, last_chunk_options);
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Provide the base64 length in bytes given the length of a binary input.
+ *
+ * @param length        the length of the input in bytes
+ * @return number of base64 bytes
+ */
+inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary(
+    size_t length, base64_options options = base64_default) noexcept {
+  return scalar::base64::base64_length_from_binary(length, options);
+}
+
+/**
+ * Provide the base64 length in bytes given the length of a binary input,
+ * taking into account line breaks.
+ *
+ * @param length        the length of the input in bytes
+ * @param line_length   the length of lines, must be at least 4 (otherwise it is
+ * interpreted as 4),
+ * @return number of base64 bytes
+ */
+inline simdutf_warn_unused simdutf_constexpr23 size_t
+base64_length_from_binary_with_lines(
+    size_t length, base64_options options = base64_default,
+    size_t line_length = default_line_length) noexcept {
+  return scalar::base64::base64_length_from_binary_with_lines(length, options,
+                                                              line_length);
+}
+
+/**
+ * Convert a binary input to a base64 output.
+ *
+ * The default option (simdutf::base64_default) uses the characters `+` and `/`
+ * as part of its alphabet. Further, it adds padding (`=`) at the end of the
+ * output to ensure that the output length is a multiple of four.
+ *
+ * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part
+ * of its alphabet. No padding is added at the end of the output.
+ *
+ * This function always succeeds.
+ *
+ * @param input         the binary to process
+ * @param length        the length of the input in bytes
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result (should be at least base64_length_from_binary(length) bytes long)
+ * @param options       the base64 options to use, can be base64_default or
+ * base64_url, is base64_default by default.
+ * @return number of written bytes, will be equal to
+ * base64_length_from_binary(length, options)
+ */
+size_t binary_to_base64(const char *input, size_t length, char *output,
+                        base64_options options = base64_default) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+binary_to_base64(const detail::input_span_of_byte_like auto &input,
+                 detail::output_span_of_byte_like auto &&binary_output,
+                 base64_options options = base64_default) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::tail_encode_base64(
+        binary_output.data(), input.data(), input.size(), options);
+  } else
+    #endif
+  {
+    return binary_to_base64(
+        reinterpret_cast<const char *>(input.data()), input.size(),
+        reinterpret_cast<char *>(binary_output.data()), options);
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Convert a binary input to a base64 output with line breaks.
+ *
+ * The default option (simdutf::base64_default) uses the characters `+` and `/`
+ * as part of its alphabet. Further, it adds padding (`=`) at the end of the
+ * output to ensure that the output length is a multiple of four.
+ *
+ * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part
+ * of its alphabet. No padding is added at the end of the output.
+ *
+ * This function always succeeds.
+ *
+ * @param input         the binary to process
+ * @param length        the length of the input in bytes
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result (should be at least base64_length_from_binary_with_lines(length,
+ * options, line_length) bytes long)
+ * @param line_length   the length of lines, must be at least 4 (otherwise it is
+ * interpreted as 4),
+ * @param options       the base64 options to use, can be base64_default or
+ * base64_url, is base64_default by default.
+ * @return number of written bytes, will be equal to
+ * base64_length_from_binary_with_lines(length, options)
+ */
+size_t
+binary_to_base64_with_lines(const char *input, size_t length, char *output,
+                            size_t line_length = simdutf::default_line_length,
+                            base64_options options = base64_default) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t
+binary_to_base64_with_lines(
+    const detail::input_span_of_byte_like auto &input,
+    detail::output_span_of_byte_like auto &&binary_output,
+    size_t line_length = simdutf::default_line_length,
+    base64_options options = base64_default) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::tail_encode_base64_impl<true>(
+        binary_output.data(), input.data(), input.size(), options, line_length);
+  } else
+    #endif
+  {
+    return binary_to_base64_with_lines(
+        reinterpret_cast<const char *>(input.data()), input.size(),
+        reinterpret_cast<char *>(binary_output.data()), line_length, options);
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+  #if SIMDUTF_ATOMIC_REF
+/**
+ * Convert a binary input to a base64 output, using atomic accesses.
+ * This function comes with a potentially significant performance
+ * penalty, but it may be useful in some cases where the input
+ * buffers are shared between threads, to avoid undefined
+ * behavior in case of data races.
+ *
+ * The function is for advanced users. Its main use case is when
+ * to silence sanitizer warnings. We have no documented use case
+ * where this function is actually necessary in terms of practical correctness.
+ *
+ * This function is only available when simdutf is compiled with
+ * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check
+ * the availability of this function by checking the macro
+ * SIMDUTF_ATOMIC_REF.
+ *
+ * The default option (simdutf::base64_default) uses the characters `+` and `/`
+ * as part of its alphabet. Further, it adds padding (`=`) at the end of the
+ * output to ensure that the output length is a multiple of four.
+ *
+ * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part
+ * of its alphabet. No padding is added at the end of the output.
+ *
+ * This function always succeeds.
+ *
+ * This function is considered experimental. It is not tested by default
+ * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested.
+ * It is not documented in the public API documentation (README). It is
+ * offered on a best effort basis. We rely on the community for further
+ * testing and feedback.
+ *
+ * @brief atomic_binary_to_base64
+ * @param input         the binary to process
+ * @param length        the length of the input in bytes
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result (should be at least base64_length_from_binary(length) bytes long)
+ * @param options       the base64 options to use, can be base64_default or
+ * base64_url, is base64_default by default.
+ * @return number of written bytes, will be equal to
+ * base64_length_from_binary(length, options)
+ */
+size_t
+atomic_binary_to_base64(const char *input, size_t length, char *output,
+                        base64_options options = base64_default) noexcept;
+    #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused size_t
+atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input,
+                        detail::output_span_of_byte_like auto &&binary_output,
+                        base64_options options = base64_default) noexcept {
+  return atomic_binary_to_base64(
+      reinterpret_cast<const char *>(input.data()), input.size(),
+      reinterpret_cast<char *>(binary_output.data()), options);
+}
+    #endif // SIMDUTF_SPAN
+  #endif   // SIMDUTF_ATOMIC_REF
+
+/**
+ * Convert a base64 input to a binary output.
+ *
+ * This function follows the WHATWG forgiving-base64 format, which means that it
+ * will ignore any ASCII spaces in the input. You may provide a padded input
+ * (with one or two equal signs at the end) or an unpadded input (without any
+ * equal signs at the end).
+ *
+ * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+ *
+ * This function will fail in case of invalid input. When last_chunk_options =
+ * loose, there are two possible reasons for failure: the input contains a
+ * number of base64 characters that when divided by 4, leaves a single remainder
+ * character (BASE64_INPUT_REMAINDER), or the input contains a character that is
+ * not a valid base64 character (INVALID_BASE64_CHARACTER).
+ *
+ * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the
+ * input where the invalid character was found. When the error is
+ * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.
+ *
+ * The default option (simdutf::base64_default) expects the characters `+` and
+ * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the
+ * characters `-` and `_` as part of its alphabet.
+ *
+ * The padding (`=`) is validated if present. There may be at most two padding
+ * characters at the end of the input. If there are any padding characters, the
+ * total number of characters (excluding spaces but including padding
+ * characters) must be divisible by four.
+ *
+ * You should call this function with a buffer that is at least
+ * maximal_binary_length_from_base64(input, length) bytes long. If you fail
+ * to provide that much space, the function may cause a buffer overflow.
+ *
+ * Advanced users may want to tailor how the last chunk is handled. By default,
+ * we use a loose (forgiving) approach but we also support a strict approach
+ * as well as a stop_before_partial approach, as per the following proposal:
+ *
+ * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+ *
+ * @param input         the base64 string to process, in ASCII stored as 16-bit
+ * units
+ * @param length        the length of the string in 16-bit units
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result (should be at least maximal_binary_length_from_base64(input, length)
+ * bytes long).
+ * @param options       the base64 options to use, can be base64_default or
+ * base64_url, is base64_default by default.
+ * @param last_chunk_options the last chunk handling options,
+ * last_chunk_handling_options::loose by default
+ * but can also be last_chunk_handling_options::strict or
+ * last_chunk_handling_options::stop_before_partial.
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and position of the
+ * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number
+ * of bytes written if successful.
+ */
+simdutf_warn_unused result
+base64_to_binary(const char16_t *input, size_t length, char *output,
+                 base64_options options = base64_default,
+                 last_chunk_handling_options last_chunk_options =
+                     last_chunk_handling_options::loose) noexcept;
+  #if SIMDUTF_SPAN
+simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result
+base64_to_binary(
+    std::span<const char16_t> input,
+    detail::output_span_of_byte_like auto &&binary_output,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options = loose) noexcept {
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    return scalar::base64::base64_to_binary_details_impl(
+        input.data(), input.size(), binary_output.data(), options,
+        last_chunk_options);
+  } else
+    #endif
+  {
+    return base64_to_binary(input.data(), input.size(),
+                            reinterpret_cast<char *>(binary_output.data()),
+                            options, last_chunk_options);
+  }
+}
+  #endif // SIMDUTF_SPAN
+
+/**
+ * Check if a character is an ignorable base64 character.
+ * Checking a large input, character by character, is not computationally
+ * efficient.
+ *
+ * @param input         the character to check
+ * @param options       the base64 options to use, is base64_default by default.
+ * @return true if the character is an ignorable base64 character, false
+ * otherwise.
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_ignorable(char input, base64_options options = base64_default) noexcept {
+  return scalar::base64::is_ignorable(input, options);
+}
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_ignorable(char16_t input,
+                 base64_options options = base64_default) noexcept {
+  return scalar::base64::is_ignorable(input, options);
+}
+
+/**
+ * Check if a character is a valid base64 character.
+ * Checking a large input, character by character, is not computationally
+ * efficient.
+ * Note that padding characters are not considered valid base64 characters in
+ * this context, nor are spaces.
+ *
+ * @param input         the character to check
+ * @param options       the base64 options to use, is base64_default by default.
+ * @return true if the character is a base64 character, false otherwise.
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_valid(char input, base64_options options = base64_default) noexcept {
+  return scalar::base64::is_base64(input, options);
+}
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_valid(char16_t input, base64_options options = base64_default) noexcept {
+  return scalar::base64::is_base64(input, options);
+}
+
+/**
+ * Check if a character is a valid base64 character or the padding character
+ * ('='). Checking a large input, character by character, is not computationally
+ * efficient.
+ *
+ * @param input         the character to check
+ * @param options       the base64 options to use, is base64_default by default.
+ * @return true if the character is a base64 character, false otherwise.
+ */
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_valid_or_padding(char input,
+                        base64_options options = base64_default) noexcept {
+  return scalar::base64::is_base64_or_padding(input, options);
+}
+simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool
+base64_valid_or_padding(char16_t input,
+                        base64_options options = base64_default) noexcept {
+  return scalar::base64::is_base64_or_padding(input, options);
+}
+
+/**
+ * Convert a base64 input to a binary output.
+ *
+ * This function follows the WHATWG forgiving-base64 format, which means that it
+ * will ignore any ASCII spaces in the input. You may provide a padded input
+ * (with one or two equal signs at the end) or an unpadded input (without any
+ * equal signs at the end).
+ *
+ * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+ *
+ * This function will fail in case of invalid input. When last_chunk_options =
+ * loose, there are three possible reasons for failure: the input contains a
+ * number of base64 characters that when divided by 4, leaves a single remainder
+ * character (BASE64_INPUT_REMAINDER), the input contains a character that is
+ * not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer
+ * is too small (OUTPUT_BUFFER_TOO_SMALL).
+ *
+ * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written
+ * and the number of units processed, see description of the parameters and
+ * returned value.
+ *
+ * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the
+ * input where the invalid character was found. When the error is
+ * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.
+ *
+ * The default option (simdutf::base64_default) expects the characters `+` and
+ * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the
+ * characters `-` and `_` as part of its alphabet.
+ *
+ * The padding (`=`) is validated if present. There may be at most two padding
+ * characters at the end of the input. If there are any padding characters, the
+ * total number of characters (excluding spaces but including padding
+ * characters) must be divisible by four.
+ *
+ * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected
+ * to discard the output unless the parameter decode_up_to_bad_char is set to
+ * true. In that case, the function will decode up to the first invalid
+ * character. Extra padding characters ('=') are considered invalid characters.
+ *
+ * Advanced users may want to tailor how the last chunk is handled. By default,
+ * we use a loose (forgiving) approach but we also support a strict approach
+ * as well as a stop_before_partial approach, as per the following proposal:
+ *
+ * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+ *
+ * @param input         the base64 string to process, in ASCII stored as 8-bit
+ * or 16-bit units
+ * @param length        the length of the string in 8-bit or 16-bit units.
+ * @param output        the pointer to a buffer that can hold the conversion
+ * result.
+ * @param outlen        the number of bytes that can be written in the output
+ * buffer. Upon return, it is modified to reflect how many bytes were written.
+ * @param options       the base64 options to use, can be base64_default or
+ * base64_url, is base64_default by default.
+ * @param last_chunk_options the last chunk handling options,
+ * last_chunk_handling_options::loose by default
+ * but can also be last_chunk_handling_options::strict or
+ * last_chunk_handling_options::stop_before_partial.
+ * @param decode_up_to_bad_char if true, the function will decode up to the
+ * first invalid character. By default (false), it is assumed that the output
+ * buffer is to be discarded. When there are multiple errors in the input,
+ * using decode_up_to_bad_char might trigger a different error.
+ * @return a result pair struct (of type simdutf::result containing the two
+ * fields error and count) with an error code and position of the
+ * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number
+ * of units processed if successful.
+ */
+simdutf_warn_unused result
+base64_to_binary_safe(const char *input, size_t length, char *output,
+                      size_t &outlen, base64_options options = base64_default,
+                      last_chunk_handling_options last_chunk_options =
+                          last_chunk_handling_options::loose,
+                      bool decode_up_to_bad_char = false) noexcept;
+// the span overload has moved to the bottom of the file
+
+simdutf_warn_unused result
+base64_to_binary_safe(const char16_t *input, size_t length, char *output,
+                      size_t &outlen, base64_options options = base64_default,
+                      last_chunk_handling_options last_chunk_options =
+                          last_chunk_handling_options::loose,
+                      bool decode_up_to_bad_char = false) noexcept;
+  // span overload moved to bottom of file
+
+  #if SIMDUTF_ATOMIC_REF
+/**
+ * Convert a base64 input to a binary output with a size limit and using atomic
+ * operations.
+ *
+ * Like `base64_to_binary_safe` but using atomic operations, this function is
+ * thread-safe for concurrent memory access, allowing the output
+ * buffers to be shared between threads without undefined behavior in case of
+ * data races.
+ *
+ * This function comes with a potentially significant performance penalty, but
+ * is useful when thread safety is needed during base64 decoding.
+ *
+ * This function is only available when simdutf is compiled with
+ * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check
+ * the availability of this function by checking the macro
+ * SIMDUTF_ATOMIC_REF.
+ *
+ * This function is considered experimental. It is not tested by default
+ * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested.
+ * It is not documented in the public API documentation (README). It is
+ * offered on a best effort basis. We rely on the community for further
+ * testing and feedback.
+ *
+ * @param input         the base64 input to decode
+ * @param length        the length of the input in bytes
+ * @param output        the pointer to buffer that can hold the conversion
+ * result
+ * @param outlen        the number of bytes that can be written in the output
+ * buffer. Upon return, it is modified to reflect how many bytes were written.
+ * @param options       the base64 options to use (default, url, etc.)
+ * @param last_chunk_options the last chunk handling options (loose, strict,
+ * stop_before_partial)
+ * @param decode_up_to_bad_char if true, the function will decode up to the
+ * first invalid character. By default (false), it is assumed that the output
+ * buffer is to be discarded. When there are multiple errors in the input,
+ * using decode_up_to_bad_char might trigger a different error.
+ * @return a result struct with an error code and count indicating error
+ * position or success
+ */
+simdutf_warn_unused result atomic_base64_to_binary_safe(
+    const char *input, size_t length, char *output, size_t &outlen,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options =
+        last_chunk_handling_options::loose,
+    bool decode_up_to_bad_char = false) noexcept;
+simdutf_warn_unused result atomic_base64_to_binary_safe(
+    const char16_t *input, size_t length, char *output, size_t &outlen,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options = loose,
+    bool decode_up_to_bad_char = false) noexcept;
+    #if SIMDUTF_SPAN
+/**
+ * @brief span overload
+ * @return a tuple of result and outlen
+ */
+simdutf_really_inline simdutf_warn_unused std::tuple<result, std::size_t>
+atomic_base64_to_binary_safe(
+    const detail::input_span_of_byte_like auto &binary_input,
+    detail::output_span_of_byte_like auto &&output,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options =
+        last_chunk_handling_options::loose,
+    bool decode_up_to_bad_char = false) noexcept {
+  size_t outlen = output.size();
+  auto ret = atomic_base64_to_binary_safe(
+      reinterpret_cast<const char *>(binary_input.data()), binary_input.size(),
+      reinterpret_cast<char *>(output.data()), outlen, options,
+      last_chunk_options, decode_up_to_bad_char);
+  return {ret, outlen};
+}
+/**
+ * @brief span overload
+ * @return a tuple of result and outlen
+ */
+simdutf_warn_unused std::tuple<result, std::size_t>
+atomic_base64_to_binary_safe(
+    std::span<const char16_t> base64_input,
+    detail::output_span_of_byte_like auto &&binary_output,
+    base64_options options = base64_default,
+    last_chunk_handling_options last_chunk_options = loose,
+    bool decode_up_to_bad_char = false) noexcept {
+  size_t outlen = binary_output.size();
+  auto ret = atomic_base64_to_binary_safe(
+      base64_input.data(), base64_input.size(),
+      reinterpret_cast<char *>(binary_output.data()), outlen, options,
+      last_chunk_options, decode_up_to_bad_char);
+  return {ret, outlen};
+}
+    #endif // SIMDUTF_SPAN
+  #endif   // SIMDUTF_ATOMIC_REF
+
+#endif // SIMDUTF_FEATURE_BASE64
+
+/**
+ * An implementation of simdutf for a particular CPU architecture.
+ *
+ * Also used to maintain the currently active implementation. The active
+ * implementation is automatically initialized on first use to the most advanced
+ * implementation supported by the host.
+ */
+class implementation {
+public:
+  /**
+   * The name of this implementation.
+   *
+   *     const implementation *impl = simdutf::active_implementation;
+   *     cout << "simdutf is optimized for " << impl->name() << "(" <<
+   * impl->description() << ")" << endl;
+   *
+   * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
+   */
+  virtual std::string name() const { return std::string(_name); }
+
+  /**
+   * The description of this implementation.
+   *
+   *     const implementation *impl = simdutf::active_implementation;
+   *     cout << "simdutf is optimized for " << impl->name() << "(" <<
+   * impl->description() << ")" << endl;
+   *
+   * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
+   */
+  virtual std::string description() const { return std::string(_description); }
+
+  /**
+   * The instruction sets this implementation is compiled against
+   * and the current CPU match. This function may poll the current CPU/system
+   * and should therefore not be called too often if performance is a concern.
+   *
+   *
+   * @return true if the implementation can be safely used on the current system
+   * (determined at runtime)
+   */
+  bool supported_by_runtime_system() const;
+
+#if SIMDUTF_FEATURE_DETECT_ENCODING
+  /**
+   * This function will try to detect the encoding
+   * @param input the string to identify
+   * @param length the length of the string in bytes.
+   * @return the encoding type detected
+   */
+  virtual encoding_type autodetect_encoding(const char *input,
+                                            size_t length) const noexcept;
+
+  /**
+   * This function will try to detect the possible encodings in one pass
+   * @param input the string to identify
+   * @param length the length of the string in bytes.
+   * @return the encoding type detected
+   */
+  virtual int detect_encodings(const char *input,
+                               size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_DETECT_ENCODING
+
+  /**
+   * @private For internal implementation use
+   *
+   * The instruction sets this implementation is compiled against.
+   *
+   * @return a mask of all required `internal::instruction_set::` values
+   */
+  virtual uint32_t required_instruction_sets() const {
+    return _required_instruction_sets;
+  }
+
+#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING
+  /**
+   * Validate the UTF-8 string.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the UTF-8 string to validate.
+   * @param len the length of the string in bytes.
+   * @return true if and only if the string is valid UTF-8.
+   */
+  simdutf_warn_unused virtual bool validate_utf8(const char *buf,
+                                                 size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF8
+  /**
+   * Validate the UTF-8 string and stop on errors.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the UTF-8 string to validate.
+   * @param len the length of the string in bytes.
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8
+
+#if SIMDUTF_FEATURE_ASCII
+  /**
+   * Validate the ASCII string.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the ASCII string to validate.
+   * @param len the length of the string in bytes.
+   * @return true if and only if the string is valid ASCII.
+   */
+  simdutf_warn_unused virtual bool
+  validate_ascii(const char *buf, size_t len) const noexcept = 0;
+
+  /**
+   * Validate the ASCII string and stop on error.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the ASCII string to validate.
+   * @param len the length of the string in bytes.
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0;
+
+#endif // SIMDUTF_FEATURE_ASCII
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII
+  /**
+   * Validate the ASCII string as a UTF-16BE sequence.
+   * An UTF-16 sequence is considered an ASCII sequence
+   * if it could be converted to an ASCII string losslessly.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the UTF-16BE string to validate.
+   * @param len the length of the string in bytes.
+   * @return true if and only if the string is valid ASCII.
+   */
+  simdutf_warn_unused virtual bool
+  validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept = 0;
+
+  /**
+   * Validate the ASCII string as a UTF-16LE sequence.
+   * An UTF-16 sequence is considered an ASCII sequence
+   * if it could be converted to an ASCII string losslessly.
+   *
+   * Overridden by each implementation.
+   *
+   * @param buf the UTF-16LE string to validate.
+   * @param len the length of the string in bytes.
+   * @return true if and only if the string is valid ASCII.
+   */
+  simdutf_warn_unused virtual bool
+  validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII
+
+#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING
+  /**
+   * Validate the UTF-16LE string.This function may be best when you expect
+   * the input to be almost always valid. Otherwise, consider using
+   * validate_utf16le_with_errors.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-16LE string to validate.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @return true if and only if the string is valid UTF-16LE.
+   */
+  simdutf_warn_unused virtual bool
+  validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF16
+  /**
+   * Validate the UTF-16BE string. This function may be best when you expect
+   * the input to be almost always valid. Otherwise, consider using
+   * validate_utf16be_with_errors.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-16BE string to validate.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @return true if and only if the string is valid UTF-16BE.
+   */
+  simdutf_warn_unused virtual bool
+  validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0;
+
+  /**
+   * Validate the UTF-16LE string and stop on error.  It might be faster than
+   * validate_utf16le when an error is expected to occur early.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-16LE string to validate.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  validate_utf16le_with_errors(const char16_t *buf,
+                               size_t len) const noexcept = 0;
+
+  /**
+   * Validate the UTF-16BE string and stop on error. It might be faster than
+   * validate_utf16be when an error is expected to occur early.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-16BE string to validate.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  validate_utf16be_with_errors(const char16_t *buf,
+                               size_t len) const noexcept = 0;
+  /**
+   * Copies the UTF-16LE string while replacing mismatched surrogates with the
+   * Unicode replacement character U+FFFD. We allow the input and output to be
+   * the same buffer so that the correction is done in-place.
+   *
+   * Overridden by each implementation.
+   *
+   * @param input the UTF-16LE string to correct.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @param output the output buffer.
+   */
+  virtual void to_well_formed_utf16le(const char16_t *input, size_t len,
+                                      char16_t *output) const noexcept = 0;
+  /**
+   * Copies the UTF-16BE string while replacing mismatched surrogates with the
+   * Unicode replacement character U+FFFD. We allow the input and output to be
+   * the same buffer so that the correction is done in-place.
+   *
+   * Overridden by each implementation.
+   *
+   * @param input the UTF-16BE string to correct.
+   * @param len the length of the string in number of 2-byte code units
+   * (char16_t).
+   * @param output the output buffer.
+   */
+  virtual void to_well_formed_utf16be(const char16_t *input, size_t len,
+                                      char16_t *output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING
+  /**
+   * Validate the UTF-32 string.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-32 string to validate.
+   * @param len the length of the string in number of 4-byte code units
+   * (char32_t).
+   * @return true if and only if the string is valid UTF-32.
+   */
+  simdutf_warn_unused virtual bool
+  validate_utf32(const char32_t *buf, size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING
+
+#if SIMDUTF_FEATURE_UTF32
+  /**
+   * Validate the UTF-32 string and stop on error.
+   *
+   * Overridden by each implementation.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param buf the UTF-32 string to validate.
+   * @param len the length of the string in number of 4-byte code units
+   * (char32_t).
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  validate_utf32_with_errors(const char32_t *buf,
+                             size_t len) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert Latin1 string into UTF-8 string.
+   *
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the Latin1 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf8_output  the pointer to buffer that can hold conversion result
+   * @return the number of written char; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_latin1_to_utf8(const char *input, size_t length,
+                         char *utf8_output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert possibly Latin1 string into UTF-16LE string.
+   *
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the Latin1  string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_latin1_to_utf16le(const char *input, size_t length,
+                            char16_t *utf16_output) const noexcept = 0;
+
+  /**
+   * Convert Latin1 string into UTF-16BE string.
+   *
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the Latin1 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_latin1_to_utf16be(const char *input, size_t length,
+                            char16_t *utf16_output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert Latin1 string into UTF-32 string.
+   *
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the Latin1 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf32_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char32_t; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_latin1_to_utf32(const char *input, size_t length,
+                          char32_t *utf32_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert possibly broken UTF-8 string into latin1 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param latin1_output  the pointer to buffer that can hold conversion result
+   * @return the number of written char; 0 if the input was not valid UTF-8
+   * string or if it cannot be represented as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf8_to_latin1(const char *input, size_t length,
+                         char *latin1_output) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-8 string into latin1 string with errors.
+   * If the string cannot be represented as Latin1, an error
+   * code is returned.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param latin1_output  the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf8_to_latin1_with_errors(const char *input, size_t length,
+                                     char *latin1_output) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-8 string into latin1 string.
+   *
+   * This function assumes that the input string is valid UTF-8 and that it can
+   * be represented as Latin1. If you violate this assumption, the result is
+   * implementation defined and may include system-dependent behavior such as
+   * crashes.
+   *
+   * This function is for expert users only and not part of our public API. Use
+   * convert_utf8_to_latin1 instead.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param latin1_output  the pointer to buffer that can hold conversion result
+   * @return the number of written char; 0 if the input was not valid UTF-8
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf8_to_latin1(const char *input, size_t length,
+                               char *latin1_output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+  /**
+   * Convert possibly broken UTF-8 string into UTF-16LE string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t; 0 if the input was not valid UTF-8
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf8_to_utf16le(const char *input, size_t length,
+                          char16_t *utf16_output) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-8 string into UTF-16BE string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t; 0 if the input was not valid UTF-8
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf8_to_utf16be(const char *input, size_t length,
+                          char16_t *utf16_output) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-8 string into UTF-16LE string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors(
+      const char *input, size_t length,
+      char16_t *utf16_output) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-8 string into UTF-16BE string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of code units validated
+   * if successful.
+   */
+  simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors(
+      const char *input, size_t length,
+      char16_t *utf16_output) const noexcept = 0;
+  /**
+   * Compute the number of bytes that this UTF-16LE string would require in
+   * UTF-8 format even when the UTF-16LE content contains mismatched
+   * surrogates that have to be replaced by the replacement character (0xFFFD).
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) where the count is the number of bytes required to
+   * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS
+   * or SURROGATE. The count is correct regardless of the error field.
+   * When SURROGATE is returned, it does not indicate an error in the case of
+   * this function: it indicates that at least one surrogate has been
+   * encountered: the surrogates may be matched or not (thus this function does
+   * not validate). If the returned error code is SUCCESS, then the input
+   * contains no surrogate, is in the Basic Multilingual Plane, and is
+   * necessarily valid.
+   */
+  virtual simdutf_warn_unused result utf8_length_from_utf16le_with_replacement(
+      const char16_t *input, size_t length) const noexcept = 0;
+
+  /**
+   * Compute the number of bytes that this UTF-16BE string would require in
+   * UTF-8 format even when the UTF-16BE content contains mismatched
+   * surrogates that have to be replaced by the replacement character (0xFFFD).
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) where the count is the number of bytes required to
+   * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS
+   * or SURROGATE. The count is correct regardless of the error field.
+   * When SURROGATE is returned, it does not indicate an error in the case of
+   * this function: it indicates that at least one surrogate has been
+   * encountered: the surrogates may be matched or not (thus this function does
+   * not validate). If the returned error code is SUCCESS, then the input
+   * contains no surrogate, is in the Basic Multilingual Plane, and is
+   * necessarily valid.
+   */
+  virtual simdutf_warn_unused result utf8_length_from_utf16be_with_replacement(
+      const char16_t *input, size_t length) const noexcept = 0;
+
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Convert possibly broken UTF-8 string into UTF-32 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf32_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t; 0 if the input was not valid UTF-8
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf8_to_utf32(const char *input, size_t length,
+                        char32_t *utf32_output) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-8 string into UTF-32 string and stop on error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf32_buffer  the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char32_t written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf8_to_utf32_with_errors(const char *input, size_t length,
+                                    char32_t *utf32_output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+  /**
+   * Convert valid UTF-8 string into UTF-16LE string.
+   *
+   * This function assumes that the input string is valid UTF-8.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf8_to_utf16le(const char *input, size_t length,
+                                char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-8 string into UTF-16BE string.
+   *
+   * This function assumes that the input string is valid UTF-8.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char16_t
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf8_to_utf16be(const char *input, size_t length,
+                                char16_t *utf16_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Convert valid UTF-8 string into UTF-32 string.
+   *
+   * This function assumes that the input string is valid UTF-8.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in bytes
+   * @param utf16_buffer  the pointer to buffer that can hold conversion result
+   * @return the number of written char32_t
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf8_to_utf32(const char *input, size_t length,
+                              char32_t *utf32_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+  /**
+   * Compute the number of 2-byte code units that this UTF-8 string would
+   * require in UTF-16LE format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-8 strings but in such cases the result is implementation defined.
+   *
+   * @param input         the UTF-8 string to process
+   * @param length        the length of the string in bytes
+   * @return the number of char16_t code units required to encode the UTF-8
+   * string as UTF-16LE
+   */
+  simdutf_warn_unused virtual size_t
+  utf16_length_from_utf8(const char *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Compute the number of 4-byte code units that this UTF-8 string would
+   * require in UTF-32 format.
+   *
+   * This function is equivalent to count_utf8. It is acceptable to pass invalid
+   * UTF-8 strings but in such cases the result is implementation defined.
+   *
+   * This function does not validate the input.
+   *
+   * @param input         the UTF-8 string to process
+   * @param length        the length of the string in bytes
+   * @return the number of char32_t code units required to encode the UTF-8
+   * string as UTF-32
+   */
+  simdutf_warn_unused virtual size_t
+  utf32_length_from_utf8(const char *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert possibly broken UTF-16LE string into Latin1 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return number of written code units; 0 if input is not a valid UTF-16LE
+   * string or if it cannot be represented as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16le_to_latin1(const char16_t *input, size_t length,
+                            char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into Latin1 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return number of written code units; 0 if input is not a valid UTF-16BE
+   * string or if it cannot be represented as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16be_to_latin1(const char16_t *input, size_t length,
+                            char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16LE string into Latin1 string.
+   * If the string cannot be represented as Latin1, an error
+   * is returned.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf16le_to_latin1_with_errors(const char16_t *input, size_t length,
+                                        char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into Latin1 string.
+   * If the string cannot be represented as Latin1, an error
+   * is returned.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf16be_to_latin1_with_errors(const char16_t *input, size_t length,
+                                        char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16LE string into Latin1 string.
+   *
+   * This function assumes that the input string is valid UTF-L16LE and that it
+   * can be represented as Latin1. If you violate this assumption, the result is
+   * implementation defined and may include system-dependent behavior such as
+   * crashes.
+   *
+   * This function is for expert users only and not part of our public API. Use
+   * convert_utf16le_to_latin1 instead.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16le_to_latin1(const char16_t *input, size_t length,
+                                  char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16BE string into Latin1 string.
+   *
+   * This function assumes that the input string is valid UTF16-BE and that it
+   * can be represented as Latin1. If you violate this assumption, the result is
+   * implementation defined and may include system-dependent behavior such as
+   * crashes.
+   *
+   * This function is for expert users only and not part of our public API. Use
+   * convert_utf16be_to_latin1 instead.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16be_to_latin1(const char16_t *input, size_t length,
+                                  char *latin1_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+  /**
+   * Convert possibly broken UTF-16LE string into UTF-8 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-16LE
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16le_to_utf8(const char16_t *input, size_t length,
+                          char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into UTF-8 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-16BE
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16be_to_utf8(const char16_t *input, size_t length,
+                          char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16LE string into UTF-8 string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf16le_to_utf8_with_errors(const char16_t *input, size_t length,
+                                      char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into UTF-8 string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length,
+                                      char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16LE string into UTF-8 string.
+   *
+   * This function assumes that the input string is valid UTF-16LE.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16le_to_utf8(const char16_t *input, size_t length,
+                                char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16BE string into UTF-8 string.
+   *
+   * This function assumes that the input string is valid UTF-16BE.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16be_to_utf8(const char16_t *input, size_t length,
+                                char *utf8_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Convert possibly broken UTF-16LE string into UTF-32 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-16LE
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16le_to_utf32(const char16_t *input, size_t length,
+                           char32_t *utf32_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into UTF-32 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-16BE
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf16be_to_utf32(const char16_t *input, size_t length,
+                           char32_t *utf32_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16LE string into UTF-32 string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char32_t written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors(
+      const char16_t *input, size_t length,
+      char32_t *utf32_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-16BE string into UTF-32 string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char32_t written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors(
+      const char16_t *input, size_t length,
+      char32_t *utf32_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16LE string into UTF-32 string.
+   *
+   * This function assumes that the input string is valid UTF-16LE.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16le_to_utf32(const char16_t *input, size_t length,
+                                 char32_t *utf32_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-16LE string into UTF-32BE string.
+   *
+   * This function assumes that the input string is valid UTF-16BE.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param utf32_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf16be_to_utf32(const char16_t *input, size_t length,
+                                 char32_t *utf32_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+  /**
+   * Compute the number of bytes that this UTF-16LE string would require in
+   * UTF-8 format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-16 strings but in such cases the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16LE string as UTF-8
+   */
+  simdutf_warn_unused virtual size_t
+  utf8_length_from_utf16le(const char16_t *input,
+                           size_t length) const noexcept = 0;
+
+  /**
+   * Compute the number of bytes that this UTF-16BE string would require in
+   * UTF-8 format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-16 strings but in such cases the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16BE string as UTF-8
+   */
+  simdutf_warn_unused virtual size_t
+  utf8_length_from_utf16be(const char16_t *input,
+                           size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert possibly broken UTF-32 string into Latin1 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return number of written code units; 0 if input is not a valid UTF-32
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf32_to_latin1(const char32_t *input, size_t length,
+                          char *latin1_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Convert possibly broken UTF-32 string into Latin1 string and stop on error.
+   * If the string cannot be represented as Latin1, an error is returned.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param latin1_buffer   the pointer to buffer that can hold conversion
+   * result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf32_to_latin1_with_errors(const char32_t *input, size_t length,
+                                      char *latin1_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-32 string into Latin1 string.
+   *
+   * This function assumes that the input string is valid UTF-32 and can be
+   * represented as Latin1. If you violate this assumption, the result is
+   * implementation defined and may include system-dependent behavior such as
+   * crashes.
+   *
+   * This function is for expert users only and not part of our public API. Use
+   * convert_utf32_to_latin1 instead.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param latin1_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf32_to_latin1(const char32_t *input, size_t length,
+                                char *latin1_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Convert possibly broken UTF-32 string into UTF-8 string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-32
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf32_to_utf8(const char32_t *input, size_t length,
+                        char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-32 string into UTF-8 string and stop on error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf8_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  convert_utf32_to_utf8_with_errors(const char32_t *input, size_t length,
+                                    char *utf8_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-32 string into UTF-8 string.
+   *
+   * This function assumes that the input string is valid UTF-32.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf8_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf32_to_utf8(const char32_t *input, size_t length,
+                              char *utf8_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Return the number of bytes that this UTF-16 string would require in Latin1
+   * format.
+   *
+   *
+   * @param input         the UTF-16 string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16 string as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  utf16_length_from_latin1(size_t length) const noexcept {
+    return length;
+  }
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Convert possibly broken UTF-32 string into UTF-16LE string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-32
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf32_to_utf16le(const char32_t *input, size_t length,
+                           char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-32 string into UTF-16BE string.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to buffer that can hold conversion result
+   * @return number of written code units; 0 if input is not a valid UTF-32
+   * string
+   */
+  simdutf_warn_unused virtual size_t
+  convert_utf32_to_utf16be(const char32_t *input, size_t length,
+                           char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-32 string into UTF-16LE string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char16_t written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors(
+      const char32_t *input, size_t length,
+      char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert possibly broken UTF-32 string into UTF-16BE string and stop on
+   * error.
+   *
+   * During the conversion also validation of the input string is done.
+   * This function is suitable to work with inputs from untrusted sources.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to buffer that can hold conversion result
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in code units) if any, or the number of char16_t written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors(
+      const char32_t *input, size_t length,
+      char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-32 string into UTF-16LE string.
+   *
+   * This function assumes that the input string is valid UTF-32.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf32_to_utf16le(const char32_t *input, size_t length,
+                                 char16_t *utf16_buffer) const noexcept = 0;
+
+  /**
+   * Convert valid UTF-32 string into UTF-16BE string.
+   *
+   * This function assumes that the input string is valid UTF-32.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @param utf16_buffer   the pointer to a buffer that can hold the conversion
+   * result
+   * @return number of written code units; 0 if conversion is not possible
+   */
+  simdutf_warn_unused virtual size_t
+  convert_valid_utf32_to_utf16be(const char32_t *input, size_t length,
+                                 char16_t *utf16_buffer) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16
+  /**
+   * Change the endianness of the input. Can be used to go from UTF-16LE to
+   * UTF-16BE or from UTF-16BE to UTF-16LE.
+   *
+   * This function does not validate the input.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16 string to process
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result
+   */
+  virtual void change_endianness_utf16(const char16_t *input, size_t length,
+                                       char16_t *output) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Return the number of bytes that this Latin1 string would require in UTF-8
+   * format.
+   *
+   * @param input         the Latin1 string to convert
+   * @param length        the length of the string bytes
+   * @return the number of bytes required to encode the Latin1 string as UTF-8
+   */
+  simdutf_warn_unused virtual size_t
+  utf8_length_from_latin1(const char *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Compute the number of bytes that this UTF-32 string would require in UTF-8
+   * format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-32 strings but in such cases the result is implementation defined.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @return the number of bytes required to encode the UTF-32 string as UTF-8
+   */
+  simdutf_warn_unused virtual size_t
+  utf8_length_from_utf32(const char32_t *input,
+                         size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Compute the number of bytes that this UTF-32 string would require in Latin1
+   * format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-32 strings but in such cases the result is implementation defined.
+   *
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @return the number of bytes required to encode the UTF-32 string as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  latin1_length_from_utf32(size_t length) const noexcept {
+    return length;
+  }
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Compute the number of bytes that this UTF-8 string would require in Latin1
+   * format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-8 strings but in such cases the result is implementation defined.
+   *
+   * @param input         the UTF-8 string to convert
+   * @param length        the length of the string in byte
+   * @return the number of bytes required to encode the UTF-8 string as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  latin1_length_from_utf8(const char *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Compute the number of bytes that this UTF-16LE/BE string would require in
+   * Latin1 format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-16 strings but in such cases the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16LE string as
+   * Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  latin1_length_from_utf16(size_t length) const noexcept {
+    return length;
+  }
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Compute the number of two-byte code units that this UTF-32 string would
+   * require in UTF-16 format.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-32 strings but in such cases the result is implementation defined.
+   *
+   * @param input         the UTF-32 string to convert
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @return the number of bytes required to encode the UTF-32 string as UTF-16
+   */
+  simdutf_warn_unused virtual size_t
+  utf16_length_from_utf32(const char32_t *input,
+                          size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+  /**
+   * Return the number of bytes that this UTF-32 string would require in Latin1
+   * format.
+   *
+   * @param length        the length of the string in 4-byte code units
+   * (char32_t)
+   * @return the number of bytes required to encode the UTF-32 string as Latin1
+   */
+  simdutf_warn_unused virtual size_t
+  utf32_length_from_latin1(size_t length) const noexcept {
+    return length;
+  }
+#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1
+
+#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+  /**
+   * Compute the number of bytes that this UTF-16LE string would require in
+   * UTF-32 format.
+   *
+   * This function is equivalent to count_utf16le.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-16 strings but in such cases the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16LE string as
+   * UTF-32
+   */
+  simdutf_warn_unused virtual size_t
+  utf32_length_from_utf16le(const char16_t *input,
+                            size_t length) const noexcept = 0;
+
+  /**
+   * Compute the number of bytes that this UTF-16BE string would require in
+   * UTF-32 format.
+   *
+   * This function is equivalent to count_utf16be.
+   *
+   * This function does not validate the input. It is acceptable to pass invalid
+   * UTF-16 strings but in such cases the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to convert
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return the number of bytes required to encode the UTF-16BE string as
+   * UTF-32
+   */
+  simdutf_warn_unused virtual size_t
+  utf32_length_from_utf16be(const char16_t *input,
+                            size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32
+
+#if SIMDUTF_FEATURE_UTF16
+  /**
+   * Count the number of code points (characters) in the string assuming that
+   * it is valid.
+   *
+   * This function assumes that the input string is valid UTF-16LE.
+   * It is acceptable to pass invalid UTF-16 strings but in such cases
+   * the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16LE string to process
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return number of code points
+   */
+  simdutf_warn_unused virtual size_t
+  count_utf16le(const char16_t *input, size_t length) const noexcept = 0;
+
+  /**
+   * Count the number of code points (characters) in the string assuming that
+   * it is valid.
+   *
+   * This function assumes that the input string is valid UTF-16BE.
+   * It is acceptable to pass invalid UTF-16 strings but in such cases
+   * the result is implementation defined.
+   *
+   * This function is not BOM-aware.
+   *
+   * @param input         the UTF-16BE string to process
+   * @param length        the length of the string in 2-byte code units
+   * (char16_t)
+   * @return number of code points
+   */
+  simdutf_warn_unused virtual size_t
+  count_utf16be(const char16_t *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF16
+
+#if SIMDUTF_FEATURE_UTF8
+  /**
+   * Count the number of code points (characters) in the string assuming that
+   * it is valid.
+   *
+   * This function assumes that the input string is valid UTF-8.
+   * It is acceptable to pass invalid UTF-8 strings but in such cases
+   * the result is implementation defined.
+   *
+   * @param input         the UTF-8 string to process
+   * @param length        the length of the string in bytes
+   * @return number of code points
+   */
+  simdutf_warn_unused virtual size_t
+  count_utf8(const char *input, size_t length) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_UTF8
+
+#if SIMDUTF_FEATURE_BASE64
+  /**
+   * Provide the maximal binary length in bytes given the base64 input.
+   * As long as the input does not contain ignorable characters (e.g., ASCII
+   * spaces or linefeed characters), the result is exact. In particular, the
+   * function checks for padding characters.
+   *
+   * The function is fast (constant time). It checks up to two characters at
+   * the end of the string. The input is not otherwise validated or read..
+   *
+   * @param input         the base64 input to process
+   * @param length        the length of the base64 input in bytes
+   * @return maximal number of binary bytes
+   */
+  simdutf_warn_unused size_t maximal_binary_length_from_base64(
+      const char *input, size_t length) const noexcept;
+
+  /**
+   * Provide the maximal binary length in bytes given the base64 input.
+   * As long as the input does not contain ignorable characters (e.g., ASCII
+   * spaces or linefeed characters), the result is exact. In particular, the
+   * function checks for padding characters.
+   *
+   * The function is fast (constant time). It checks up to two characters at
+   * the end of the string. The input is not otherwise validated or read.
+   *
+   * @param input         the base64 input to process, in ASCII stored as 16-bit
+   * units
+   * @param length        the length of the base64 input in 16-bit units
+   * @return maximal number of binary bytes
+   */
+  simdutf_warn_unused size_t maximal_binary_length_from_base64(
+      const char16_t *input, size_t length) const noexcept;
+
+  /**
+   * Convert a base64 input to a binary output.
+   *
+   * This function follows the WHATWG forgiving-base64 format, which means that
+   * it will ignore any ASCII spaces in the input. You may provide a padded
+   * input (with one or two equal signs at the end) or an unpadded input
+   * (without any equal signs at the end).
+   *
+   * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+   *
+   * This function will fail in case of invalid input. When last_chunk_options =
+   * loose, there are two possible reasons for failure: the input contains a
+   * number of base64 characters that when divided by 4, leaves a single
+   * remainder character (BASE64_INPUT_REMAINDER), or the input contains a
+   * character that is not a valid base64 character (INVALID_BASE64_CHARACTER).
+   *
+   * You should call this function with a buffer that is at least
+   * maximal_binary_length_from_base64(input, length) bytes long. If you fail to
+   * provide that much space, the function may cause a buffer overflow.
+   *
+   * @param input         the base64 string to process
+   * @param length        the length of the string in bytes
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least maximal_binary_length_from_base64(input, length)
+   * bytes long).
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and either position of the error
+   * (in the input in bytes) if any, or the number of bytes written if
+   * successful.
+   */
+  simdutf_warn_unused virtual result
+  base64_to_binary(const char *input, size_t length, char *output,
+                   base64_options options = base64_default,
+                   last_chunk_handling_options last_chunk_options =
+                       last_chunk_handling_options::loose) const noexcept = 0;
+
+  /**
+   * Convert a base64 input to a binary output while returning more details
+   * than base64_to_binary.
+   *
+   * This function follows the WHATWG forgiving-base64 format, which means that
+   * it will ignore any ASCII spaces in the input. You may provide a padded
+   * input (with one or two equal signs at the end) or an unpadded input
+   * (without any equal signs at the end).
+   *
+   * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+   *
+   * This function will fail in case of invalid input. When last_chunk_options =
+   * loose, there are two possible reasons for failure: the input contains a
+   * number of base64 characters that when divided by 4, leaves a single
+   * remainder character (BASE64_INPUT_REMAINDER), or the input contains a
+   * character that is not a valid base64 character (INVALID_BASE64_CHARACTER).
+   *
+   * You should call this function with a buffer that is at least
+   * maximal_binary_length_from_base64(input, length) bytes long. If you fail to
+   * provide that much space, the function may cause a buffer overflow.
+   *
+   * @param input         the base64 string to process
+   * @param length        the length of the string in bytes
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least maximal_binary_length_from_base64(input, length)
+   * bytes long).
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return a full_result pair struct (of type simdutf::result containing the
+   * three fields error, input_count and output_count).
+   */
+  simdutf_warn_unused virtual full_result base64_to_binary_details(
+      const char *input, size_t length, char *output,
+      base64_options options = base64_default,
+      last_chunk_handling_options last_chunk_options =
+          last_chunk_handling_options::loose) const noexcept = 0;
+
+  /**
+   * Convert a base64 input to a binary output.
+   *
+   * This function follows the WHATWG forgiving-base64 format, which means that
+   * it will ignore any ASCII spaces in the input. You may provide a padded
+   * input (with one or two equal signs at the end) or an unpadded input
+   * (without any equal signs at the end).
+   *
+   * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+   *
+   * This function will fail in case of invalid input. When last_chunk_options =
+   * loose, there are two possible reasons for failure: the input contains a
+   * number of base64 characters that when divided by 4, leaves a single
+   * remainder character (BASE64_INPUT_REMAINDER), or the input contains a
+   * character that is not a valid base64 character (INVALID_BASE64_CHARACTER).
+   *
+   * You should call this function with a buffer that is at least
+   * maximal_binary_length_from_base64(input, length) bytes long. If you
+   * fail to provide that much space, the function may cause a buffer overflow.
+   *
+   * @param input         the base64 string to process, in ASCII stored as
+   * 16-bit units
+   * @param length        the length of the string in 16-bit units
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least maximal_binary_length_from_base64(input, length)
+   * bytes long).
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return a result pair struct (of type simdutf::result containing the two
+   * fields error and count) with an error code and position of the
+   * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the
+   * number of bytes written if successful.
+   */
+  simdutf_warn_unused virtual result
+  base64_to_binary(const char16_t *input, size_t length, char *output,
+                   base64_options options = base64_default,
+                   last_chunk_handling_options last_chunk_options =
+                       last_chunk_handling_options::loose) const noexcept = 0;
+
+  /**
+   * Convert a base64 input to a binary output while returning more details
+   * than base64_to_binary.
+   *
+   * This function follows the WHATWG forgiving-base64 format, which means that
+   * it will ignore any ASCII spaces in the input. You may provide a padded
+   * input (with one or two equal signs at the end) or an unpadded input
+   * (without any equal signs at the end).
+   *
+   * See https://infra.spec.whatwg.org/#forgiving-base64-decode
+   *
+   * This function will fail in case of invalid input. When last_chunk_options =
+   * loose, there are two possible reasons for failure: the input contains a
+   * number of base64 characters that when divided by 4, leaves a single
+   * remainder character (BASE64_INPUT_REMAINDER), or the input contains a
+   * character that is not a valid base64 character (INVALID_BASE64_CHARACTER).
+   *
+   * You should call this function with a buffer that is at least
+   * maximal_binary_length_from_base64(input, length) bytes long. If you fail to
+   * provide that much space, the function may cause a buffer overflow.
+   *
+   * @param input         the base64 string to process
+   * @param length        the length of the string in bytes
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least maximal_binary_length_from_base64(input, length)
+   * bytes long).
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return a full_result pair struct (of type simdutf::result containing the
+   * three fields error, input_count and output_count).
+   */
+  simdutf_warn_unused virtual full_result base64_to_binary_details(
+      const char16_t *input, size_t length, char *output,
+      base64_options options = base64_default,
+      last_chunk_handling_options last_chunk_options =
+          last_chunk_handling_options::loose) const noexcept = 0;
+
+  /**
+   * Provide the base64 length in bytes given the length of a binary input.
+   *
+   * @param length        the length of the input in bytes
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return number of base64 bytes
+   */
+  simdutf_warn_unused size_t base64_length_from_binary(
+      size_t length, base64_options options = base64_default) const noexcept;
+
+  /**
+   * Convert a binary input to a base64 output.
+   *
+   * The default option (simdutf::base64_default) uses the characters `+` and
+   * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of
+   * the output to ensure that the output length is a multiple of four.
+   *
+   * The URL option (simdutf::base64_url) uses the characters `-` and `_` as
+   * part of its alphabet. No padding is added at the end of the output.
+   *
+   * This function always succeeds.
+   *
+   * @param input         the binary to process
+   * @param length        the length of the input in bytes
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least base64_length_from_binary(length) bytes long)
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return number of written bytes, will be equal to
+   * base64_length_from_binary(length, options)
+   */
+  virtual size_t
+  binary_to_base64(const char *input, size_t length, char *output,
+                   base64_options options = base64_default) const noexcept = 0;
+
+  /**
+   * Convert a binary input to a base64 output with lines of given length.
+   * Lines are separated by a single linefeed character.
+   *
+   * The default option (simdutf::base64_default) uses the characters `+` and
+   * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of
+   * the output to ensure that the output length is a multiple of four.
+   *
+   * The URL option (simdutf::base64_url) uses the characters `-` and `_` as
+   * part of its alphabet. No padding is added at the end of the output.
+   *
+   * This function always succeeds.
+   *
+   * @param input         the binary to process
+   * @param length        the length of the input in bytes
+   * @param output        the pointer to a buffer that can hold the conversion
+   * result (should be at least base64_length_from_binary_with_lines(length,
+   * options, line_length) bytes long)
+   * @param line_length   the length of each line, values smaller than 4 are
+   * interpreted as 4
+   * @param options       the base64 options to use, can be base64_default or
+   * base64_url, is base64_default by default.
+   * @return number of written bytes, will be equal to
+   * base64_length_from_binary_with_lines(length, options, line_length)
+   */
+  virtual size_t binary_to_base64_with_lines(
+      const char *input, size_t length, char *output,
+      size_t line_length = simdutf::default_line_length,
+      base64_options options = base64_default) const noexcept = 0;
+
+  /**
+   * Find the first occurrence of a character in a string. If the character is
+   * not found, return a pointer to the end of the string.
+   * @param start        the start of the string
+   * @param end          the end of the string
+   * @param character    the character to find
+   * @return a pointer to the first occurrence of the character in the string,
+   * or a pointer to the end of the string if the character is not found.
+   *
+   */
+  virtual const char *find(const char *start, const char *end,
+                           char character) const noexcept = 0;
+  virtual const char16_t *find(const char16_t *start, const char16_t *end,
+                               char16_t character) const noexcept = 0;
+#endif // SIMDUTF_FEATURE_BASE64
+
+#ifdef SIMDUTF_INTERNAL_TESTS
+  // This method is exported only in developer mode, its purpose
+  // is to expose some internal test procedures from the given
+  // implementation and then use them through our standard test
+  // framework.
+  //
+  // Regular users should not use it, the tests of the public
+  // API are enough.
+
+  struct TestProcedure {
+    // display name
+    std::string name;
+
+    // procedure should return whether given test pass or not
+    void (*procedure)(const implementation &);
+  };
+
+  virtual std::vector<TestProcedure> internal_tests() const;
+#endif
+
+protected:
+  /** @private Construct an implementation with the given name and description.
+   * For subclasses. */
+  simdutf_really_inline implementation(const char *name,
+                                       const char *description,
+                                       uint32_t required_instruction_sets)
+      : _name(name), _description(description),
+        _required_instruction_sets(required_instruction_sets) {}
+
+protected:
+  ~implementation() = default;
+
+private:
+  /**
+   * The name of this implementation.
+   */
+  const char *_name;
+
+  /**
+   * The description of this implementation.
+   */
+  const char *_description;
+
+  /**
+   * Instruction sets required for this implementation.
+   */
+  const uint32_t _required_instruction_sets;
+};
+
+/** @private */
+namespace internal {
+
+/**
+ * The list of available implementations compiled into simdutf.
+ */
+class available_implementation_list {
+public:
+  /** Get the list of available implementations compiled into simdutf */
+  simdutf_really_inline available_implementation_list() {}
+  /** Number of implementations */
+  size_t size() const noexcept;
+  /** STL const begin() iterator */
+  const implementation *const *begin() const noexcept;
+  /** STL const end() iterator */
+  const implementation *const *end() const noexcept;
+
+  /**
+   * Get the implementation with the given name.
+   *
+   * Case sensitive.
+   *
+   *     const implementation *impl =
+   * simdutf::available_implementations["westmere"]; if (!impl) { exit(1); } if
+   * (!imp->supported_by_runtime_system()) { exit(1); }
+   *     simdutf::active_implementation = impl;
+   *
+   * @param name the implementation to find, e.g. "westmere", "haswell", "arm64"
+   * @return the implementation, or nullptr if the parse failed.
+   */
+  const implementation *operator[](const std::string &name) const noexcept {
+    for (const implementation *impl : *this) {
+      if (impl->name() == name) {
+        return impl;
+      }
+    }
+    return nullptr;
+  }
+
+  /**
+   * Detect the most advanced implementation supported by the current host.
+   *
+   * This is used to initialize the implementation on startup.
+   *
+   *     const implementation *impl =
+   * simdutf::available_implementation::detect_best_supported();
+   *     simdutf::active_implementation = impl;
+   *
+   * @return the most advanced supported implementation for the current host, or
+   * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no
+   * supported implementation. Will never return nullptr.
+   */
+  const implementation *detect_best_supported() const noexcept;
+};
+
+template <typename T> class atomic_ptr {
+public:
+  atomic_ptr(T *_ptr) : ptr{_ptr} {}
+
+#if defined(SIMDUTF_NO_THREADS)
+  operator const T *() const { return ptr; }
+  const T &operator*() const { return *ptr; }
+  const T *operator->() const { return ptr; }
+
+  operator T *() { return ptr; }
+  T &operator*() { return *ptr; }
+  T *operator->() { return ptr; }
+  atomic_ptr &operator=(T *_ptr) {
+    ptr = _ptr;
+    return *this;
+  }
+
+#else
+  operator const T *() const { return ptr.load(); }
+  const T &operator*() const { return *ptr; }
+  const T *operator->() const { return ptr.load(); }
+
+  operator T *() { return ptr.load(); }
+  T &operator*() { return *ptr; }
+  T *operator->() { return ptr.load(); }
+  atomic_ptr &operator=(T *_ptr) {
+    ptr = _ptr;
+    return *this;
+  }
+
+#endif
+
+private:
+#if defined(SIMDUTF_NO_THREADS)
+  T *ptr;
+#else
+  std::atomic<T *> ptr;
+#endif
+};
+
+class detect_best_supported_implementation_on_first_use;
+
+} // namespace internal
+
+/**
+ * The list of available implementations compiled into simdutf.
+ */
+extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list &
+get_available_implementations();
+
+/**
+ * The active implementation.
+ *
+ * Automatically initialized on first use to the most advanced implementation
+ * supported by this hardware.
+ */
+extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> &
+get_active_implementation();
+
+} // namespace simdutf
+
+#if SIMDUTF_FEATURE_BASE64
+  // this header is not part of the public api
+/* begin file include/simdutf/base64_implementation.h */
+#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H
+#define SIMDUTF_BASE64_IMPLEMENTATION_H
+
+// this is not part of the public api
+
+namespace simdutf {
+
+template <typename chartype>
+simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl(
+    const chartype *input, size_t length, char *output, size_t &outlen,
+    base64_options options,
+    last_chunk_handling_options last_chunk_options) noexcept {
+  const bool ignore_garbage = (options & base64_default_accept_garbage) != 0;
+  auto ri = simdutf::scalar::base64::find_end(input, length, options);
+  size_t equallocation = ri.equallocation;
+  size_t equalsigns = ri.equalsigns;
+  length = ri.srclen;
+  size_t full_input_length = ri.full_input_length;
+  (void)full_input_length;
+  if (length == 0) {
+    outlen = 0;
+    if (!ignore_garbage && equalsigns > 0) {
+      return {INVALID_BASE64_CHARACTER, equallocation};
+    }
+    return {SUCCESS, 0};
+  }
+
+  // The parameters of base64_tail_decode_safe are:
+  // - dst: the output buffer
+  // - outlen: the size of the output buffer
+  // - srcr: the input buffer
+  // - length: the size of the input buffer
+  // - padded_characters: the number of padding characters
+  // - options: the options for the base64 decoder
+  // - last_chunk_options: the options for the last chunk
+  // The function will return the number of bytes written to the output buffer
+  // and the number of bytes read from the input buffer.
+  // The function will also return an error code if the input buffer is not
+  // valid base64.
+  full_result r = scalar::base64::base64_tail_decode_safe(
+      output, outlen, input, length, equalsigns, options, last_chunk_options);
+  r = scalar::base64::patch_tail_result(r, 0, 0, equallocation,
+                                        full_input_length, last_chunk_options);
+  outlen = r.output_count;
+  if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS &&
+      equalsigns > 0) {
+    // additional checks
+    if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) {
+      r.error = error_code::INVALID_BASE64_CHARACTER;
+    }
+  }
+  return {r.error, r.input_count}; // we cannot return r itself because it gets
+                                   // converted to error/output_count
+}
+
+template <typename chartype>
+simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl(
+    const chartype *input, size_t length, char *output, size_t &outlen,
+    base64_options options,
+    last_chunk_handling_options last_chunk_handling_options,
+    bool decode_up_to_bad_char) noexcept {
+  static_assert(std::is_same<chartype, char>::value ||
+                    std::is_same<chartype, char16_t>::value,
+                "Only char and char16_t are supported.");
+  size_t remaining_input_length = length;
+  size_t remaining_output_length = outlen;
+  size_t input_position = 0;
+  size_t output_position = 0;
+
+  // We also do a first pass using the fast path to decode as much as possible
+  size_t safe_input = (std::min)(
+      remaining_input_length,
+      base64_length_from_binary(remaining_output_length / 3 * 3, options));
+  bool done_with_partial = (safe_input == remaining_input_length);
+  simdutf::full_result r;
+
+#if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    r = scalar::base64::base64_to_binary_details_impl(
+        input + input_position, safe_input, output + output_position, options,
+        done_with_partial
+            ? last_chunk_handling_options
+            : simdutf::last_chunk_handling_options::only_full_chunks);
+  } else
+#endif
+  {
+    r = get_active_implementation()->base64_to_binary_details(
+        input + input_position, safe_input, output + output_position, options,
+        done_with_partial
+            ? last_chunk_handling_options
+            : simdutf::last_chunk_handling_options::only_full_chunks);
+  }
+  simdutf_log_assert(r.input_count <= safe_input,
+                     "You should not read more than safe_input");
+  simdutf_log_assert(r.output_count <= remaining_output_length,
+                     "You should not write more than remaining_output_length");
+  // Technically redundant, but we want to be explicit about it.
+  input_position += r.input_count;
+  output_position += r.output_count;
+  remaining_input_length -= r.input_count;
+  remaining_output_length -= r.output_count;
+  if (r.error != simdutf::error_code::SUCCESS) {
+    // There is an error. We return.
+    if (decode_up_to_bad_char &&
+        r.error == error_code::INVALID_BASE64_CHARACTER) {
+      return slow_base64_to_binary_safe_impl(
+          input, length, output, outlen, options, last_chunk_handling_options);
+    }
+    outlen = output_position;
+    return {r.error, input_position};
+  }
+
+  if (done_with_partial) {
+    // We are done. We have decoded everything.
+    outlen = output_position;
+    return {simdutf::error_code::SUCCESS, input_position};
+  }
+  // We have decoded some data, but we still have some data to decode.
+  // We need to decode the rest of the input buffer.
+  r = simdutf::scalar::base64::base64_to_binary_details_safe_impl(
+      input + input_position, remaining_input_length, output + output_position,
+      remaining_output_length, options, last_chunk_handling_options);
+  input_position += r.input_count;
+  output_position += r.output_count;
+  remaining_input_length -= r.input_count;
+  remaining_output_length -= r.output_count;
+
+  if (r.error != simdutf::error_code::SUCCESS) {
+    // There is an error. We return.
+    if (decode_up_to_bad_char &&
+        r.error == error_code::INVALID_BASE64_CHARACTER) {
+      return slow_base64_to_binary_safe_impl(
+          input, length, output, outlen, options, last_chunk_handling_options);
+    }
+    outlen = output_position;
+    return {r.error, input_position};
+  }
+  if (input_position < length) {
+    // We cannot process the entire input in one go, so we need to
+    // process it in two steps: first the fast path, then the slow path.
+    // In some cases, the processing might 'eat up' trailing ignorable
+    // characters in the fast path, but that can be a problem.
+    // suppose we have just white space followed by a single base64 character.
+    // If we first process the white space with the fast path, it will
+    // eat all of it. But, by the JavaScript standard, we should consume
+    // no character. See
+    // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64
+    while (input_position > 0 &&
+           base64_ignorable(input[input_position - 1], options)) {
+      input_position--;
+    }
+  }
+  outlen = output_position;
+  return {simdutf::error_code::SUCCESS, input_position};
+}
+
+} // namespace simdutf
+#endif // SIMDUTF_BASE64_IMPLEMENTATION_H
+/* end file include/simdutf/base64_implementation.h */
+
+namespace simdutf {
+  #if SIMDUTF_SPAN
+/**
+ * @brief span overload
+ * @return a tuple of result and outlen
+ */
+simdutf_really_inline
+    simdutf_constexpr23 simdutf_warn_unused std::tuple<result, std::size_t>
+    base64_to_binary_safe(
+        const detail::input_span_of_byte_like auto &input,
+        detail::output_span_of_byte_like auto &&binary_output,
+        base64_options options = base64_default,
+        last_chunk_handling_options last_chunk_options = loose,
+        bool decode_up_to_bad_char = false) noexcept {
+  size_t outlen = binary_output.size();
+    #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    using CInput = std::decay_t<decltype(*input.data())>;
+    static_assert(std::is_same_v<CInput, char>,
+                  "sorry, the constexpr implementation is for now limited to "
+                  "input of type char");
+    using COutput = std::decay_t<decltype(*binary_output.data())>;
+    static_assert(std::is_same_v<COutput, char>,
+                  "sorry, the constexpr implementation is for now limited to "
+                  "output of type char");
+    auto r = base64_to_binary_safe_impl(
+        input.data(), input.size(), binary_output.data(), outlen, options,
+        last_chunk_options, decode_up_to_bad_char);
+    return {r, outlen};
+  } else
+    #endif
+  {
+    auto r = base64_to_binary_safe_impl<char>(
+        reinterpret_cast<const char *>(input.data()), input.size(),
+        reinterpret_cast<char *>(binary_output.data()), outlen, options,
+        last_chunk_options, decode_up_to_bad_char);
+    return {r, outlen};
+  }
+}
+
+    #if SIMDUTF_SPAN
+/**
+ * @brief span overload
+ * @return a tuple of result and outlen
+ */
+simdutf_really_inline
+    simdutf_warn_unused simdutf_constexpr23 std::tuple<result, std::size_t>
+    base64_to_binary_safe(
+        std::span<const char16_t> input,
+        detail::output_span_of_byte_like auto &&binary_output,
+        base64_options options = base64_default,
+        last_chunk_handling_options last_chunk_options = loose,
+        bool decode_up_to_bad_char = false) noexcept {
+  size_t outlen = binary_output.size();
+      #if SIMDUTF_CPLUSPLUS23
+  if consteval {
+    auto r = base64_to_binary_safe_impl(
+        input.data(), input.size(), binary_output.data(), outlen, options,
+        last_chunk_options, decode_up_to_bad_char);
+    return {r, outlen};
+  } else
+      #endif
+  {
+    auto r = base64_to_binary_safe(
+        input.data(), input.size(),
+        reinterpret_cast<char *>(binary_output.data()), outlen, options,
+        last_chunk_options, decode_up_to_bad_char);
+    return {r, outlen};
+  }
+}
+    #endif // SIMDUTF_SPAN
+
+  #endif // SIMDUTF_SPAN
+} // namespace simdutf
+
+#endif // SIMDUTF_FEATURE_BASE64
+
+#endif // SIMDUTF_IMPLEMENTATION_H
+/* end file include/simdutf/implementation.h */
+
+// Implementation-internal files (must be included before the implementations
+// themselves, to keep amalgamation working--otherwise, the first time a file is
+// included, it might be put inside the #ifdef
+// SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other
+// implementations can't compile unless that implementation is turned on).
+
+SIMDUTF_POP_DISABLE_WARNINGS
+
+#endif // SIMDUTF_H
+/* end file include/simdutf.h */
+ simdutf/simdutf_c.h view
@@ -0,0 +1,339 @@+/***
+ * simdutf_c.h.h - C API for simdutf
+ * This is currently experimental.
+ * We are committed to keeping the C API, but there might be mistakes in our
+ * implementation. Please report any issues you find.
+ */
+
+#ifndef SIMDUTF_C_H
+#define SIMDUTF_C_H
+
+#include <stddef.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __has_include
+  #if __has_include(<uchar.h>)
+    #include <uchar.h>
+  #else // __has_include(<uchar.h>)
+    #define char16_t uint16_t
+    #define char32_t uint32_t
+  #endif // __has_include(<uchar.h>)
+#else    // __has_include(<uchar.h>)
+  #define char16_t uint16_t
+  #define char32_t uint32_t
+#endif // __has_include
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* C-friendly subset of simdutf errors */
+typedef enum simdutf_error_code {
+  SIMDUTF_ERROR_SUCCESS = 0,
+  SIMDUTF_ERROR_HEADER_BITS,
+  SIMDUTF_ERROR_TOO_SHORT,
+  SIMDUTF_ERROR_TOO_LONG,
+  SIMDUTF_ERROR_OVERLONG,
+  SIMDUTF_ERROR_TOO_LARGE,
+  SIMDUTF_ERROR_SURROGATE,
+  SIMDUTF_ERROR_INVALID_BASE64_CHARACTER,
+  SIMDUTF_ERROR_BASE64_INPUT_REMAINDER,
+  SIMDUTF_ERROR_BASE64_EXTRA_BITS,
+  SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL,
+  SIMDUTF_ERROR_OTHER
+} simdutf_error_code;
+
+typedef struct simdutf_result {
+  simdutf_error_code error;
+  size_t count; /* position of error or number of code units validated */
+} simdutf_result;
+
+typedef enum simdutf_encoding_type {
+  SIMDUTF_ENCODING_UNSPECIFIED = 0,
+  SIMDUTF_ENCODING_UTF8 = 1,
+  SIMDUTF_ENCODING_UTF16_LE = 2,
+  SIMDUTF_ENCODING_UTF16_BE = 4,
+  SIMDUTF_ENCODING_UTF32_LE = 8,
+  SIMDUTF_ENCODING_UTF32_BE = 16
+} simdutf_encoding_type;
+
+/* Validate UTF-8: returns true iff input is valid UTF-8 */
+bool simdutf_validate_utf8(const char *buf, size_t len);
+
+/* Validate UTF-8 with detailed result */
+simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len);
+
+/* Encoding detection */
+simdutf_encoding_type simdutf_autodetect_encoding(const char *input,
+                                                  size_t length);
+int simdutf_detect_encodings(const char *input, size_t length);
+
+/* ASCII validation */
+bool simdutf_validate_ascii(const char *buf, size_t len);
+simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len);
+
+/* UTF-16 ASCII checks */
+bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len);
+bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len);
+bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len);
+
+/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */
+bool simdutf_validate_utf16(const char16_t *buf, size_t len);
+bool simdutf_validate_utf16le(const char16_t *buf, size_t len);
+bool simdutf_validate_utf16be(const char16_t *buf, size_t len);
+simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf,
+                                                  size_t len);
+simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf,
+                                                    size_t len);
+simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf,
+                                                    size_t len);
+
+bool simdutf_validate_utf32(const char32_t *buf, size_t len);
+simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf,
+                                                  size_t len);
+
+/* to_well_formed UTF-16 helpers */
+void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len,
+                                    char16_t *output);
+void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len,
+                                    char16_t *output);
+void simdutf_to_well_formed_utf16(const char16_t *input, size_t len,
+                                  char16_t *output);
+
+/* Counting */
+size_t simdutf_count_utf16(const char16_t *input, size_t length);
+size_t simdutf_count_utf16le(const char16_t *input, size_t length);
+size_t simdutf_count_utf16be(const char16_t *input, size_t length);
+size_t simdutf_count_utf8(const char *input, size_t length);
+
+/* Length estimators */
+size_t simdutf_utf8_length_from_latin1(const char *input, size_t length);
+size_t simdutf_latin1_length_from_utf8(const char *input, size_t length);
+size_t simdutf_latin1_length_from_utf16(size_t length);
+size_t simdutf_latin1_length_from_utf32(size_t length);
+size_t simdutf_utf16_length_from_utf8(const char *input, size_t length);
+size_t simdutf_utf32_length_from_utf8(const char *input, size_t length);
+size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length);
+simdutf_result
+simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input,
+                                                size_t length);
+size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length);
+size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length);
+simdutf_result
+simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input,
+                                                  size_t length);
+simdutf_result
+simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input,
+                                                  size_t length);
+
+/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */
+size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length,
+                                      char *output);
+size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length,
+                                           char *output, size_t utf8_len);
+size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length,
+                                         char16_t *output);
+size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length,
+                                         char16_t *output);
+size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length,
+                                       char32_t *output);
+
+size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length,
+                                      char *output);
+size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length,
+                                       char16_t *output);
+size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length,
+                                       char16_t *output);
+size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length,
+                                     char16_t *output);
+
+size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length,
+                                     char32_t *output);
+simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input,
+                                                          size_t length,
+                                                          char *output);
+simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input,
+                                                         size_t length,
+                                                         char16_t *output);
+simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input,
+                                                           size_t length,
+                                                           char16_t *output);
+simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input,
+                                                           size_t length,
+                                                           char16_t *output);
+simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input,
+                                                         size_t length,
+                                                         char32_t *output);
+
+/* Conversions assuming valid input */
+size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length,
+                                            char *output);
+size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length,
+                                             char16_t *output);
+size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length,
+                                             char16_t *output);
+size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length,
+                                           char32_t *output);
+
+/* UTF-16 -> UTF-8 and related conversions */
+size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length,
+                                     char *output);
+size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length,
+                                       char *output);
+size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length,
+                                       char *output);
+size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length,
+                                          char *output, size_t utf8_len);
+size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length,
+                                       char *output);
+size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length,
+                                         char *output);
+size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length,
+                                         char *output);
+simdutf_result
+simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input,
+                                            size_t length, char *output);
+simdutf_result
+simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input,
+                                              size_t length, char *output);
+simdutf_result
+simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input,
+                                              size_t length, char *output);
+
+simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input,
+                                                         size_t length,
+                                                         char *output);
+simdutf_result
+simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input,
+                                            size_t length, char *output);
+simdutf_result
+simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input,
+                                            size_t length, char *output);
+
+size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length,
+                                           char *output);
+size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input,
+                                             size_t length, char *output);
+size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input,
+                                               size_t length, char *output);
+size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input,
+                                               size_t length, char *output);
+
+size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input,
+                                             size_t length, char *output);
+size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input,
+                                             size_t length, char *output);
+
+/* UTF-16 <-> UTF-32 conversions */
+size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length,
+                                      char32_t *output);
+size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length,
+                                        char32_t *output);
+size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length,
+                                        char32_t *output);
+simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input,
+                                                          size_t length,
+                                                          char32_t *output);
+simdutf_result
+simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input,
+                                             size_t length, char32_t *output);
+simdutf_result
+simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input,
+                                             size_t length, char32_t *output);
+
+/* Valid UTF-16 conversions */
+size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input,
+                                            size_t length, char32_t *output);
+size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input,
+                                              size_t length, char32_t *output);
+size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input,
+                                              size_t length, char32_t *output);
+
+/* UTF-32 -> ... conversions */
+size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length,
+                                     char *output);
+simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input,
+                                                         size_t length,
+                                                         char *output);
+size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length,
+                                           char *output);
+
+size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length,
+                                      char16_t *output);
+size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length,
+                                        char16_t *output);
+size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length,
+                                        char16_t *output);
+simdutf_result
+simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input,
+                                            size_t length, char *output);
+
+/* --- Find helpers --- */
+const char *simdutf_find(const char *start, const char *end, char character);
+const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end,
+                                   char16_t character);
+
+/* --- Base64 enums and helpers --- */
+typedef enum simdutf_base64_options {
+  SIMDUTF_BASE64_DEFAULT = 0,
+  SIMDUTF_BASE64_URL = 1,
+  SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2,
+  SIMDUTF_BASE64_URL_WITH_PADDING = 3,
+  SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4,
+  SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5,
+  SIMDUTF_BASE64_DEFAULT_OR_URL = 8,
+  SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12
+} simdutf_base64_options;
+
+typedef enum simdutf_last_chunk_handling_options {
+  SIMDUTF_LAST_CHUNK_LOOSE = 0,
+  SIMDUTF_LAST_CHUNK_STRICT = 1,
+  SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2,
+  SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3
+} simdutf_last_chunk_handling_options;
+
+/* maximal binary length estimators */
+size_t simdutf_maximal_binary_length_from_base64(const char *input,
+                                                 size_t length);
+size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input,
+                                                       size_t length);
+
+/* base64 decoding/encoding */
+simdutf_result simdutf_base64_to_binary(
+    const char *input, size_t length, char *output,
+    simdutf_base64_options options,
+    simdutf_last_chunk_handling_options last_chunk_options);
+simdutf_result simdutf_base64_to_binary_utf16(
+    const char16_t *input, size_t length, char *output,
+    simdutf_base64_options options,
+    simdutf_last_chunk_handling_options last_chunk_options);
+
+size_t simdutf_base64_length_from_binary(size_t length,
+                                         simdutf_base64_options options);
+size_t simdutf_base64_length_from_binary_with_lines(
+    size_t length, simdutf_base64_options options, size_t line_length);
+
+size_t simdutf_binary_to_base64(const char *input, size_t length, char *output,
+                                simdutf_base64_options options);
+size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length,
+                                           char *output, size_t line_length,
+                                           simdutf_base64_options options);
+
+/* safe decoding that provides an in/out outlen parameter */
+simdutf_result simdutf_base64_to_binary_safe(
+    const char *input, size_t length, char *output, size_t *outlen,
+    simdutf_base64_options options,
+    simdutf_last_chunk_handling_options last_chunk_options,
+    bool decode_up_to_bad_char);
+simdutf_result simdutf_base64_to_binary_safe_utf16(
+    const char16_t *input, size_t length, char *output, size_t *outlen,
+    simdutf_base64_options options,
+    simdutf_last_chunk_handling_options last_chunk_options,
+    bool decode_up_to_bad_char);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* SIMDUTF_C_H */
+ src/Data/Text.hs view
@@ -0,0 +1,2276 @@+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-}
+{-# LANGUAGE TemplateHaskellQuotes #-}
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE UnliftedFFITypes #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE PartialTypeSignatures #-}
+{-# LANGUAGE PatternSynonyms #-}
+{-# LANGUAGE ViewPatterns #-}
+
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# OPTIONS_GHC -Wno-partial-type-signatures #-}
+
+-- |
+-- Module      : Data.Text
+-- Copyright   : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan,
+--               (c) 2009 Duncan Coutts,
+--               (c) 2008, 2009 Tom Harper
+--               (c) 2021 Andrew Lelechenko
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : GHC
+--
+-- A time and space-efficient implementation of Unicode text.
+-- Suitable for performance critical use, both in terms of large data
+-- quantities and high speed.
+--
+-- /Note/: Read below the synopsis for important notes on the use of
+-- this module.
+--
+-- This module is intended to be imported @qualified@, to avoid name
+-- clashes with "Prelude" functions, e.g.
+--
+-- > import qualified Data.Text as T
+--
+-- To use an extended and very rich family of functions for working
+-- with Unicode text (including normalization, regular expressions,
+-- non-standard encodings, text breaking, and locales), see the
+-- <http://hackage.haskell.org/package/text-icu text-icu package >.
+--
+
+module Data.Text
+    (
+    -- * Strict vs lazy types
+    -- $strict
+
+    -- * Acceptable data
+    -- $replacement
+
+    -- * Definition of character
+    -- $character_definition
+
+    -- * Fusion
+    -- $fusion
+
+    -- * Types
+      Text
+    , StrictText
+
+    -- * Creation and elimination
+    , pack
+    , unpack
+    , singleton
+    , empty
+
+    -- * Pattern matching
+    , pattern Empty
+    , pattern (:<)
+    , pattern (:>)
+
+    -- * Basic interface
+    , cons
+    , snoc
+    , append
+    , uncons
+    , unsnoc
+    , head
+    , last
+    , tail
+    , init
+    , null
+    , length
+    , compareLength
+
+    -- * Transformations
+    , map
+    , intercalate
+    , intersperse
+    , transpose
+    , reverse
+    , replace
+
+    -- ** Case conversion
+    -- $case
+    , toCaseFold
+    , toLower
+    , toUpper
+    , toTitle
+
+    -- ** Justification
+    , justifyLeft
+    , justifyRight
+    , center
+
+    -- * Folds
+    , foldl
+    , foldl'
+    , foldl1
+    , foldl1'
+    , foldr
+    , foldr'
+    , foldr1
+    , foldlM'
+
+    -- ** Special folds
+    , concat
+    , concatMap
+    , any
+    , all
+    , maximum
+    , minimum
+    , isAscii
+
+    -- * Construction
+
+    -- ** Scans
+    , scanl
+    , scanl1
+    , scanr
+    , scanr1
+
+    -- ** Accumulating maps
+    , mapAccumL
+    , mapAccumR
+
+    -- ** Generation and unfolding
+    , replicate
+    , unfoldr
+    , unfoldrN
+
+    -- * Substrings
+
+    -- ** Breaking strings
+    , take
+    , takeEnd
+    , drop
+    , dropEnd
+    , takeWhile
+    , takeWhileEnd
+    , dropWhile
+    , dropWhileEnd
+    , dropAround
+    , strip
+    , stripStart
+    , stripEnd
+    , splitAt
+    , breakOn
+    , breakOnEnd
+    , break
+    , span
+    , spanM
+    , spanEndM
+    , group
+    , groupBy
+    , inits
+    , initsNE
+    , tails
+    , tailsNE
+
+    -- ** Breaking into many substrings
+    -- $split
+    , splitOn
+    , split
+    , chunksOf
+
+    -- ** Breaking into lines and words
+    , lines
+    --, lines'
+    , words
+    , unlines
+    , unwords
+
+    -- * Predicates
+    , isPrefixOf
+    , isSuffixOf
+    , isInfixOf
+
+    -- ** View patterns
+    , stripPrefix
+    , stripSuffix
+    , commonPrefixes
+
+    -- * Searching
+    , filter
+    , breakOnAll
+    , find
+    , elem
+    , partition
+
+    -- , findSubstring
+
+    -- * Indexing
+    -- $index
+    , index
+    , findIndex
+    , count
+
+    -- * Zipping
+    , zip
+    , zipWith
+
+    -- * Showing values
+    , show
+
+    -- -* Ordered text
+    -- , sort
+
+    -- * Low level operations
+    , copy
+    , unpackCString#
+    , unpackCStringAscii#
+
+    , measureOff
+    ) where
+
+import Prelude (Char, Bool(..), Int, Maybe(..), String,
+                Eq, (==), (/=), Ord(..), Ordering(..), (++),
+                Monad(..), pure, Read(..), Show,
+                (&&), (||), (+), (-), (.), ($), ($!), (>>),
+                not, return, otherwise, quot)
+import Control.DeepSeq (NFData(rnf))
+#if defined(ASSERTS)
+import Control.Exception (assert)
+#endif
+import Data.Bits ((.&.))
+import qualified Data.Char as Char
+import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,
+                  Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))
+import Control.Monad (foldM)
+import Control.Monad.ST (ST, runST)
+import qualified Data.Text.Array as A
+import qualified Data.List as L hiding (head, tail)
+import qualified Data.List.NonEmpty as NonEmptyList
+import Data.Binary (Binary(get, put))
+import Data.Binary.Put (putBuilder)
+import Data.Monoid (Monoid(..))
+import Data.Semigroup (Semigroup(..))
+import Data.String (IsString(..))
+import Data.Text.Internal.ArrayUtils (memchr)
+import Data.Text.Internal.IsAscii (isAscii)
+import Data.Text.Internal.Reverse (reverse)
+import Data.Text.Internal.Measure (measure_off)
+import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4)
+import qualified Data.Text.Internal.Fusion as S
+import qualified Data.Text.Internal.Fusion.Common as S
+import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder)
+import Data.Text.Internal.Fusion (stream, unstream)
+import Data.Text.Internal.Private (span_)
+import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack)
+import Data.Text.Internal.Unsafe.Char (unsafeWrite)
+import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#)
+import qualified Prelude as P
+import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter,
+                         reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray)
+import Data.Text.Internal.Search (indices)
+import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_)
+#if defined(__HADDOCK__)
+import Data.ByteString (ByteString)
+import qualified Data.Text.Lazy as L
+#endif
+import Data.Word (Word8)
+import Foreign.C.Types
+import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)
+import qualified GHC.Exts as Exts
+import GHC.Int (Int8)
+import GHC.Stack (HasCallStack)
+#if __GLASGOW_HASKELL__ >= 914
+import qualified Language.Haskell.TH.Lift as TH
+#else
+import qualified Language.Haskell.TH.Lib as TH
+import qualified Language.Haskell.TH.Syntax as TH
+#endif
+import Text.Printf (PrintfArg, formatArg, formatString)
+import System.Posix.Types (CSsize(..))
+
+#if __GLASGOW_HASKELL__ >= 810
+import Data.Text.Foreign (asForeignPtr)
+import System.IO.Unsafe (unsafePerformIO)
+#endif
+
+-- $setup
+-- >>> :set -package transformers
+-- >>> import Control.Monad.Trans.State
+-- >>> import Data.Text
+-- >>> import qualified Data.Text as T
+-- >>> :seti -XOverloadedStrings
+
+-- $character_definition
+--
+-- This package uses the term /character/ to denote Unicode /code points/.
+--
+-- Note that this is not the same thing as a grapheme (e.g. a
+-- composition of code points that form one visual symbol). For
+-- instance, consider the grapheme \"&#x00e4;\". This symbol has two
+-- Unicode representations: a single code-point representation
+-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),
+-- and a two code point representation @U+0061@ (the \"@A@\" code
+-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).
+
+-- $strict
+--
+-- This package provides both strict and lazy 'Text' types.  The
+-- strict type is provided by the "Data.Text" module, while the lazy
+-- type is provided by the "Data.Text.Lazy" module. Internally, the
+-- lazy @Text@ type consists of a list of strict chunks.
+--
+-- The strict 'Text' type requires that an entire string fit into
+-- memory at once.  The lazy 'Data.Text.Lazy.Text' type is capable of
+-- streaming strings that are larger than memory using a small memory
+-- footprint.  In many cases, the overhead of chunked streaming makes
+-- the lazy 'Data.Text.Lazy.Text' type slower than its strict
+-- counterpart, but this is not always the case.  Sometimes, the time
+-- complexity of a function in one module may be different from the
+-- other, due to their differing internal structures.
+--
+-- Each module provides an almost identical API, with the main
+-- difference being that the strict module uses 'Int' values for
+-- lengths and counts, while the lazy module uses 'Data.Int.Int64'
+-- lengths.
+
+-- $replacement
+--
+-- A 'Text' value is a sequence of Unicode scalar values, as defined
+-- in
+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.
+-- As such, a 'Text' cannot contain values in the range U+D800 to
+-- U+DFFF inclusive. Haskell implementations admit all Unicode code
+-- points
+-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)
+-- as 'Char' values, including code points from this invalid range.
+-- This means that there are some 'Char' values
+-- (corresponding to 'Data.Char.Surrogate' category) that are not valid
+-- Unicode scalar values, and the functions in this module must handle
+-- those cases.
+--
+-- Within this module, many functions construct a 'Text' from one or
+-- more 'Char' values. Those functions will substitute 'Char' values
+-- that are not valid Unicode scalar values with the replacement
+-- character \"&#xfffd;\" (U+FFFD).  Functions that perform this
+-- inspection and replacement are documented with the phrase
+-- \"Performs replacement on invalid scalar values\". The functions replace
+-- invalid scalar values, instead of dropping them, as a security
+-- measure. For details, see
+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)
+
+-- $fusion
+--
+-- Starting from @text-1.3@ fusion is no longer implicit,
+-- and pipelines of transformations usually allocate intermediate 'Text' values.
+-- Users, who observe significant changes to performances,
+-- are encouraged to use fusion framework explicitly, employing
+-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".
+
+instance Eq Text where
+    Text arrA offA lenA == Text arrB offB lenB
+        | lenA == lenB = A.equal arrA offA arrB offB lenA
+        | otherwise    = False
+    {-# INLINE (==) #-}
+
+instance Ord Text where
+    compare = compareText
+
+instance Read Text where
+    readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]
+
+-- | @since 1.2.2.0
+--
+-- Beware: @stimes@ will crash if the given number does not fit into
+-- an @Int@.
+instance Semigroup Text where
+    (<>) = append
+
+    stimes howManyTimes
+      | howManyTimes < 0 = P.error "Data.Text.stimes: given number is negative!"
+      | otherwise =
+        let howManyTimesInt = P.fromIntegral howManyTimes :: Int
+        in  if P.fromIntegral howManyTimesInt == howManyTimes && howManyTimesInt >= 0
+            then replicate howManyTimesInt
+            else P.error "Data.Text.stimes: given number does not fit into an Int!"
+
+    sconcat = concat . NonEmptyList.toList
+
+instance Monoid Text where
+    mempty  = empty
+    mappend = (<>)
+    mconcat = concat
+
+-- | Performs replacement on invalid scalar values:
+--
+-- >>> :set -XOverloadedStrings
+-- >>> "\55555" :: Text
+-- "\65533"
+instance IsString Text where
+    fromString = pack
+
+-- | Performs replacement on invalid scalar values:
+--
+-- >>> :set -XOverloadedLists
+-- >>> ['\55555'] :: Text
+-- "\65533"
+--
+-- @since 1.2.0.0
+instance Exts.IsList Text where
+    type Item Text = Char
+    fromList       = pack
+    toList         = unpack
+
+instance NFData Text where rnf !_ = ()
+
+-- | @since 1.2.1.0
+instance Binary Text where
+    put t = do
+      -- This needs to be in sync with the Binary instance for ByteString
+      -- in the binary package.
+      put (lengthWord8 t)
+      putBuilder (encodeUtf8Builder t)
+    get   = do
+      bs <- get
+      case decodeUtf8' bs of
+        P.Left exn -> P.fail (P.show exn)
+        P.Right a -> P.return a
+
+-- | This instance preserves data abstraction at the cost of inefficiency.
+-- We omit reflection services for the sake of data abstraction.
+--
+-- This instance was created by copying the updated behavior of
+-- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you
+-- feel a mistake has been made, please feel free to submit
+-- improvements.
+--
+-- The original discussion is archived here:
+-- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? >
+--
+-- The followup discussion that changed the behavior of 'Data.Set.Set'
+-- and 'Data.Map.Map' is archived here:
+-- <https://mail.haskell.org/pipermail/libraries/2012-August/018366.html Proposal: Allow gunfold for Data.Map, ... >
+
+instance Data Text where
+  gfoldl f z txt = z pack `f` (unpack txt)
+  toConstr _ = packConstr
+  gunfold k z c = case constrIndex c of
+    1 -> k (z pack)
+    _ -> P.error "gunfold"
+  dataTypeOf _ = textDataType
+
+-- | @since 1.2.4.0
+instance TH.Lift Text where
+#if __GLASGOW_HASKELL__ >= 914
+  lift txt = do
+    let (ptr, len) = unsafePerformIO $ asForeignPtr txt
+    case len of
+        0 -> [| empty |]
+        _ ->
+          let
+            bytesQ = TH.liftAddrCompat ptr 0 (P.fromIntegral len)
+            lenQ = TH.liftIntCompat (P.fromIntegral len)
+          in [| unpackCStringLen# $bytesQ $lenQ |]
+#elif __GLASGOW_HASKELL__ >= 810
+  lift txt = do
+    let (ptr, len) = unsafePerformIO $ asForeignPtr txt
+    case len of
+        0 -> TH.varE 'empty
+        _ ->
+          let
+            bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len)
+            lenQ = liftInt (P.fromIntegral len)
+            liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n)))
+          in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ
+#else
+  lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack
+#endif
+#if __GLASGOW_HASKELL__ >= 914
+  liftTyped = TH.defaultLiftTyped
+#elif __GLASGOW_HASKELL__ >= 900
+  liftTyped = TH.unsafeCodeCoerce . TH.lift
+#elif __GLASGOW_HASKELL__ >= 810
+  liftTyped = TH.unsafeTExpCoerce . TH.lift
+#endif
+
+#if __GLASGOW_HASKELL__ >= 810
+unpackCStringLen# :: Exts.Addr# -> Int -> Text
+unpackCStringLen# addr# l = Text ba 0 l
+  where
+    ba = runST $ do
+      marr <- A.new l
+      A.copyFromPointer marr 0 (Exts.Ptr addr#) l
+      A.unsafeFreeze marr
+{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat
+#endif
+
+-- | @since 1.2.2.0
+instance PrintfArg Text where
+  formatArg txt = formatString $ unpack txt
+
+packConstr :: Constr
+packConstr = mkConstr textDataType "pack" [] Prefix
+
+textDataType :: DataType
+textDataType = mkDataType "Data.Text.Text" [packConstr]
+
+-- | /O(n)/ Compare two 'Text' values lexicographically.
+compareText :: Text -> Text -> Ordering
+compareText (Text arrA offA lenA) (Text arrB offB lenB) =
+    A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB
+-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),
+-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering
+-- of underlying bytearrays, no decoding is needed.
+
+-- -----------------------------------------------------------------------------
+-- * Basic functions
+
+-- | /O(n)/ Adds a character to the front of a 'Text'.  This function
+-- is more costly than its 'List' counterpart because it requires
+-- copying a new array.  Performs replacement on
+-- invalid scalar values.
+cons :: Char -> Text -> Text
+cons c (Text srcArr srcOff srcLen) = runST $ do
+  let ch = safe c
+      chLen = utf8Length ch
+      totalLen = chLen + srcLen
+  marr <- A.new totalLen
+  _ <- unsafeWrite marr 0 ch
+  A.copyI srcLen marr chLen srcArr srcOff
+  arr <- A.unsafeFreeze marr
+  pure $ Text arr 0 totalLen
+{-# INLINE [1] cons #-}
+
+infixr 5 `cons`
+
+-- | /O(n)/ Adds a character to the end of a 'Text'.  This copies the
+-- entire array in the process.
+-- Performs replacement on invalid scalar values.
+snoc :: Text -> Char -> Text
+snoc (Text srcArr srcOff srcLen) c = runST $ do
+  let ch = safe c
+      chLen = utf8Length ch
+      totalLen = srcLen + chLen
+  marr <- A.new totalLen
+  A.copyI srcLen marr 0 srcArr srcOff
+  _ <- unsafeWrite marr srcLen ch
+  arr <- A.unsafeFreeze marr
+  pure $ Text arr 0 totalLen
+{-# INLINE snoc #-}
+
+-- | /O(1)/ Returns the first character of a 'Text', which must be
+-- non-empty. This is a partial function, consider using 'uncons' instead.
+head :: HasCallStack => Text -> Char
+head t
+  | null t = emptyError "head"
+  | otherwise = let Iter c _ = iter t 0 in c
+{-# INLINE head #-}
+
+-- | /O(1)/ Returns the first character and rest of a 'Text', or
+-- 'Nothing' if empty.
+uncons :: Text -> Maybe (Char, Text)
+uncons t@(Text arr off len)
+    | len <= 0  = Nothing
+    | otherwise = Just $ let !(Iter c d) = iter t 0
+                         in (c, text arr (off+d) (len-d))
+{-# INLINE [1] uncons #-}
+
+-- | /O(1)/ Returns the last character of a 'Text', which must be
+-- non-empty. This is a partial function, consider using 'unsnoc' instead.
+last :: HasCallStack => Text -> Char
+last t@(Text _ _ len)
+    | null t = emptyError "last"
+    | otherwise = let Iter c _ = reverseIter t (len - 1) in c
+{-# INLINE [1] last #-}
+
+-- | /O(1)/ Returns all characters after the head of a 'Text', which
+-- must be non-empty. This is a partial function, consider using 'uncons' instead.
+tail :: HasCallStack => Text -> Text
+tail t@(Text arr off len)
+    | null t = emptyError "tail"
+    | otherwise = text arr (off+d) (len-d)
+    where d = iter_ t 0
+{-# INLINE [1] tail #-}
+
+-- | /O(1)/ Returns all but the last character of a 'Text', which must
+-- be non-empty. This is a partial function, consider using 'unsnoc' instead.
+init :: HasCallStack => Text -> Text
+init t@(Text arr off len)
+    | null t = emptyError "init"
+    | otherwise = text arr off (len + reverseIter_ t (len - 1))
+{-# INLINE [1] init #-}
+
+-- | /O(1)/ Returns all but the last character and the last character of a
+-- 'Text', or 'Nothing' if empty.
+--
+-- @since 1.2.3.0
+unsnoc :: Text -> Maybe (Text, Char)
+unsnoc t@(Text arr off len)
+    | null t = Nothing
+    | otherwise = Just (text arr off (len + d), c)
+        where
+            Iter c d = reverseIter t (len - 1)
+{-# INLINE [1] unsnoc #-}
+
+-- | /O(1)/ Tests whether a 'Text' is empty or not.
+null :: Text -> Bool
+null (Text _arr _off len) =
+#if defined(ASSERTS)
+    assert (len >= 0) $
+#endif
+    len <= 0
+{-# INLINE [1] null #-}
+
+{-# RULES
+ "TEXT null/empty -> True" null empty = True
+#-}
+
+-- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/),
+-- to be used together with '(:<)' or '(:>)'.
+--
+-- @since 2.1.2
+pattern Empty :: Text
+pattern Empty <- (null -> True) where
+  Empty = empty
+
+-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/),
+-- to be used together with 'Empty'.
+--
+-- @since 2.1.2
+pattern (:<) :: Char -> Text -> Text
+pattern x :< xs <- (uncons -> Just (x, xs)) where
+  (:<) = cons
+infixr 5 :<
+{-# COMPLETE Empty, (:<) #-}
+
+-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/)
+-- to be used together with 'Empty'.
+--
+-- @since 2.1.2
+pattern (:>) :: Text -> Char -> Text
+pattern xs :> x <- (unsnoc -> Just (xs, x)) where
+  (:>) = snoc
+infixl 5 :>
+{-# COMPLETE Empty, (:>) #-}
+
+-- | /O(1)/ Tests whether a 'Text' contains exactly one character.
+isSingleton :: Text -> Bool
+isSingleton (Text arr off len) =
+  len /= 0 && len == utf8LengthByLeader (A.unsafeIndex arr off)
+{-# INLINE isSingleton #-}
+
+-- | /O(n)/ Returns the number of characters in a 'Text'.
+length ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Text -> Int
+length = P.negate . measureOff P.maxBound
+{-# INLINE [1] length #-}
+-- length needs to be phased after the compareN/length rules otherwise
+-- it may inline before the rules have an opportunity to fire.
+
+{-# RULES
+"TEXT length/filter -> S.length/S.filter" forall p t.
+    length (filter p t) = S.length (S.filter p (stream t))
+"TEXT length/unstream -> S.length" forall t.
+    length (unstream t) = S.length t
+"TEXT length/pack -> P.length" forall t.
+    length (pack t) = P.length t
+"TEXT length/map -> length" forall f t.
+    length (map f t) = length t
+"TEXT length/zipWith -> length" forall f t1 t2.
+    length (zipWith f t1 t2) = min (length t1) (length t2)
+"TEXT length/replicate -> n" forall n t.
+    length (replicate n t) = mul (max 0 n) (length t)
+"TEXT length/cons -> length+1" forall c t.
+    length (cons c t) = 1 + length t
+"TEXT length/intersperse -> 2*length-1" forall c t.
+    length (intersperse c t) = max 0 (mul 2 (length t) - 1)
+"TEXT length/intercalate -> n*length" forall s ts.
+    length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)
+"TEXT length/empty -> 0"
+    length empty = 0
+  #-}
+
+-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number.
+--
+-- @
+-- 'compareLength' t c = 'P.compare' ('length' t) c
+-- @
+--
+-- This function gives the same answer as comparing against the result
+-- of 'length', but can short circuit if the count of characters is
+-- greater than the number, and hence be more efficient.
+compareLength :: Text -> Int -> Ordering
+compareLength t c = S.compareLengthI (stream t) c
+{-# INLINE [1] compareLength #-}
+
+{-# RULES
+"TEXT compareN/length -> compareLength" [~1] forall t n.
+    compare (length t) n = compareLength t n
+  #-}
+
+{-# RULES
+"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.
+    eqInt (length t) n = compareLength t n == EQ
+  #-}
+
+{-# RULES
+"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.
+    neInt (length t) n = compareLength t n /= EQ
+  #-}
+
+{-# RULES
+"TEXT <N/length -> compareLength/==LT" [~1] forall t n.
+    ltInt (length t) n = compareLength t n == LT
+  #-}
+
+{-# RULES
+"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.
+    leInt (length t) n = compareLength t n /= GT
+  #-}
+
+{-# RULES
+"TEXT >N/length -> compareLength/==GT" [~1] forall t n.
+    gtInt (length t) n = compareLength t n == GT
+  #-}
+
+{-# RULES
+"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.
+    geInt (length t) n = compareLength t n /= LT
+  #-}
+
+-- -----------------------------------------------------------------------------
+-- * Transformations
+-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
+-- each element of @t@.
+--
+-- Example:
+--
+-- >>> let message = pack "I am not angry. Not at all."
+-- >>> T.map (\c -> if c == '.' then '!' else c) message
+-- "I am not angry! Not at all!"
+--
+-- Performs replacement on invalid scalar values.
+map :: (Char -> Char) -> Text -> Text
+map f = \t -> if null t then empty else mapNonEmpty f t
+{-# INLINE [1] map #-}
+
+{-# RULES
+"TEXT map/map -> map" forall f g t.
+    map f (map g t) = map (f . safe . g) t
+#-}
+
+-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of
+-- 'Text's and concatenates the list after interspersing the first
+-- argument between each element of the list.
+--
+-- Example:
+--
+-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]
+-- "WeNI!seekNI!theNI!HolyNI!Grail"
+intercalate :: Text -> [Text] -> Text
+intercalate t = concat . L.intersperse t
+{-# INLINE [1] intercalate #-}
+
+-- | /O(n)/ The 'intersperse' function takes a character and places it
+-- between the characters of a 'Text'.
+--
+-- Example:
+--
+-- >>> T.intersperse '.' "SHIELD"
+-- "S.H.I.E.L.D"
+--
+-- Performs replacement on invalid scalar values.
+intersperse :: Char -> Text -> Text
+intersperse c t@(Text src o l) = if null t then empty else runST $ do
+    let !cLen = utf8Length c
+        dstLen = l + length t P.* cLen
+
+    dst <- A.new dstLen
+
+    let writeSep = case cLen of
+          1 -> \dstOff ->
+            A.unsafeWrite dst dstOff (ord8 c)
+          2 -> let (c0, c1) = ord2 c in \dstOff -> do
+            A.unsafeWrite dst dstOff c0
+            A.unsafeWrite dst (dstOff + 1) c1
+          3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do
+            A.unsafeWrite dst dstOff c0
+            A.unsafeWrite dst (dstOff + 1) c1
+            A.unsafeWrite dst (dstOff + 2) c2
+          _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do
+            A.unsafeWrite dst dstOff c0
+            A.unsafeWrite dst (dstOff + 1) c1
+            A.unsafeWrite dst (dstOff + 2) c2
+            A.unsafeWrite dst (dstOff + 3) c3
+    let go !srcOff !dstOff = if srcOff >= o + l then return () else do
+          let m0 = A.unsafeIndex src srcOff
+              m1 = A.unsafeIndex src (srcOff + 1)
+              m2 = A.unsafeIndex src (srcOff + 2)
+              m3 = A.unsafeIndex src (srcOff + 3)
+              !d = utf8LengthByLeader m0
+          case d of
+            1 -> do
+              A.unsafeWrite dst dstOff m0
+              writeSep (dstOff + 1)
+              go (srcOff + 1) (dstOff + 1 + cLen)
+            2 -> do
+              A.unsafeWrite dst dstOff m0
+              A.unsafeWrite dst (dstOff + 1) m1
+              writeSep (dstOff + 2)
+              go (srcOff + 2) (dstOff + 2 + cLen)
+            3 -> do
+              A.unsafeWrite dst dstOff m0
+              A.unsafeWrite dst (dstOff + 1) m1
+              A.unsafeWrite dst (dstOff + 2) m2
+              writeSep (dstOff + 3)
+              go (srcOff + 3) (dstOff + 3 + cLen)
+            _ -> do
+              A.unsafeWrite dst dstOff m0
+              A.unsafeWrite dst (dstOff + 1) m1
+              A.unsafeWrite dst (dstOff + 2) m2
+              A.unsafeWrite dst (dstOff + 3) m3
+              writeSep (dstOff + 4)
+              go (srcOff + 4) (dstOff + 4 + cLen)
+
+    go o 0
+    arr <- A.unsafeFreeze dst
+    return (Text arr 0 (dstLen - cLen))
+{-# INLINE [1] intersperse #-}
+
+-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in
+-- @haystack@ with @replacement@.
+--
+-- This function behaves as though it was defined as follows:
+--
+-- @
+-- replace needle replacement haystack =
+--   'intercalate' replacement ('splitOn' needle haystack)
+-- @
+--
+-- As this suggests, each occurrence is replaced exactly once.  So if
+-- @needle@ occurs in @replacement@, that occurrence will /not/ itself
+-- be replaced recursively:
+--
+-- >>> replace "oo" "foo" "oo"
+-- "foo"
+--
+-- In cases where several instances of @needle@ overlap, only the
+-- first one will be replaced:
+--
+-- >>> replace "ofo" "bar" "ofofo"
+-- "barfo"
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+replace :: HasCallStack
+        => Text
+        -- ^ @needle@ to search for.  If this string is empty, an
+        -- error will occur.
+        -> Text
+        -- ^ @replacement@ to replace @needle@ with.
+        -> Text
+        -- ^ @haystack@ in which to search.
+        -> Text
+replace needle@(Text _      _      neeLen)
+               (Text repArr repOff repLen)
+      haystack@(Text hayArr hayOff hayLen)
+  | neeLen == 0 = emptyError "replace"
+  | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it
+  | L.null ixs  = haystack
+  | otherwise   = Text (A.run x) 0 len
+  where
+    ixs = indices needle haystack
+    len = hayLen - (neeLen - repLen) `mul` L.length ixs
+    x :: ST s (A.MArray s)
+    x = do
+      marr <- A.new len
+      let loop (i:is) o d = do
+            let d0 = d + i - o
+                d1 = d0 + repLen
+            A.copyI (i - o) marr d  hayArr (hayOff+o)
+            A.copyI repLen  marr d0 repArr repOff
+            loop is (i + neeLen) d1
+          loop []     o d = A.copyI (len - d) marr d hayArr (hayOff+o)
+      loop ixs 0 0
+      return marr
+
+-- ----------------------------------------------------------------------------
+-- ** Case conversions (folds)
+
+-- $case
+--
+-- When case converting 'Text' values, do not use combinators like
+-- @map toUpper@ to case convert each character of a string
+-- individually, as this gives incorrect results according to the
+-- rules of some writing systems.  The whole-string case conversion
+-- functions from this module, such as @toUpper@, obey the correct
+-- case conversion rules.  As a result, these functions may map one
+-- input character to two or three output characters. For examples,
+-- see the documentation of each function.
+--
+-- /Note/: In some languages, case conversion is a locale- and
+-- context-dependent operation. The case conversion functions in this
+-- module are /not/ locale sensitive. Programs that require locale
+-- sensitivity should use appropriate versions of the
+-- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >.
+
+-- | /O(n)/ Convert a string to folded case.
+--
+-- This function is mainly useful for performing caseless (also known
+-- as case insensitive) string comparisons.
+--
+-- A string @x@ is a caseless match for a string @y@ if and only if:
+--
+-- @toCaseFold x == toCaseFold y@
+--
+-- The result string may be longer than the input string, and may
+-- differ from applying 'toLower' to the input string.  For instance,
+-- the Armenian small ligature \"&#xfb13;\" (men now, U+FB13) is case
+-- folded to the sequence \"&#x574;\" (men, U+0574) followed by
+-- \"&#x576;\" (now, U+0576), while the Greek \"&#xb5;\" (micro sign,
+-- U+00B5) is case folded to \"&#x3bc;\" (small letter mu, U+03BC)
+-- instead of itself.
+toCaseFold :: Text -> Text
+toCaseFold = \t ->
+    if null t then empty
+    else toCaseFoldNonEmpty t
+{-# INLINE toCaseFold #-}
+
+-- | /O(n)/ Convert a string to lower case, using simple case
+-- conversion.
+--
+-- The result string may be longer than the input string.  For
+-- instance, \"&#x130;\" (Latin capital letter I with dot above,
+-- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069)
+-- followed by \" &#x307;\" (combining dot above, U+0307).
+toLower :: Text -> Text
+toLower = \t ->
+  if null t then empty
+  else toLowerNonEmpty t
+{-# INLINE toLower #-}
+
+-- | /O(n)/ Convert a string to upper case, using simple case
+-- conversion.
+--
+-- The result string may be longer than the input string.  For
+-- instance, the German \"&#xdf;\" (eszett, U+00DF) maps to the
+-- two-letter sequence \"SS\".
+toUpper :: Text -> Text
+toUpper = \t ->
+  if null t then empty
+  else toUpperNonEmpty t
+{-# INLINE toUpper #-}
+
+-- | /O(n)/ Convert a string to title case, using simple case
+-- conversion.
+--
+-- The first letter (as determined by 'Data.Char.isLetter')
+-- of the input is converted to title case, as is
+-- every subsequent letter that immediately follows a non-letter.
+-- Every letter that immediately follows another letter is converted
+-- to lower case.
+--
+-- This function is not idempotent.
+-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).
+-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input
+-- is converted to title case, becoming two letters.
+-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter
+-- and as such is recognised as a letter by 'Data.Char.isLetter',
+-- so 'T.toTitle' @"ʼN"@ = @"'n"@.
+--
+-- The result string may be longer than the input string. For example,
+-- the Latin small ligature &#xfb02; (U+FB02) is converted to the
+-- sequence Latin capital letter F (U+0046) followed by Latin small
+-- letter l (U+006C).
+--
+-- /Note/: this function does not take language or culture specific
+-- rules into account. For instance, in English, different style
+-- guides disagree on whether the book name \"The Hill of the Red
+-- Fox\" is correctly title cased&#x2014;but this function will
+-- capitalize /every/ word.
+--
+-- @since 1.0.0.0
+toTitle :: Text -> Text
+toTitle = \t ->
+  if null t then empty
+  else toTitleNonEmpty t
+{-# INLINE toTitle #-}
+
+-- | /O(n)/ Left-justify a string to the given length, using the
+-- specified fill character on the right.
+-- Performs replacement on invalid scalar values.
+--
+-- Examples:
+--
+-- >>> justifyLeft 7 'x' "foo"
+-- "fooxxxx"
+--
+-- >>> justifyLeft 3 'x' "foobar"
+-- "foobar"
+justifyLeft :: Int -> Char -> Text -> Text
+justifyLeft k c t
+    | len >= k  = t
+    | otherwise = t `append` replicateChar (k-len) c
+  where len = length t
+{-# INLINE [1] justifyLeft #-}
+
+-- | /O(n)/ Right-justify a string to the given length, using the
+-- specified fill character on the left.  Performs replacement on
+-- invalid scalar values.
+--
+-- Examples:
+--
+-- >>> justifyRight 7 'x' "bar"
+-- "xxxxbar"
+--
+-- >>> justifyRight 3 'x' "foobar"
+-- "foobar"
+justifyRight :: Int -> Char -> Text -> Text
+justifyRight k c t
+    | len >= k  = t
+    | otherwise = replicateChar (k-len) c `append` t
+  where len = length t
+{-# INLINE justifyRight #-}
+
+-- | /O(n)/ Center a string to the given length, using the specified
+-- fill character on either side.  Performs replacement on invalid
+-- scalar values.
+--
+-- Examples:
+--
+-- >>> center 8 'x' "HS"
+-- "xxxHSxxx"
+center :: Int -> Char -> Text -> Text
+center k c t
+    | len >= k  = t
+    | otherwise = replicateChar l c `append` t `append` replicateChar r c
+  where len = length t
+        d   = k - len
+        r   = d `quot` 2
+        l   = d - r
+{-# INLINE center #-}
+
+-- | /O(n)/ The 'transpose' function transposes the rows and columns
+-- of its 'Text' argument.  Note that this function uses 'pack',
+-- 'unpack', and the list version of transpose, and is thus not very
+-- efficient.
+--
+-- Examples:
+--
+-- >>> transpose ["green","orange"]
+-- ["go","rr","ea","en","ng","e"]
+--
+-- >>> transpose ["blue","red"]
+-- ["br","le","ud","e"]
+transpose :: [Text] -> [Text]
+transpose ts = P.map pack (L.transpose (P.map unpack ts))
+
+-- -----------------------------------------------------------------------------
+-- * Reducing 'Text's (folds)
+
+-- | /O(n)/ 'foldl', applied to a binary operator, a starting value
+-- (typically the left-identity of the operator), and a 'Text',
+-- reduces the 'Text' using the binary operator, from left to right.
+foldl :: (a -> Char -> a) -> a -> Text -> a
+foldl f z (Text arr off len) = go (off + len - 1)
+  where
+    go !i
+      | i < off = z
+      | otherwise = let !(Iter c l) = reverseIterArray arr i in f (go (i + l)) c
+{-# INLINE foldl #-}
+
+-- | /O(n)/ A strict version of 'foldl'.
+foldl' :: (a -> Char -> a) -> a -> Text -> a
+foldl' f z (Text arr off len) = go off z
+  where
+    go !i !acc
+      | i >= off + len = acc
+      | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) (f acc c)
+{-# INLINE foldl' #-}
+
+-- | /O(n)/ A variant of 'foldl' that has no starting value argument,
+-- and thus must be applied to a non-empty 'Text'.
+foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldl1 f t = case uncons t of
+  Nothing -> emptyError "foldl"
+  Just (c, t') -> foldl f c t'
+{-# INLINE foldl1 #-}
+
+-- | /O(n)/ A strict version of 'foldl1'.
+foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldl1' f t = case uncons t of
+  Nothing -> emptyError "foldl'"
+  Just (c, t') -> foldl' f c t'
+{-# INLINE foldl1' #-}
+
+-- | /O(n)/ A monadic version of 'foldl''.
+--
+-- @since 2.1.2
+foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a
+foldlM' f z (Text arr off len) = go off z
+  where
+    go !i !acc
+      | i >= off + len = pure acc
+      | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) P.=<< f acc c
+{-# INLINE foldlM' #-}
+
+-- | /O(n)/ 'foldr', applied to a binary operator, a starting value
+-- (typically the right-identity of the operator), and a 'Text',
+-- reduces the 'Text' using the binary operator, from right to left.
+--
+-- If the binary operator is strict in its second argument, use 'foldr''
+-- instead.
+--
+-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually
+-- traverses the 'Text' from left to right, only as far as it needs to.
+--
+-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr':
+--
+-- @
+-- head :: Text -> Char
+-- head = foldr const (error "head empty")
+-- @
+--
+-- Searches from left to right with short-circuiting behavior can
+-- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem').
+foldr :: (Char -> a -> a) -> a -> Text -> a
+foldr f z (Text arr off len) = go off
+  where
+    go !i
+      | i >= off + len = z
+      | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l))
+{-# INLINE foldr #-}
+
+-- | /O(n)/ A variant of 'foldr' that has no starting value argument,
+-- and thus must be applied to a non-empty 'Text'.
+foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldr1 f t = case unsnoc t of
+  Nothing -> emptyError "foldr1"
+  Just (t', c) -> foldr f c t'
+{-# INLINE foldr1 #-}
+
+-- | /O(n)/ A strict version of 'foldr'.
+--
+-- 'foldr'' evaluates as a right-to-left traversal using constant stack space.
+--
+-- @since 2.0.1
+foldr' :: (Char -> a -> a) -> a -> Text -> a
+foldr' f z (Text arr off len) = go (off + len - 1) z
+  where
+    go !i !acc
+      | i < off = acc
+      | otherwise = let !(Iter c l) = reverseIterArray arr i in go (i + l) (f c acc)
+{-# INLINE foldr' #-}
+
+-- -----------------------------------------------------------------------------
+-- ** Special folds
+
+-- | /O(n)/ Concatenate a list of 'Text's.
+concat :: [Text] -> Text
+concat ts = case ts of
+    [] -> empty
+    [t] -> t
+    _ | len == 0 -> empty
+      | otherwise -> Text (A.run go) 0 len
+  where
+    len = sumP "concat" $ L.map lengthWord8 ts
+    go :: ST s (A.MArray s)
+    go = do
+      arr <- A.new len
+      let step i (Text a o l) = A.copyI l arr i a o >> return (i + l)
+      foldM step 0 ts >> return arr
+
+-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and
+-- concatenate the results.
+concatMap :: (Char -> Text) -> Text -> Text
+concatMap f = concat . foldr ((:) . f) []
+{-# INLINE concatMap #-}
+
+-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
+-- 'Text' @t@ satisfies the predicate @p@.
+any :: (Char -> Bool) -> Text -> Bool
+any p = foldr (\c acc -> p c || acc) False
+{-# INLINE any #-}
+
+-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
+-- 'Text' @t@ satisfy the predicate @p@.
+all :: (Char -> Bool) -> Text -> Bool
+all p = foldr (\c acc -> p c && acc) True
+{-# INLINE all #-}
+
+-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
+-- must be non-empty.
+maximum :: HasCallStack => Text -> Char
+maximum = foldl1' max
+-- This could be implemented faster: look for the longest
+-- and largest UTF-8 sequence, then decode it to Char only once,
+-- instead of decoding all characters, but I doubt anyone cares
+-- about the performance of 'maximum' much.
+{-# INLINE maximum #-}
+
+-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
+-- must be non-empty.
+minimum :: HasCallStack => Text -> Char
+minimum = foldl1' min
+-- This could be implemented faster, see the comment for 'maximum' above.
+{-# INLINE minimum #-}
+
+-- -----------------------------------------------------------------------------
+-- * Building 'Text's
+-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of
+-- successive reduced values from the left.
+-- Performs replacement on invalid scalar values.
+--
+-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
+--
+-- __Properties__
+--
+-- @'head' ('scanl' f z xs) = z@
+--
+-- @'last' ('scanl' f z xs) = 'foldl' f z xs@
+scanl :: (Char -> Char -> Char) -> Char -> Text -> Text
+scanl f c0 (Text src o l) = runST $ do
+  let l' = l + 4
+      c0' = safe c0
+  marr <- A.new l'
+  d' <- unsafeWrite marr 0 c0'
+  outer marr l' o d' c0'
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> Int -> Char -> ST s Text
+    outer !dst !dstLen = inner
+      where
+        inner !srcOff !dstOff !c
+          | srcOff >= l + o = do
+            A.shrinkM dst dstOff
+            arr <- A.unsafeFreeze dst
+            pure $ Text arr 0 dstOff
+          | dstOff + 4 > dstLen = do
+            let !dstLen' = dstLen + (l + o) - srcOff + 4
+            dst' <- A.resizeM dst dstLen'
+            outer dst' dstLen' srcOff dstOff c
+          | otherwise = do
+            let !(Iter c' d) = iterArray src srcOff
+                c'' = safe $ f c c'
+            d' <- unsafeWrite dst dstOff c''
+            inner (srcOff + d) (dstOff + d') c''
+
+-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting
+-- value argument. Performs replacement on invalid scalar values.
+--
+-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]
+scanl1 :: (Char -> Char -> Char) -> Text -> Text
+scanl1 f t | null t    = empty
+           | otherwise = scanl f (unsafeHead t) (unsafeTail t)
+{-# INLINE scanl1 #-}
+
+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs
+-- replacement on invalid scalar values.
+--
+-- > scanr f v == reverse . scanl (flip f) v . reverse
+scanr :: (Char -> Char -> Char) -> Char -> Text -> Text
+scanr f c0 (Text src o l) = runST $ do
+  let l' = l + 4
+      c0' = safe c0
+      !d' = utf8Length c0'
+  marr <- A.new l'
+  _ <- unsafeWrite marr (l' - d') c0'
+  outer marr (l + o - 1) (l' - d' - 1) c0'
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> Char -> ST s Text
+    outer !dst = inner
+      where
+        inner !srcOff !dstOff !c
+          | srcOff < o = do
+            dstLen <- A.getSizeofMArray dst
+            arr <- A.unsafeFreeze dst
+            pure $ Text arr (dstOff + 1) (dstLen - dstOff - 1)
+          | dstOff < 3 = do
+            dstLen <- A.getSizeofMArray dst
+            let !dstLen' = dstLen + (srcOff - o) + 4
+            dst' <- A.new dstLen'
+            A.copyM dst' (dstLen' - dstLen) dst 0 dstLen
+            outer dst' srcOff (dstOff + dstLen' - dstLen) c
+          | otherwise = do
+            let !(Iter c' d) = reverseIterArray src srcOff
+                c'' = safe $ f c' c
+                !d' = utf8Length c''
+                dstOff' = dstOff - d'
+            _ <- unsafeWrite dst (dstOff' + 1) c''
+            inner (srcOff + d) dstOff' c''
+
+-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting
+-- value argument. Performs replacement on invalid scalar values.
+scanr1 :: (Char -> Char -> Char) -> Text -> Text
+scanr1 f t | null t    = empty
+           | otherwise = scanr f (last t) (init t)
+{-# INLINE scanr1 #-}
+
+-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a
+-- function to each element of a 'Text', passing an accumulating
+-- parameter from left to right, and returns a final 'Text'.  Performs
+-- replacement on invalid scalar values.
+mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)
+mapAccumL f z0 (Text src o l) = runST $ do
+  marr <- A.new (l + 4)
+  outer marr (l + 4) o 0 z0
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text)
+    outer !dst !dstLen = inner
+      where
+        inner !srcOff !dstOff !z
+          | srcOff >= l + o = do
+            A.shrinkM dst dstOff
+            arr <- A.unsafeFreeze dst
+            return (z, Text arr 0 dstOff)
+          | dstOff + 4 > dstLen = do
+            let !dstLen' = dstLen + (l + o) - srcOff + 4
+            dst' <- A.resizeM dst dstLen'
+            outer dst' dstLen' srcOff dstOff z
+          | otherwise = do
+            let !(Iter c d) = iterArray src srcOff
+                (z', c') = f z c
+            d' <- unsafeWrite dst dstOff (safe c')
+            inner (srcOff + d) (dstOff + d') z'
+
+-- | The 'mapAccumR' function behaves like a combination of 'map' and
+-- a strict 'foldr'; it applies a function to each element of a
+-- 'Text', passing an accumulating parameter from right to left, and
+-- returning a final value of this accumulator together with the new
+-- 'Text'.
+-- Performs replacement on invalid scalar values.
+mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)
+mapAccumR f z0 (Text src o l) = runST $ do
+  marr <- A.new (l + 4)
+  outer marr (l + o - 1) (l + 4 - 1) z0
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text)
+    outer !dst = inner
+      where
+        inner !srcOff !dstOff !z
+          | srcOff < o = do
+            dstLen <- A.getSizeofMArray dst
+            arr <- A.unsafeFreeze dst
+            return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1))
+          | dstOff < 3 = do
+            dstLen <- A.getSizeofMArray dst
+            let !dstLen' = dstLen + (srcOff - o) + 4
+            dst' <- A.new dstLen'
+            A.copyM dst' (dstLen' - dstLen) dst 0 dstLen
+            outer dst' srcOff (dstOff + dstLen' - dstLen) z
+          | otherwise = do
+            let !(Iter c d) = reverseIterArray src srcOff
+                (z', c') = f z c
+                c'' = safe c'
+                !d' = utf8Length c''
+                dstOff' = dstOff - d'
+            _ <- unsafeWrite dst (dstOff' + 1) c''
+            inner (srcOff + d) dstOff' z'
+
+-- -----------------------------------------------------------------------------
+-- ** Generating and unfolding 'Text's
+
+-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input
+-- @t@ repeated @n@ times.
+replicate :: Int -> Text -> Text
+replicate n t@(Text a o l)
+    | n <= 0 || l <= 0       = empty
+    | n == 1                 = t
+    | isSingleton t          = replicateChar n (unsafeHead t)
+    | otherwise              = runST $ do
+        let totalLen = n `mul` l
+        marr <- A.new totalLen
+        A.copyI l marr 0 a o
+        A.tile marr l
+        arr  <- A.unsafeFreeze marr
+        return $ Text arr 0 totalLen
+{-# INLINE [1] replicate #-}
+
+{-# RULES
+"TEXT replicate/singleton -> replicateChar" [~1] forall n c.
+    replicate n (singleton c) = replicateChar n c
+  #-}
+
+-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the
+-- value of every element.
+replicateChar :: Int -> Char -> Text
+replicateChar !len !c'
+  | len <= 0  = empty
+  | Char.isAscii c = runST $ do
+    marr <- A.newFilled len (Char.ord c)
+    arr  <- A.unsafeFreeze marr
+    return $ Text arr 0 len
+  | otherwise = runST $ do
+    let cLen = utf8Length c
+        totalLen = cLen P.* len
+    marr <- A.new totalLen
+    _ <- unsafeWrite marr 0 c
+    A.tile marr cLen
+    arr  <- A.unsafeFreeze marr
+    return $ Text arr 0 totalLen
+  where
+    c = safe c'
+{-# INLINE replicateChar #-}
+
+-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'
+-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a
+-- 'Text' from a seed value. The function takes the element and
+-- returns 'Nothing' if it is done producing the 'Text', otherwise
+-- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the
+-- string, and @b@ is the seed value for further production.
+-- Performs replacement on invalid scalar values.
+unfoldr     :: (a -> Maybe (Char,a)) -> a -> Text
+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)
+{-# INLINE unfoldr #-}
+
+-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
+-- value. However, the length of the result should be limited by the
+-- first argument to 'unfoldrN'. This function is more efficient than
+-- 'unfoldr' when the maximum length of the result is known and
+-- correct, otherwise its performance is similar to 'unfoldr'.
+-- Performs replacement on invalid scalar values.
+unfoldrN     :: Int -> (a -> Maybe (Char,a)) -> a -> Text
+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)
+{-# INLINE unfoldrN #-}
+
+-- -----------------------------------------------------------------------------
+-- * Substrings
+
+-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the
+-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than
+-- the length of the Text.
+take :: Int -> Text -> Text
+take n t@(Text arr off len)
+    | n <= 0    = empty
+    | n >= len || m >= len || m < 0  = t
+    | otherwise = Text arr off m
+  where
+    m = measureOff n t
+{-# INLINE [1] take #-}
+
+-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@
+-- returns a non-negative number, measuring their size in 'Word8'. Otherwise,
+-- if @t@ is shorter, return a non-positive number, which is a negated total count
+-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0.
+--
+-- This function is used to implement 'take', 'drop', 'splitAt' and 'length'
+-- and is useful on its own in streaming and parsing libraries.
+--
+-- @since 2.0
+measureOff :: Int -> Text -> Int
+measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else
+  cSsizeToInt $
+    measure_off arr (intToCSize off) (intToCSize len) (intToCSize n)
+
+-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after
+-- taking @n@ characters from the end of @t@.
+--
+-- Examples:
+--
+-- >>> takeEnd 3 "foobar"
+-- "bar"
+--
+-- @since 1.1.1.0
+takeEnd :: Int -> Text -> Text
+takeEnd n t@(Text arr off len)
+    | n <= 0    = empty
+    | n >= len  = t
+    | otherwise = text arr (off+i) (len-i)
+  where i = iterNEnd n t
+
+iterNEnd :: Int -> Text -> Int
+iterNEnd n t@(Text _arr _off len) = loop (len-1) n
+  where loop i !m
+          | m <= 0    = i+1
+          | i <= 0    = 0
+          | otherwise = loop (i+d) (m-1)
+          where d = reverseIter_ t i
+
+-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the
+-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@
+-- is greater than the length of the 'Text'.
+drop :: Int -> Text -> Text
+drop n t@(Text arr off len)
+    | n <= 0    = t
+    | n >= len || m >= len || m < 0 = empty
+    | otherwise = Text arr (off+m) (len-m)
+  where m = measureOff n t
+{-# INLINE [1] drop #-}
+
+-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after
+-- dropping @n@ characters from the end of @t@.
+--
+-- Examples:
+--
+-- >>> dropEnd 3 "foobar"
+-- "foo"
+--
+-- @since 1.1.1.0
+dropEnd :: Int -> Text -> Text
+dropEnd n t@(Text arr off len)
+    | n <= 0    = t
+    | n >= len  = empty
+    | otherwise = text arr off (iterNEnd n t)
+
+-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',
+-- returns the longest prefix (possibly empty) of elements that
+-- satisfy @p@.
+takeWhile :: (Char -> Bool) -> Text -> Text
+takeWhile p t@(Text arr off len) = loop 0
+  where loop !i | i >= len    = t
+                | p c         = loop (i+d)
+                | otherwise   = text arr off i
+            where Iter c d    = iter t i
+{-# INLINE [1] takeWhile #-}
+
+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',
+-- returns the longest suffix (possibly empty) of elements that
+-- satisfy @p@.
+-- Examples:
+--
+-- >>> takeWhileEnd (=='o') "foo"
+-- "oo"
+--
+-- @since 1.2.2.0
+takeWhileEnd :: (Char -> Bool) -> Text -> Text
+takeWhileEnd p t@(Text arr off len) = loop (len-1) len
+  where loop !i !l | l <= 0    = t
+                   | p c       = loop (i+d) (l+d)
+                   | otherwise = text arr (off+l) (len-l)
+            where Iter c d     = reverseIter t i
+{-# INLINE [1] takeWhileEnd #-}
+
+-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
+-- 'takeWhile' @p@ @t@.
+dropWhile :: (Char -> Bool) -> Text -> Text
+dropWhile p t@(Text arr off len) = loop 0 0
+  where loop !i !l | l >= len  = empty
+                   | p c       = loop (i+d) (l+d)
+                   | otherwise = Text arr (off+i) (len-l)
+            where Iter c d     = iter t i
+{-# INLINE [1] dropWhile #-}
+
+-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after
+-- dropping characters that satisfy the predicate @p@ from the end of
+-- @t@.
+--
+-- Examples:
+--
+-- >>> dropWhileEnd (=='.') "foo..."
+-- "foo"
+dropWhileEnd :: (Char -> Bool) -> Text -> Text
+dropWhileEnd p t@(Text arr off len) = loop (len-1) len
+  where loop !i !l | l <= 0    = empty
+                   | p c       = loop (i+d) (l+d)
+                   | otherwise = Text arr off l
+            where Iter c d     = reverseIter t i
+{-# INLINE [1] dropWhileEnd #-}
+
+-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after
+-- dropping characters that satisfy the predicate @p@ from both the
+-- beginning and end of @t@.
+dropAround :: (Char -> Bool) -> Text -> Text
+dropAround p = dropWhile p . dropWhileEnd p
+{-# INLINE [1] dropAround #-}
+
+-- | /O(n)/ Remove leading white space from a string.  Equivalent to:
+--
+-- > dropWhile isSpace
+stripStart :: Text -> Text
+stripStart = dropWhile Char.isSpace
+{-# INLINE stripStart #-}
+
+-- | /O(n)/ Remove trailing white space from a string.  Equivalent to:
+--
+-- > dropWhileEnd isSpace
+stripEnd :: Text -> Text
+stripEnd = dropWhileEnd Char.isSpace
+{-# INLINE [1] stripEnd #-}
+
+-- | /O(n)/ Remove leading and trailing white space from a string.
+-- Equivalent to:
+--
+-- > dropAround isSpace
+strip :: Text -> Text
+strip = dropAround Char.isSpace
+{-# INLINE [1] strip #-}
+
+-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a
+-- prefix of @t@ of length @n@, and whose second is the remainder of
+-- the string. It is equivalent to @('take' n t, 'drop' n t)@.
+splitAt :: Int -> Text -> (Text, Text)
+splitAt n t@(Text arr off len)
+    | n <= 0    = (empty, t)
+    | n >= len || m >= len || m < 0  = (t, empty)
+    | otherwise = (Text arr off m, Text arr (off+m) (len-m))
+  where
+    m = measureOff n t
+
+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns
+-- a pair whose first element is the longest prefix (possibly empty)
+-- of @t@ of elements that satisfy @p@, and whose second is the
+-- remainder of the text.
+--
+-- >>> T.span (=='0') "000AB"
+-- ("000","AB")
+span :: (Char -> Bool) -> Text -> (Text, Text)
+span p t = case span_ p t of
+             (# hd,tl #) -> (hd,tl)
+{-# INLINE span #-}
+
+-- | /O(n)/ 'break' is like 'span', but the prefix returned is
+-- over elements that fail the predicate @p@.
+--
+-- >>> T.break (=='c') "180cm"
+-- ("180","cm")
+break :: (Char -> Bool) -> Text -> (Text, Text)
+break p = span (not . p)
+{-# INLINE break #-}
+
+-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@,
+-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of
+-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text.
+--
+-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97
+-- (("abc","efg"),101)
+--
+-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity':
+--
+-- @
+-- -- for all p :: Char -> Bool
+-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p)
+-- @
+--
+-- @since 2.0.1
+spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)
+spanM p t@(Text arr off len) = go 0
+  where
+    go !i | i < len = case iterArray arr (off+i) of
+        Iter c l -> do
+            continue <- p c
+            if continue then go (i+l)
+            else pure (text arr off i, text arr (off+i) (len-i))
+    go _ = pure (t, empty)
+{-# INLINE spanM #-}
+
+-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@,
+-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of
+-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text.
+--
+-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122
+-- (("tuv","xyz"),118)
+--
+-- @
+-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p
+-- @
+--
+-- @since 2.0.1
+spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)
+spanEndM p t@(Text arr off len) = go (len-1)
+  where
+    go !i | 0 <= i = case reverseIterArray arr (off+i) of
+        Iter c l -> do
+            continue <- p c
+            if continue then go (i+l)
+            else pure (text arr off (i+1), text arr (off+i+1) (len-i-1))
+    go _ = pure (empty, t)
+{-# INLINE spanEndM #-}
+
+-- | /O(n)/ Group characters in a string according to a predicate.
+groupBy :: (Char -> Char -> Bool) -> Text -> [Text]
+groupBy p = loop
+  where
+    loop t@(Text arr off len)
+        | null t    = []
+        | otherwise = text arr off n : loop (text arr (off+n) (len-n))
+        where Iter c d = iter t 0
+              n     = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d))
+
+-- | Returns the /array/ index (in units of 'Word8') at which a
+-- character may be found.  This is /not/ the same as the logical
+-- index returned by e.g. 'findIndex'.
+findAIndexOrEnd :: (Char -> Bool) -> Text -> Int
+findAIndexOrEnd q t@(Text _arr _off len) = go 0
+    where go !i | i >= len || q c       = i
+                | otherwise             = go (i+d)
+                where Iter c d          = iter t i
+
+-- | /O(n)/ Group characters in a string by equality.
+group :: Text -> [Text]
+group = groupBy (==)
+
+-- | /O(n)/ Return all initial segments of the given 'Text', shortest
+-- first.
+inits :: Text -> [Text]
+inits = (NonEmptyList.toList $!) . initsNE
+
+-- | /O(n)/ Return all initial segments of the given 'Text', shortest
+-- first.
+--
+-- @since 2.1.2
+initsNE :: Text -> NonEmptyList.NonEmpty Text
+initsNE t = empty NonEmptyList.:| case t of
+  Text arr off len ->
+    let loop i
+          | i >= len = []
+          | otherwise = let !j = i + iter_ t i in Text arr off j : loop j
+    in loop 0
+
+-- | /O(n)/ Return all final segments of the given 'Text', longest
+-- first.
+tails :: Text -> [Text]
+tails = (NonEmptyList.toList $!) . tailsNE
+
+-- | /O(n)/ Return all final segments of the given 'Text', longest
+-- first.
+--
+-- @since 2.1.2
+tailsNE :: Text -> NonEmptyList.NonEmpty Text
+tailsNE t
+  | null t = empty NonEmptyList.:| []
+  | otherwise = t NonEmptyList.:| tails (unsafeTail t)
+
+-- $split
+--
+-- Splitting functions in this library do not perform character-wise
+-- copies to create substrings; they just construct new 'Text's that
+-- are slices of the original.
+
+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'
+-- argument (which cannot be empty), consuming the delimiter. An empty
+-- delimiter is invalid, and will cause an error to be raised.
+--
+-- Examples:
+--
+-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"
+-- ["a","b","d","e"]
+--
+-- >>> splitOn "aaa"  "aaaXaaaXaaaXaaa"
+-- ["","X","X","X",""]
+--
+-- >>> splitOn "x"    "x"
+-- ["",""]
+--
+-- and
+--
+-- > intercalate s . splitOn s         == id
+-- > splitOn (singleton c)             == split (==c)
+--
+-- (Note: the string @s@ to split on above cannot be empty.)
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+splitOn :: HasCallStack
+        => Text
+        -- ^ String to split on. If this string is empty, an error
+        -- will occur.
+        -> Text
+        -- ^ Input text.
+        -> [Text]
+splitOn pat@(Text _ _ l) src@(Text arr off len)
+    | l <= 0          = emptyError "splitOn"
+    | isSingleton pat = split (== unsafeHead pat) src
+    | otherwise       = go 0 (indices pat src)
+  where
+    go !s (x:xs) =  text arr (s+off) (x-s) : go (x+l) xs
+    go  s _      = [text arr (s+off) (len-s)]
+{-# INLINE [1] splitOn #-}
+
+{-# RULES
+"TEXT splitOn/singleton -> split/==" [~1] forall c t.
+    splitOn (singleton c) t = split (==c) t
+  #-}
+
+-- | /O(n)/ Splits a 'Text' into components delimited by separators,
+-- where the predicate returns True for a separator element.  The
+-- resulting components do not contain the separators.  Two adjacent
+-- separators result in an empty component in the output.  eg.
+--
+-- >>> split (=='a') "aabbaca"
+-- ["","","bb","c",""]
+--
+-- >>> split (=='a') ""
+-- [""]
+split :: (Char -> Bool) -> Text -> [Text]
+split p t
+    | null t = [empty]
+    | otherwise = loop t
+    where loop s | null s'   = [l]
+                 | otherwise = l : loop (unsafeTail s')
+              where (# l, s' #) = span_ (not . p) s
+{-# INLINE split #-}
+
+-- | /O(n)/ Splits a 'Text' into components of length @k@.  The last
+-- element may be shorter than the other chunks, depending on the
+-- length of the input. Examples:
+--
+-- >>> chunksOf 3 "foobarbaz"
+-- ["foo","bar","baz"]
+--
+-- >>> chunksOf 4 "haskell.org"
+-- ["hask","ell.","org"]
+chunksOf :: Int -> Text -> [Text]
+chunksOf k = go
+  where
+    go t = case splitAt k t of
+             (a,b) | null a    -> []
+                   | otherwise -> a : go b
+{-# INLINE chunksOf #-}
+
+-- ----------------------------------------------------------------------------
+-- * Searching
+
+-------------------------------------------------------------------------------
+-- ** Searching with a predicate
+
+-- | /O(n)/ The 'elem' function takes a character and a 'Text', and
+-- returns 'True' if the element is found in the given 'Text', or
+-- 'False' otherwise.
+elem :: Char -> Text -> Bool
+elem = any . (==)
+-- TODO This can be implemented much faster: there is no need to decode
+-- any UTF-8 sequences at all.
+{-# INLINE elem #-}
+
+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and
+-- returns the first element matching the predicate, or 'Nothing' if
+-- there is no such element.
+find :: (Char -> Bool) -> Text -> Maybe Char
+find p = foldr (\c acc -> if p c then Just c else acc) Nothing
+{-# INLINE find #-}
+
+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',
+-- and returns the pair of 'Text's with elements which do and do not
+-- satisfy the predicate, respectively; i.e.
+--
+-- > partition p t == (filter p t, filter (not . p) t)
+partition :: (Char -> Bool) -> Text -> (Text, Text)
+partition p t = (filter p t, filter (not . p) t)
+{-# INLINE partition #-}
+
+-- | /O(n)/ 'filter', applied to a predicate and a 'Text',
+-- returns a 'Text' containing those characters that satisfy the
+-- predicate.
+filter :: (Char -> Bool) -> Text -> Text
+filter p = filter_ text p
+{-# INLINE [1] filter #-}
+
+{-# RULES
+"TEXT filter/filter -> filter" forall p q t.
+    filter p (filter q t) = filter (\c -> q c && p c) t
+#-}
+
+-- | /O(n+m)/ Find the first instance of @needle@ (which must be
+-- non-'null') in @haystack@.  The first element of the returned tuple
+-- is the prefix of @haystack@ before @needle@ is matched.  The second
+-- is the remainder of @haystack@, starting with the match.
+--
+-- Examples:
+--
+-- >>> breakOn "::" "a::b::c"
+-- ("a","::b::c")
+--
+-- >>> breakOn "/" "foobar"
+-- ("foobar","")
+--
+-- Laws:
+--
+-- > append prefix match == haystack
+-- >   where (prefix, match) = breakOn needle haystack
+--
+-- If you need to break a string by a substring repeatedly (e.g. you
+-- want to break on every instance of a substring), use 'breakOnAll'
+-- instead, as it has lower startup overhead.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+breakOn :: HasCallStack => Text -> Text -> (Text, Text)
+breakOn pat src@(Text arr off len)
+    | null pat  = emptyError "breakOn"
+    | otherwise = case indices pat src of
+                    []    -> (src, empty)
+                    (x:_) -> (text arr off x, text arr (off+x) (len-x))
+{-# INLINE breakOn #-}
+
+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the
+-- string.
+--
+-- The first element of the returned tuple is the prefix of @haystack@
+-- up to and including the last match of @needle@.  The second is the
+-- remainder of @haystack@, following the match.
+--
+-- >>> breakOnEnd "::" "a::b::c"
+-- ("a::b::","c")
+breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)
+breakOnEnd pat src = (reverse b, reverse a)
+    where (a,b) = breakOn (reverse pat) (reverse src)
+{-# INLINE breakOnEnd #-}
+
+-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in
+-- @haystack@.  Each element of the returned list consists of a pair:
+--
+-- * The entire string prior to the /k/th match (i.e. the prefix)
+--
+-- * The /k/th match, followed by the remainder of the string
+--
+-- Examples:
+--
+-- >>> breakOnAll "::" ""
+-- []
+--
+-- >>> breakOnAll "/" "a/b/c/"
+-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+--
+-- The @needle@ parameter may not be empty.
+breakOnAll :: HasCallStack
+           => Text              -- ^ @needle@ to search for
+           -> Text              -- ^ @haystack@ in which to search
+           -> [(Text, Text)]
+breakOnAll pat src@(Text arr off slen)
+    | null pat  = emptyError "breakOnAll"
+    | otherwise = L.map step (indices pat src)
+  where
+    step       x = (chunk 0 x, chunk x (slen-x))
+    chunk !n !l  = text arr (n+off) l
+{-# INLINE breakOnAll #-}
+
+-------------------------------------------------------------------------------
+-- ** Indexing 'Text's
+
+-- $index
+--
+-- If you think of a 'Text' value as an array of 'Char' values (which
+-- it is not), you run the risk of writing inefficient code.
+--
+-- An idiom that is common in some languages is to find the numeric
+-- offset of a character or substring, then use that number to split
+-- or trim the searched string.  With a 'Text' value, this approach
+-- would require two /O(n)/ operations: one to perform the search, and
+-- one to operate from wherever the search ended.
+--
+-- For example, suppose you have a string that you want to split on
+-- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of
+-- searching for the index of @\"::\"@ and taking the substrings
+-- before and after that index, you would instead use @breakOnAll \"::\"@.
+
+-- | /O(n)/ 'Text' index (subscript) operator, starting from 0.
+index :: HasCallStack => Text -> Int -> Char
+index t@(Text _ _ lenInBytes) ix
+  | ix < 0
+  = P.error $ "Data.Text.index: negative index " ++ P.show ix
+  | off < 0 || off == lenInBytes
+  = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large"
+  | otherwise = ch
+  where
+    off = measureOff ix t
+    Iter ch _ = iter t off
+{-# INLINE index #-}
+
+-- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text'
+-- and returns the index of the first element in the 'Text' satisfying
+-- the predicate.
+findIndex :: (Char -> Bool) -> Text -> Maybe Int
+findIndex p t = S.findIndex p (stream t)
+{-# INLINE findIndex #-}
+
+-- | /O(n+m)/ The 'count' function returns the number of times the
+-- query string appears in the given 'Text'. An empty query string is
+-- invalid, and will cause an error to be raised.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+count :: HasCallStack => Text -> Text -> Int
+count pat
+    | null pat        = emptyError "count"
+    | isSingleton pat = countChar (unsafeHead pat)
+    | otherwise       = L.length . indices pat
+{-# INLINE [1] count #-}
+
+{-# RULES
+"TEXT count/singleton -> countChar" [~1] forall c t.
+    count (singleton c) t = countChar c t
+  #-}
+
+-- | /O(n)/ The 'countChar' function returns the number of times the
+-- query element appears in the given 'Text'.
+countChar :: Char -> Text -> Int
+countChar c = foldl' (\acc c' -> if c == c' then acc + 1 else acc) 0
+{-# INLINE countChar #-}
+
+-------------------------------------------------------------------------------
+-- * Zipping
+
+-- | /O(n)/ 'zip' takes two 'Text's and returns a list of
+-- corresponding pairs of bytes. If one input 'Text' is short,
+-- excess elements of the longer 'Text' are discarded. This is
+-- equivalent to a pair of 'unpack' operations.
+zip :: Text -> Text -> [(Char,Char)]
+zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
+{-# INLINE zip #-}
+
+-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
+-- given as the first argument, instead of a tupling function.
+-- Performs replacement on invalid scalar values.
+zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))
+    where g a b = safe (f a b)
+{-# INLINE [1] zipWith #-}
+
+-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's
+-- representing white space.
+words :: Text -> [Text]
+words (Text arr off len) = loop 0 0
+  where
+    loop !start !n
+        | n >= len = if start == n
+                     then []
+                     else [Text arr (start + off) (n - start)]
+        -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20
+        | isAsciiSpace w0 =
+            if start == n
+            then loop (n + 1) (n + 1)
+            else Text arr (start + off) (n - start) : loop (n + 1) (n + 1)
+        | w0 < 0x80 = loop start (n + 1)
+        -- or 2 bytes for 0xA0
+        | w0 == 0xC2, w1 == 0xA0 =
+            if start == n
+            then loop (n + 2) (n + 2)
+            else Text arr (start + off) (n - start) : loop (n + 2) (n + 2)
+        | w0 < 0xE0 = loop start (n + 2)
+        -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000
+        |  w0 == 0xE1 && w1 == 0x9A && w2 == 0x80
+        || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F)
+        || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 =
+            if start == n
+            then loop (n + 3) (n + 3)
+            else Text arr (start + off) (n - start) : loop (n + 3) (n + 3)
+        | otherwise = loop start (n + utf8LengthByLeader w0)
+        where
+            w0 = A.unsafeIndex arr (off + n)
+            w1 = A.unsafeIndex arr (off + n + 1)
+            w2 = A.unsafeIndex arr (off + n + 2)
+{-# INLINE words #-}
+
+-- Adapted from Data.ByteString.Internal.isSpaceWord8
+isAsciiSpace :: Word8 -> Bool
+isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5)
+{-# INLINE isAsciiSpace #-}
+
+-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters
+-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.
+--
+-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.
+lines :: Text -> [Text]
+lines (Text arr@(A.ByteArray arr#) off len) = go off
+  where
+    go !n
+      | n >= len + off = []
+      | delta < 0 = [Text arr n (len + off - n)]
+      | otherwise = Text arr n delta : go (n + delta + 1)
+      where
+        delta = memchr arr# n (len + off - n) 0x0A
+{-# INLINE lines #-}
+
+-- | /O(n)/ Joins lines, after appending a terminating newline to
+-- each.
+unlines :: [Text] -> Text
+unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []
+{-# INLINE unlines #-}
+
+-- | /O(n)/ Joins words using single space characters.
+unwords :: [Text] -> Text
+unwords = intercalate (singleton ' ')
+{-# INLINE unwords #-}
+
+-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is a prefix of the second.
+isPrefixOf :: Text -> Text -> Bool
+isPrefixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) =
+    d >= 0 && a == b'
+  where d              = bLen - aLen
+        b' | d == 0    = b
+           | otherwise = Text bArr bOff aLen
+{-# INLINE [1] isPrefixOf #-}
+
+-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is a suffix of the second.
+isSuffixOf :: Text -> Text -> Bool
+isSuffixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) =
+    d >= 0 && a == b'
+  where d              = bLen - aLen
+        b' | d == 0    = b
+           | otherwise = Text bArr (bOff+d) aLen
+{-# INLINE isSuffixOf #-}
+
+-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is contained, wholly and intact, anywhere
+-- within the second.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+isInfixOf ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Text -> Text -> Bool
+isInfixOf needle haystack
+    | null needle        = True
+    | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack
+    | otherwise          = not . L.null . indices needle $ haystack
+{-# INLINE [1] isInfixOf #-}
+
+-------------------------------------------------------------------------------
+-- * View patterns
+
+-- | /O(n)/ Return the suffix of the second string if its prefix
+-- matches the entire first string.
+--
+-- Examples:
+--
+-- >>> stripPrefix "foo" "foobar"
+-- Just "bar"
+--
+-- >>> stripPrefix ""    "baz"
+-- Just "baz"
+--
+-- >>> stripPrefix "foo" "quux"
+-- Nothing
+--
+-- This is particularly useful with the @ViewPatterns@ extension to
+-- GHC, as follows:
+--
+-- > {-# LANGUAGE ViewPatterns #-}
+-- > import Data.Text as T
+-- >
+-- > fnordLength :: Text -> Int
+-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf
+-- > fnordLength _                                 = -1
+stripPrefix :: Text -> Text -> Maybe Text
+stripPrefix p@(Text _arr _off plen) t@(Text arr off len)
+    | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen)
+    | otherwise        = Nothing
+
+-- | /O(n)/ Find the longest non-empty common prefix of two strings
+-- and return it, along with the suffixes of each string at which they
+-- no longer match.
+--
+-- If the strings do not have a common prefix or either one is empty,
+-- this function returns 'Nothing'.
+--
+-- Examples:
+--
+-- >>> commonPrefixes "foobar" "fooquux"
+-- Just ("foo","bar","quux")
+--
+-- >>> commonPrefixes "veeble" "fetzer"
+-- Nothing
+--
+-- >>> commonPrefixes "" "baz"
+-- Nothing
+commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text)
+commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1)
+  | len0 == 0 = Nothing
+  | len1 == 0 = Nothing
+  | otherwise = go 0 0
+  where
+    go !i !j
+      | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i))
+      | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty)
+      | a == b = go (i + 1) k
+      | k > 0 = Just (Text arr0 off0 k,
+                      Text arr0 (off0 + k) (len0 - k),
+                      Text arr1 (off1 + k) (len1 - k))
+      | otherwise = Nothing
+      where
+        a = A.unsafeIndex arr0 (off0 + i)
+        b = A.unsafeIndex arr1 (off1 + i)
+        isLeader = word8ToInt8 a >= -64
+        k = if isLeader then i else j
+{-# INLINE commonPrefixes #-}
+
+-- | /O(n)/ Return the prefix of the second string if its suffix
+-- matches the entire first string.
+--
+-- Examples:
+--
+-- >>> stripSuffix "bar" "foobar"
+-- Just "foo"
+--
+-- >>> stripSuffix ""    "baz"
+-- Just "baz"
+--
+-- >>> stripSuffix "foo" "quux"
+-- Nothing
+--
+-- This is particularly useful with the @ViewPatterns@ extension to
+-- GHC, as follows:
+--
+-- > {-# LANGUAGE ViewPatterns #-}
+-- > import Data.Text as T
+-- >
+-- > quuxLength :: Text -> Int
+-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre
+-- > quuxLength _                                = -1
+stripSuffix :: Text -> Text -> Maybe Text
+stripSuffix p@(Text _arr _off plen) t@(Text arr off len)
+    | p `isSuffixOf` t = Just $! text arr off (len-plen)
+    | otherwise        = Nothing
+
+-- | Add a list of non-negative numbers.  Errors out on overflow.
+sumP :: String -> [Int] -> Int
+sumP fun = L.foldl' add 0
+  where add a x
+            | ax >= 0   = ax
+            | otherwise = overflowError fun
+          where ax = a + x
+{-# INLINE sumP #-} -- Use foldl' and inline for fusion.
+
+emptyError :: HasCallStack => String -> a
+emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input"
+
+overflowError :: HasCallStack => String -> a
+overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow"
+
+-- | Convert a value to 'Text'.
+--
+-- @since 2.1.2
+show :: Show a => a -> Text
+show = pack . P.show
+
+-- | /O(n)/ Make a distinct copy of the given string, sharing no
+-- storage with the original string.
+--
+-- As an example, suppose you read a large string, of which you need
+-- only a small portion.  If you do not use 'copy', the entire original
+-- array will be kept alive in memory by the smaller string. Making a
+-- copy \"breaks the link\" to the original array, allowing it to be
+-- garbage collected if there are no other live references to it.
+copy :: Text -> Text
+copy t@(Text arr off len)
+  | null t = empty
+  | otherwise = Text (A.run go) 0 len
+  where
+    go :: ST s (A.MArray s)
+    go = do
+      marr <- A.new len
+      A.copyI len marr 0 arr off
+      return marr
+
+ord8 :: Char -> Word8
+ord8 = P.fromIntegral . Char.ord
+
+intToCSize :: Int -> CSize
+intToCSize = P.fromIntegral
+
+cSsizeToInt :: CSsize -> Int
+cSsizeToInt = P.fromIntegral
+
+word8ToInt8 :: Word8 -> Int8
+word8ToInt8 = P.fromIntegral
+
+-------------------------------------------------
+-- NOTE: the named chunk below used by doctest;
+--       verify the doctests via `doctest -fobject-code Data/Text.hs`
+
+-- $setup
+-- >>> :set -XOverloadedStrings
+-- >>> import qualified Data.Text as T
+ src/Data/Text/Array.hs view
@@ -0,0 +1,361 @@+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE MagicHash #-}
+{-# LANGUAGE PatternSynonyms #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE UnboxedTuples #-}
+{-# LANGUAGE UnliftedFFITypes #-}
+
+{-# OPTIONS_GHC -fno-warn-unused-matches #-}
+-- |
+-- Module      : Data.Text.Array
+-- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : portable
+--
+-- Packed, unboxed, heap-resident arrays.  Suitable for performance
+-- critical use, both in terms of large data quantities and high
+-- speed.
+--
+-- This module is intended to be imported @qualified@, to avoid name
+-- clashes with "Prelude" functions, e.g.
+--
+-- > import qualified Data.Text.Array as A
+--
+-- The names in this module resemble those in the 'Data.Array' family
+-- of modules, but are shorter due to the assumption of qualified
+-- naming.
+module Data.Text.Array
+    (
+    -- * Types
+      Array
+    , pattern ByteArray
+    , MArray
+    , pattern MutableByteArray
+    -- * Functions
+    , resizeM
+    , shrinkM
+    , copyM
+    , copyI
+    , copyFromPointer
+    , copyToPointer
+    , empty
+    , equal
+    , compare
+    , run
+    , run2
+    , toList
+    , unsafeFreeze
+    , unsafeIndex
+    , new
+    , newPinned
+    , newFilled
+    , unsafeWrite
+    , tile
+    , getSizeofMArray
+    ) where
+
+#if defined(ASSERTS)
+import GHC.Stack (HasCallStack)
+#endif
+#if !MIN_VERSION_base(4,11,0)
+import Foreign.C.Types (CInt(..))
+#endif
+import GHC.Exts hiding (toList)
+import GHC.ST (ST(..), runST)
+import GHC.Word (Word8(..))
+import qualified Prelude
+import Prelude hiding (length, read, compare)
+import Data.Array.Byte (ByteArray(..), MutableByteArray(..))
+
+-- | Immutable array type.
+type Array = ByteArray
+
+-- | Mutable array type, for use in the ST monad.
+type MArray = MutableByteArray
+
+-- | Create an uninitialized mutable array.
+new :: forall s. Int -> ST s (MArray s)
+new (I# len#)
+#if defined(ASSERTS)
+  | I# len# < 0 = error "Data.Text.Array.new: size overflow"
+#endif
+  | otherwise = ST $ \s1# ->
+    case newByteArray# len# s1# of
+      (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)
+{-# INLINE new #-}
+
+-- | Create an uninitialized mutable pinned array.
+--
+-- @since 2.0
+newPinned :: forall s. Int -> ST s (MArray s)
+newPinned (I# len#)
+#if defined(ASSERTS)
+  | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow"
+#endif
+  | otherwise = ST $ \s1# ->
+    case newPinnedByteArray# len# s1# of
+      (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)
+{-# INLINE newPinned #-}
+
+-- | @since 2.0
+newFilled :: Int -> Int -> ST s (MArray s)
+newFilled (I# len#) (I# c#) = ST $ \s1# ->
+  case newByteArray# len# s1# of
+    (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of
+      s3# -> (# s3#, MutableByteArray marr# #)
+{-# INLINE newFilled #-}
+
+-- | @since 2.0
+tile :: MArray s -> Int -> ST s ()
+tile marr tileLen = do
+  totalLen <- getSizeofMArray marr
+  let go l
+        | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l)
+        | otherwise = copyM marr l marr 0 l >> go (2 * l)
+  go tileLen
+{-# INLINE tile #-}
+
+-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!
+unsafeFreeze :: MArray s -> ST s Array
+unsafeFreeze (MutableByteArray marr) = ST $ \s1# ->
+    case unsafeFreezeByteArray# marr s1# of
+        (# s2#, ba# #) -> (# s2#, ByteArray ba# #)
+{-# INLINE unsafeFreeze #-}
+
+-- | Unchecked read of an immutable array.  May return garbage or
+-- crash on an out-of-bounds access.
+unsafeIndex ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Array -> Int -> Word8
+unsafeIndex (ByteArray arr) i@(I# i#) =
+#if defined(ASSERTS)
+  let word8len = I# (sizeofByteArray# arr) in
+  if i < 0 || i >= word8len
+  then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len)
+  else
+#endif
+  case indexWord8Array# arr i# of r# -> (W8# r#)
+{-# INLINE unsafeIndex #-}
+
+-- | @since 2.0
+getSizeofMArray :: MArray s -> ST s Int
+getSizeofMArray (MutableByteArray marr) = ST $ \s0# ->
+  -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is
+  -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'.
+  case getSizeofMutableByteArray# marr s0# of
+    (# s1#, word8len# #) -> (# s1#, I# word8len# #)
+
+#if defined(ASSERTS)
+checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s ()
+checkBoundsM ma i elSize = do
+  len <- getSizeofMArray ma
+  if i < 0 || i + elSize > len
+    then error ("bounds error, offset " ++ show i ++ ", length " ++ show len)
+    else return ()
+#endif
+
+-- | Unchecked write of a mutable array.  May return garbage or crash
+-- on an out-of-bounds access.
+unsafeWrite ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  MArray s -> Int -> Word8 -> ST s ()
+unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) =
+#if defined(ASSERTS)
+  checkBoundsM ma i 1 >>
+#endif
+  (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of
+    s2# -> (# s2#, () #))
+{-# INLINE unsafeWrite #-}
+
+-- | Convert an immutable array to a list.
+toList :: Array -> Int -> Int -> [Word8]
+toList ary off len = loop 0
+    where loop i | i < len   = unsafeIndex ary (off+i) : loop (i+1)
+                 | otherwise = []
+
+-- | An empty immutable array.
+empty :: Array
+empty = runST (new 0 >>= unsafeFreeze)
+
+-- | Run an action in the ST monad and return an immutable array of
+-- its result.
+run :: (forall s. ST s (MArray s)) -> Array
+run k = runST (k >>= unsafeFreeze)
+
+-- | Run an action in the ST monad and return an immutable array of
+-- its result paired with whatever else the action returns.
+run2 :: (forall s. ST s (MArray s, a)) -> (Array, a)
+run2 k = runST (do
+                 (marr,b) <- k
+                 arr <- unsafeFreeze marr
+                 return (arr,b))
+{-# INLINE run2 #-}
+
+-- | @since 2.0
+resizeM :: MArray s -> Int -> ST s (MArray s)
+resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# ->
+  case resizeMutableByteArray# ma i# s1# of
+    (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #)
+{-# INLINE resizeM #-}
+
+-- | @since 2.0
+shrinkM ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  MArray s -> Int -> ST s ()
+shrinkM (MutableByteArray marr) i@(I# newSize) = do
+#if defined(ASSERTS)
+  oldSize <- getSizeofMArray (MutableByteArray marr)
+  if I# newSize > oldSize
+    then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize)
+    else return ()
+#endif
+  ST $ \s1# ->
+    case shrinkMutableByteArray# marr newSize s1# of
+      s2# -> (# s2#, () #)
+{-# INLINE shrinkM #-}
+
+-- | Copy some elements of a mutable array.
+copyM :: MArray s               -- ^ Destination
+      -> Int                    -- ^ Destination offset
+      -> MArray s               -- ^ Source
+      -> Int                    -- ^ Source offset
+      -> Int                    -- ^ Count
+      -> ST s ()
+copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#)
+#if defined(ASSERTS)
+  | count < 0 = error $
+    "copyM: count must be >= 0, but got " ++ show count
+#endif
+    | otherwise = do
+#if defined(ASSERTS)
+    srcLen <- getSizeofMArray src
+    dstLen <- getSizeofMArray dst
+    if srcOff + count > srcLen
+      then error "copyM: source is too short"
+      else return ()
+    if dstOff + count > dstLen
+      then error "copyM: destination is too short"
+      else return ()
+#endif
+    ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of
+      s2# -> (# s2#, () #)
+{-# INLINE copyM #-}
+
+-- | Copy some elements of an immutable array.
+copyI :: Int                    -- ^ Count
+      -> MArray s               -- ^ Destination
+      -> Int                    -- ^ Destination offset
+      -> Array                  -- ^ Source
+      -> Int                    -- ^ Source offset
+      -> ST s ()
+copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#)
+#if defined(ASSERTS)
+  | count < 0 = error $
+    "copyI: count must be >= 0, but got " ++ show count
+#endif
+  | otherwise = ST $ \s1# ->
+    case copyByteArray# src# srcOff# dst# dstOff# count# s1# of
+      s2# -> (# s2#, () #)
+{-# INLINE copyI #-}
+
+-- | Copy from pointer.
+--
+-- @since 2.0
+copyFromPointer
+  :: MArray s               -- ^ Destination
+  -> Int                    -- ^ Destination offset
+  -> Ptr Word8              -- ^ Source
+  -> Int                    -- ^ Count
+  -> ST s ()
+copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#)
+#if defined(ASSERTS)
+  | count < 0 = error $
+    "copyFromPointer: count must be >= 0, but got " ++ show count
+#endif
+  | otherwise = ST $ \s1# ->
+    case copyAddrToByteArray# src# dst# dstOff# count# s1# of
+      s2# -> (# s2#, () #)
+{-# INLINE copyFromPointer #-}
+
+-- | Copy to pointer.
+--
+-- @since 2.0
+copyToPointer
+  :: Array                  -- ^ Source
+  -> Int                    -- ^ Source offset
+  -> Ptr Word8              -- ^ Destination
+  -> Int                    -- ^ Count
+  -> ST s ()
+copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#)
+#if defined(ASSERTS)
+  | count < 0 = error $
+    "copyToPointer: count must be >= 0, but got " ++ show count
+#endif
+  | otherwise = ST $ \s1# ->
+    case copyByteArrayToAddr# src# srcOff# dst# count# s1# of
+      s2# -> (# s2#, () #)
+{-# INLINE copyToPointer #-}
+
+-- | Compare portions of two arrays for equality.  No bounds checking
+-- is performed.
+equal
+  :: Array
+  -- ^ First array
+  -> Int
+  -- ^ Offset in the first array
+  -> Array
+  -- ^ Second array
+  -> Int
+  -- ^ Offset in the second array
+  -> Int
+  -- ^ How many bytes to compare?
+  -> Bool
+equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0
+{-# INLINE equal #-}
+
+-- | Compare portions of two arrays. No bounds checking is performed.
+--
+-- @since 2.0
+compare
+  :: Array
+  -- ^ First array
+  -> Int
+  -- ^ Offset in the first array
+  -> Array
+  -- ^ Second array
+  -> Int
+  -- ^ Offset in the second array
+  -> Int
+  -- ^ How many bytes to compare?
+  -> Ordering
+compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0
+{-# INLINE compare #-}
+
+compareInternal
+      :: Array                  -- ^ First
+      -> Int                    -- ^ Offset into first
+      -> Array                  -- ^ Second
+      -> Int                    -- ^ Offset into second
+      -> Int                    -- ^ Count
+      -> Int
+compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i
+  where
+#if MIN_VERSION_base(4,11,0)
+    i = I# (compareByteArrays# src1# off1# src2# off2# count#)
+#else
+    i = fromIntegral (memcmp src1# off1# src2# off2# count#)
+
+foreign import ccall unsafe "_hs_text_memcmp2" memcmp
+    :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt
+#endif
+{-# INLINE compareInternal #-}
+ src/Data/Text/Encoding.hs view
@@ -0,0 +1,588 @@+{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash,
+    UnliftedFFITypes #-}
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE TypeApplications #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+-- |
+-- Module      : Data.Text.Encoding
+-- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan,
+--               (c) 2009 Duncan Coutts,
+--               (c) 2008, 2009 Tom Harper
+--               (c) 2021 Andrew Lelechenko
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : portable
+--
+-- Functions for converting 'Text' values to and from 'ByteString',
+-- using several standard encodings.
+--
+-- To gain access to a much larger family of encodings, use the
+-- <http://hackage.haskell.org/package/text-icu text-icu package>.
+
+module Data.Text.Encoding
+    (
+    -- * Decoding ByteStrings to Text
+    -- $strict
+
+    -- ** Total Functions #total#
+    -- $total
+      decodeLatin1
+    , decodeASCIIPrefix
+    , decodeUtf8Lenient
+    , decodeUtf8'
+    , decodeASCII'
+
+    -- *** Controllable error handling
+    , decodeUtf8With
+    , decodeUtf16LEWith
+    , decodeUtf16BEWith
+    , decodeUtf32LEWith
+    , decodeUtf32BEWith
+
+    -- *** Incremental UTF-8 decoding
+    -- $incremental
+    , decodeUtf8Chunk
+    , decodeUtf8More
+    , Utf8State
+    , startUtf8State
+    , StrictBuilder
+    , StrictTextBuilder
+    , strictBuilderToText
+    , textToStrictBuilder
+
+    -- ** Partial Functions
+    -- $partial
+    , decodeASCII
+    , decodeUtf8
+    , decodeUtf16LE
+    , decodeUtf16BE
+    , decodeUtf32LE
+    , decodeUtf32BE
+
+    -- ** Stream oriented decoding
+    -- $stream
+    , streamDecodeUtf8
+    , streamDecodeUtf8With
+    , Decoding(..)
+
+    -- * Encoding Text to ByteStrings
+    , encodeUtf8
+    , encodeUtf16LE
+    , encodeUtf16BE
+    , encodeUtf32LE
+    , encodeUtf32BE
+
+    -- * Encoding Text using ByteString Builders
+    , encodeUtf8Builder
+    , encodeUtf8BuilderEscaped
+
+    -- * ByteString validation
+    -- $validation
+    , validateUtf8Chunk
+    , validateUtf8More
+    ) where
+
+import Control.Exception (evaluate, try)
+import Data.Word (Word8)
+import GHC.Exts (byteArrayContents#, unsafeCoerce#)
+import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr))
+import Data.ByteString (ByteString)
+#if defined(PURE_HASKELL)
+import Control.Monad.ST.Unsafe (unsafeSTToIO)
+import Data.ByteString.Char8 (unpack)
+import Data.Text.Internal (pack)
+import Foreign.Ptr (minusPtr, plusPtr)
+import Foreign.Storable (poke)
+#else
+import Control.Monad.ST (runST)
+import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
+import Data.Bits (shiftR, (.&.))
+import Data.Text.Internal.ByteStringCompat (withBS)
+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)
+import Foreign.C.Types (CSize(..))
+import Foreign.Ptr (Ptr, minusPtr, plusPtr)
+import Foreign.Storable (poke, peekByteOff)
+#endif
+import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode)
+import Data.Text.Internal (Text(..), empty)
+import Data.Text.Internal.Encoding
+import Data.Text.Internal.IsAscii (asciiPrefixLength)
+import Data.Text.Unsafe (unsafeDupablePerformIO)
+import Data.Text.Show ()
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Internal as B
+import qualified Data.ByteString.Builder as B
+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)
+import qualified Data.ByteString.Builder.Prim as BP
+import qualified Data.ByteString.Builder.Prim.Internal as BP
+import qualified Data.ByteString.Short.Internal as SBS
+import qualified Data.Text.Array as A
+import qualified Data.Text.Internal.Encoding.Fusion as E
+import qualified Data.Text.Internal.Fusion as F
+#if defined(ASSERTS)
+import GHC.Stack (HasCallStack)
+#endif
+
+-- $validation
+-- These functions are for validating 'ByteString's as encoded text.
+
+-- $strict
+--
+-- All of the single-parameter functions for decoding bytestrings
+-- encoded in one of the Unicode Transformation Formats (UTF) operate
+-- in a /strict/ mode: each will throw an exception if given invalid
+-- input.
+--
+-- Each function has a variant, whose name is suffixed with -'With',
+-- that gives greater control over the handling of decoding errors.
+-- For instance, 'decodeUtf8' will throw an exception, but
+-- 'decodeUtf8With' allows the programmer to determine what to do on a
+-- decoding error.
+
+-- $total
+--
+-- These functions facilitate total decoding and should be preferred
+-- over their partial counterparts.
+
+-- $partial
+--
+-- These functions are partial and should only be used with great caution
+-- (preferably not at all). See "Data.Text.Encoding#g:total" for better
+-- solutions.
+
+-- | Decode a 'ByteString' containing ASCII text.
+--
+-- This is a total function which returns a pair of the longest ASCII prefix
+-- as 'Text', and the remaining suffix as 'ByteString'.
+--
+-- Important note: the pair is lazy. This lets you check for errors by testing
+-- whether the second component is empty, without forcing the first component
+-- (which does a copy).
+-- To drop references to the input bytestring, force the prefix
+-- (using 'seq' or @BangPatterns@) and drop references to the suffix.
+--
+-- === Properties
+--
+-- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@.
+-- - Either @suffix@ is empty, or @'B.head' suffix > 127@.
+--
+-- @since 2.0.2
+decodeASCIIPrefix :: ByteString -> (Text, ByteString)
+decodeASCIIPrefix bs = if B.null bs
+  then (empty, B.empty)
+  else
+    let len = asciiPrefixLength bs
+        prefix =
+          let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in
+          Text (A.ByteArray arr) 0 len
+        suffix = B.drop len bs in
+    (prefix, suffix)
+{-# INLINE decodeASCIIPrefix #-}
+
+-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
+--
+-- This is a total function which returns either the 'ByteString' converted to a
+-- 'Text' containing ASCII text, or 'Nothing'.
+--
+-- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid
+-- input instead of discarding it.
+--
+-- @since 2.0.2
+decodeASCII' :: ByteString -> Maybe Text
+decodeASCII' bs =
+  let (prefix, suffix) = decodeASCIIPrefix bs in
+  if B.null suffix then Just prefix else Nothing
+{-# INLINE decodeASCII' #-}
+
+-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.
+--
+-- This is a partial function: it checks that input does not contain
+-- anything except ASCII and copies buffer or throws an error otherwise.
+decodeASCII :: ByteString -> Text
+decodeASCII bs =
+  let (prefix, suffix) = decodeASCIIPrefix bs in
+  case B.uncons suffix of
+    Nothing -> prefix
+    Just (word, _) ->
+      let !errPos = B.length bs - B.length suffix in
+      error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos
+
+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
+--
+-- 'decodeLatin1' is semantically equivalent to
+--  @'Data.Text.pack' . 'Data.ByteString.Char8.unpack'@
+--
+-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII)
+-- characters to UTf-8 requires actual work and is not just buffer copying.
+--
+decodeLatin1 ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  ByteString -> Text
+#if defined(PURE_HASKELL)
+decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs)
+#else
+decodeLatin1 bs = withBS bs $ \fp len -> runST $ do
+  dst <- A.new (2 * len)
+  let inner srcOff dstOff = if srcOff >= len then return dstOff else do
+        asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->
+          c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len)
+        if asciiPrefixLen == 0
+        then do
+          byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff
+          A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6))
+          A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F))
+          inner (srcOff + 1) (dstOff + 2)
+        else do
+          unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->
+            unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen
+          inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen)
+  actualLen <- inner 0 0
+  dst' <- A.resizeM dst actualLen
+  arr <- A.unsafeFreeze dst'
+  return $ Text arr 0 actualLen
+#endif
+
+#if !defined(PURE_HASKELL)
+foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii
+    :: Ptr Word8 -> Ptr Word8 -> IO CSize
+#endif
+
+-- $stream
+--
+-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept
+-- a strict 'ByteString' that represents a possibly incomplete input (e.g. a
+-- packet from a network stream) that may not end on a UTF-8 boundary
+-- and return 'Decoding', which consists of:
+--
+-- *  The maximal prefix of 'Text' that could be decoded from the
+--    given input.
+--
+-- *  The suffix of the 'ByteString' that could not be decoded due to
+--    insufficient input.
+--
+-- *  A function that accepts another 'ByteString'.  That string will
+--    be assumed to directly follow the string that was passed as
+--    input to the original function, and it will in turn be decoded.
+--
+-- To help understand the use of these functions, consider the Unicode
+-- string @\"hi &#9731;\"@. If encoded as UTF-8, this becomes @\"hi
+-- \\xe2\\x98\\x83\"@; the final @\'&#9731;\'@ is encoded as 3 bytes.
+--
+-- Now suppose that we receive this encoded string as 3 packets that
+-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",
+-- \"\\x83\"]@. We cannot decode the entire Unicode string until we
+-- have received all three packets, but we would like to make progress
+-- as we receive each one.
+--
+-- @
+-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\"
+-- ghci> s0
+-- 'Some' \"hi \" \"\\xe2\" _
+-- @
+--
+-- We use the continuation @f0@ to decode our second packet.
+--
+-- @
+-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\"
+-- ghci> s1
+-- 'Some' \"\" \"\\xe2\\x98\"
+-- @
+--
+-- We could not give @f0@ enough input to decode anything, so it
+-- returned an empty string. Once we feed our second continuation @f1@
+-- the last byte of input, it will make progress.
+--
+-- @
+-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\"
+-- ghci> s2
+-- 'Some' \"\\x2603\" \"\" _
+-- @
+--
+-- If given invalid input, an exception will be thrown by the function
+-- or continuation where it is encountered.
+
+-- | A stream-oriented decoding result (see 'streamDecodeUtf8' and 'streamDecodeUtf8With').
+--
+-- @since 1.0.0.0
+data Decoding = Some
+  !Text
+  -- ^ The maximal prefix that could be decoded from the given input.
+  !ByteString
+  -- ^ The remaining suffix of the input that could not be decoded
+  -- (usually because the input breaks in the middle of UTF-8 character)
+  (ByteString -> Decoding)
+  -- ^ The continuation call which should be fed with the next
+  -- chunk of the input.
+
+instance Show Decoding where
+    showsPrec d (Some t bs _) = showParen (d > prec) $
+                                showString "Some " . showsPrec prec' t .
+                                showChar ' ' . showsPrec prec' bs .
+                                showString " _"
+      where prec = 10; prec' = prec + 1
+
+-- | Initiate a stream-oriented decoding
+-- with a strict 'ByteString' containing UTF-8 data that is known to be valid.
+--
+-- If the input contains any invalid UTF-8 data, an exception will be
+-- thrown (either by this function or a continuation) that cannot be
+-- caught in pure code.  For more control over the handling of invalid
+-- data, use 'streamDecodeUtf8With'.
+--
+-- @since 1.0.0.0
+streamDecodeUtf8 ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  ByteString -> Decoding
+streamDecodeUtf8 = streamDecodeUtf8With strictDecode
+
+-- | Initiate a stream-oriented decoding
+-- with a strict 'ByteString' containing UTF-8 data.
+--
+-- @since 1.0.0.0
+streamDecodeUtf8With ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  OnDecodeError -> ByteString -> Decoding
+streamDecodeUtf8With onErr = loop startUtf8State
+  where
+    loop s chunk =
+      let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk
+      in Some (strictBuilderToText builder) undecoded (loop s')
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text.
+--
+-- Surrogate code points in replacement character returned by 'OnDecodeError'
+-- will be automatically remapped to the replacement char @U+FFFD@.
+decodeUtf8With ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  OnDecodeError -> ByteString -> Text
+decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg
+
+invalidUtf8Msg :: String
+invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream"
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
+-- to be valid.
+--
+-- If the input contains any invalid UTF-8 data, an exception will be
+-- thrown that cannot be caught in pure code.  For more control over
+-- the handling of invalid data, use 'decodeUtf8'' or
+-- 'decodeUtf8With'.
+--
+-- This is a partial function: it checks that input is a well-formed
+-- UTF-8 sequence and copies buffer or throws an error otherwise.
+--
+decodeUtf8 :: ByteString -> Text
+decodeUtf8 = decodeUtf8With strictDecode
+{-# INLINE[0] decodeUtf8 #-}
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text.
+--
+-- If the input contains any invalid UTF-8 data, the relevant
+-- exception will be returned, otherwise the decoded text.
+decodeUtf8' ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  ByteString -> Either UnicodeException Text
+decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
+{-# INLINE decodeUtf8' #-}
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text.
+--
+-- Any invalid input bytes will be replaced with the Unicode replacement
+-- character U+FFFD.
+--
+-- @since 2.0
+decodeUtf8Lenient :: ByteString -> Text
+decodeUtf8Lenient = decodeUtf8With lenientDecode
+
+-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
+--
+-- @since 1.1.0.0
+encodeUtf8Builder :: Text -> B.Builder
+encodeUtf8Builder =
+    -- manual eta-expansion to ensure inlining works as expected
+    \txt -> B.builder (step txt)
+  where
+    step txt@(Text arr off len) !k br@(B.BufferRange op ope)
+      -- Ensure that the common case is not recursive and therefore yields
+      -- better code.
+      | op' <= ope = do
+          unsafeSTToIO $ A.copyToPointer arr off op len
+          k (B.BufferRange op' ope)
+      | otherwise = textCopyStep txt k br
+      where
+        op' = op `plusPtr` len
+{-# INLINE encodeUtf8Builder #-}
+
+textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a
+textCopyStep (Text arr off len) k =
+    go off (off + len)
+  where
+    go !ip !ipe (B.BufferRange op ope)
+      | inpRemaining <= outRemaining = do
+          unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining
+          let !br = B.BufferRange (op `plusPtr` inpRemaining) ope
+          k br
+      | otherwise = do
+          unsafeSTToIO $ A.copyToPointer arr ip op outRemaining
+          let !ip' = ip + outRemaining
+          return $ B.bufferFull 1 ope (go ip' ipe)
+      where
+        outRemaining = ope `minusPtr` op
+        inpRemaining = ipe - ip
+
+-- | Encode text using UTF-8 encoding and escape the ASCII characters using
+-- a 'BP.BoundedPrim'.
+--
+-- Use this function is to implement efficient encoders for text-based formats
+-- like JSON or HTML.
+--
+-- @since 1.1.0.0
+{-# INLINE encodeUtf8BuilderEscaped #-}
+-- TODO: Extend documentation with references to source code in @blaze-html@
+-- or @aeson@ that uses this function.
+encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
+encodeUtf8BuilderEscaped be =
+    -- manual eta-expansion to ensure inlining works as expected
+    \txt -> B.builder (mkBuildstep txt)
+  where
+    bound = max 4 $ BP.sizeBound be
+
+    mkBuildstep (Text arr off len) !k =
+        outerLoop off
+      where
+        iend = off + len
+
+        outerLoop !i0 !br@(B.BufferRange op0 ope)
+          | i0 >= iend       = k br
+          | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)
+          -- TODO: Use a loop with an integrated bound's check if outRemaining
+          -- is smaller than 8, as this will save on divisions.
+          | otherwise        = return $ B.bufferFull bound op0 (outerLoop i0)
+          where
+            outRemaining = (ope `minusPtr` op0) `quot` bound
+            inpRemaining = iend - i0
+
+            goPartial !iendTmp = go i0 op0
+              where
+                go !i !op
+                  | i < iendTmp = do
+                    let w = A.unsafeIndex arr i
+                    if w < 0x80
+                      then BP.runB be w op >>= go (i + 1)
+                      else poke op w >> go (i + 1) (op `plusPtr` 1)
+                  | otherwise = outerLoop i (B.BufferRange op ope)
+
+-- | Encode text using UTF-8 encoding.
+encodeUtf8 :: Text -> ByteString
+encodeUtf8 (Text arr off len)
+  | len == 0  = B.empty
+  -- It would be easier to use Data.ByteString.Short.fromShort and slice later,
+  -- but this is undesirable when len is significantly smaller than length arr.
+  | otherwise = unsafeDupablePerformIO $ do
+    marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len
+    unsafeSTToIO $ A.copyI len marr 0 arr off
+    let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba))
+                        (PlainPtr mba)
+    pure $ B.fromForeignPtr fp 0 len
+
+-- | Decode text from little endian UTF-16 encoding.
+decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
+decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
+{-# INLINE decodeUtf16LEWith #-}
+
+-- | Decode text from little endian UTF-16 encoding.
+--
+-- If the input contains any invalid little endian UTF-16 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf16LEWith'.
+decodeUtf16LE :: ByteString -> Text
+decodeUtf16LE = decodeUtf16LEWith strictDecode
+{-# INLINE decodeUtf16LE #-}
+
+-- | Decode text from big endian UTF-16 encoding.
+decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text
+decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
+{-# INLINE decodeUtf16BEWith #-}
+
+-- | Decode text from big endian UTF-16 encoding.
+--
+-- If the input contains any invalid big endian UTF-16 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf16BEWith'.
+decodeUtf16BE :: ByteString -> Text
+decodeUtf16BE = decodeUtf16BEWith strictDecode
+{-# INLINE decodeUtf16BE #-}
+
+-- | Encode text using little endian UTF-16 encoding.
+encodeUtf16LE :: Text -> ByteString
+encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))
+{-# INLINE encodeUtf16LE #-}
+
+-- | Encode text using big endian UTF-16 encoding.
+encodeUtf16BE :: Text -> ByteString
+encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))
+{-# INLINE encodeUtf16BE #-}
+
+-- | Decode text from little endian UTF-32 encoding.
+decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text
+decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
+{-# INLINE decodeUtf32LEWith #-}
+
+-- | Decode text from little endian UTF-32 encoding.
+--
+-- If the input contains any invalid little endian UTF-32 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf32LEWith'.
+decodeUtf32LE :: ByteString -> Text
+decodeUtf32LE = decodeUtf32LEWith strictDecode
+{-# INLINE decodeUtf32LE #-}
+
+-- | Decode text from big endian UTF-32 encoding.
+decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text
+decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
+{-# INLINE decodeUtf32BEWith #-}
+
+-- | Decode text from big endian UTF-32 encoding.
+--
+-- If the input contains any invalid big endian UTF-32 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf32BEWith'.
+decodeUtf32BE :: ByteString -> Text
+decodeUtf32BE = decodeUtf32BEWith strictDecode
+{-# INLINE decodeUtf32BE #-}
+
+-- | Encode text using little endian UTF-32 encoding.
+encodeUtf32LE :: Text -> ByteString
+encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))
+{-# INLINE encodeUtf32LE #-}
+
+-- | Encode text using big endian UTF-32 encoding.
+encodeUtf32BE :: Text -> ByteString
+encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))
+{-# INLINE encodeUtf32BE #-}
+
+-- $incremental
+-- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more
+-- control for error-handling and streaming.
+--
+-- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error.
+--   For example this lets you insert or delete arbitrary text, or do some
+--   stateful operations before resuming, such as keeping track of error locations.
+--   In contrast, the older stream-oriented interface only lets you substitute
+--   a single fixed 'Char' for each invalid byte in 'OnDecodeError'.
+-- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks
+--   before doing the copying work to construct a 'Text', or you can
+--   output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'.
+--
+-- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'.
+ src/Data/Text/Encoding/Error.hs view
@@ -0,0 +1,119 @@+{-# LANGUAGE CPP #-}
+{-# LANGUAGE Safe #-}
+-- |
+-- Module      : Data.Text.Encoding.Error
+-- Copyright   : (c) Bryan O'Sullivan 2009
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : GHC
+--
+-- Types and functions for dealing with encoding and decoding errors
+-- in Unicode text.
+--
+-- The standard functions for encoding and decoding text are strict,
+-- which is to say that they throw exceptions on invalid input.  This
+-- is often unhelpful on real world input, so alternative functions
+-- exist that accept custom handlers for dealing with invalid inputs.
+-- These 'OnError' handlers are normal Haskell functions.  You can use
+-- one of the presupplied functions in this module, or you can write a
+-- custom handler of your own.
+
+module Data.Text.Encoding.Error
+    (
+    -- * Error handling types
+      UnicodeException(..)
+    , OnError
+    , OnDecodeError
+    , OnEncodeError
+    -- * Useful error handling functions
+    , lenientDecode
+    , strictDecode
+    , strictEncode
+    , ignore
+    , replace
+    ) where
+
+import Control.DeepSeq (NFData (..))
+import Control.Exception (Exception, throw)
+import Data.Word (Word8)
+import Numeric (showHex)
+
+-- | Function type for handling a coding error.  It is supplied with
+-- two inputs:
+--
+-- * A 'String' that describes the error.
+--
+-- * The input value that caused the error.  If the error arose
+--   because the end of input was reached or could not be identified
+--   precisely, this value will be 'Nothing'.
+--
+-- If the handler returns a value wrapped with 'Just', that value will
+-- be used in the output as the replacement for the invalid input.  If
+-- it returns 'Nothing', no value will be used in the output.
+--
+-- Should the handler need to abort processing, it should use 'error'
+-- or 'throw' an exception (preferably a 'UnicodeException').  It may
+-- use the description provided to construct a more helpful error
+-- report.
+type OnError a b = String -> Maybe a -> Maybe b
+
+-- | A handler for a decoding error.
+type OnDecodeError = OnError Word8 Char
+
+-- | A handler for an encoding error.
+{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-}
+type OnEncodeError = OnError Char Word8
+
+-- | An exception type for representing Unicode encoding errors.
+data UnicodeException =
+    DecodeError String (Maybe Word8)
+    -- ^ Could not decode a byte sequence because it was invalid under
+    -- the given encoding, or ran out of input in mid-decode.
+  | EncodeError String (Maybe Char)
+    -- ^ Tried to encode a character that could not be represented
+    -- under the given encoding, or ran out of input in mid-encode.
+    deriving (Eq)
+
+{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-}
+
+showUnicodeException :: UnicodeException -> String
+showUnicodeException (DecodeError desc (Just w))
+    = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)
+showUnicodeException (DecodeError desc Nothing)
+    = "Cannot decode input: " ++ desc
+showUnicodeException (EncodeError desc (Just c))
+    = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc)
+showUnicodeException (EncodeError desc Nothing)
+    = "Cannot encode input: " ++ desc
+
+instance Show UnicodeException where
+    show = showUnicodeException
+
+instance Exception UnicodeException
+
+instance NFData UnicodeException where
+    rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` ()
+    rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` ()
+
+-- | Throw a 'UnicodeException' if decoding fails.
+strictDecode :: OnDecodeError
+strictDecode desc c = throw (DecodeError desc c)
+
+-- | Replace an invalid input byte with the Unicode replacement
+-- character U+FFFD.
+lenientDecode :: OnDecodeError
+lenientDecode _ _ = Just '\xfffd'
+
+-- | Throw a 'UnicodeException' if encoding fails.
+{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-}
+strictEncode :: OnEncodeError
+strictEncode desc c = throw (EncodeError desc c)
+
+-- | Ignore an invalid input, substituting nothing in the output.
+ignore :: OnError a b
+ignore _ _ = Nothing
+
+-- | Replace an invalid input with a valid output.
+replace :: b -> OnError a b
+replace c _ _ = Just c
+ src/Data/Text/Foreign.hs view
@@ -0,0 +1,212 @@+{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-}
+-- |
+-- Module      : Data.Text.Foreign
+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : GHC
+--
+-- Support for using 'Text' data with native code via the Haskell
+-- foreign function interface.
+
+module Data.Text.Foreign
+    (
+    -- * Interoperability with native code
+    -- $interop
+      I8
+    -- * Pointer conversion functions
+    , fromPtr
+    , fromPtr0
+    , useAsPtr
+    , asForeignPtr
+    -- ** Encoding as UTF-8
+    , peekCString
+    , withCString
+    , peekCStringLen
+    , withCStringLen
+    -- * Low-level manipulation
+    -- $lowlevel
+    , dropWord8
+    , takeWord8
+    , lengthWord8
+    , unsafeCopyToPtr
+    ) where
+
+import Control.Monad.ST.Unsafe (unsafeSTToIO)
+import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen)
+import Data.Text.Encoding (decodeUtf8, encodeUtf8)
+import Data.Text.Internal (Text(..), empty)
+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)
+import Data.Text.Show (addrLen)
+import Data.Text.Unsafe (lengthWord8)
+import Data.Word (Word8)
+import Foreign.C.String (CString, CStringLen)
+import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray)
+import Foreign.Marshal.Alloc (allocaBytes)
+import Foreign.Ptr (Ptr, castPtr)
+import Foreign.Storable (pokeByteOff)
+import GHC.Exts (Ptr(..))
+import qualified Data.Text.Array as A
+
+-- $interop
+--
+-- The 'Text' type is implemented using arrays that are not guaranteed
+-- to have a fixed address in the Haskell heap. All communication with
+-- native code must thus occur by copying data back and forth.
+--
+-- The 'Text' type's internal representation is UTF-8.
+-- To interoperate with native libraries that use different
+-- internal representations, such as UTF-16 or UTF-32, consider using
+-- the functions in the 'Data.Text.Encoding' module.
+
+-- | A type representing a number of UTF-8 code units.
+--
+-- @since 2.0
+newtype I8 = I8 Int
+    deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show)
+
+-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the
+-- contents of the array.
+--
+-- __This function is unsafe.__ The source array must contain a valid
+-- UTF-8 string of the given length. There are no guarantees about what
+-- happens otherwise.
+fromPtr :: Ptr Word8           -- ^ source array
+        -> I8                  -- ^ length of source array (in 'Word8' units)
+        -> IO Text
+fromPtr _   (I8 0)   = pure empty
+fromPtr ptr (I8 len) = unsafeSTToIO $ do
+  dst <- A.new len
+  A.copyFromPointer dst 0 ptr len
+  arr <- A.unsafeFreeze dst
+  return $! Text arr 0 len
+
+-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the
+-- contents of the NUL-terminated array.
+--
+-- __This function is unsafe.__ The source array must contain a NULL-terminated
+-- valid UTF-8 string. There are no guarantees about what happens otherwise.
+--
+-- @since 2.0.1
+fromPtr0 :: Ptr Word8           -- ^ source array
+         -> IO Text
+fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#))
+
+-- $lowlevel
+--
+-- Foreign functions that use UTF-8 internally may return indices in
+-- units of 'Word8' instead of characters.  These functions may
+-- safely be used with such indices, as they will adjust offsets if
+-- necessary to preserve the validity of a Unicode string.
+
+-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in
+-- length.
+--
+-- If @n@ would cause the 'Text' to end inside a code point, the
+-- end of the prefix will be advanced by several additional 'Word8' units
+-- to maintain its validity.
+--
+-- @since 2.0
+takeWord8 :: I8 -> Text -> Text
+takeWord8 = (fst .) . splitAtWord8
+
+-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units
+-- dropped from its beginning.
+--
+-- If @n@ would cause the 'Text' to begin inside a code point, the
+-- beginning of the suffix will be advanced by several additional 'Word8'
+-- unit to maintain its validity.
+--
+-- @since 2.0
+dropWord8 :: I8 -> Text -> Text
+dropWord8 = (snd .) . splitAtWord8
+
+splitAtWord8 :: I8 -> Text -> (Text, Text)
+splitAtWord8 (I8 n) t@(Text arr off len)
+    | n <= 0               = (empty, t)
+    | n >= len || m >= len = (t, empty)
+    | otherwise            = (Text arr off m, Text arr (off+m) (len-m))
+  where
+    m | w0 <  0x80 = n   -- last char is ASCII
+      | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence
+      | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence
+      | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence
+      | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence
+      | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence
+      | w1 >= 0xC0 = n   -- pre-last char starts 2-byte sequence
+      | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence
+      | otherwise  = n   -- pre-pre-last char starts 3-byte sequence
+    w0 = A.unsafeIndex arr (off+n-1)
+    w1 = A.unsafeIndex arr (off+n-2)
+    w2 = A.unsafeIndex arr (off+n-3)
+
+-- | /O(n)/ Copy a 'Text' to an array.  The array is assumed to be big
+-- enough to hold the contents of the entire 'Text'.
+unsafeCopyToPtr :: Text -> Ptr Word8 -> IO ()
+unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len
+
+-- | /O(n)/ Perform an action on a temporary, mutable copy of a
+-- 'Text'.  The copy is freed as soon as the action returns.
+useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a
+useAsPtr t@(Text _arr _off len) action =
+    allocaBytes len $ \buf -> do
+      unsafeCopyToPtr t buf
+      action (castPtr buf) (I8 len)
+
+-- | /O(n)/ Make a mutable copy of a 'Text'.
+asForeignPtr :: Text -> IO (ForeignPtr Word8, I8)
+asForeignPtr t@(Text _arr _off len) = do
+  fp <- mallocForeignPtrArray len
+  unsafeWithForeignPtr fp $ unsafeCopyToPtr t
+  return (fp, I8 len)
+
+-- | Marshal a 'Text' into a C string with a trailing NUL byte,
+-- encoded as UTF-8 in temporary storage.
+--
+-- The 'Text' itself must not contain any NUL bytes, this precondition
+-- is not checked. Cf. 'withCStringLen'.
+--
+-- The temporary storage is freed when the subcomputation terminates
+-- (either normally or via an exception), so the pointer to the
+-- temporary storage must /not/ be used after this function returns.
+--
+-- @since 2.0.1
+withCString :: Text -> (CString -> IO a) -> IO a
+withCString t@(Text _arr _off len) action =
+  allocaBytes (len + 1) $ \buf -> do
+    unsafeCopyToPtr t buf
+    pokeByteOff buf len (0 :: Word8)
+    action (castPtr buf)
+
+-- | /O(n)/ Decode a C string with explicit length, which is assumed
+-- to have been encoded as UTF-8. If decoding fails, a
+-- 'UnicodeException' is thrown.
+--
+-- @since 1.0.0.0
+peekCStringLen :: CStringLen -> IO Text
+peekCStringLen cs = do
+  bs <- unsafePackCStringLen cs
+  return $! decodeUtf8 bs
+
+-- | /O(n)/ Decode a null-terminated C string, which is assumed
+-- to have been encoded as UTF-8. If decoding fails, a
+-- 'UnicodeException' is thrown.
+--
+-- @since 2.1.2
+peekCString :: CString -> IO Text
+peekCString cs = do
+  bs <- unsafePackCString cs
+  return $! decodeUtf8 bs
+
+-- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary
+-- storage, with explicit length information. The encoded string may
+-- contain NUL bytes, and is not followed by a trailing NUL byte.
+--
+-- The temporary storage is freed when the subcomputation terminates
+-- (either normally or via an exception), so the pointer to the
+-- temporary storage must /not/ be used after this function returns.
+--
+-- @since 1.0.0.0
+withCStringLen :: Text -> (CStringLen -> IO a) -> IO a
+withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act
+ src/Data/Text/IO.hs view
@@ -0,0 +1,211 @@+{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-}
+{-# LANGUAGE Trustworthy #-}
+-- |
+-- Module      : Data.Text.IO
+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,
+--               (c) 2009 Simon Marlow
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : GHC
+--
+-- Efficient locale-sensitive support for text I\/O.
+--
+-- The functions in this module obey the runtime system's locale,
+-- character set encoding, and line ending conversion settings.
+--
+-- If you want to do I\/O using the UTF-8 encoding, use "Data.Text.IO.Utf8",
+-- which is faster than this module.
+--
+-- If you know in advance that you will be working with data that has
+-- a specific encoding, and your application is highly
+-- performance sensitive, you may find that it is faster to perform
+-- I\/O with bytestrings and to encode and decode yourself than to use
+-- the functions in this module.
+
+module Data.Text.IO
+    (
+    -- * File-at-a-time operations
+      readFile
+    , writeFile
+    , appendFile
+    -- * Operations on handles
+    , hGetContents
+    , hGetChunk
+    , hGetLine
+    , hPutStr
+    , hPutStrLn
+    -- * Special cases for standard input and output
+    , interact
+    , getContents
+    , getLine
+    , putStr
+    , putStrLn
+    ) where
+
+import Data.Text (Text)
+import Prelude hiding (appendFile, getContents, getLine, interact,
+                       putStr, putStrLn, readFile, writeFile)
+import System.IO (Handle, IOMode(..), openFile, stdin, stdout,
+                  withFile)
+import qualified Control.Exception as E
+import Control.Monad (liftM2, when)
+import Data.IORef (readIORef)
+import qualified Data.Text as T
+import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn)
+import GHC.IO.Buffer (CharBuffer, isEmptyBuffer)
+import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType))
+import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle)
+import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..))
+import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)
+import System.IO.Error (isEOFError)
+
+-- | The 'readFile' function reads a file and returns the contents of
+-- the file as a string.  The entire file is read strictly, as with
+-- 'getContents'.
+--
+-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent.
+-- Unexpected system locale may cause your application to read corrupted data or
+-- throw runtime exceptions about "invalid argument (invalid byte sequence)"
+-- or "invalid argument (invalid character)". This is also slow, because GHC
+-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8.
+--
+-- If your data is UTF-8,
+-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile'
+-- is a much faster and safer alternative.
+readFile :: FilePath -> IO Text
+readFile name = openFile name ReadMode >>= hGetContents
+
+-- | Write a string to a file.  The file is truncated to zero length
+-- before writing begins.
+writeFile :: FilePath -> Text -> IO ()
+writeFile p = withFile p WriteMode . flip hPutStr
+
+-- | Write a string to the end of a file.
+appendFile :: FilePath -> Text -> IO ()
+appendFile p = withFile p AppendMode . flip hPutStr
+
+catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool)
+catchError caller h Handle__{..} err
+    | isEOFError err = do
+        buf <- readIORef haCharBuffer
+        return $ if isEmptyBuffer buf
+                 then (T.empty, True)
+                 else (T.singleton '\r', True)
+    | otherwise = E.throwIO (augmentIOError err caller h)
+
+-- | Wrap readChunk and return a value indicating if we're reached the EOF.
+-- This is needed because unpack_nl is unable to discern the difference
+-- between a buffer with just \r due to EOF or because not enough data was left
+-- for decoding. e.g. the final character decoded from the byte buffer was \r.
+readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool)
+readChunkEof hh buf = do t <- readChunk hh buf
+                         return (t, False)
+
+-- | Read a single chunk of strict text from a
+-- 'Handle'. The size of the chunk depends on the amount of input
+-- currently buffered.
+--
+-- This function blocks only if there is no data available, and EOF
+-- has not yet been reached. Once EOF is reached, this function
+-- returns an empty string instead of throwing an exception.
+--
+-- === Behavior
+--
+-- Unlike byte-oriented functions, 'hGetChunk' operates on complete UTF-8
+-- characters. Since UTF-8 characters can occupy 1 to 4 bytes, this function
+-- cannot guarantee reading an exact number of bytes. Instead, it reads
+-- complete characters up to the handle's internal buffer limit.
+--
+-- === Buffer Size
+--
+-- The maximum chunk size is determined by the handle's internal character
+-- buffer, which is set to 8192 bytes (2048 characters) by the GHC runtime
+-- constant @dEFAULT_CHAR_BUFFER_SIZE@. This buffer size cannot be modified
+-- through any public API.
+--
+-- === UTF-8 Considerations
+--
+-- When working with UTF-8 encoded text:
+--
+-- * The function will never return a partial character
+-- * The actual number of bytes read may vary depending on the character
+--   encoding (ASCII characters = 1 byte, other Unicode characters = 2-4 bytes)
+hGetChunk :: Handle -> IO Text
+hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk
+ where
+  readSingleChunk hh@Handle__{..} = do
+    buf <- readIORef haCharBuffer
+    (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh
+    return (hh, t)
+
+-- | Read the remaining contents of a 'Handle' as a string.  The
+-- 'Handle' is closed once the contents have been read, or if an
+-- exception is thrown.
+--
+-- Internally, this function reads a chunk at a time from the
+-- lower-level buffering abstraction, and concatenates the chunks into
+-- a single string once the entire file has been read.
+--
+-- As a result, it requires approximately twice as much memory as its
+-- result to construct its result.  For files more than a half of
+-- available RAM in size, this may result in memory exhaustion.
+hGetContents :: Handle -> IO Text
+hGetContents h = do
+  chooseGoodBuffering h
+  wantReadableHandle "hGetContents" h readAll
+ where
+  readAll hh@Handle__{..} = do
+    let readChunks = do
+          buf <- readIORef haCharBuffer
+          (t, eof) <- readChunkEof hh buf
+                         `E.catch` catchError "hGetContents" h hh
+          if eof
+            then return [t]
+            else (t:) `fmap` readChunks
+    ts <- readChunks
+    (hh', _) <- hClose_help hh
+    return (hh'{haType=ClosedHandle}, T.concat ts)
+
+-- | Use a more efficient buffer size if we're reading in
+-- block-buffered mode with the default buffer size.  When we can
+-- determine the size of the handle we're reading, set the buffer size
+-- to that, so that we can read the entire file in one chunk.
+-- Otherwise, use a buffer size of at least 16KB.
+chooseGoodBuffering :: Handle -> IO ()
+chooseGoodBuffering h = do
+  bufMode <- hGetBuffering h
+  case bufMode of
+    BlockBuffering Nothing -> do
+      d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) ->
+           if ioe_type e == InappropriateType
+           then return 16384 -- faster than the 2KB default
+           else E.throwIO e
+      when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d
+    _ -> return ()
+
+-- | Read a single line from a handle.
+hGetLine :: Handle -> IO Text
+hGetLine = hGetLineWith T.concat
+
+-- | The 'interact' function takes a function of type @Text -> Text@
+-- as its argument. The entire input from the standard input device is
+-- passed to this function as its argument, and the resulting string
+-- is output on the standard output device.
+interact :: (Text -> Text) -> IO ()
+interact f = putStr . f =<< getContents
+
+-- | Read all user input on 'stdin' as a single string.
+getContents :: IO Text
+getContents = hGetContents stdin
+
+-- | Read a single line of user input from 'stdin'.
+getLine :: IO Text
+getLine = hGetLine stdin
+
+-- | Write a string to 'stdout'.
+putStr :: Text -> IO ()
+putStr = hPutStr stdout
+
+-- | Write a string to 'stdout', followed by a newline.
+putStrLn :: Text -> IO ()
+putStrLn = hPutStrLn stdout
+ src/Data/Text/IO/Utf8.hs view
@@ -0,0 +1,94 @@+-- |
+-- Module      : Data.Text.IO.Utf8
+-- License     : BSD-style
+-- Portability : GHC
+--
+-- Efficient UTF-8 support for text I\/O.
+-- Unlike "Data.Text.IO", these functions do not depend on the locale
+-- and do not do line ending conversion.
+module Data.Text.IO.Utf8
+    (
+    -- * File-at-a-time operations
+      readFile
+    , writeFile
+    , appendFile
+    -- * Operations on handles
+    , hGetContents
+    , hGetLine
+    , hPutStr
+    , hPutStrLn
+    -- * Special cases for standard input and output
+    , interact
+    , getContents
+    , getLine
+    , putStr
+    , putStrLn
+    ) where
+
+import Prelude ()
+import Control.Exception (evaluate)
+import Control.Monad ((<=<), (=<<))
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as B
+import Data.Function ((.))
+import Data.Text (Text)
+import Data.Text.Encoding (decodeUtf8, encodeUtf8)
+import GHC.IO.Handle (Handle)
+import System.IO (IO, FilePath)
+
+decodeUtf8IO :: ByteString -> IO Text
+decodeUtf8IO = evaluate . decodeUtf8
+
+-- | The 'readFile' function reads a file and returns the contents of
+-- the file as a string.  The entire file is read strictly, as with
+-- 'getContents'.
+readFile :: FilePath -> IO Text
+readFile = decodeUtf8IO <=< B.readFile
+
+-- | Write a string to a file.  The file is truncated to zero length
+-- before writing begins.
+writeFile :: FilePath -> Text -> IO ()
+writeFile fp = B.writeFile fp . encodeUtf8
+
+-- | Write a string to the end of a file.
+appendFile :: FilePath -> Text -> IO ()
+appendFile fp = B.appendFile fp . encodeUtf8
+
+-- | Read the remaining contents of a 'Handle' as a string.
+hGetContents :: Handle -> IO Text
+hGetContents = decodeUtf8IO <=< B.hGetContents
+
+-- | Read a single line from a handle.
+hGetLine :: Handle -> IO Text
+hGetLine = decodeUtf8IO <=< B.hGetLine
+
+-- | Write a string to a handle.
+hPutStr :: Handle -> Text -> IO ()
+hPutStr h = B.hPutStr h . encodeUtf8
+
+-- | Write a string to a handle, followed by a newline.
+hPutStrLn :: Handle -> Text -> IO ()
+hPutStrLn h = B.hPutStrLn h . encodeUtf8
+
+-- | The 'interact' function takes a function of type @Text -> Text@
+-- as its argument. The entire input from the standard input device is
+-- passed to this function as its argument, and the resulting string
+-- is output on the standard output device.
+interact :: (Text -> Text) -> IO ()
+interact f = putStr . f =<< getContents
+
+-- | Read all user input on 'stdin' as a single string.
+getContents :: IO Text
+getContents = decodeUtf8IO =<< B.getContents
+
+-- | Read a single line of user input from 'stdin'.
+getLine :: IO Text
+getLine = decodeUtf8IO =<< B.getLine
+
+-- | Write a string to 'stdout'.
+putStr :: Text -> IO ()
+putStr = B.putStr . encodeUtf8
+
+-- | Write a string to 'stdout', followed by a newline.
+putStrLn :: Text -> IO ()
+putStrLn = B.putStrLn . encodeUtf8
+ src/Data/Text/Internal.hs view
@@ -0,0 +1,274 @@+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE UnboxedTuples #-}
+{-# OPTIONS_HADDOCK not-home #-}
+
+-- |
+-- Module      : Data.Text.Internal
+-- Copyright   : (c) 2008, 2009 Tom Harper,
+--               (c) 2009, 2010 Bryan O'Sullivan,
+--               (c) 2009 Duncan Coutts
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- A module containing private 'Text' internals. This exposes the
+-- 'Text' representation and low level construction functions.
+-- Modules which extend the 'Text' system may need to use this module.
+--
+-- You should not use this module unless you are determined to monkey
+-- with the internals, as the functions here do just about nothing to
+-- preserve data invariants.  You have been warned!
+
+module Data.Text.Internal
+    (
+    -- * Types
+    -- $internals
+      Text(..)
+    , StrictText
+    -- * Construction
+    , text
+    , textP
+    -- * Safety
+    , safe
+    -- * Code that must be here for accessibility
+    , empty
+    , append
+    -- * Utilities
+    , firstf
+    -- * Checked multiplication
+    , mul
+    , mul32
+    , mul64
+    -- * Debugging
+    , showText
+    -- * Conversions
+    , pack
+    ) where
+
+#if defined(ASSERTS)
+import Control.Exception (assert)
+import GHC.Stack (HasCallStack)
+#endif
+import Control.Monad.ST (ST, runST)
+import Data.Bits
+import Data.Int (Int32, Int64)
+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
+import qualified Data.Text.Array as A
+
+-- | A space efficient, packed, unboxed Unicode text type.
+data Text = Text
+    {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8
+    {-# UNPACK #-} !Int     -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence
+    {-# UNPACK #-} !Int     -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence
+
+-- | Type synonym for the strict flavour of 'Text'.
+--
+-- @since 2.1.1
+type StrictText = Text
+
+-- | Smart constructor.
+text_ ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+     A.Array -- ^ bytearray encoded as UTF-8
+  -> Int     -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence
+  -> Int     -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence
+  -> Text
+text_ arr off len =
+#if defined(ASSERTS)
+  let c    = A.unsafeIndex arr off
+  in assert (len >= 0) .
+     assert (off >= 0) .
+     assert (len == 0 || c < 0x80 || c >= 0xC0) $
+#endif
+     Text arr off len
+{-# INLINE text_ #-}
+
+-- | /O(1)/ The empty 'Text'.
+empty :: Text
+empty = Text A.empty 0 0
+{-# NOINLINE empty #-}
+
+-- | /O(n)/ Appends one 'Text' to the other by copying both of them
+-- into a new 'Text'.
+append :: Text -> Text -> Text
+append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2)
+    | len1 == 0 = b
+    | len2 == 0 = a
+    | len > 0   = Text (A.run x) 0 len
+    | otherwise = error $ "Data.Text.append: size overflow"
+    where
+      len = len1+len2
+      x :: ST s (A.MArray s)
+      x = do
+        arr <- A.new len
+        A.copyI len1 arr 0 arr1 off1
+        A.copyI len2 arr len1 arr2 off2
+        return arr
+{-# NOINLINE append #-}
+
+-- | Construct a 'Text' without invisibly pinning its byte array in
+-- memory if its length has dwindled to zero.
+-- It ensures that empty 'Text' values are shared.
+text ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+     A.Array -- ^ bytearray encoded as UTF-8
+  -> Int     -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence
+  -> Int     -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence
+  -> Text
+text arr off len | len == 0  = empty
+                 | otherwise = text_ arr off len
+{-# INLINE [0] text #-}
+
+textP :: A.Array -> Int -> Int -> Text
+{-# DEPRECATED textP "Use text instead" #-}
+textP = text
+
+-- | A useful 'show'-like function for debugging purposes.
+showText :: Text -> String
+showText (Text arr off len) =
+    "Text " ++ show (A.toList arr off len) ++ ' ' :
+            show off ++ ' ' : show len
+
+-- | Map a 'Char' to a 'Text'-safe value.
+--
+-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode
+-- scalar values, but are unfortunately admitted as valid 'Char'
+-- values by Haskell.  They cannot be represented in a 'Text'.  This
+-- function remaps those code points to the Unicode replacement
+-- character (U+FFFD, \'&#xfffd;\'), and leaves other code points
+-- unchanged.
+safe :: Char -> Char
+safe c
+    | ord c .&. 0x1ff800 /= 0xd800 = c
+    | otherwise                    = '\xfffd'
+{-# INLINE [0] safe #-}
+
+-- | Apply a function to the first element of an optional pair.
+firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)
+firstf f (Just (a, b)) = Just (f a, b)
+firstf _  Nothing      = Nothing
+
+-- | Checked multiplication.  Calls 'error' if the result would
+-- overflow.
+mul :: Int -> Int -> Int
+mul a b
+  | finiteBitSize (0 :: Word) == 64
+  = int64ToInt $ intToInt64 a `mul64` intToInt64 b
+  | otherwise
+  = int32ToInt $ intToInt32 a `mul32` intToInt32 b
+{-# INLINE mul #-}
+infixl 7 `mul`
+
+-- | Checked multiplication.  Calls 'error' if the result would
+-- overflow.
+mul64 :: Int64 -> Int64 -> Int64
+mul64 a b
+  | a >= 0 && b >= 0 =  mul64_ a b
+  | a >= 0           = -mul64_ a (-b)
+  | b >= 0           = -mul64_ (-a) b
+  | otherwise        =  mul64_ (-a) (-b)
+{-# INLINE mul64 #-}
+infixl 7 `mul64`
+
+mul64_ :: Int64 -> Int64 -> Int64
+mul64_ a b
+  | ahi > 0 && bhi > 0 = error "overflow"
+  | top > 0x7fffffff   = error "overflow"
+  | total < 0          = error "overflow"
+  | otherwise          = total
+  where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #)
+        (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #)
+        top            = ahi * blo + alo * bhi
+        total          = (top `shiftL` 32) + alo * blo
+{-# INLINE mul64_ #-}
+
+-- | Checked multiplication.  Calls 'error' if the result would
+-- overflow.
+mul32 :: Int32 -> Int32 -> Int32
+mul32 a b = case int32ToInt64 a * int32ToInt64 b of
+              ab | ab < min32 || ab > max32 -> error "overflow"
+                 | otherwise                -> int64ToInt32 ab
+  where min32 = -0x80000000 :: Int64
+        max32 =  0x7fffffff
+{-# INLINE mul32 #-}
+infixl 7 `mul32`
+
+intToInt64 :: Int -> Int64
+intToInt64 = fromIntegral
+
+int64ToInt :: Int64 -> Int
+int64ToInt = fromIntegral
+
+intToInt32 :: Int -> Int32
+intToInt32 = fromIntegral
+
+int32ToInt :: Int32 -> Int
+int32ToInt = fromIntegral
+
+int32ToInt64 :: Int32 -> Int64
+int32ToInt64 = fromIntegral
+
+int64ToInt32 :: Int64 -> Int32
+int64ToInt32 = fromIntegral
+
+-- $internals
+--
+-- Internally, the 'Text' type is represented as an array of 'Word8'
+-- UTF-8 code units. The offset and length fields in the constructor
+-- are in these units, /not/ units of 'Char'.
+--
+-- Invariants that all functions must maintain:
+--
+-- * Since the 'Text' type uses UTF-8 internally, it cannot represent
+--   characters in the reserved surrogate code point range U+D800 to
+--   U+DFFF. To maintain this invariant, the 'safe' function maps
+--   'Char' values in this range to the replacement character (U+FFFD,
+--   \'&#xfffd;\').
+--
+-- * Offset and length must point to a valid UTF-8 sequence of bytes.
+--   Violation of this may cause memory access violation and divergence.
+
+-- -----------------------------------------------------------------------------
+-- * Conversion to/from 'Text'
+
+-- | /O(n)/ Convert a 'String' into a 'Text'.
+-- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id':
+--
+-- >>> Data.Text.unpack (pack "\55555")
+-- "\65533"
+pack :: String -> Text
+pack [] = empty
+pack xs = runST $ do
+  -- It's tempting to allocate a buffer of 4 * length xs bytes,
+  -- but not only it's wasteful for predominantly ASCII arguments,
+  -- the computation of length xs would force allocation of the entire xs at once.
+  let dstLen = 64
+  dst <- A.new dstLen
+  outer dst dstLen 0 xs
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text
+    outer !dst !dstLen = inner
+      where
+        inner !dstOff [] = do
+          A.shrinkM dst dstOff
+          arr <- A.unsafeFreeze dst
+          return (Text arr 0 dstOff)
+        inner !dstOff ccs@(c : cs)
+          -- Each 'Char' takes up to 4 bytes
+          | dstOff + 4 > dstLen = do
+            -- Double size of the buffer
+            let !dstLen' = dstLen * 2
+            dst' <- A.resizeM dst dstLen'
+            outer dst' dstLen' dstOff ccs
+          | otherwise = do
+            d <- unsafeWrite dst dstOff (safe c)
+            inner (dstOff + d) cs
+{-# NOINLINE [0] pack #-}
+ src/Data/Text/Internal/ArrayUtils.hs view
@@ -0,0 +1,33 @@+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}+{-# LANGUAGE CPP #-}++module Data.Text.Internal.ArrayUtils (memchr) where++#if defined(PURE_HASKELL)+import qualified Data.Text.Array as A+import Data.List (elemIndex)+#else+import Foreign.C.Types+import System.Posix.Types (CSsize(..))+#endif+import GHC.Exts (ByteArray#)+import Data.Word (Word8)++memchr :: ByteArray# -> Int -> Int -> Word8 -> Int+#if defined(PURE_HASKELL)+memchr arr# off len w =+    let tempBa = A.ByteArray arr#+    in case elemIndex w (A.toList tempBa off len) of+        Nothing -> -1+        Just i -> i+#else+memchr arr# off len w = fromIntegral $ c_memchr arr# (intToCSize off) (intToCSize len) w++intToCSize :: Int -> CSize+intToCSize = fromIntegral+++foreign import ccall unsafe "_hs_text_memchr" c_memchr+    :: ByteArray# -> CSize -> CSize -> Word8 -> CSsize+#endif
+ src/Data/Text/Internal/Builder.hs view
@@ -0,0 +1,350 @@+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-}
+{-# OPTIONS_HADDOCK not-home #-}
+
+-----------------------------------------------------------------------------
+-- |
+-- Module      : Data.Text.Internal.Builder
+-- Copyright   : (c) 2013 Bryan O'Sullivan
+--               (c) 2010 Johan Tibell
+-- License     : BSD-style (see LICENSE)
+--
+-- Maintainer  : Johan Tibell <johan.tibell@gmail.com>
+-- Stability   : experimental
+-- Portability : portable to Hugs and GHC
+--
+-- /Warning/: this is an internal module, and does not have a stable
+-- API or name. Functions in this module may not check or enforce
+-- preconditions expected by public modules. Use at your own risk!
+--
+-- Efficient construction of lazy @Text@ values.  The principal
+-- operations on a @Builder@ are @singleton@, @fromText@, and
+-- @fromLazyText@, which construct new builders, and 'mappend', which
+-- concatenates two builders.
+--
+-- To get maximum performance when building lazy @Text@ values using a
+-- builder, associate @mappend@ calls to the right.  For example,
+-- prefer
+--
+-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')
+--
+-- to
+--
+-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'
+--
+-- as the latter associates @mappend@ to the left.
+--
+-----------------------------------------------------------------------------
+
+module Data.Text.Internal.Builder
+   ( -- * Public API
+     -- ** The Builder type
+     Builder
+   , LazyTextBuilder
+   , toLazyText
+   , toLazyTextWith
+
+     -- ** Constructing Builders
+   , singleton
+   , fromText
+   , fromLazyText
+   , fromString
+
+     -- ** Flushing the buffer state
+   , flush
+
+     -- * Internal functions
+   , append'
+   , ensureFree
+   , writeN
+   ) where
+
+import Control.Monad.ST (ST, runST)
+import Data.Monoid (Monoid(..))
+#if !MIN_VERSION_base(4,11,0)
+import Data.Semigroup (Semigroup(..))
+#endif
+import Data.Text.Internal (Text(..), safe)
+import Data.Text.Internal.Lazy (smallChunkSize)
+import Data.Text.Unsafe (inlineInterleaveST)
+import Data.Text.Internal.Unsafe.Char (unsafeWrite)
+import Prelude hiding (map, putChar)
+
+import qualified Data.String as String
+import qualified Data.Text as S
+import qualified Data.Text.Array as A
+import qualified Data.Text.Lazy as L
+
+#if defined(ASSERTS)
+import GHC.Stack (HasCallStack)
+#endif
+
+------------------------------------------------------------------------
+
+-- | A @Builder@ is an efficient way to build lazy @Text@ values.
+-- There are several functions for constructing builders, but only one
+-- to inspect them: to extract any data, you have to turn them into
+-- lazy @Text@ values using @toLazyText@.
+--
+-- Internally, a builder constructs a lazy @Text@ by filling arrays
+-- piece by piece.  As each buffer is filled, it is \'popped\' off, to
+-- become a new chunk of the resulting lazy @Text@.  All this is
+-- hidden from the user of the @Builder@.
+newtype Builder = Builder {
+     -- Invariant (from Data.Text.Lazy):
+     --      The lists include no null Texts.
+     runBuilder :: forall s. (Buffer s -> ST s [S.Text])
+                -> Buffer s
+                -> ST s [S.Text]
+   }
+
+-- | @since 2.1.2
+type LazyTextBuilder = Builder
+
+instance Semigroup Builder where
+   (<>) = append
+   {-# INLINE (<>) #-}
+
+instance Monoid Builder where
+   mempty  = empty
+   {-# INLINE mempty #-}
+   mappend = (<>)
+   {-# INLINE mappend #-}
+   mconcat = foldr mappend Data.Monoid.mempty
+   {-# INLINE mconcat #-}
+
+-- | Performs replacement on invalid scalar values:
+--
+-- >>> :set -XOverloadedStrings
+-- >>> "\55555" :: Builder
+-- "\65533"
+instance String.IsString Builder where
+    fromString = fromString
+    {-# INLINE fromString #-}
+
+instance Show Builder where
+    show = show . toLazyText
+
+instance Eq Builder where
+    a == b = toLazyText a == toLazyText b
+
+instance Ord Builder where
+    a <= b = toLazyText a <= toLazyText b
+
+------------------------------------------------------------------------
+
+-- | /O(1)./ The empty @Builder@, satisfying
+--
+--  * @'toLazyText' 'empty' = 'L.empty'@
+--
+empty :: Builder
+empty = Builder (\ k buf -> k buf)
+{-# INLINE empty #-}
+
+-- | /O(1)./ A @Builder@ taking a single character, satisfying
+--
+--  * @'toLazyText' ('singleton' c) = 'L.singleton' c@
+--
+singleton ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Char -> Builder
+singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c)
+{-# INLINE singleton #-}
+
+------------------------------------------------------------------------
+
+-- | /O(1)./ The concatenation of two builders, an associative
+-- operation with identity 'empty', satisfying
+--
+--  * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@
+--
+append :: Builder -> Builder -> Builder
+append (Builder f) (Builder g) = Builder (f . g)
+{-# INLINE [0] append #-}
+
+-- TODO: Experiment to find the right threshold.
+copyLimit :: Int
+copyLimit = 128
+
+-- This function attempts to merge small @Text@ values instead of
+-- treating each value as its own chunk.  We may not always want this.
+
+-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying
+--
+--  * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@
+--
+fromText :: S.Text -> Builder
+fromText t@(Text arr off l)
+    | S.null t       = empty
+    | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off
+    | otherwise      = flush `append` mapBuilder (t :)
+{-# INLINE [1] fromText #-}
+
+{-# RULES
+"fromText/pack" forall s .
+        fromText (S.pack s) = fromString s
+ #-}
+
+-- | /O(1)./ A Builder taking a @String@, satisfying
+--
+--  * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@
+--
+-- Performs replacement on invalid scalar values:
+--
+-- >>> fromString "\55555"
+-- "\65533"
+--
+-- @since 1.2.0.0
+fromString :: String -> Builder
+fromString str = Builder $ \k (Buffer p0 o0 u0) -> do
+    len <- A.getSizeofMArray p0
+    -- `end` is 3 bytes before the actual end of `marr`
+    -- to make sure there's room for a 4-byte UTF-8 code point.
+    let loop !marr !o !u !_ [] = k (Buffer marr o u)
+        loop marr o u end s@(c:cs)
+            | u >= end = do
+                A.shrinkM marr (o + u)
+                arr <- A.unsafeFreeze marr
+                let !t = Text arr o u
+                marr' <- A.new chunkSize
+                ts <- inlineInterleaveST (loop marr' 0 0 (chunkSize - 3) s)
+                return $ t : ts
+            | otherwise = do
+                n <- unsafeWrite marr (o+u) (safe c)
+                loop marr o (u+n) end cs
+    loop p0 o0 u0 (len - o0 - 3) str
+  where
+    chunkSize = smallChunkSize
+{-# INLINEABLE fromString #-}
+
+-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying
+--
+--  * @'toLazyText' ('fromLazyText' t) = t@
+--
+fromLazyText :: L.Text -> Builder
+fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++)
+{-# INLINE fromLazyText #-}
+
+------------------------------------------------------------------------
+
+-- Our internal buffer type
+data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s)
+                       {-# UNPACK #-} !Int  -- offset
+                       {-# UNPACK #-} !Int  -- used units
+
+------------------------------------------------------------------------
+
+-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default
+-- buffer size.  The construction work takes place if and when the
+-- relevant part of the lazy @Text@ is demanded.
+toLazyText :: Builder -> L.Text
+toLazyText = toLazyTextWith smallChunkSize
+
+-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given
+-- size for the initial buffer.  The construction work takes place if
+-- and when the relevant part of the lazy @Text@ is demanded.
+--
+-- If the initial buffer is too small to hold all data, subsequent
+-- buffers will be the default buffer size.
+toLazyTextWith :: Int -> Builder -> L.Text
+toLazyTextWith chunkSize m = L.fromChunks (runST $
+  newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return [])))
+
+-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any,
+-- yielding a new chunk in the result lazy @Text@.
+flush :: Builder
+flush = Builder $ \ k buf@(Buffer p o u) ->
+    if u == 0
+    then k buf
+    else do arr <- A.unsafeFreeze p
+            let !b = Buffer p (o+u) 0
+                !t = Text arr o u
+            ts <- inlineInterleaveST (k b)
+            return $! t : ts
+{-# INLINE [1] flush #-}
+-- defer inlining so that flush/flush rule may fire.
+
+------------------------------------------------------------------------
+
+-- | Sequence an ST operation on the buffer
+withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder
+withBuffer f = Builder $ \k buf -> f buf >>= k
+{-# INLINE withBuffer #-}
+
+-- | Get the size of the buffer
+withSize :: (Int -> Builder) -> Builder
+withSize f = Builder $ \ k buf@(Buffer arr offset used) -> do
+    len <- A.getSizeofMArray arr
+    runBuilder (f (len - offset - used)) k buf
+{-# INLINE withSize #-}
+
+-- | Map the resulting list of texts.
+mapBuilder :: ([S.Text] -> [S.Text]) -> Builder
+mapBuilder f = Builder (fmap f .)
+
+------------------------------------------------------------------------
+
+-- | Ensure that there are at least @n@ many elements available.
+ensureFree :: Int -> Builder
+ensureFree !n = withSize $ \ l ->
+    if n <= l
+    then empty
+    else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize)))
+{-# INLINE [0] ensureFree #-}
+
+writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder
+writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f)
+{-# INLINE [0] writeAtMost #-}
+
+-- | Ensure that @n@ many elements are available, and then use @f@ to
+-- write some elements into the memory.
+writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder
+writeN n f = writeAtMost n (\ p o -> f p o >> return n)
+{-# INLINE writeN #-}
+
+writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s)
+writeBuffer f (Buffer p o u) = do
+    n <- f p (o+u)
+    return $! Buffer p o (u+n)
+{-# INLINE writeBuffer #-}
+
+newBuffer :: Int -> ST s (Buffer s)
+newBuffer size = do
+    arr <- A.new size
+    return $! Buffer arr 0 0
+{-# INLINE newBuffer #-}
+
+------------------------------------------------------------------------
+-- Some nice rules for Builder
+
+-- This function makes GHC understand that 'writeN' and 'ensureFree'
+-- are *not* recursive in the presence of the rewrite rules below.
+-- This is not needed with GHC 7+.
+append' :: Builder -> Builder -> Builder
+append' (Builder f) (Builder g) = Builder (f . g)
+{-# INLINE append' #-}
+
+{-# RULES
+
+"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)
+                           (g::forall s. A.MArray s -> Int -> ST s Int) ws.
+    append (writeAtMost a f) (append (writeAtMost b g) ws) =
+        append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->
+                                    g marr (o+n) >>= \ m ->
+                                    let s = n+m in s `seq` return s)) ws
+
+"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)
+                           (g::forall s. A.MArray s -> Int -> ST s Int).
+    append (writeAtMost a f) (writeAtMost b g) =
+        writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->
+                            g marr (o+n) >>= \ m ->
+                            let s = n+m in s `seq` return s)
+
+"ensureFree/ensureFree" forall a b .
+    append (ensureFree a) (ensureFree b) = ensureFree (max a b)
+
+"flush/flush"
+    append flush flush = flush
+
+ #-}
+ src/Data/Text/Internal/Builder/Functions.hs view
@@ -0,0 +1,40 @@+{-# LANGUAGE MagicHash #-}++-- |+-- Module      : Data.Text.Internal.Builder.Functions+-- Copyright   : (c) 2011 MailRank, Inc.+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Useful functions and combinators.++module Data.Text.Internal.Builder.Functions+    (+      (<>)+    , i2d+    ) where++import Data.Monoid (mappend)+import Data.Text.Lazy.Builder (Builder)+import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#))+import Prelude ()++-- | Unsafe conversion for decimal digits.+{-# INLINE i2d #-}+i2d :: Int -> Char+i2d (I# i#) = C# (chr# (ord# '0'# +# i#))++-- | The normal 'mappend' function with right associativity instead of+-- left.+(<>) :: Builder -> Builder -> Builder+(<>) = mappend+{-# INLINE (<>) #-}++infixr 4 <>
+ src/Data/Text/Internal/Builder/Int/Digits.hs view
@@ -0,0 +1,25 @@+-- Module:      Data.Text.Internal.Builder.Int.Digits+-- Copyright:   (c) 2013 Bryan O'Sullivan+-- License:     BSD-style+-- Maintainer:  Bryan O'Sullivan <bos@serpentine.com>+-- Stability:   experimental+-- Portability: portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- This module exists because the C preprocessor does things that we+-- shall not speak of when confronted with Haskell multiline strings.++module Data.Text.Internal.Builder.Int.Digits (digits) where++import Data.ByteString.Char8 (ByteString, pack)++digits :: ByteString+digits = pack+  "0001020304050607080910111213141516171819\+  \2021222324252627282930313233343536373839\+  \4041424344454647484950515253545556575859\+  \6061626364656667686970717273747576777879\+  \8081828384858687888990919293949596979899"
+ src/Data/Text/Internal/Builder/RealFloat/Functions.hs view
@@ -0,0 +1,35 @@+{-# LANGUAGE CPP #-}++-- |+-- Module:    Data.Text.Internal.Builder.RealFloat.Functions+-- Copyright: (c) The University of Glasgow 1994-2002+-- License:   see libraries/base/LICENSE+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!++module Data.Text.Internal.Builder.RealFloat.Functions+    (+      roundTo+    ) where++roundTo :: Int -> [Int] -> (Int,[Int])+roundTo d is =+  case f d True is of+    x@(0,_) -> x+    (1,xs)  -> (1, 1:xs)+    _       -> error "roundTo: bad Value"+ where+  b2 = base `quot` 2++  f n _ []     = (0, replicate n 0)+  f 0 e (x:xs) | x == b2 && e && all (== 0) xs = (0, [])   -- Round to even when at exactly half the base+               | otherwise = (if x >= b2 then 1 else 0, [])+  f n _ (i:xs)+     | i' == base = (1,0:ds)+     | otherwise  = (0,i':ds)+      where+       (c,ds) = f (n-1) (even i) xs+       i'     = c + i+  base = 10
+ src/Data/Text/Internal/ByteStringCompat.hs view
@@ -0,0 +1,28 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+module Data.Text.Internal.ByteStringCompat (mkBS, withBS) where++import Data.ByteString.Internal (ByteString (..))+import Data.Word (Word8)+import Foreign.ForeignPtr (ForeignPtr)++#if !MIN_VERSION_bytestring(0,11,0)+import GHC.ForeignPtr (plusForeignPtr)+#endif++mkBS :: ForeignPtr Word8 -> Int -> ByteString+#if MIN_VERSION_bytestring(0,11,0)+mkBS dfp n = BS dfp n+#else+mkBS dfp n = PS dfp 0 n+#endif+{-# INLINE mkBS #-}++withBS :: ByteString -> (ForeignPtr Word8 -> Int -> r) -> r+#if MIN_VERSION_bytestring(0,11,0)+withBS (BS !sfp !slen)       kont = kont sfp slen+#else+withBS (PS !sfp !soff !slen) kont = kont (plusForeignPtr sfp soff) slen+#endif+{-# INLINE withBS #-}
+ src/Data/Text/Internal/Encoding.hs view
@@ -0,0 +1,535 @@+{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash,+    UnliftedFFITypes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE Trustworthy #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE ViewPatterns #-}++-- |+-- Module      : Data.Text.Internal.Builder+-- License     : BSD-style (see LICENSE)+-- Stability   : experimental+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Internals of "Data.Text.Encoding".+--+-- @since 2.0.2+module Data.Text.Internal.Encoding+  ( validateUtf8Chunk+  , validateUtf8More+  , decodeUtf8Chunk+  , decodeUtf8More+  , decodeUtf8With1+  , decodeUtf8With2+  , Utf8State+  , startUtf8State+  , StrictTextBuilder()+  , StrictBuilder+  , strictBuilderToText+  , textToStrictBuilder++    -- * Internal+  , skipIncomplete+  , getCompleteLen+  , getPartialUtf8+  ) where++#if defined(ASSERTS)+import Control.Exception (assert)+#endif+import Data.Bits ((.&.), shiftL, shiftR)+import Data.ByteString (ByteString)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup (Semigroup(..))+#endif+import Data.Word (Word32, Word8)+import Foreign.Storable (pokeElemOff)+import Data.Text.Encoding.Error (OnDecodeError)+import Data.Text.Internal (Text(..))+import Data.Text.Internal.Encoding.Utf8+  (DecoderState, utf8AcceptState, utf8RejectState, updateDecoderState)+import Data.Text.Internal.StrictBuilder (StrictBuilder, StrictTextBuilder)+import qualified Data.ByteString as B+import qualified Data.ByteString.Internal as BI+import qualified Data.ByteString.Short.Internal as SBS+import qualified Data.Text.Array as A+import qualified Data.Text.Internal.StrictBuilder as SB+#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif++#ifdef SIMDUTF+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Foreign.C.Types (CSize(..))+import Foreign.C.Types (CInt(..))+import Foreign.Ptr (Ptr)+#endif++-- | Use 'StrictBuilder' to build 'Text'.+--+-- @since 2.0.2+strictBuilderToText :: StrictTextBuilder -> Text+strictBuilderToText = SB.toText++-- | Copy 'Text' in a 'StrictBuilder'+--+-- @since 2.0.2+textToStrictBuilder :: Text -> StrictTextBuilder+textToStrictBuilder = SB.fromText++-- | State of decoding a 'ByteString' in UTF-8.+-- Enables incremental decoding ('validateUtf8Chunk', 'validateUtf8More',+-- 'decodeUtf8Chunk', 'decodeUtf8More').+--+-- @since 2.0.2++-- Internal invariant:+-- the first component is the initial state if and only if+-- the second component is empty.+--+-- @+-- 'utf9CodePointState' s = 'utf8StartState'+-- <=>+-- 'partialUtf8CodePoint' s = 'PartialUtf8CodePoint' 0+-- @+data Utf8State = Utf8State+  { -- | State of the UTF-8 state machine.+    utf8CodePointState :: {-# UNPACK #-} !DecoderState+    -- | Bytes of the currently incomplete code point (if any).+  , partialUtf8CodePoint :: {-# UNPACK #-} !PartialUtf8CodePoint+  }+  deriving (Eq, Show)++-- | Initial 'Utf8State'.+--+-- @since 2.0.2+startUtf8State :: Utf8State+startUtf8State = Utf8State utf8AcceptState partUtf8Empty++-- | Prefix of a UTF-8 code point encoded in 4 bytes,+-- possibly empty.+--+-- - The most significant byte contains the number of bytes,+--   between 0 and 3.+-- - The remaining bytes hold the incomplete code point.+-- - Unused bytes must be 0.+--+-- All of operations available on it are the functions below.+-- The constructor should never be used outside of those.+--+-- @since 2.0.2+newtype PartialUtf8CodePoint = PartialUtf8CodePoint Word32+  deriving (Eq, Show)++-- | Empty prefix.+partUtf8Empty :: PartialUtf8CodePoint+partUtf8Empty = PartialUtf8CodePoint 0++-- | Length of the partial code point, stored in the most significant byte.+partUtf8Len :: PartialUtf8CodePoint -> Int+partUtf8Len (PartialUtf8CodePoint w) = fromIntegral $ w `shiftR` 24++-- | Length of the code point once completed (it is known in the first byte).+-- 0 if empty.+partUtf8CompleteLen :: PartialUtf8CodePoint -> Int+partUtf8CompleteLen c@(PartialUtf8CodePoint w)+  | partUtf8Len c == 0 = 0+  | 0xf0 <= firstByte = 4+  | 0xe0 <= firstByte = 3+  | 0xc2 <= firstByte = 2+  | otherwise = 0+  where+    firstByte = (w `shiftR` 16) .&. 255++-- | Get the @n@-th byte, assuming it is within bounds: @0 <= n < partUtf8Len c@.+--+-- Unsafe: no bounds checking.+partUtf8UnsafeIndex ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  PartialUtf8CodePoint -> Int -> Word8+partUtf8UnsafeIndex _c@(PartialUtf8CodePoint w) n =+#if defined(ASSERTS)+  assert (0 <= n && n < partUtf8Len _c) $+#endif+  fromIntegral $ w `shiftR` (16 - 8 * n)++-- | Append some bytes.+--+-- Unsafe: no bounds checking.+partUtf8UnsafeAppend ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  PartialUtf8CodePoint -> ByteString -> PartialUtf8CodePoint+partUtf8UnsafeAppend c@(PartialUtf8CodePoint word) bs =+#if defined(ASSERTS)+  assert (lenc + lenbs <= 3) $+#endif+  PartialUtf8CodePoint $+    tryPush 0 $ tryPush 1 $ tryPush 2 $ word + (fromIntegral lenbs `shiftL` 24)+  where+    lenc = partUtf8Len c+    lenbs = B.length bs+    tryPush i w =+      if i < lenbs+      then w + (fromIntegral (B.index bs i) `shiftL` fromIntegral (16 - 8 * (lenc + i)))+      else w++-- | Fold a 'PartialUtf8CodePoint'. This avoids recursion so it can unfold to straightline code.+{-# INLINE partUtf8Foldr #-}+partUtf8Foldr :: (Word8 -> a -> a) -> a -> PartialUtf8CodePoint -> a+partUtf8Foldr f x0 c = case partUtf8Len c of+    0 -> x0+    1 -> build 0 x0+    2 -> build 0 (build 1 x0)+    _ -> build 0 (build 1 (build 2 x0))+  where+    build i x = f (partUtf8UnsafeIndex c i) x++-- | Convert 'PartialUtf8CodePoint' to 'ByteString'.+partUtf8ToByteString :: PartialUtf8CodePoint -> B.ByteString+partUtf8ToByteString c = BI.unsafeCreate (partUtf8Len c) $ \ptr ->+  partUtf8Foldr (\w k i -> pokeElemOff ptr i w >> k (i+1)) (\_ -> pure ()) c 0++-- | Exported for testing.+getCompleteLen :: Utf8State -> Int+getCompleteLen = partUtf8CompleteLen . partialUtf8CodePoint++-- | Exported for testing.+getPartialUtf8 :: Utf8State -> B.ByteString+getPartialUtf8 = partUtf8ToByteString . partialUtf8CodePoint++#ifdef SIMDUTF+foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8+    :: Ptr Word8 -> CSize -> IO CInt+#endif++-- | Validate a 'ByteString' as UTF-8-encoded text. To be continued using 'validateUtf8More'.+--+-- See also 'validateUtf8More' for details on the result of this function.+--+-- @+-- 'validateUtf8Chunk' = 'validateUtf8More' 'startUtf8State'+-- @+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- 'validateUtf8Chunk' chunk = (n, ms)+-- @+--+-- - The prefix is valid UTF-8. In particular, it should be accepted+--   by this validation:+--+--     @+--     'validateUtf8Chunk' ('Data.ByteString.take' n chunk) = (n, Just 'startUtf8State')+--     @+validateUtf8Chunk :: ByteString -> (Int, Maybe Utf8State)+validateUtf8Chunk bs = validateUtf8ChunkFrom 0 bs (,)++-- Assume bytes up to offset @ofs@ have been validated already.+--+-- Using CPS lets us inline the continuation and avoid allocating a @Maybe@+-- in the @decode...@ functions.+{-# INLINE validateUtf8ChunkFrom #-}+validateUtf8ChunkFrom :: forall r. Int -> ByteString -> (Int -> Maybe Utf8State -> r) -> r+validateUtf8ChunkFrom ofs bs k+  -- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0.+  -- MIN_VERSION_bytestring does not allow us to differentiate+  -- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+.+#if defined(SIMDUTF) || MIN_VERSION_bytestring(0,12,1)+  | guessUtf8Boundary > 0 &&+    -- the rest of the bytestring is valid utf-8 up to the boundary+    (+#ifdef SIMDUTF+      withBS (B.drop ofs bs) $ \ fp _ -> unsafeDupablePerformIO $+        unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$>+          c_is_valid_utf8 ptr (fromIntegral guessUtf8Boundary)+#else+      B.isValidUtf8 $ B.take guessUtf8Boundary (B.drop ofs bs)+#endif+    ) = slowValidateUtf8ChunkFrom (ofs + guessUtf8Boundary)+    -- No+  | otherwise = slowValidateUtf8ChunkFrom ofs+  where+    len = B.length bs - ofs+    isBoundary n p = len >= n && p (B.index bs (ofs + len - n))+    guessUtf8Boundary+      | isBoundary 1 (<= 0x80) = len      -- last char is ASCII (common short-circuit)+      | isBoundary 1 (0xc2 <=) = len - 1  -- last char starts a two-(or more-)byte code point+      | isBoundary 2 (0xe0 <=) = len - 2  -- pre-last char starts a three-or-four-byte code point+      | isBoundary 3 (0xf0 <=) = len - 3  -- third to last char starts a four-byte code point+      | otherwise = len+#else+  = slowValidateUtf8ChunkFrom ofs+  where+#endif+    -- A pure Haskell implementation of validateUtf8More.+    -- Ideally the primitives 'B.isValidUtf8' or 'c_is_valid_utf8' should give us+    -- indices to let us avoid this function.+    slowValidateUtf8ChunkFrom :: Int -> r+    slowValidateUtf8ChunkFrom ofs1 = slowLoop ofs1 ofs1 utf8AcceptState++    slowLoop !utf8End i s+      | i < B.length bs =+          case updateDecoderState (B.index bs i) s of+            s' | s' == utf8RejectState -> k utf8End Nothing+               | s' == utf8AcceptState -> slowLoop (i + 1) (i + 1) s'+               | otherwise -> slowLoop utf8End (i + 1) s'+      | otherwise = k utf8End (Just (Utf8State s (partUtf8UnsafeAppend partUtf8Empty (B.drop utf8End bs))))++-- | Validate another 'ByteString' chunk in an ongoing stream of UTF-8-encoded text.+--+-- Returns a pair:+--+-- 1. The first component @n@ is the end position, relative to the current+--    chunk, of the longest prefix of the accumulated bytestring which is valid UTF-8.+--    @n@ may be negative: that happens when an incomplete code point started in+--    a previous chunk and is not completed by the current chunk (either+--    that code point is still incomplete, or it is broken by an invalid byte).+--+-- 2. The second component @ms@ indicates the following:+--+--     - if @ms = Nothing@, the remainder of the chunk contains an invalid byte,+--       within four bytes from position @n@;+--     - if @ms = Just s'@, you can carry on validating another chunk+--       by calling 'validateUtf8More' with the new state @s'@.+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- 'validateUtf8More' s chunk = (n, ms)+-- @+--+-- - If the chunk is invalid, it cannot be extended to be valid.+--+--     @+--     ms = Nothing+--     ==> 'validateUtf8More' s (chunk '<>' more) = (n, Nothing)+--     @+--+-- - Validating two chunks sequentially is the same as validating them+--   together at once:+--+--     @+--     ms = Just s'+--     ==> 'validateUtf8More' s (chunk '<>' more) = 'Data.Bifunctor.first' ('Data.ByteString.length' chunk '+') ('validateUtf8More' s' more)+--     @+validateUtf8More :: Utf8State -> ByteString -> (Int, Maybe Utf8State)+validateUtf8More st bs = validateUtf8MoreCont st bs (,)++-- CPS: inlining the continuation lets us make more tail calls and avoid+-- allocating a @Maybe@ in @decodeWith1/2@.+{-# INLINE validateUtf8MoreCont #-}+validateUtf8MoreCont :: Utf8State -> ByteString -> (Int -> Maybe Utf8State -> r) -> r+validateUtf8MoreCont st@(Utf8State s0 part) bs k+  | len > 0 = loop 0 s0+  | otherwise = k (- partUtf8Len part) (Just st)+  where+    len = B.length bs+    -- Complete an incomplete code point (if there is one)+    -- and then jump to validateUtf8ChunkFrom+    loop !i s+      | s == utf8AcceptState = validateUtf8ChunkFrom i bs k+      | i < len =+        case updateDecoderState (B.index bs i) s of+          s' | s' == utf8RejectState -> k (- partUtf8Len part) Nothing+             | otherwise -> loop (i + 1) s'+      | otherwise = k (- partUtf8Len part) (Just (Utf8State s (partUtf8UnsafeAppend part bs)))++-- Eta-expanded to inline partUtf8Foldr+partUtf8ToStrictBuilder :: PartialUtf8CodePoint -> StrictTextBuilder+partUtf8ToStrictBuilder c =+  partUtf8Foldr ((<>) . SB.unsafeFromWord8) mempty c++utf8StateToStrictBuilder :: Utf8State -> StrictTextBuilder+utf8StateToStrictBuilder = partUtf8ToStrictBuilder . partialUtf8CodePoint++-- | Decode another chunk in an ongoing UTF-8 stream.+--+-- Returns a triple:+--+-- 1. A 'StrictBuilder' for the decoded chunk of text. You can accumulate+--    chunks with @('<>')@ or output them with 'SB.toText'.+-- 2. The undecoded remainder of the given chunk, for diagnosing errors+--    and resuming (presumably after skipping some bytes).+-- 3. 'Just' the new state, or 'Nothing' if an invalid byte was encountered+--    (it will be within the first 4 bytes of the undecoded remainder).+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- (pre, suf, ms) = 'decodeUtf8More' s chunk+-- @+--+-- 1. If the output @pre@ is nonempty (alternatively, if @length chunk > length suf@)+--+--     @+--     s2b pre \`'Data.ByteString.append'\` suf = p2b s \`'Data.ByteString.append'\` chunk+--     @+--+--     where+--+--     @+--     s2b = 'Data.Text.Encoding.encodeUtf8' . 'Data.Text.Encoding.toText'+--     p2b = 'Data.Text.Internal.Encoding.partUtf8ToByteString'+--     @+--+-- 2. If the output @pre@ is empty (alternatively, if @length chunk = length suf@)+--+--     @suf = chunk@+--+-- 3. Decoding chunks separately is equivalent to decoding their concatenation.+--+--     Given:+--+--     @+--     (pre1, suf1, Just s1) = 'decodeUtf8More' s chunk1+--     (pre2, suf2,     ms2) = 'decodeUtf8More' s1 chunk2+--     (pre3, suf3,     ms3) = 'decodeUtf8More' s (chunk1 \`B.append\` chunk2)+--     @+--+--     we have:+--+--     @+--     s2b (pre1 '<>' pre2) = s2b pre3+--     ms2 = ms3+--     @+decodeUtf8More :: Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State)+decodeUtf8More s bs =+  validateUtf8MoreCont s bs $ \len ms ->+    let builder | len <= 0 = mempty+                | otherwise = utf8StateToStrictBuilder s+                  <> SB.unsafeFromByteString (B.take len bs)+    in (builder, B.drop len bs, ms)++-- | Decode a chunk of UTF-8 text. To be continued with 'decodeUtf8More'.+--+-- See 'decodeUtf8More' for details on the result.+--+-- @since 2.0.2+--+-- === Properties+--+-- @+-- 'decodeUtf8Chunk' = 'decodeUtf8More' 'startUtf8State'+-- @+--+-- Given:+--+-- @+-- 'decodeUtf8Chunk' chunk = (builder, rest, ms)+-- @+--+-- @builder@ is a prefix and @rest@ is a suffix of @chunk@.+--+-- @+-- 'Data.Text.Encoding.encodeUtf8' ('Data.Text.Encoding.strictBuilderToText' builder) '<>' rest = chunk+-- @+decodeUtf8Chunk :: ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State)+decodeUtf8Chunk = decodeUtf8More startUtf8State++-- | Call the error handler on each byte of the partial code point stored in+-- 'Utf8State' and append the results.+--+-- Exported for use in lazy 'Data.Text.Lazy.Encoding.decodeUtf8With'.+--+-- @since 2.0.2+{-# INLINE skipIncomplete #-}+skipIncomplete :: OnDecodeError -> String -> Utf8State -> StrictTextBuilder+skipIncomplete onErr msg s =+  partUtf8Foldr+    ((<>) . handleUtf8Error onErr msg)+    mempty (partialUtf8CodePoint s)++{-# INLINE handleUtf8Error #-}+handleUtf8Error :: OnDecodeError -> String -> Word8 -> StrictTextBuilder+handleUtf8Error onErr msg w = case onErr msg (Just w) of+  Just c -> SB.fromChar c+  Nothing -> mempty++-- | Helper for 'Data.Text.Encoding.decodeUtf8With'.+--+-- @since 2.0.2++-- This could be shorter by calling 'decodeUtf8With2' directly, but we make the+-- first call validateUtf8Chunk directly to return even faster in successful+-- cases.+decodeUtf8With1 ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  OnDecodeError -> String -> ByteString -> Text+decodeUtf8With1 onErr msg bs = validateUtf8ChunkFrom 0 bs $ \len ms -> case ms of+    Just s+      | len == B.length bs ->+        let !(SBS.SBS arr) = SBS.toShort bs in+        Text (A.ByteArray arr) 0 len+      | otherwise -> SB.toText $+          SB.unsafeFromByteString (B.take len bs) <> skipIncomplete onErr msg s+    Nothing ->+       let (builder, _, s) = decodeUtf8With2 onErr msg startUtf8State (B.drop (len + 1) bs) in+       SB.toText $+         SB.unsafeFromByteString (B.take len bs) <>+         handleUtf8Error onErr msg (B.index bs len) <>+         builder <>+         skipIncomplete onErr msg s++-- | Helper for 'Data.Text.Encoding.decodeUtf8With',+-- 'Data.Text.Encoding.streamDecodeUtf8With', and lazy+-- 'Data.Text.Lazy.Encoding.decodeUtf8With',+-- which use an 'OnDecodeError' to process bad bytes.+--+-- See 'decodeUtf8Chunk' for a more flexible alternative.+--+-- @since 2.0.2+decodeUtf8With2 ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  OnDecodeError -> String -> Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Utf8State)+decodeUtf8With2 onErr msg s0 bs = loop s0 0 mempty+  where+    loop s i !builder =+      let nonEmptyPrefix len = builder+            <> utf8StateToStrictBuilder s+            <> SB.unsafeFromByteString (B.take len (B.drop i bs))+      in validateUtf8MoreCont s (B.drop i bs) $ \len ms -> case ms of+        Nothing ->+          if len < 0+          then+            -- If the first byte cannot complete the partial code point in s,+            -- retry from startUtf8State.+            let builder' = builder <> skipIncomplete onErr msg s+            -- Note: loop is strict on builder, so if onErr raises an error it will+            -- be forced here, short-circuiting the loop as desired.+            in loop startUtf8State i builder'+          else+            let builder' = nonEmptyPrefix len+                  <> handleUtf8Error onErr msg (B.index bs (i + len))+            in loop startUtf8State (i + len + 1) builder'+        Just s' ->+          let builder' = if len <= 0 then builder else nonEmptyPrefix len+              undecoded = if B.length bs >= partUtf8Len (partialUtf8CodePoint s')+                then B.drop (i + len) bs  -- Reuse bs if possible+                else partUtf8ToByteString (partialUtf8CodePoint s')+          in (builder', undecoded, s')
+ src/Data/Text/Internal/Encoding/Fusion.hs view
@@ -0,0 +1,217 @@+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-}++-- |+-- Module      : Data.Text.Internal.Encoding.Fusion+-- Copyright   : (c) Tom Harper 2008-2009,+--               (c) Bryan O'Sullivan 2009,+--               (c) Duncan Coutts 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Fusible 'Stream'-oriented functions for converting between 'Text'+-- and several common encodings.++module Data.Text.Internal.Encoding.Fusion+    (+    -- * Streaming+      streamASCII+    , streamUtf8+    , streamUtf16LE+    , streamUtf16BE+    , streamUtf32LE+    , streamUtf32BE++    -- * Unstreaming+    , unstream++    , module Data.Text.Internal.Encoding.Fusion.Common+    ) where++#if defined(ASSERTS)+import Control.Exception (assert)+#endif+import Data.Bits (shiftL, shiftR)+import Data.ByteString.Internal (ByteString(..), mallocByteString)+import Data.Text.Internal.Fusion (Step(..), Stream(..))+import Data.Text.Internal.Fusion.Size+import Data.Text.Encoding.Error+import Data.Text.Internal.Encoding.Fusion.Common+import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Word (Word8, Word16, Word32)+import Foreign.ForeignPtr (ForeignPtr)+import Foreign.Marshal.Utils (copyBytes)+import Foreign.Storable (pokeByteOff)+import qualified Data.ByteString as B+import qualified Data.ByteString.Unsafe as B+import qualified Data.Text.Internal.Encoding.Utf8 as U8+import qualified Data.Text.Internal.Encoding.Utf16 as U16+import qualified Data.Text.Internal.Encoding.Utf32 as U32+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Data.Text.Internal.ByteStringCompat++streamASCII :: ByteString -> Stream Char+streamASCII bs = Stream next 0 (maxSize l)+    where+      l = B.length bs+      {-# INLINE next #-}+      next i+          | i >= l    = Done+          | otherwise = Yield (unsafeChr8 x1) (i+1)+          where+            x1 = B.unsafeIndex bs i+{-# DEPRECATED streamASCII "Do not use this function" #-}+{-# INLINE [0] streamASCII #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8+-- encoding.+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char+streamUtf8 onErr bs = Stream next 0 (maxSize l)+    where+      l = B.length bs+      next i+          | i >= l = Done+          | U8.validate1 x1 = Yield (unsafeChr8 x1) (i+1)+          | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (i+2)+          | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (i+3)+          | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (i+4)+          | otherwise = decodeError "streamUtf8" "UTF-8" onErr (Just x1) (i+1)+          where+            x1 = idx i+            x2 = idx (i + 1)+            x3 = idx (i + 2)+            x4 = idx (i + 3)+            idx = B.unsafeIndex bs+{-# INLINE [0] streamUtf8 #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little+-- endian UTF-16 encoding.+streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16LE onErr bs = Stream next 0 (maxSize (l `shiftR` 1))+    where+      l = B.length bs+      {-# INLINE next #-}+      next i+          | i >= l                         = Done+          | i+1 < l && U16.validate1 x1    = Yield (unsafeChr16 x1) (i+2)+          | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)+          | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1)+          where+            x1    = idx i       + (idx (i + 1) `shiftL` 8)+            x2    = idx (i + 2) + (idx (i + 3) `shiftL` 8)+            idx = word8ToWord16 . B.unsafeIndex bs :: Int -> Word16+{-# INLINE [0] streamUtf16LE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big+-- endian UTF-16 encoding.+streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16BE onErr bs = Stream next 0 (maxSize (l `shiftR` 1))+    where+      l = B.length bs+      {-# INLINE next #-}+      next i+          | i >= l                         = Done+          | i+1 < l && U16.validate1 x1    = Yield (unsafeChr16 x1) (i+2)+          | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)+          | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1)+          where+            x1    = (idx i `shiftL` 8)       + idx (i + 1)+            x2    = (idx (i + 2) `shiftL` 8) + idx (i + 3)+            idx = word8ToWord16 . B.unsafeIndex bs :: Int -> Word16+{-# INLINE [0] streamUtf16BE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big+-- endian UTF-32 encoding.+streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32BE onErr bs = Stream next 0 (maxSize (l `shiftR` 2))+    where+      l = B.length bs+      {-# INLINE next #-}+      next i+          | i >= l                    = Done+          | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)+          | otherwise = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (i+1)+          where+            x     = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4+            x1    = idx i+            x2    = idx (i+1)+            x3    = idx (i+2)+            x4    = idx (i+3)+            idx = word8ToWord32 . B.unsafeIndex bs :: Int -> Word32+{-# INLINE [0] streamUtf32BE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little+-- endian UTF-32 encoding.+streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32LE onErr bs = Stream next 0 (maxSize (l `shiftR` 2))+    where+      l = B.length bs+      {-# INLINE next #-}+      next i+          | i >= l                    = Done+          | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)+          | otherwise = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (i+1)+          where+            x     = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1+            x1    = idx i+            x2    = idx $ i+1+            x3    = idx $ i+2+            x4    = idx $ i+3+            idx = word8ToWord32 . B.unsafeIndex bs :: Int -> Word32+{-# INLINE [0] streamUtf32LE #-}++-- | /O(n)/ Convert a 'Stream' 'Word8' to a 'ByteString'.+unstream :: Stream Word8 -> ByteString+unstream (Stream next s0 len) = unsafeDupablePerformIO $ do+    let mlen = upperBound 4 len+    mallocByteString mlen >>= loop mlen 0 s0+    where+      loop !n !off !s fp = case next s of+          Done -> trimUp fp n off+          Skip s' -> loop n off s' fp+          Yield x s'+              | off == n -> realloc fp n off s' x+              | otherwise -> do+            unsafeWithForeignPtr fp $ \p -> pokeByteOff p off x+            loop n (off+1) s' fp+      {-# NOINLINE realloc #-}+      realloc fp n off s x = do+        let n' = n+n+        fp' <- copy0 fp n n'+        unsafeWithForeignPtr fp' $ \p -> pokeByteOff p off x+        loop n' (off+1) s fp'+      {-# NOINLINE trimUp #-}+      trimUp fp _ off = return $! mkBS fp off+      copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)+      copy0 !src !srcLen !destLen =+#if defined(ASSERTS)+        assert (srcLen <= destLen) $+#endif+        do+          dest <- mallocByteString destLen+          unsafeWithForeignPtr src  $ \src'  ->+              unsafeWithForeignPtr dest $ \dest' ->+                  copyBytes dest' src' srcLen+          return dest++decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8+            -> s -> Step s Char+decodeError func kind onErr mb i =+    case onErr desc mb of+      Nothing -> Skip i+      Just c  -> Yield c i+    where desc = "Data.Text.Internal.Encoding.Fusion." ++ func ++ ": Invalid " +++                 kind ++ " stream"++word8ToWord16 :: Word8 -> Word16+word8ToWord16 = fromIntegral++word8ToWord32 :: Word8 -> Word32+word8ToWord32 = fromIntegral
+ src/Data/Text/Internal/Encoding/Fusion/Common.hs view
@@ -0,0 +1,125 @@+{-# LANGUAGE BangPatterns #-}++-- |+-- Module      : Data.Text.Internal.Encoding.Fusion.Common+-- Copyright   : (c) Tom Harper 2008-2009,+--               (c) Bryan O'Sullivan 2009,+--               (c) Duncan Coutts 2009,+--               (c) Jasper Van der Jeugt 2011+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Use at your own risk!+--+-- Fusible 'Stream'-oriented functions for converting between 'Text'+-- and several common encodings.++module Data.Text.Internal.Encoding.Fusion.Common+    (+    -- * Restreaming+    -- Restreaming is the act of converting from one 'Stream'+    -- representation to another.+      restreamUtf16LE+    , restreamUtf16BE+    , restreamUtf32LE+    , restreamUtf32BE+    ) where++import Data.Bits ((.&.), shiftR)+import Data.Text.Internal.Fusion (Step(..), Stream(..))+import Data.Text.Internal.Fusion.Types (RS(..))+import Data.Text.Internal.Unsafe.Char (ord)+import Data.Word (Word8)++restreamUtf16BE :: Stream Char -> Stream Word8+restreamUtf16BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)+  where+    next (RS0 s) = case next0 s of+        Done -> Done+        Skip s' -> Skip (RS0 s')+        Yield x s'+            | n < 0x10000 -> Yield (intToWord8 $ n `shiftR` 8) $+                             RS1 s' (intToWord8 n)+            | otherwise   -> Yield c1 $ RS3 s' c2 c3 c4+            where+              n  = ord x+              n1 = n - 0x10000+              c1 = intToWord8 (n1 `shiftR` 18 + 0xD8)+              c2 = intToWord8 (n1 `shiftR` 10)+              n2 = n1 .&. 0x3FF+              c3 = intToWord8 (n2 `shiftR` 8 + 0xDC)+              c4 = intToWord8 n2+    next (RS1 s x2)       = Yield x2 (RS0 s)+    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)+    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)+    {-# INLINE next #-}+{-# INLINE restreamUtf16BE #-}++restreamUtf16LE :: Stream Char -> Stream Word8+restreamUtf16LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)+  where+    next (RS0 s) = case next0 s of+        Done -> Done+        Skip s' -> Skip (RS0 s')+        Yield x s'+            | n < 0x10000 -> Yield (intToWord8 n) $+                             RS1 s' (intToWord8 $ shiftR n 8)+            | otherwise   -> Yield c1 $ RS3 s' c2 c3 c4+          where+            n  = ord x+            n1 = n - 0x10000+            c2 = intToWord8 (shiftR n1 18 + 0xD8)+            c1 = intToWord8 (shiftR n1 10)+            n2 = n1 .&. 0x3FF+            c4 = intToWord8 (shiftR n2 8 + 0xDC)+            c3 = intToWord8 n2+    next (RS1 s x2)       = Yield x2 (RS0 s)+    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)+    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)+    {-# INLINE next #-}+{-# INLINE restreamUtf16LE #-}++restreamUtf32BE :: Stream Char -> Stream Word8+restreamUtf32BE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)+  where+    next (RS0 s) = case next0 s of+        Done       -> Done+        Skip s'    -> Skip (RS0 s')+        Yield x s' -> Yield c1 (RS3 s' c2 c3 c4)+          where+            n  = ord x+            c1 = intToWord8 $ shiftR n 24+            c2 = intToWord8 $ shiftR n 16+            c3 = intToWord8 $ shiftR n 8+            c4 = intToWord8 n+    next (RS1 s x2)       = Yield x2 (RS0 s)+    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)+    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)+    {-# INLINE next #-}+{-# INLINE restreamUtf32BE #-}++restreamUtf32LE :: Stream Char -> Stream Word8+restreamUtf32LE (Stream next0 s0 len) = Stream next (RS0 s0) (len * 2)+  where+    next (RS0 s) = case next0 s of+        Done       -> Done+        Skip s'    -> Skip (RS0 s')+        Yield x s' -> Yield c1 (RS3 s' c2 c3 c4)+          where+            n  = ord x+            c4 = intToWord8 $ shiftR n 24+            c3 = intToWord8 $ shiftR n 16+            c2 = intToWord8 $ shiftR n 8+            c1 = intToWord8 n+    next (RS1 s x2)       = Yield x2 (RS0 s)+    next (RS2 s x2 x3)    = Yield x2 (RS1 s x3)+    next (RS3 s x2 x3 x4) = Yield x2 (RS2 s x3 x4)+    {-# INLINE next #-}+{-# INLINE restreamUtf32LE #-}++intToWord8 :: Int -> Word8+intToWord8 = fromIntegral
+ src/Data/Text/Internal/Encoding/Utf16.hs view
@@ -0,0 +1,52 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE BangPatterns #-}++-- |+-- Module      : Data.Text.Internal.Encoding.Utf16+-- Copyright   : (c) 2008, 2009 Tom Harper,+--               (c) 2009 Bryan O'Sullivan,+--               (c) 2009 Duncan Coutts+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Basic UTF-16 validation and character manipulation.+module Data.Text.Internal.Encoding.Utf16+    (+      chr2+    , validate1+    , validate2+    ) where++import GHC.Exts+import GHC.Word (Word16(..))++#if !MIN_VERSION_base(4,16,0)+-- harmless to import, except for warnings that it is unused.+import Data.Text.Internal.PrimCompat ( word16ToWord# )+#endif++chr2 :: Word16 -> Word16 -> Char+chr2 (W16# a#) (W16# b#) = C# (chr# (upper# +# lower# +# 0x10000#))+    where+      !x# = word2Int# (word16ToWord# a#)+      !y# = word2Int# (word16ToWord# b#)+      !upper# = uncheckedIShiftL# (x# -# 0xD800#) 10#+      !lower# = y# -# 0xDC00#+{-# INLINE chr2 #-}++validate1    :: Word16 -> Bool+validate1 x1 = x1 < 0xD800 || x1 > 0xDFFF+{-# INLINE validate1 #-}++validate2       ::  Word16 -> Word16 -> Bool+validate2 x1 x2 = x1 >= 0xD800 && x1 <= 0xDBFF &&+                  x2 >= 0xDC00 && x2 <= 0xDFFF+{-# INLINE validate2 #-}
+ src/Data/Text/Internal/Encoding/Utf32.hs view
@@ -0,0 +1,26 @@+-- |+-- Module      : Data.Text.Internal.Encoding.Utf32+-- Copyright   : (c) 2008, 2009 Tom Harper,+--               (c) 2009, 2010 Bryan O'Sullivan,+--               (c) 2009 Duncan Coutts+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Basic UTF-32 validation.+module Data.Text.Internal.Encoding.Utf32+    (+      validate+    ) where++import Data.Word (Word32)++validate    :: Word32 -> Bool+validate x1 = x1 < 0xD800 || (x1 > 0xDFFF && x1 <= 0x10FFFF)+{-# INLINE validate #-}
+ src/Data/Text/Internal/Encoding/Utf8.hs view
@@ -0,0 +1,318 @@+{-# LANGUAGE CPP, MagicHash, BangPatterns #-}
+
+-- |
+-- Module      : Data.Text.Internal.Encoding.Utf8
+-- Copyright   : (c) 2008, 2009 Tom Harper,
+--               (c) 2009, 2010 Bryan O'Sullivan,
+--               (c) 2009 Duncan Coutts
+--               (c) 2021 Andrew Lelechenko
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- /Warning/: this is an internal module, and does not have a stable
+-- API or name. Functions in this module may not check or enforce
+-- preconditions expected by public modules. Use at your own risk!
+--
+-- Basic UTF-8 validation and character manipulation.
+module Data.Text.Internal.Encoding.Utf8
+    ( utf8Length
+    , utf8LengthByLeader
+    -- Decomposition
+    , ord2
+    , ord3
+    , ord4
+    -- Construction
+    , chr2
+    , chr3
+    , chr4
+    -- * Validation
+    , validate1
+    , validate2
+    , validate3
+    , validate4
+    -- * Naive decoding
+    , DecoderState(..)
+    , utf8AcceptState
+    , utf8RejectState
+    , updateDecoderState
+    , DecoderResult(..)
+    , CodePoint(..)
+    , utf8DecodeStart
+    , utf8DecodeContinue
+    ) where
+
+#if defined(ASSERTS)
+import Control.Exception (assert)
+import GHC.Stack (HasCallStack)
+#endif
+import Data.Bits (Bits(..))
+import Data.Char (ord, chr)
+import GHC.Exts
+import GHC.Word (Word8(..))
+
+#if !MIN_VERSION_base(4,16,0)
+-- harmless to import, except for warnings that it is unused.
+import Data.Text.Internal.PrimCompat (word8ToWord#)
+#endif
+
+default(Int)
+
+between :: Word8                -- ^ byte to check
+        -> Word8                -- ^ lower bound
+        -> Word8                -- ^ upper bound
+        -> Bool
+between x y z = x >= y && x <= z
+{-# INLINE between #-}
+
+-- This is a branchless version of
+-- utf8Length c
+--   | ord c < 0x80    = 1
+--   | ord c < 0x800   = 2
+--   | ord c < 0x10000 = 3
+--   | otherwise       = 4
+-- Implementation suggested by Alex Mason.
+
+-- | Measure byte length of UTF-8 encoding for a given character.
+--
+-- @since 2.0
+utf8Length :: Char -> Int
+utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#)))
+{-# INLINE utf8Length #-}
+
+-- | Measure byte length of UTF-8 encoding for characters,
+-- starting with a given byte.
+--
+-- @since 2.0
+utf8LengthByLeader :: Word8 -> Int
+utf8LengthByLeader w
+  | w < 0x80  = 1
+  | w < 0xE0  = 2
+  | w < 0xF0  = 3
+  | otherwise = 4
+{-# INLINE utf8LengthByLeader #-}
+
+-- | Encode a character as UTF-8 bytes assuming that exactly 2 are needed.
+-- This precondition is not checked.
+--
+-- @since 1.1.0.0
+ord2 ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Char -> (Word8,Word8)
+ord2 c =
+#if defined(ASSERTS)
+    assert (n >= 0x80 && n <= 0x07ff)
+#endif
+    (x1,x2)
+    where
+      n  = ord c
+      x1 = intToWord8 $ (n `shiftR` 6) + 0xC0
+      x2 = intToWord8 $ (n .&. 0x3F)   + 0x80
+{-# INLINE ord2 #-}
+
+-- | Encode a character as UTF-8 bytes assuming that exactly 3 are needed.
+-- This precondition is not checked.
+--
+-- @since 1.1.0.0
+ord3 ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Char -> (Word8,Word8,Word8)
+ord3 c =
+#if defined(ASSERTS)
+    assert (n >= 0x0800 && n <= 0xffff)
+#endif
+    (x1,x2,x3)
+    where
+      n  = ord c
+      x1 = intToWord8 $ (n `shiftR` 12) + 0xE0
+      x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80
+      x3 = intToWord8 $ (n .&. 0x3F) + 0x80
+{-# INLINE ord3 #-}
+
+-- | Encode a character as UTF-8 bytes assuming that exactly 4 are needed.
+-- This precondition is not checked.
+--
+-- @since 1.1.0.0
+ord4 ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Char -> (Word8,Word8,Word8,Word8)
+ord4 c =
+#if defined(ASSERTS)
+    assert (n >= 0x10000)
+#endif
+    (x1,x2,x3,x4)
+    where
+      n  = ord c
+      x1 = intToWord8 $ (n `shiftR` 18) + 0xF0
+      x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80
+      x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80
+      x4 = intToWord8 $ (n .&. 0x3F) + 0x80
+{-# INLINE ord4 #-}
+
+-- | @since 1.1.0.0
+chr2 :: Word8 -> Word8 -> Char
+chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#))
+    where
+      !y1# = word2Int# (word8ToWord# x1#)
+      !y2# = word2Int# (word8ToWord# x2#)
+      !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6#
+      !z2# = y2# -# 0x80#
+{-# INLINE chr2 #-}
+
+-- | @since 1.1.0.0
+chr3 :: Word8 -> Word8 -> Word8 -> Char
+chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#))
+    where
+      !y1# = word2Int# (word8ToWord# x1#)
+      !y2# = word2Int# (word8ToWord# x2#)
+      !y3# = word2Int# (word8ToWord# x3#)
+      !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12#
+      !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6#
+      !z3# = y3# -# 0x80#
+{-# INLINE chr3 #-}
+
+-- | @since 1.1.0.0
+chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char
+chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) =
+    C# (chr# (z1# +# z2# +# z3# +# z4#))
+    where
+      !y1# = word2Int# (word8ToWord# x1#)
+      !y2# = word2Int# (word8ToWord# x2#)
+      !y3# = word2Int# (word8ToWord# x3#)
+      !y4# = word2Int# (word8ToWord# x4#)
+      !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18#
+      !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12#
+      !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6#
+      !z4# = y4# -# 0x80#
+{-# INLINE chr4 #-}
+
+-- | @since 1.1.0.0
+validate1 :: Word8 -> Bool
+validate1 x1 = x1 <= 0x7F
+{-# INLINE validate1 #-}
+
+-- | @since 1.1.0.0
+validate2 :: Word8 -> Word8 -> Bool
+validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF
+{-# INLINE validate2 #-}
+
+-- | @since 1.1.0.0
+validate3 :: Word8 -> Word8 -> Word8 -> Bool
+{-# INLINE validate3 #-}
+validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4
+  where
+    validate3_1 = (x1 == 0xE0) &&
+                  between x2 0xA0 0xBF &&
+                  between x3 0x80 0xBF
+    validate3_2 = between x1 0xE1 0xEC &&
+                  between x2 0x80 0xBF &&
+                  between x3 0x80 0xBF
+    validate3_3 = x1 == 0xED &&
+                  between x2 0x80 0x9F &&
+                  between x3 0x80 0xBF
+    validate3_4 = between x1 0xEE 0xEF &&
+                  between x2 0x80 0xBF &&
+                  between x3 0x80 0xBF
+
+-- | @since 1.1.0.0
+validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool
+{-# INLINE validate4 #-}
+validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3
+  where
+    validate4_1 = x1 == 0xF0 &&
+                  between x2 0x90 0xBF &&
+                  between x3 0x80 0xBF &&
+                  between x4 0x80 0xBF
+    validate4_2 = between x1 0xF1 0xF3 &&
+                  between x2 0x80 0xBF &&
+                  between x3 0x80 0xBF &&
+                  between x4 0x80 0xBF
+    validate4_3 = x1 == 0xF4 &&
+                  between x2 0x80 0x8F &&
+                  between x3 0x80 0xBF &&
+                  between x4 0x80 0xBF
+
+intToWord8 :: Int -> Word8
+intToWord8 = fromIntegral
+
+word8ToInt :: Word8 -> Int
+word8ToInt = fromIntegral
+
+-------------------------------------------------------------------------------
+-- Naive UTF8 decoder.
+-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine.
+
+newtype ByteClass = ByteClass Word8
+
+byteToClass :: Word8 -> ByteClass
+byteToClass n = ByteClass (W8# el#)
+  where
+    !(I# n#) = word8ToInt n
+    el# = indexWord8OffAddr# table# n#
+
+    table# :: Addr#
+    table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"#
+
+-- | @since 2.0
+newtype DecoderState = DecoderState Word8
+  deriving (Eq, Show)
+
+-- | @since 2.0.2
+utf8AcceptState :: DecoderState
+utf8AcceptState = DecoderState 0
+
+-- | @since 2.0.2
+utf8RejectState :: DecoderState
+utf8RejectState = DecoderState 12
+
+updateState :: ByteClass -> DecoderState -> DecoderState
+updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#)
+  where
+    !(I# n#) = word8ToInt (c + s)
+    el# = indexWord8OffAddr# table# n#
+
+    table# :: Addr#
+    table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"#
+
+-- | @since 2.0.2
+updateDecoderState :: Word8 -> DecoderState -> DecoderState
+updateDecoderState b s = updateState (byteToClass b) s
+
+-- | @since 2.0
+newtype CodePoint = CodePoint Int
+
+-- | @since 2.0
+data DecoderResult
+  = Accept !Char
+  | Incomplete !DecoderState !CodePoint
+  | Reject
+
+-- | @since 2.0
+utf8DecodeStart :: Word8 -> DecoderResult
+utf8DecodeStart !w
+  | st == utf8AcceptState = Accept (chr (word8ToInt w))
+  | st == utf8RejectState = Reject
+  | otherwise             = Incomplete st (CodePoint cp)
+  where
+    cl@(ByteClass cl') = byteToClass w
+    st = updateState cl utf8AcceptState
+    cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w
+
+-- | @since 2.0
+utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult
+utf8DecodeContinue !w !st (CodePoint !cp)
+  | st' == utf8AcceptState = Accept (chr cp')
+  | st' == utf8RejectState = Reject
+  | otherwise              = Incomplete st' (CodePoint cp')
+  where
+    cl  = byteToClass w
+    st' = updateState cl st
+    cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f)
+ src/Data/Text/Internal/Fusion.hs view
@@ -0,0 +1,326 @@+{-# LANGUAGE BangPatterns, CPP, MagicHash #-}++-- |+-- Module      : Data.Text.Internal.Fusion+-- Copyright   : (c) Tom Harper 2008-2009,+--               (c) Bryan O'Sullivan 2009-2010,+--               (c) Duncan Coutts 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Text manipulation functions represented as fusible operations over+-- streams.+module Data.Text.Internal.Fusion+    (+    -- * Types+      Stream(..)+    , Step(..)++    -- * Creation and elimination+    , stream+    , streamLn+    , unstream+    , reverseStream++    , length++    -- * Transformations+    , reverse++    -- * Construction+    -- ** Scans+    , reverseScanr++    -- ** Accumulating maps+    , mapAccumL++    -- ** Generation and unfolding+    , unfoldrN++    -- * Indexing+    , index+    , findIndex+    , countChar+    ) where++import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int,+                Num(..), Ord(..), ($), (&&),+                otherwise)+import Data.Bits (shiftL, shiftR)+import Data.Text.Internal (Text(..))+import Data.Text.Internal.Private (runText)+import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeWrite)+import qualified Data.Text.Array as A+import qualified Data.Text.Internal.Fusion.Common as S+import Data.Text.Internal.Fusion.Types+import Data.Text.Internal.Fusion.Size+import qualified Data.Text.Internal as I+import qualified Data.Text.Internal.Encoding.Utf8 as U8+import GHC.Stack (HasCallStack)++default(Int)++-- | /O(n)/ Convert 'Text' into a 'Stream' 'Char'.+--+-- __Properties__+--+-- @'unstream' . 'stream' = 'Data.Function.id'@+--+-- @'stream' . 'unstream' = 'Data.Function.id' @+stream ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Stream Char+stream = stream' False+{-# INLINE [0] stream #-}++-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@+--+-- @since 2.1.2+streamLn ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Stream Char+streamLn = stream' True++-- | Shared implementation of 'stream' and 'streamLn'.+stream' ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Bool -> Text -> Stream Char+stream' addNl (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) maxLen)+    where+      maxLen = if addNl then len + 1 else len+      !end = off+len+      next !i+          | i < end = Yield chr (i + l)+          | addNl && i == end = Yield '\n' (i + 1)+          | otherwise = Done+          where+            n0 = A.unsafeIndex arr i+            n1 = A.unsafeIndex arr (i + 1)+            n2 = A.unsafeIndex arr (i + 2)+            n3 = A.unsafeIndex arr (i + 3)++            l  = U8.utf8LengthByLeader n0+            chr = case l of+              1 -> unsafeChr8 n0+              2 -> U8.chr2 n0 n1+              3 -> U8.chr3 n0 n1 n2+              _ -> U8.chr4 n0 n1 n2 n3+{-# INLINE [0] stream' #-}++-- | /O(n)/ Converts 'Text' into a 'Stream' 'Char', but iterates+-- backwards through the text.+--+-- __Properties__+--+-- @'unstream' . 'reverseStream' = 'Data.Text.reverse' @+reverseStream :: Text -> Stream Char+reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 2) len)+    where+      {-# INLINE next #-}+      next !i+          | i < off    = Done+          | n0 <  0x80 = Yield (unsafeChr8 n0)       (i - 1)+          | n1 >= 0xC0 = Yield (U8.chr2 n1 n0)       (i - 2)+          | n2 >= 0xC0 = Yield (U8.chr3 n2 n1 n0)    (i - 3)+          | otherwise  = Yield (U8.chr4 n3 n2 n1 n0) (i - 4)+          where+            n0 = A.unsafeIndex arr i+            n1 = A.unsafeIndex arr (i - 1)+            n2 = A.unsafeIndex arr (i - 2)+            n3 = A.unsafeIndex arr (i - 3)+{-# INLINE [0] reverseStream #-}++-- | /O(n)/ Convert 'Stream' 'Char' into a 'Text'.+--+-- __Properties__+--+-- @'unstream' . 'stream' = 'Data.Function.id'@+--+-- @'stream' . 'unstream' = 'Data.Function.id' @+unstream :: Stream Char -> Text+unstream (Stream next0 s0 len) = runText $ \done -> do+  -- Before encoding each char we perform a buffer realloc check assuming+  -- worst case encoding size of four 8-bit units for the char. Just add an+  -- extra space to the buffer so that we do not end up reallocating even when+  -- all the chars are encoded as single unit.+  let mlen = upperBound 4 len + 3+  arr0 <- A.new mlen+  let outer !arr !maxi = encode+       where+        -- keep the common case loop as small as possible+        encode !si !di =+            case next0 si of+                Done        -> done arr di+                Skip si'    -> encode si' di+                Yield c si'+                    -- simply check for the worst case+                    | maxi < di + 3 -> realloc si di+                    | otherwise -> do+                            n <- unsafeWrite arr di c+                            encode si' (di + n)++        -- keep uncommon case separate from the common case code+        {-# NOINLINE realloc #-}+        realloc !si !di = do+            let newlen = (maxi + 1) * 2+            arr' <- A.resizeM arr newlen+            outer arr' (newlen - 1) si di++  outer arr0 (mlen - 1) s0 0+{-# INLINE [0] unstream #-}+{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}+++-- ----------------------------------------------------------------------------+-- * Basic stream functions++-- | /O(n)/ Returns the number of characters in a 'Stream'.+--+-- __Properties__+--+-- @'length' . 'stream' = 'Data.Text.length' @+length :: Stream Char -> Int+length = S.lengthI+{-# INLINE[0] length #-}++-- | /O(n)/ Reverse the characters of a 'Stream' returning 'Text'.+--+-- __Properties__+--+-- @'reverse' . 'stream' = 'Data.Text.reverse' @+reverse ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Stream Char -> Text+reverse (Stream next s len0)+    | isEmpty len0 = I.empty+    | otherwise    = I.text arr off' len'+  where+    len0' = upperBound 4 (larger len0 4)+    (arr, (off', len')) = A.run2 (A.new len0' >>= loop s (len0'-1) len0')+    loop !s0 !i !len marr =+        case next s0 of+          Done -> return (marr, (j, len-j))+              where j = i + 1+          Skip s1    -> loop s1 i len marr+          Yield x s1 | i < least -> {-# SCC "reverse/resize" #-} do+                       let newLen = len `shiftL` 1+                       marr' <- A.new newLen+                       A.copyM marr' (newLen-len) marr 0 len+                       _ <- unsafeWrite marr' (len + i - least) x+                       loop s1 (len + i - least - 1) newLen marr'+                     | otherwise -> do+                       _ <- unsafeWrite marr (i - least) x+                       loop s1 (i - least - 1) len marr+            where least = U8.utf8Length x - 1+{-# INLINE [0] reverse #-}++-- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with+-- the input and result reversed.+--+-- __Properties__+--+-- @'reverse' . 'reverseScanr' f c . 'reverseStream' = 'Data.Text.scanr' f c @+reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char+reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low+  where+    {-# INLINE next #-}+    next (Scan1 z s) = Yield z (Scan2 z s)+    next (Scan2 z s) = case next0 s of+                         Yield x s' -> let !x' = f x z+                                       in Yield x' (Scan2 x' s')+                         Skip s'    -> Skip (Scan2 z s')+                         Done       -> Done+{-# INLINE reverseScanr #-}++-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed+-- value. However, the length of the result is limited by the+-- first argument to 'unfoldrN'. This function is more efficient than+-- 'unfoldr' when the length of the result is known.+--+-- __Properties__+--+-- @'unstream' ('unfoldrN' n f a) = 'Data.Text.unfoldrN' n f a @+unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Stream Char+unfoldrN n = S.unfoldrNI n+{-# INLINE [0] unfoldrN #-}++-------------------------------------------------------------------------------+-- ** Indexing streams++-- | /O(n)/ stream index (subscript) operator, starting from 0.+--+-- __Properties__+--+-- @'index' ('stream' t) n  = 'Data.Text.index' t n @+index :: HasCallStack => Stream Char -> Int -> Char+index = S.indexI+{-# INLINE [0] index #-}++-- | The 'findIndex' function takes a predicate and a stream and+-- returns the index of the first element in the stream+-- satisfying the predicate.+--+-- __Properties__+--+-- @'findIndex' p . 'stream'  = 'Data.Text.findIndex' p @+findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int+findIndex = S.findIndexI+{-# INLINE [0] findIndex #-}++-- | /O(n)/ The 'count' function returns the number of times the query+-- element appears in the given stream.+--+-- __Properties__+--+-- @'countChar' c . 'stream'  = 'Data.Text.countChar' c @+countChar :: Char -> Stream Char -> Int+countChar = S.countCharI+{-# INLINE [0] countChar #-}++-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a+-- function to each element of a 'Text', passing an accumulating+-- parameter from left to right, and returns a final 'Text'.+--+-- __Properties__+--+-- @'mapAccumL' g z0 . 'stream' = 'Data.Text.mapAccumL' g z0@+mapAccumL ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  (a -> Char -> (a,Char)) -> a -> Stream Char -> (a, Text)+mapAccumL f z0 (Stream next0 s0 len) = (nz, I.text na 0 nl)+  where+    (na,(nz,nl)) = A.run2 (A.new mlen >>= \arr -> outer arr mlen z0 s0 0)+      where mlen = upperBound 4 len+    outer arr top = loop+      where+        loop !z !s !i =+            case next0 s of+              Done          -> return (arr, (z,i))+              Skip s'       -> loop z s' i+              Yield x s'+                | j >= top  -> {-# SCC "mapAccumL/resize" #-} do+                               let top' = (top + 1) `shiftL` 1+                               arr' <- A.resizeM arr top'+                               outer arr' top' z s i+                | otherwise -> do d <- unsafeWrite arr i c+                                  loop z' s' (i+d)+                where (z',c) = f z x+                      j = i + U8.utf8Length c - 1+{-# INLINE [0] mapAccumL #-}
+ src/Data/Text/Internal/Fusion/CaseMapping.hs view
@@ -0,0 +1,7984 @@+-- AUTOMATICALLY GENERATED - DO NOT EDIT
+-- Generated by scripts/CaseMapping.hs
+-- CaseFolding-17.0.0.txt
+-- Date: 2025-07-30, 23:54:36 GMT
+-- SpecialCasing-17.0.0.txt
+-- Date: 2025-07-31, 22:11:55 GMT
+
+{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}
+{-# OPTIONS_GHC -Wno-partial-type-signatures #-}
+module Data.Text.Internal.Fusion.CaseMapping where
+import GHC.Int
+import GHC.Exts
+import Data.Version (Version, makeVersion)
+unicodeVersion :: Version
+unicodeVersion = makeVersion [17, 0, 0]
+unI64 :: Int64 -> _ {- unboxed Int64 -}
+unI64 (I64# n) = n
+
+upperMapping :: Char# -> _ {- unboxed Int64 -}
+{-# NOINLINE upperMapping #-}
+upperMapping = \case
+  -- LATIN SMALL LETTER SHARP S
+  '\x00df'# -> unI64 174063699
+  -- LATIN SMALL LIGATURE FF
+  '\xfb00'# -> unI64 146800710
+  -- LATIN SMALL LIGATURE FI
+  '\xfb01'# -> unI64 153092166
+  -- LATIN SMALL LIGATURE FL
+  '\xfb02'# -> unI64 159383622
+  -- LATIN SMALL LIGATURE FFI
+  '\xfb03'# -> unI64 321057542111302
+  -- LATIN SMALL LIGATURE FFL
+  '\xfb04'# -> unI64 334251681644614
+  -- LATIN SMALL LIGATURE LONG S T
+  '\xfb05'# -> unI64 176160851
+  -- LATIN SMALL LIGATURE ST
+  '\xfb06'# -> unI64 176160851
+  -- ARMENIAN SMALL LIGATURE ECH YIWN
+  '\x0587'# -> unI64 2856322357
+  -- ARMENIAN SMALL LIGATURE MEN NOW
+  '\xfb13'# -> unI64 2831156548
+  -- ARMENIAN SMALL LIGATURE MEN ECH
+  '\xfb14'# -> unI64 2795504964
+  -- ARMENIAN SMALL LIGATURE MEN INI
+  '\xfb15'# -> unI64 2808087876
+  -- ARMENIAN SMALL LIGATURE VEW NOW
+  '\xfb16'# -> unI64 2831156558
+  -- ARMENIAN SMALL LIGATURE MEN XEH
+  '\xfb17'# -> unI64 2812282180
+  -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+  '\x0149'# -> unI64 163578556
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+  '\x0390'# -> unI64 3382099394429849
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+  '\x03b0'# -> unI64 3382099394429861
+  -- LATIN SMALL LETTER J WITH CARON
+  '\x01f0'# -> unI64 1635778634
+  -- LATIN SMALL LETTER H WITH LINE BELOW
+  '\x1e96'# -> unI64 1713373256
+  -- LATIN SMALL LETTER T WITH DIAERESIS
+  '\x1e97'# -> unI64 1627390036
+  -- LATIN SMALL LETTER W WITH RING ABOVE
+  '\x1e98'# -> unI64 1631584343
+  -- LATIN SMALL LETTER Y WITH RING ABOVE
+  '\x1e99'# -> unI64 1631584345
+  -- LATIN SMALL LETTER A WITH RIGHT HALF RING
+  '\x1e9a'# -> unI64 1472200769
+  -- GREEK SMALL LETTER UPSILON WITH PSILI
+  '\x1f50'# -> unI64 1650459557
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+  '\x1f52'# -> unI64 3377701370987429
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+  '\x1f54'# -> unI64 3382099417498533
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+  '\x1f56'# -> unI64 3667972440720293
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+  '\x1fb6'# -> unI64 1749025681
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI
+  '\x1fc6'# -> unI64 1749025687
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+  '\x1fd2'# -> unI64 3377701347918745
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+  '\x1fd3'# -> unI64 3382099394429849
+  -- GREEK SMALL LETTER IOTA WITH PERISPOMENI
+  '\x1fd6'# -> unI64 1749025689
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+  '\x1fd7'# -> unI64 3667972417651609
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+  '\x1fe2'# -> unI64 3377701347918757
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+  '\x1fe3'# -> unI64 3382099394429861
+  -- GREEK SMALL LETTER RHO WITH PSILI
+  '\x1fe4'# -> unI64 1650459553
+  -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+  '\x1fe6'# -> unI64 1749025701
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+  '\x1fe7'# -> unI64 3667972417651621
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+  '\x1ff6'# -> unI64 1749025705
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f80'# -> unI64 1931484936
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f81'# -> unI64 1931484937
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f82'# -> unI64 1931484938
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f83'# -> unI64 1931484939
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f84'# -> unI64 1931484940
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f85'# -> unI64 1931484941
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f86'# -> unI64 1931484942
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f87'# -> unI64 1931484943
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f88'# -> unI64 1931484936
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f89'# -> unI64 1931484937
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f8a'# -> unI64 1931484938
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f8b'# -> unI64 1931484939
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f8c'# -> unI64 1931484940
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f8d'# -> unI64 1931484941
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8e'# -> unI64 1931484942
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8f'# -> unI64 1931484943
+  -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f90'# -> unI64 1931484968
+  -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f91'# -> unI64 1931484969
+  -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f92'# -> unI64 1931484970
+  -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f93'# -> unI64 1931484971
+  -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f94'# -> unI64 1931484972
+  -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f95'# -> unI64 1931484973
+  -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f96'# -> unI64 1931484974
+  -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f97'# -> unI64 1931484975
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f98'# -> unI64 1931484968
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f99'# -> unI64 1931484969
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f9a'# -> unI64 1931484970
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f9b'# -> unI64 1931484971
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f9c'# -> unI64 1931484972
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f9d'# -> unI64 1931484973
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9e'# -> unI64 1931484974
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9f'# -> unI64 1931484975
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+  '\x1fa0'# -> unI64 1931485032
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+  '\x1fa1'# -> unI64 1931485033
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1fa2'# -> unI64 1931485034
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1fa3'# -> unI64 1931485035
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1fa4'# -> unI64 1931485036
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1fa5'# -> unI64 1931485037
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa6'# -> unI64 1931485038
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa7'# -> unI64 1931485039
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+  '\x1fa8'# -> unI64 1931485032
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+  '\x1fa9'# -> unI64 1931485033
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1faa'# -> unI64 1931485034
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1fab'# -> unI64 1931485035
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1fac'# -> unI64 1931485036
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1fad'# -> unI64 1931485037
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1fae'# -> unI64 1931485038
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1faf'# -> unI64 1931485039
+  -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+  '\x1fb3'# -> unI64 1931477905
+  -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+  '\x1fbc'# -> unI64 1931477905
+  -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+  '\x1fc3'# -> unI64 1931477911
+  -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+  '\x1fcc'# -> unI64 1931477911
+  -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+  '\x1ff3'# -> unI64 1931477929
+  -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+  '\x1ffc'# -> unI64 1931477929
+  -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fb2'# -> unI64 1931485114
+  -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fb4'# -> unI64 1931477894
+  -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fc2'# -> unI64 1931485130
+  -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fc4'# -> unI64 1931477897
+  -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+  '\x1ff2'# -> unI64 1931485178
+  -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+  '\x1ff4'# -> unI64 1931477903
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fb7'# -> unI64 4050602585752465
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fc7'# -> unI64 4050602585752471
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1ff7'# -> unI64 4050602585752489
+  '\x0061'# -> unI64 65
+  '\x0062'# -> unI64 66
+  '\x0063'# -> unI64 67
+  '\x0064'# -> unI64 68
+  '\x0065'# -> unI64 69
+  '\x0066'# -> unI64 70
+  '\x0067'# -> unI64 71
+  '\x0068'# -> unI64 72
+  '\x0069'# -> unI64 73
+  '\x006a'# -> unI64 74
+  '\x006b'# -> unI64 75
+  '\x006c'# -> unI64 76
+  '\x006d'# -> unI64 77
+  '\x006e'# -> unI64 78
+  '\x006f'# -> unI64 79
+  '\x0070'# -> unI64 80
+  '\x0071'# -> unI64 81
+  '\x0072'# -> unI64 82
+  '\x0073'# -> unI64 83
+  '\x0074'# -> unI64 84
+  '\x0075'# -> unI64 85
+  '\x0076'# -> unI64 86
+  '\x0077'# -> unI64 87
+  '\x0078'# -> unI64 88
+  '\x0079'# -> unI64 89
+  '\x007a'# -> unI64 90
+  '\x00b5'# -> unI64 924
+  '\x00e0'# -> unI64 192
+  '\x00e1'# -> unI64 193
+  '\x00e2'# -> unI64 194
+  '\x00e3'# -> unI64 195
+  '\x00e4'# -> unI64 196
+  '\x00e5'# -> unI64 197
+  '\x00e6'# -> unI64 198
+  '\x00e7'# -> unI64 199
+  '\x00e8'# -> unI64 200
+  '\x00e9'# -> unI64 201
+  '\x00ea'# -> unI64 202
+  '\x00eb'# -> unI64 203
+  '\x00ec'# -> unI64 204
+  '\x00ed'# -> unI64 205
+  '\x00ee'# -> unI64 206
+  '\x00ef'# -> unI64 207
+  '\x00f0'# -> unI64 208
+  '\x00f1'# -> unI64 209
+  '\x00f2'# -> unI64 210
+  '\x00f3'# -> unI64 211
+  '\x00f4'# -> unI64 212
+  '\x00f5'# -> unI64 213
+  '\x00f6'# -> unI64 214
+  '\x00f8'# -> unI64 216
+  '\x00f9'# -> unI64 217
+  '\x00fa'# -> unI64 218
+  '\x00fb'# -> unI64 219
+  '\x00fc'# -> unI64 220
+  '\x00fd'# -> unI64 221
+  '\x00fe'# -> unI64 222
+  '\x00ff'# -> unI64 376
+  '\x0101'# -> unI64 256
+  '\x0103'# -> unI64 258
+  '\x0105'# -> unI64 260
+  '\x0107'# -> unI64 262
+  '\x0109'# -> unI64 264
+  '\x010b'# -> unI64 266
+  '\x010d'# -> unI64 268
+  '\x010f'# -> unI64 270
+  '\x0111'# -> unI64 272
+  '\x0113'# -> unI64 274
+  '\x0115'# -> unI64 276
+  '\x0117'# -> unI64 278
+  '\x0119'# -> unI64 280
+  '\x011b'# -> unI64 282
+  '\x011d'# -> unI64 284
+  '\x011f'# -> unI64 286
+  '\x0121'# -> unI64 288
+  '\x0123'# -> unI64 290
+  '\x0125'# -> unI64 292
+  '\x0127'# -> unI64 294
+  '\x0129'# -> unI64 296
+  '\x012b'# -> unI64 298
+  '\x012d'# -> unI64 300
+  '\x012f'# -> unI64 302
+  '\x0131'# -> unI64 73
+  '\x0133'# -> unI64 306
+  '\x0135'# -> unI64 308
+  '\x0137'# -> unI64 310
+  '\x013a'# -> unI64 313
+  '\x013c'# -> unI64 315
+  '\x013e'# -> unI64 317
+  '\x0140'# -> unI64 319
+  '\x0142'# -> unI64 321
+  '\x0144'# -> unI64 323
+  '\x0146'# -> unI64 325
+  '\x0148'# -> unI64 327
+  '\x014b'# -> unI64 330
+  '\x014d'# -> unI64 332
+  '\x014f'# -> unI64 334
+  '\x0151'# -> unI64 336
+  '\x0153'# -> unI64 338
+  '\x0155'# -> unI64 340
+  '\x0157'# -> unI64 342
+  '\x0159'# -> unI64 344
+  '\x015b'# -> unI64 346
+  '\x015d'# -> unI64 348
+  '\x015f'# -> unI64 350
+  '\x0161'# -> unI64 352
+  '\x0163'# -> unI64 354
+  '\x0165'# -> unI64 356
+  '\x0167'# -> unI64 358
+  '\x0169'# -> unI64 360
+  '\x016b'# -> unI64 362
+  '\x016d'# -> unI64 364
+  '\x016f'# -> unI64 366
+  '\x0171'# -> unI64 368
+  '\x0173'# -> unI64 370
+  '\x0175'# -> unI64 372
+  '\x0177'# -> unI64 374
+  '\x017a'# -> unI64 377
+  '\x017c'# -> unI64 379
+  '\x017e'# -> unI64 381
+  '\x017f'# -> unI64 83
+  '\x0180'# -> unI64 579
+  '\x0183'# -> unI64 386
+  '\x0185'# -> unI64 388
+  '\x0188'# -> unI64 391
+  '\x018c'# -> unI64 395
+  '\x0192'# -> unI64 401
+  '\x0195'# -> unI64 502
+  '\x0199'# -> unI64 408
+  '\x019a'# -> unI64 573
+  '\x019b'# -> unI64 42972
+  '\x019e'# -> unI64 544
+  '\x01a1'# -> unI64 416
+  '\x01a3'# -> unI64 418
+  '\x01a5'# -> unI64 420
+  '\x01a8'# -> unI64 423
+  '\x01ad'# -> unI64 428
+  '\x01b0'# -> unI64 431
+  '\x01b4'# -> unI64 435
+  '\x01b6'# -> unI64 437
+  '\x01b9'# -> unI64 440
+  '\x01bd'# -> unI64 444
+  '\x01bf'# -> unI64 503
+  '\x01c5'# -> unI64 452
+  '\x01c6'# -> unI64 452
+  '\x01c8'# -> unI64 455
+  '\x01c9'# -> unI64 455
+  '\x01cb'# -> unI64 458
+  '\x01cc'# -> unI64 458
+  '\x01ce'# -> unI64 461
+  '\x01d0'# -> unI64 463
+  '\x01d2'# -> unI64 465
+  '\x01d4'# -> unI64 467
+  '\x01d6'# -> unI64 469
+  '\x01d8'# -> unI64 471
+  '\x01da'# -> unI64 473
+  '\x01dc'# -> unI64 475
+  '\x01dd'# -> unI64 398
+  '\x01df'# -> unI64 478
+  '\x01e1'# -> unI64 480
+  '\x01e3'# -> unI64 482
+  '\x01e5'# -> unI64 484
+  '\x01e7'# -> unI64 486
+  '\x01e9'# -> unI64 488
+  '\x01eb'# -> unI64 490
+  '\x01ed'# -> unI64 492
+  '\x01ef'# -> unI64 494
+  '\x01f2'# -> unI64 497
+  '\x01f3'# -> unI64 497
+  '\x01f5'# -> unI64 500
+  '\x01f9'# -> unI64 504
+  '\x01fb'# -> unI64 506
+  '\x01fd'# -> unI64 508
+  '\x01ff'# -> unI64 510
+  '\x0201'# -> unI64 512
+  '\x0203'# -> unI64 514
+  '\x0205'# -> unI64 516
+  '\x0207'# -> unI64 518
+  '\x0209'# -> unI64 520
+  '\x020b'# -> unI64 522
+  '\x020d'# -> unI64 524
+  '\x020f'# -> unI64 526
+  '\x0211'# -> unI64 528
+  '\x0213'# -> unI64 530
+  '\x0215'# -> unI64 532
+  '\x0217'# -> unI64 534
+  '\x0219'# -> unI64 536
+  '\x021b'# -> unI64 538
+  '\x021d'# -> unI64 540
+  '\x021f'# -> unI64 542
+  '\x0223'# -> unI64 546
+  '\x0225'# -> unI64 548
+  '\x0227'# -> unI64 550
+  '\x0229'# -> unI64 552
+  '\x022b'# -> unI64 554
+  '\x022d'# -> unI64 556
+  '\x022f'# -> unI64 558
+  '\x0231'# -> unI64 560
+  '\x0233'# -> unI64 562
+  '\x023c'# -> unI64 571
+  '\x023f'# -> unI64 11390
+  '\x0240'# -> unI64 11391
+  '\x0242'# -> unI64 577
+  '\x0247'# -> unI64 582
+  '\x0249'# -> unI64 584
+  '\x024b'# -> unI64 586
+  '\x024d'# -> unI64 588
+  '\x024f'# -> unI64 590
+  '\x0250'# -> unI64 11375
+  '\x0251'# -> unI64 11373
+  '\x0252'# -> unI64 11376
+  '\x0253'# -> unI64 385
+  '\x0254'# -> unI64 390
+  '\x0256'# -> unI64 393
+  '\x0257'# -> unI64 394
+  '\x0259'# -> unI64 399
+  '\x025b'# -> unI64 400
+  '\x025c'# -> unI64 42923
+  '\x0260'# -> unI64 403
+  '\x0261'# -> unI64 42924
+  '\x0263'# -> unI64 404
+  '\x0264'# -> unI64 42955
+  '\x0265'# -> unI64 42893
+  '\x0266'# -> unI64 42922
+  '\x0268'# -> unI64 407
+  '\x0269'# -> unI64 406
+  '\x026a'# -> unI64 42926
+  '\x026b'# -> unI64 11362
+  '\x026c'# -> unI64 42925
+  '\x026f'# -> unI64 412
+  '\x0271'# -> unI64 11374
+  '\x0272'# -> unI64 413
+  '\x0275'# -> unI64 415
+  '\x027d'# -> unI64 11364
+  '\x0280'# -> unI64 422
+  '\x0282'# -> unI64 42949
+  '\x0283'# -> unI64 425
+  '\x0287'# -> unI64 42929
+  '\x0288'# -> unI64 430
+  '\x0289'# -> unI64 580
+  '\x028a'# -> unI64 433
+  '\x028b'# -> unI64 434
+  '\x028c'# -> unI64 581
+  '\x0292'# -> unI64 439
+  '\x029d'# -> unI64 42930
+  '\x029e'# -> unI64 42928
+  '\x0345'# -> unI64 921
+  '\x0371'# -> unI64 880
+  '\x0373'# -> unI64 882
+  '\x0377'# -> unI64 886
+  '\x037b'# -> unI64 1021
+  '\x037c'# -> unI64 1022
+  '\x037d'# -> unI64 1023
+  '\x03ac'# -> unI64 902
+  '\x03ad'# -> unI64 904
+  '\x03ae'# -> unI64 905
+  '\x03af'# -> unI64 906
+  '\x03b1'# -> unI64 913
+  '\x03b2'# -> unI64 914
+  '\x03b3'# -> unI64 915
+  '\x03b4'# -> unI64 916
+  '\x03b5'# -> unI64 917
+  '\x03b6'# -> unI64 918
+  '\x03b7'# -> unI64 919
+  '\x03b8'# -> unI64 920
+  '\x03b9'# -> unI64 921
+  '\x03ba'# -> unI64 922
+  '\x03bb'# -> unI64 923
+  '\x03bc'# -> unI64 924
+  '\x03bd'# -> unI64 925
+  '\x03be'# -> unI64 926
+  '\x03bf'# -> unI64 927
+  '\x03c0'# -> unI64 928
+  '\x03c1'# -> unI64 929
+  '\x03c2'# -> unI64 931
+  '\x03c3'# -> unI64 931
+  '\x03c4'# -> unI64 932
+  '\x03c5'# -> unI64 933
+  '\x03c6'# -> unI64 934
+  '\x03c7'# -> unI64 935
+  '\x03c8'# -> unI64 936
+  '\x03c9'# -> unI64 937
+  '\x03ca'# -> unI64 938
+  '\x03cb'# -> unI64 939
+  '\x03cc'# -> unI64 908
+  '\x03cd'# -> unI64 910
+  '\x03ce'# -> unI64 911
+  '\x03d0'# -> unI64 914
+  '\x03d1'# -> unI64 920
+  '\x03d5'# -> unI64 934
+  '\x03d6'# -> unI64 928
+  '\x03d7'# -> unI64 975
+  '\x03d9'# -> unI64 984
+  '\x03db'# -> unI64 986
+  '\x03dd'# -> unI64 988
+  '\x03df'# -> unI64 990
+  '\x03e1'# -> unI64 992
+  '\x03e3'# -> unI64 994
+  '\x03e5'# -> unI64 996
+  '\x03e7'# -> unI64 998
+  '\x03e9'# -> unI64 1000
+  '\x03eb'# -> unI64 1002
+  '\x03ed'# -> unI64 1004
+  '\x03ef'# -> unI64 1006
+  '\x03f0'# -> unI64 922
+  '\x03f1'# -> unI64 929
+  '\x03f2'# -> unI64 1017
+  '\x03f3'# -> unI64 895
+  '\x03f5'# -> unI64 917
+  '\x03f8'# -> unI64 1015
+  '\x03fb'# -> unI64 1018
+  '\x0430'# -> unI64 1040
+  '\x0431'# -> unI64 1041
+  '\x0432'# -> unI64 1042
+  '\x0433'# -> unI64 1043
+  '\x0434'# -> unI64 1044
+  '\x0435'# -> unI64 1045
+  '\x0436'# -> unI64 1046
+  '\x0437'# -> unI64 1047
+  '\x0438'# -> unI64 1048
+  '\x0439'# -> unI64 1049
+  '\x043a'# -> unI64 1050
+  '\x043b'# -> unI64 1051
+  '\x043c'# -> unI64 1052
+  '\x043d'# -> unI64 1053
+  '\x043e'# -> unI64 1054
+  '\x043f'# -> unI64 1055
+  '\x0440'# -> unI64 1056
+  '\x0441'# -> unI64 1057
+  '\x0442'# -> unI64 1058
+  '\x0443'# -> unI64 1059
+  '\x0444'# -> unI64 1060
+  '\x0445'# -> unI64 1061
+  '\x0446'# -> unI64 1062
+  '\x0447'# -> unI64 1063
+  '\x0448'# -> unI64 1064
+  '\x0449'# -> unI64 1065
+  '\x044a'# -> unI64 1066
+  '\x044b'# -> unI64 1067
+  '\x044c'# -> unI64 1068
+  '\x044d'# -> unI64 1069
+  '\x044e'# -> unI64 1070
+  '\x044f'# -> unI64 1071
+  '\x0450'# -> unI64 1024
+  '\x0451'# -> unI64 1025
+  '\x0452'# -> unI64 1026
+  '\x0453'# -> unI64 1027
+  '\x0454'# -> unI64 1028
+  '\x0455'# -> unI64 1029
+  '\x0456'# -> unI64 1030
+  '\x0457'# -> unI64 1031
+  '\x0458'# -> unI64 1032
+  '\x0459'# -> unI64 1033
+  '\x045a'# -> unI64 1034
+  '\x045b'# -> unI64 1035
+  '\x045c'# -> unI64 1036
+  '\x045d'# -> unI64 1037
+  '\x045e'# -> unI64 1038
+  '\x045f'# -> unI64 1039
+  '\x0461'# -> unI64 1120
+  '\x0463'# -> unI64 1122
+  '\x0465'# -> unI64 1124
+  '\x0467'# -> unI64 1126
+  '\x0469'# -> unI64 1128
+  '\x046b'# -> unI64 1130
+  '\x046d'# -> unI64 1132
+  '\x046f'# -> unI64 1134
+  '\x0471'# -> unI64 1136
+  '\x0473'# -> unI64 1138
+  '\x0475'# -> unI64 1140
+  '\x0477'# -> unI64 1142
+  '\x0479'# -> unI64 1144
+  '\x047b'# -> unI64 1146
+  '\x047d'# -> unI64 1148
+  '\x047f'# -> unI64 1150
+  '\x0481'# -> unI64 1152
+  '\x048b'# -> unI64 1162
+  '\x048d'# -> unI64 1164
+  '\x048f'# -> unI64 1166
+  '\x0491'# -> unI64 1168
+  '\x0493'# -> unI64 1170
+  '\x0495'# -> unI64 1172
+  '\x0497'# -> unI64 1174
+  '\x0499'# -> unI64 1176
+  '\x049b'# -> unI64 1178
+  '\x049d'# -> unI64 1180
+  '\x049f'# -> unI64 1182
+  '\x04a1'# -> unI64 1184
+  '\x04a3'# -> unI64 1186
+  '\x04a5'# -> unI64 1188
+  '\x04a7'# -> unI64 1190
+  '\x04a9'# -> unI64 1192
+  '\x04ab'# -> unI64 1194
+  '\x04ad'# -> unI64 1196
+  '\x04af'# -> unI64 1198
+  '\x04b1'# -> unI64 1200
+  '\x04b3'# -> unI64 1202
+  '\x04b5'# -> unI64 1204
+  '\x04b7'# -> unI64 1206
+  '\x04b9'# -> unI64 1208
+  '\x04bb'# -> unI64 1210
+  '\x04bd'# -> unI64 1212
+  '\x04bf'# -> unI64 1214
+  '\x04c2'# -> unI64 1217
+  '\x04c4'# -> unI64 1219
+  '\x04c6'# -> unI64 1221
+  '\x04c8'# -> unI64 1223
+  '\x04ca'# -> unI64 1225
+  '\x04cc'# -> unI64 1227
+  '\x04ce'# -> unI64 1229
+  '\x04cf'# -> unI64 1216
+  '\x04d1'# -> unI64 1232
+  '\x04d3'# -> unI64 1234
+  '\x04d5'# -> unI64 1236
+  '\x04d7'# -> unI64 1238
+  '\x04d9'# -> unI64 1240
+  '\x04db'# -> unI64 1242
+  '\x04dd'# -> unI64 1244
+  '\x04df'# -> unI64 1246
+  '\x04e1'# -> unI64 1248
+  '\x04e3'# -> unI64 1250
+  '\x04e5'# -> unI64 1252
+  '\x04e7'# -> unI64 1254
+  '\x04e9'# -> unI64 1256
+  '\x04eb'# -> unI64 1258
+  '\x04ed'# -> unI64 1260
+  '\x04ef'# -> unI64 1262
+  '\x04f1'# -> unI64 1264
+  '\x04f3'# -> unI64 1266
+  '\x04f5'# -> unI64 1268
+  '\x04f7'# -> unI64 1270
+  '\x04f9'# -> unI64 1272
+  '\x04fb'# -> unI64 1274
+  '\x04fd'# -> unI64 1276
+  '\x04ff'# -> unI64 1278
+  '\x0501'# -> unI64 1280
+  '\x0503'# -> unI64 1282
+  '\x0505'# -> unI64 1284
+  '\x0507'# -> unI64 1286
+  '\x0509'# -> unI64 1288
+  '\x050b'# -> unI64 1290
+  '\x050d'# -> unI64 1292
+  '\x050f'# -> unI64 1294
+  '\x0511'# -> unI64 1296
+  '\x0513'# -> unI64 1298
+  '\x0515'# -> unI64 1300
+  '\x0517'# -> unI64 1302
+  '\x0519'# -> unI64 1304
+  '\x051b'# -> unI64 1306
+  '\x051d'# -> unI64 1308
+  '\x051f'# -> unI64 1310
+  '\x0521'# -> unI64 1312
+  '\x0523'# -> unI64 1314
+  '\x0525'# -> unI64 1316
+  '\x0527'# -> unI64 1318
+  '\x0529'# -> unI64 1320
+  '\x052b'# -> unI64 1322
+  '\x052d'# -> unI64 1324
+  '\x052f'# -> unI64 1326
+  '\x0561'# -> unI64 1329
+  '\x0562'# -> unI64 1330
+  '\x0563'# -> unI64 1331
+  '\x0564'# -> unI64 1332
+  '\x0565'# -> unI64 1333
+  '\x0566'# -> unI64 1334
+  '\x0567'# -> unI64 1335
+  '\x0568'# -> unI64 1336
+  '\x0569'# -> unI64 1337
+  '\x056a'# -> unI64 1338
+  '\x056b'# -> unI64 1339
+  '\x056c'# -> unI64 1340
+  '\x056d'# -> unI64 1341
+  '\x056e'# -> unI64 1342
+  '\x056f'# -> unI64 1343
+  '\x0570'# -> unI64 1344
+  '\x0571'# -> unI64 1345
+  '\x0572'# -> unI64 1346
+  '\x0573'# -> unI64 1347
+  '\x0574'# -> unI64 1348
+  '\x0575'# -> unI64 1349
+  '\x0576'# -> unI64 1350
+  '\x0577'# -> unI64 1351
+  '\x0578'# -> unI64 1352
+  '\x0579'# -> unI64 1353
+  '\x057a'# -> unI64 1354
+  '\x057b'# -> unI64 1355
+  '\x057c'# -> unI64 1356
+  '\x057d'# -> unI64 1357
+  '\x057e'# -> unI64 1358
+  '\x057f'# -> unI64 1359
+  '\x0580'# -> unI64 1360
+  '\x0581'# -> unI64 1361
+  '\x0582'# -> unI64 1362
+  '\x0583'# -> unI64 1363
+  '\x0584'# -> unI64 1364
+  '\x0585'# -> unI64 1365
+  '\x0586'# -> unI64 1366
+  '\x10d0'# -> unI64 7312
+  '\x10d1'# -> unI64 7313
+  '\x10d2'# -> unI64 7314
+  '\x10d3'# -> unI64 7315
+  '\x10d4'# -> unI64 7316
+  '\x10d5'# -> unI64 7317
+  '\x10d6'# -> unI64 7318
+  '\x10d7'# -> unI64 7319
+  '\x10d8'# -> unI64 7320
+  '\x10d9'# -> unI64 7321
+  '\x10da'# -> unI64 7322
+  '\x10db'# -> unI64 7323
+  '\x10dc'# -> unI64 7324
+  '\x10dd'# -> unI64 7325
+  '\x10de'# -> unI64 7326
+  '\x10df'# -> unI64 7327
+  '\x10e0'# -> unI64 7328
+  '\x10e1'# -> unI64 7329
+  '\x10e2'# -> unI64 7330
+  '\x10e3'# -> unI64 7331
+  '\x10e4'# -> unI64 7332
+  '\x10e5'# -> unI64 7333
+  '\x10e6'# -> unI64 7334
+  '\x10e7'# -> unI64 7335
+  '\x10e8'# -> unI64 7336
+  '\x10e9'# -> unI64 7337
+  '\x10ea'# -> unI64 7338
+  '\x10eb'# -> unI64 7339
+  '\x10ec'# -> unI64 7340
+  '\x10ed'# -> unI64 7341
+  '\x10ee'# -> unI64 7342
+  '\x10ef'# -> unI64 7343
+  '\x10f0'# -> unI64 7344
+  '\x10f1'# -> unI64 7345
+  '\x10f2'# -> unI64 7346
+  '\x10f3'# -> unI64 7347
+  '\x10f4'# -> unI64 7348
+  '\x10f5'# -> unI64 7349
+  '\x10f6'# -> unI64 7350
+  '\x10f7'# -> unI64 7351
+  '\x10f8'# -> unI64 7352
+  '\x10f9'# -> unI64 7353
+  '\x10fa'# -> unI64 7354
+  '\x10fd'# -> unI64 7357
+  '\x10fe'# -> unI64 7358
+  '\x10ff'# -> unI64 7359
+  '\x13f8'# -> unI64 5104
+  '\x13f9'# -> unI64 5105
+  '\x13fa'# -> unI64 5106
+  '\x13fb'# -> unI64 5107
+  '\x13fc'# -> unI64 5108
+  '\x13fd'# -> unI64 5109
+  '\x1c80'# -> unI64 1042
+  '\x1c81'# -> unI64 1044
+  '\x1c82'# -> unI64 1054
+  '\x1c83'# -> unI64 1057
+  '\x1c84'# -> unI64 1058
+  '\x1c85'# -> unI64 1058
+  '\x1c86'# -> unI64 1066
+  '\x1c87'# -> unI64 1122
+  '\x1c88'# -> unI64 42570
+  '\x1c8a'# -> unI64 7305
+  '\x1d79'# -> unI64 42877
+  '\x1d7d'# -> unI64 11363
+  '\x1d8e'# -> unI64 42950
+  '\x1e01'# -> unI64 7680
+  '\x1e03'# -> unI64 7682
+  '\x1e05'# -> unI64 7684
+  '\x1e07'# -> unI64 7686
+  '\x1e09'# -> unI64 7688
+  '\x1e0b'# -> unI64 7690
+  '\x1e0d'# -> unI64 7692
+  '\x1e0f'# -> unI64 7694
+  '\x1e11'# -> unI64 7696
+  '\x1e13'# -> unI64 7698
+  '\x1e15'# -> unI64 7700
+  '\x1e17'# -> unI64 7702
+  '\x1e19'# -> unI64 7704
+  '\x1e1b'# -> unI64 7706
+  '\x1e1d'# -> unI64 7708
+  '\x1e1f'# -> unI64 7710
+  '\x1e21'# -> unI64 7712
+  '\x1e23'# -> unI64 7714
+  '\x1e25'# -> unI64 7716
+  '\x1e27'# -> unI64 7718
+  '\x1e29'# -> unI64 7720
+  '\x1e2b'# -> unI64 7722
+  '\x1e2d'# -> unI64 7724
+  '\x1e2f'# -> unI64 7726
+  '\x1e31'# -> unI64 7728
+  '\x1e33'# -> unI64 7730
+  '\x1e35'# -> unI64 7732
+  '\x1e37'# -> unI64 7734
+  '\x1e39'# -> unI64 7736
+  '\x1e3b'# -> unI64 7738
+  '\x1e3d'# -> unI64 7740
+  '\x1e3f'# -> unI64 7742
+  '\x1e41'# -> unI64 7744
+  '\x1e43'# -> unI64 7746
+  '\x1e45'# -> unI64 7748
+  '\x1e47'# -> unI64 7750
+  '\x1e49'# -> unI64 7752
+  '\x1e4b'# -> unI64 7754
+  '\x1e4d'# -> unI64 7756
+  '\x1e4f'# -> unI64 7758
+  '\x1e51'# -> unI64 7760
+  '\x1e53'# -> unI64 7762
+  '\x1e55'# -> unI64 7764
+  '\x1e57'# -> unI64 7766
+  '\x1e59'# -> unI64 7768
+  '\x1e5b'# -> unI64 7770
+  '\x1e5d'# -> unI64 7772
+  '\x1e5f'# -> unI64 7774
+  '\x1e61'# -> unI64 7776
+  '\x1e63'# -> unI64 7778
+  '\x1e65'# -> unI64 7780
+  '\x1e67'# -> unI64 7782
+  '\x1e69'# -> unI64 7784
+  '\x1e6b'# -> unI64 7786
+  '\x1e6d'# -> unI64 7788
+  '\x1e6f'# -> unI64 7790
+  '\x1e71'# -> unI64 7792
+  '\x1e73'# -> unI64 7794
+  '\x1e75'# -> unI64 7796
+  '\x1e77'# -> unI64 7798
+  '\x1e79'# -> unI64 7800
+  '\x1e7b'# -> unI64 7802
+  '\x1e7d'# -> unI64 7804
+  '\x1e7f'# -> unI64 7806
+  '\x1e81'# -> unI64 7808
+  '\x1e83'# -> unI64 7810
+  '\x1e85'# -> unI64 7812
+  '\x1e87'# -> unI64 7814
+  '\x1e89'# -> unI64 7816
+  '\x1e8b'# -> unI64 7818
+  '\x1e8d'# -> unI64 7820
+  '\x1e8f'# -> unI64 7822
+  '\x1e91'# -> unI64 7824
+  '\x1e93'# -> unI64 7826
+  '\x1e95'# -> unI64 7828
+  '\x1e9b'# -> unI64 7776
+  '\x1ea1'# -> unI64 7840
+  '\x1ea3'# -> unI64 7842
+  '\x1ea5'# -> unI64 7844
+  '\x1ea7'# -> unI64 7846
+  '\x1ea9'# -> unI64 7848
+  '\x1eab'# -> unI64 7850
+  '\x1ead'# -> unI64 7852
+  '\x1eaf'# -> unI64 7854
+  '\x1eb1'# -> unI64 7856
+  '\x1eb3'# -> unI64 7858
+  '\x1eb5'# -> unI64 7860
+  '\x1eb7'# -> unI64 7862
+  '\x1eb9'# -> unI64 7864
+  '\x1ebb'# -> unI64 7866
+  '\x1ebd'# -> unI64 7868
+  '\x1ebf'# -> unI64 7870
+  '\x1ec1'# -> unI64 7872
+  '\x1ec3'# -> unI64 7874
+  '\x1ec5'# -> unI64 7876
+  '\x1ec7'# -> unI64 7878
+  '\x1ec9'# -> unI64 7880
+  '\x1ecb'# -> unI64 7882
+  '\x1ecd'# -> unI64 7884
+  '\x1ecf'# -> unI64 7886
+  '\x1ed1'# -> unI64 7888
+  '\x1ed3'# -> unI64 7890
+  '\x1ed5'# -> unI64 7892
+  '\x1ed7'# -> unI64 7894
+  '\x1ed9'# -> unI64 7896
+  '\x1edb'# -> unI64 7898
+  '\x1edd'# -> unI64 7900
+  '\x1edf'# -> unI64 7902
+  '\x1ee1'# -> unI64 7904
+  '\x1ee3'# -> unI64 7906
+  '\x1ee5'# -> unI64 7908
+  '\x1ee7'# -> unI64 7910
+  '\x1ee9'# -> unI64 7912
+  '\x1eeb'# -> unI64 7914
+  '\x1eed'# -> unI64 7916
+  '\x1eef'# -> unI64 7918
+  '\x1ef1'# -> unI64 7920
+  '\x1ef3'# -> unI64 7922
+  '\x1ef5'# -> unI64 7924
+  '\x1ef7'# -> unI64 7926
+  '\x1ef9'# -> unI64 7928
+  '\x1efb'# -> unI64 7930
+  '\x1efd'# -> unI64 7932
+  '\x1eff'# -> unI64 7934
+  '\x1f00'# -> unI64 7944
+  '\x1f01'# -> unI64 7945
+  '\x1f02'# -> unI64 7946
+  '\x1f03'# -> unI64 7947
+  '\x1f04'# -> unI64 7948
+  '\x1f05'# -> unI64 7949
+  '\x1f06'# -> unI64 7950
+  '\x1f07'# -> unI64 7951
+  '\x1f10'# -> unI64 7960
+  '\x1f11'# -> unI64 7961
+  '\x1f12'# -> unI64 7962
+  '\x1f13'# -> unI64 7963
+  '\x1f14'# -> unI64 7964
+  '\x1f15'# -> unI64 7965
+  '\x1f20'# -> unI64 7976
+  '\x1f21'# -> unI64 7977
+  '\x1f22'# -> unI64 7978
+  '\x1f23'# -> unI64 7979
+  '\x1f24'# -> unI64 7980
+  '\x1f25'# -> unI64 7981
+  '\x1f26'# -> unI64 7982
+  '\x1f27'# -> unI64 7983
+  '\x1f30'# -> unI64 7992
+  '\x1f31'# -> unI64 7993
+  '\x1f32'# -> unI64 7994
+  '\x1f33'# -> unI64 7995
+  '\x1f34'# -> unI64 7996
+  '\x1f35'# -> unI64 7997
+  '\x1f36'# -> unI64 7998
+  '\x1f37'# -> unI64 7999
+  '\x1f40'# -> unI64 8008
+  '\x1f41'# -> unI64 8009
+  '\x1f42'# -> unI64 8010
+  '\x1f43'# -> unI64 8011
+  '\x1f44'# -> unI64 8012
+  '\x1f45'# -> unI64 8013
+  '\x1f51'# -> unI64 8025
+  '\x1f53'# -> unI64 8027
+  '\x1f55'# -> unI64 8029
+  '\x1f57'# -> unI64 8031
+  '\x1f60'# -> unI64 8040
+  '\x1f61'# -> unI64 8041
+  '\x1f62'# -> unI64 8042
+  '\x1f63'# -> unI64 8043
+  '\x1f64'# -> unI64 8044
+  '\x1f65'# -> unI64 8045
+  '\x1f66'# -> unI64 8046
+  '\x1f67'# -> unI64 8047
+  '\x1f70'# -> unI64 8122
+  '\x1f71'# -> unI64 8123
+  '\x1f72'# -> unI64 8136
+  '\x1f73'# -> unI64 8137
+  '\x1f74'# -> unI64 8138
+  '\x1f75'# -> unI64 8139
+  '\x1f76'# -> unI64 8154
+  '\x1f77'# -> unI64 8155
+  '\x1f78'# -> unI64 8184
+  '\x1f79'# -> unI64 8185
+  '\x1f7a'# -> unI64 8170
+  '\x1f7b'# -> unI64 8171
+  '\x1f7c'# -> unI64 8186
+  '\x1f7d'# -> unI64 8187
+  '\x1fb0'# -> unI64 8120
+  '\x1fb1'# -> unI64 8121
+  '\x1fbe'# -> unI64 921
+  '\x1fd0'# -> unI64 8152
+  '\x1fd1'# -> unI64 8153
+  '\x1fe0'# -> unI64 8168
+  '\x1fe1'# -> unI64 8169
+  '\x1fe5'# -> unI64 8172
+  '\x214e'# -> unI64 8498
+  '\x2170'# -> unI64 8544
+  '\x2171'# -> unI64 8545
+  '\x2172'# -> unI64 8546
+  '\x2173'# -> unI64 8547
+  '\x2174'# -> unI64 8548
+  '\x2175'# -> unI64 8549
+  '\x2176'# -> unI64 8550
+  '\x2177'# -> unI64 8551
+  '\x2178'# -> unI64 8552
+  '\x2179'# -> unI64 8553
+  '\x217a'# -> unI64 8554
+  '\x217b'# -> unI64 8555
+  '\x217c'# -> unI64 8556
+  '\x217d'# -> unI64 8557
+  '\x217e'# -> unI64 8558
+  '\x217f'# -> unI64 8559
+  '\x2184'# -> unI64 8579
+  '\x24d0'# -> unI64 9398
+  '\x24d1'# -> unI64 9399
+  '\x24d2'# -> unI64 9400
+  '\x24d3'# -> unI64 9401
+  '\x24d4'# -> unI64 9402
+  '\x24d5'# -> unI64 9403
+  '\x24d6'# -> unI64 9404
+  '\x24d7'# -> unI64 9405
+  '\x24d8'# -> unI64 9406
+  '\x24d9'# -> unI64 9407
+  '\x24da'# -> unI64 9408
+  '\x24db'# -> unI64 9409
+  '\x24dc'# -> unI64 9410
+  '\x24dd'# -> unI64 9411
+  '\x24de'# -> unI64 9412
+  '\x24df'# -> unI64 9413
+  '\x24e0'# -> unI64 9414
+  '\x24e1'# -> unI64 9415
+  '\x24e2'# -> unI64 9416
+  '\x24e3'# -> unI64 9417
+  '\x24e4'# -> unI64 9418
+  '\x24e5'# -> unI64 9419
+  '\x24e6'# -> unI64 9420
+  '\x24e7'# -> unI64 9421
+  '\x24e8'# -> unI64 9422
+  '\x24e9'# -> unI64 9423
+  '\x2c30'# -> unI64 11264
+  '\x2c31'# -> unI64 11265
+  '\x2c32'# -> unI64 11266
+  '\x2c33'# -> unI64 11267
+  '\x2c34'# -> unI64 11268
+  '\x2c35'# -> unI64 11269
+  '\x2c36'# -> unI64 11270
+  '\x2c37'# -> unI64 11271
+  '\x2c38'# -> unI64 11272
+  '\x2c39'# -> unI64 11273
+  '\x2c3a'# -> unI64 11274
+  '\x2c3b'# -> unI64 11275
+  '\x2c3c'# -> unI64 11276
+  '\x2c3d'# -> unI64 11277
+  '\x2c3e'# -> unI64 11278
+  '\x2c3f'# -> unI64 11279
+  '\x2c40'# -> unI64 11280
+  '\x2c41'# -> unI64 11281
+  '\x2c42'# -> unI64 11282
+  '\x2c43'# -> unI64 11283
+  '\x2c44'# -> unI64 11284
+  '\x2c45'# -> unI64 11285
+  '\x2c46'# -> unI64 11286
+  '\x2c47'# -> unI64 11287
+  '\x2c48'# -> unI64 11288
+  '\x2c49'# -> unI64 11289
+  '\x2c4a'# -> unI64 11290
+  '\x2c4b'# -> unI64 11291
+  '\x2c4c'# -> unI64 11292
+  '\x2c4d'# -> unI64 11293
+  '\x2c4e'# -> unI64 11294
+  '\x2c4f'# -> unI64 11295
+  '\x2c50'# -> unI64 11296
+  '\x2c51'# -> unI64 11297
+  '\x2c52'# -> unI64 11298
+  '\x2c53'# -> unI64 11299
+  '\x2c54'# -> unI64 11300
+  '\x2c55'# -> unI64 11301
+  '\x2c56'# -> unI64 11302
+  '\x2c57'# -> unI64 11303
+  '\x2c58'# -> unI64 11304
+  '\x2c59'# -> unI64 11305
+  '\x2c5a'# -> unI64 11306
+  '\x2c5b'# -> unI64 11307
+  '\x2c5c'# -> unI64 11308
+  '\x2c5d'# -> unI64 11309
+  '\x2c5e'# -> unI64 11310
+  '\x2c5f'# -> unI64 11311
+  '\x2c61'# -> unI64 11360
+  '\x2c65'# -> unI64 570
+  '\x2c66'# -> unI64 574
+  '\x2c68'# -> unI64 11367
+  '\x2c6a'# -> unI64 11369
+  '\x2c6c'# -> unI64 11371
+  '\x2c73'# -> unI64 11378
+  '\x2c76'# -> unI64 11381
+  '\x2c81'# -> unI64 11392
+  '\x2c83'# -> unI64 11394
+  '\x2c85'# -> unI64 11396
+  '\x2c87'# -> unI64 11398
+  '\x2c89'# -> unI64 11400
+  '\x2c8b'# -> unI64 11402
+  '\x2c8d'# -> unI64 11404
+  '\x2c8f'# -> unI64 11406
+  '\x2c91'# -> unI64 11408
+  '\x2c93'# -> unI64 11410
+  '\x2c95'# -> unI64 11412
+  '\x2c97'# -> unI64 11414
+  '\x2c99'# -> unI64 11416
+  '\x2c9b'# -> unI64 11418
+  '\x2c9d'# -> unI64 11420
+  '\x2c9f'# -> unI64 11422
+  '\x2ca1'# -> unI64 11424
+  '\x2ca3'# -> unI64 11426
+  '\x2ca5'# -> unI64 11428
+  '\x2ca7'# -> unI64 11430
+  '\x2ca9'# -> unI64 11432
+  '\x2cab'# -> unI64 11434
+  '\x2cad'# -> unI64 11436
+  '\x2caf'# -> unI64 11438
+  '\x2cb1'# -> unI64 11440
+  '\x2cb3'# -> unI64 11442
+  '\x2cb5'# -> unI64 11444
+  '\x2cb7'# -> unI64 11446
+  '\x2cb9'# -> unI64 11448
+  '\x2cbb'# -> unI64 11450
+  '\x2cbd'# -> unI64 11452
+  '\x2cbf'# -> unI64 11454
+  '\x2cc1'# -> unI64 11456
+  '\x2cc3'# -> unI64 11458
+  '\x2cc5'# -> unI64 11460
+  '\x2cc7'# -> unI64 11462
+  '\x2cc9'# -> unI64 11464
+  '\x2ccb'# -> unI64 11466
+  '\x2ccd'# -> unI64 11468
+  '\x2ccf'# -> unI64 11470
+  '\x2cd1'# -> unI64 11472
+  '\x2cd3'# -> unI64 11474
+  '\x2cd5'# -> unI64 11476
+  '\x2cd7'# -> unI64 11478
+  '\x2cd9'# -> unI64 11480
+  '\x2cdb'# -> unI64 11482
+  '\x2cdd'# -> unI64 11484
+  '\x2cdf'# -> unI64 11486
+  '\x2ce1'# -> unI64 11488
+  '\x2ce3'# -> unI64 11490
+  '\x2cec'# -> unI64 11499
+  '\x2cee'# -> unI64 11501
+  '\x2cf3'# -> unI64 11506
+  '\x2d00'# -> unI64 4256
+  '\x2d01'# -> unI64 4257
+  '\x2d02'# -> unI64 4258
+  '\x2d03'# -> unI64 4259
+  '\x2d04'# -> unI64 4260
+  '\x2d05'# -> unI64 4261
+  '\x2d06'# -> unI64 4262
+  '\x2d07'# -> unI64 4263
+  '\x2d08'# -> unI64 4264
+  '\x2d09'# -> unI64 4265
+  '\x2d0a'# -> unI64 4266
+  '\x2d0b'# -> unI64 4267
+  '\x2d0c'# -> unI64 4268
+  '\x2d0d'# -> unI64 4269
+  '\x2d0e'# -> unI64 4270
+  '\x2d0f'# -> unI64 4271
+  '\x2d10'# -> unI64 4272
+  '\x2d11'# -> unI64 4273
+  '\x2d12'# -> unI64 4274
+  '\x2d13'# -> unI64 4275
+  '\x2d14'# -> unI64 4276
+  '\x2d15'# -> unI64 4277
+  '\x2d16'# -> unI64 4278
+  '\x2d17'# -> unI64 4279
+  '\x2d18'# -> unI64 4280
+  '\x2d19'# -> unI64 4281
+  '\x2d1a'# -> unI64 4282
+  '\x2d1b'# -> unI64 4283
+  '\x2d1c'# -> unI64 4284
+  '\x2d1d'# -> unI64 4285
+  '\x2d1e'# -> unI64 4286
+  '\x2d1f'# -> unI64 4287
+  '\x2d20'# -> unI64 4288
+  '\x2d21'# -> unI64 4289
+  '\x2d22'# -> unI64 4290
+  '\x2d23'# -> unI64 4291
+  '\x2d24'# -> unI64 4292
+  '\x2d25'# -> unI64 4293
+  '\x2d27'# -> unI64 4295
+  '\x2d2d'# -> unI64 4301
+  '\xa641'# -> unI64 42560
+  '\xa643'# -> unI64 42562
+  '\xa645'# -> unI64 42564
+  '\xa647'# -> unI64 42566
+  '\xa649'# -> unI64 42568
+  '\xa64b'# -> unI64 42570
+  '\xa64d'# -> unI64 42572
+  '\xa64f'# -> unI64 42574
+  '\xa651'# -> unI64 42576
+  '\xa653'# -> unI64 42578
+  '\xa655'# -> unI64 42580
+  '\xa657'# -> unI64 42582
+  '\xa659'# -> unI64 42584
+  '\xa65b'# -> unI64 42586
+  '\xa65d'# -> unI64 42588
+  '\xa65f'# -> unI64 42590
+  '\xa661'# -> unI64 42592
+  '\xa663'# -> unI64 42594
+  '\xa665'# -> unI64 42596
+  '\xa667'# -> unI64 42598
+  '\xa669'# -> unI64 42600
+  '\xa66b'# -> unI64 42602
+  '\xa66d'# -> unI64 42604
+  '\xa681'# -> unI64 42624
+  '\xa683'# -> unI64 42626
+  '\xa685'# -> unI64 42628
+  '\xa687'# -> unI64 42630
+  '\xa689'# -> unI64 42632
+  '\xa68b'# -> unI64 42634
+  '\xa68d'# -> unI64 42636
+  '\xa68f'# -> unI64 42638
+  '\xa691'# -> unI64 42640
+  '\xa693'# -> unI64 42642
+  '\xa695'# -> unI64 42644
+  '\xa697'# -> unI64 42646
+  '\xa699'# -> unI64 42648
+  '\xa69b'# -> unI64 42650
+  '\xa723'# -> unI64 42786
+  '\xa725'# -> unI64 42788
+  '\xa727'# -> unI64 42790
+  '\xa729'# -> unI64 42792
+  '\xa72b'# -> unI64 42794
+  '\xa72d'# -> unI64 42796
+  '\xa72f'# -> unI64 42798
+  '\xa733'# -> unI64 42802
+  '\xa735'# -> unI64 42804
+  '\xa737'# -> unI64 42806
+  '\xa739'# -> unI64 42808
+  '\xa73b'# -> unI64 42810
+  '\xa73d'# -> unI64 42812
+  '\xa73f'# -> unI64 42814
+  '\xa741'# -> unI64 42816
+  '\xa743'# -> unI64 42818
+  '\xa745'# -> unI64 42820
+  '\xa747'# -> unI64 42822
+  '\xa749'# -> unI64 42824
+  '\xa74b'# -> unI64 42826
+  '\xa74d'# -> unI64 42828
+  '\xa74f'# -> unI64 42830
+  '\xa751'# -> unI64 42832
+  '\xa753'# -> unI64 42834
+  '\xa755'# -> unI64 42836
+  '\xa757'# -> unI64 42838
+  '\xa759'# -> unI64 42840
+  '\xa75b'# -> unI64 42842
+  '\xa75d'# -> unI64 42844
+  '\xa75f'# -> unI64 42846
+  '\xa761'# -> unI64 42848
+  '\xa763'# -> unI64 42850
+  '\xa765'# -> unI64 42852
+  '\xa767'# -> unI64 42854
+  '\xa769'# -> unI64 42856
+  '\xa76b'# -> unI64 42858
+  '\xa76d'# -> unI64 42860
+  '\xa76f'# -> unI64 42862
+  '\xa77a'# -> unI64 42873
+  '\xa77c'# -> unI64 42875
+  '\xa77f'# -> unI64 42878
+  '\xa781'# -> unI64 42880
+  '\xa783'# -> unI64 42882
+  '\xa785'# -> unI64 42884
+  '\xa787'# -> unI64 42886
+  '\xa78c'# -> unI64 42891
+  '\xa791'# -> unI64 42896
+  '\xa793'# -> unI64 42898
+  '\xa794'# -> unI64 42948
+  '\xa797'# -> unI64 42902
+  '\xa799'# -> unI64 42904
+  '\xa79b'# -> unI64 42906
+  '\xa79d'# -> unI64 42908
+  '\xa79f'# -> unI64 42910
+  '\xa7a1'# -> unI64 42912
+  '\xa7a3'# -> unI64 42914
+  '\xa7a5'# -> unI64 42916
+  '\xa7a7'# -> unI64 42918
+  '\xa7a9'# -> unI64 42920
+  '\xa7b5'# -> unI64 42932
+  '\xa7b7'# -> unI64 42934
+  '\xa7b9'# -> unI64 42936
+  '\xa7bb'# -> unI64 42938
+  '\xa7bd'# -> unI64 42940
+  '\xa7bf'# -> unI64 42942
+  '\xa7c1'# -> unI64 42944
+  '\xa7c3'# -> unI64 42946
+  '\xa7c8'# -> unI64 42951
+  '\xa7ca'# -> unI64 42953
+  '\xa7cd'# -> unI64 42956
+  '\xa7cf'# -> unI64 42958
+  '\xa7d1'# -> unI64 42960
+  '\xa7d3'# -> unI64 42962
+  '\xa7d5'# -> unI64 42964
+  '\xa7d7'# -> unI64 42966
+  '\xa7d9'# -> unI64 42968
+  '\xa7db'# -> unI64 42970
+  '\xa7f6'# -> unI64 42997
+  '\xab53'# -> unI64 42931
+  '\xab70'# -> unI64 5024
+  '\xab71'# -> unI64 5025
+  '\xab72'# -> unI64 5026
+  '\xab73'# -> unI64 5027
+  '\xab74'# -> unI64 5028
+  '\xab75'# -> unI64 5029
+  '\xab76'# -> unI64 5030
+  '\xab77'# -> unI64 5031
+  '\xab78'# -> unI64 5032
+  '\xab79'# -> unI64 5033
+  '\xab7a'# -> unI64 5034
+  '\xab7b'# -> unI64 5035
+  '\xab7c'# -> unI64 5036
+  '\xab7d'# -> unI64 5037
+  '\xab7e'# -> unI64 5038
+  '\xab7f'# -> unI64 5039
+  '\xab80'# -> unI64 5040
+  '\xab81'# -> unI64 5041
+  '\xab82'# -> unI64 5042
+  '\xab83'# -> unI64 5043
+  '\xab84'# -> unI64 5044
+  '\xab85'# -> unI64 5045
+  '\xab86'# -> unI64 5046
+  '\xab87'# -> unI64 5047
+  '\xab88'# -> unI64 5048
+  '\xab89'# -> unI64 5049
+  '\xab8a'# -> unI64 5050
+  '\xab8b'# -> unI64 5051
+  '\xab8c'# -> unI64 5052
+  '\xab8d'# -> unI64 5053
+  '\xab8e'# -> unI64 5054
+  '\xab8f'# -> unI64 5055
+  '\xab90'# -> unI64 5056
+  '\xab91'# -> unI64 5057
+  '\xab92'# -> unI64 5058
+  '\xab93'# -> unI64 5059
+  '\xab94'# -> unI64 5060
+  '\xab95'# -> unI64 5061
+  '\xab96'# -> unI64 5062
+  '\xab97'# -> unI64 5063
+  '\xab98'# -> unI64 5064
+  '\xab99'# -> unI64 5065
+  '\xab9a'# -> unI64 5066
+  '\xab9b'# -> unI64 5067
+  '\xab9c'# -> unI64 5068
+  '\xab9d'# -> unI64 5069
+  '\xab9e'# -> unI64 5070
+  '\xab9f'# -> unI64 5071
+  '\xaba0'# -> unI64 5072
+  '\xaba1'# -> unI64 5073
+  '\xaba2'# -> unI64 5074
+  '\xaba3'# -> unI64 5075
+  '\xaba4'# -> unI64 5076
+  '\xaba5'# -> unI64 5077
+  '\xaba6'# -> unI64 5078
+  '\xaba7'# -> unI64 5079
+  '\xaba8'# -> unI64 5080
+  '\xaba9'# -> unI64 5081
+  '\xabaa'# -> unI64 5082
+  '\xabab'# -> unI64 5083
+  '\xabac'# -> unI64 5084
+  '\xabad'# -> unI64 5085
+  '\xabae'# -> unI64 5086
+  '\xabaf'# -> unI64 5087
+  '\xabb0'# -> unI64 5088
+  '\xabb1'# -> unI64 5089
+  '\xabb2'# -> unI64 5090
+  '\xabb3'# -> unI64 5091
+  '\xabb4'# -> unI64 5092
+  '\xabb5'# -> unI64 5093
+  '\xabb6'# -> unI64 5094
+  '\xabb7'# -> unI64 5095
+  '\xabb8'# -> unI64 5096
+  '\xabb9'# -> unI64 5097
+  '\xabba'# -> unI64 5098
+  '\xabbb'# -> unI64 5099
+  '\xabbc'# -> unI64 5100
+  '\xabbd'# -> unI64 5101
+  '\xabbe'# -> unI64 5102
+  '\xabbf'# -> unI64 5103
+  '\xff41'# -> unI64 65313
+  '\xff42'# -> unI64 65314
+  '\xff43'# -> unI64 65315
+  '\xff44'# -> unI64 65316
+  '\xff45'# -> unI64 65317
+  '\xff46'# -> unI64 65318
+  '\xff47'# -> unI64 65319
+  '\xff48'# -> unI64 65320
+  '\xff49'# -> unI64 65321
+  '\xff4a'# -> unI64 65322
+  '\xff4b'# -> unI64 65323
+  '\xff4c'# -> unI64 65324
+  '\xff4d'# -> unI64 65325
+  '\xff4e'# -> unI64 65326
+  '\xff4f'# -> unI64 65327
+  '\xff50'# -> unI64 65328
+  '\xff51'# -> unI64 65329
+  '\xff52'# -> unI64 65330
+  '\xff53'# -> unI64 65331
+  '\xff54'# -> unI64 65332
+  '\xff55'# -> unI64 65333
+  '\xff56'# -> unI64 65334
+  '\xff57'# -> unI64 65335
+  '\xff58'# -> unI64 65336
+  '\xff59'# -> unI64 65337
+  '\xff5a'# -> unI64 65338
+  '\x10428'# -> unI64 66560
+  '\x10429'# -> unI64 66561
+  '\x1042a'# -> unI64 66562
+  '\x1042b'# -> unI64 66563
+  '\x1042c'# -> unI64 66564
+  '\x1042d'# -> unI64 66565
+  '\x1042e'# -> unI64 66566
+  '\x1042f'# -> unI64 66567
+  '\x10430'# -> unI64 66568
+  '\x10431'# -> unI64 66569
+  '\x10432'# -> unI64 66570
+  '\x10433'# -> unI64 66571
+  '\x10434'# -> unI64 66572
+  '\x10435'# -> unI64 66573
+  '\x10436'# -> unI64 66574
+  '\x10437'# -> unI64 66575
+  '\x10438'# -> unI64 66576
+  '\x10439'# -> unI64 66577
+  '\x1043a'# -> unI64 66578
+  '\x1043b'# -> unI64 66579
+  '\x1043c'# -> unI64 66580
+  '\x1043d'# -> unI64 66581
+  '\x1043e'# -> unI64 66582
+  '\x1043f'# -> unI64 66583
+  '\x10440'# -> unI64 66584
+  '\x10441'# -> unI64 66585
+  '\x10442'# -> unI64 66586
+  '\x10443'# -> unI64 66587
+  '\x10444'# -> unI64 66588
+  '\x10445'# -> unI64 66589
+  '\x10446'# -> unI64 66590
+  '\x10447'# -> unI64 66591
+  '\x10448'# -> unI64 66592
+  '\x10449'# -> unI64 66593
+  '\x1044a'# -> unI64 66594
+  '\x1044b'# -> unI64 66595
+  '\x1044c'# -> unI64 66596
+  '\x1044d'# -> unI64 66597
+  '\x1044e'# -> unI64 66598
+  '\x1044f'# -> unI64 66599
+  '\x104d8'# -> unI64 66736
+  '\x104d9'# -> unI64 66737
+  '\x104da'# -> unI64 66738
+  '\x104db'# -> unI64 66739
+  '\x104dc'# -> unI64 66740
+  '\x104dd'# -> unI64 66741
+  '\x104de'# -> unI64 66742
+  '\x104df'# -> unI64 66743
+  '\x104e0'# -> unI64 66744
+  '\x104e1'# -> unI64 66745
+  '\x104e2'# -> unI64 66746
+  '\x104e3'# -> unI64 66747
+  '\x104e4'# -> unI64 66748
+  '\x104e5'# -> unI64 66749
+  '\x104e6'# -> unI64 66750
+  '\x104e7'# -> unI64 66751
+  '\x104e8'# -> unI64 66752
+  '\x104e9'# -> unI64 66753
+  '\x104ea'# -> unI64 66754
+  '\x104eb'# -> unI64 66755
+  '\x104ec'# -> unI64 66756
+  '\x104ed'# -> unI64 66757
+  '\x104ee'# -> unI64 66758
+  '\x104ef'# -> unI64 66759
+  '\x104f0'# -> unI64 66760
+  '\x104f1'# -> unI64 66761
+  '\x104f2'# -> unI64 66762
+  '\x104f3'# -> unI64 66763
+  '\x104f4'# -> unI64 66764
+  '\x104f5'# -> unI64 66765
+  '\x104f6'# -> unI64 66766
+  '\x104f7'# -> unI64 66767
+  '\x104f8'# -> unI64 66768
+  '\x104f9'# -> unI64 66769
+  '\x104fa'# -> unI64 66770
+  '\x104fb'# -> unI64 66771
+  '\x10597'# -> unI64 66928
+  '\x10598'# -> unI64 66929
+  '\x10599'# -> unI64 66930
+  '\x1059a'# -> unI64 66931
+  '\x1059b'# -> unI64 66932
+  '\x1059c'# -> unI64 66933
+  '\x1059d'# -> unI64 66934
+  '\x1059e'# -> unI64 66935
+  '\x1059f'# -> unI64 66936
+  '\x105a0'# -> unI64 66937
+  '\x105a1'# -> unI64 66938
+  '\x105a3'# -> unI64 66940
+  '\x105a4'# -> unI64 66941
+  '\x105a5'# -> unI64 66942
+  '\x105a6'# -> unI64 66943
+  '\x105a7'# -> unI64 66944
+  '\x105a8'# -> unI64 66945
+  '\x105a9'# -> unI64 66946
+  '\x105aa'# -> unI64 66947
+  '\x105ab'# -> unI64 66948
+  '\x105ac'# -> unI64 66949
+  '\x105ad'# -> unI64 66950
+  '\x105ae'# -> unI64 66951
+  '\x105af'# -> unI64 66952
+  '\x105b0'# -> unI64 66953
+  '\x105b1'# -> unI64 66954
+  '\x105b3'# -> unI64 66956
+  '\x105b4'# -> unI64 66957
+  '\x105b5'# -> unI64 66958
+  '\x105b6'# -> unI64 66959
+  '\x105b7'# -> unI64 66960
+  '\x105b8'# -> unI64 66961
+  '\x105b9'# -> unI64 66962
+  '\x105bb'# -> unI64 66964
+  '\x105bc'# -> unI64 66965
+  '\x10cc0'# -> unI64 68736
+  '\x10cc1'# -> unI64 68737
+  '\x10cc2'# -> unI64 68738
+  '\x10cc3'# -> unI64 68739
+  '\x10cc4'# -> unI64 68740
+  '\x10cc5'# -> unI64 68741
+  '\x10cc6'# -> unI64 68742
+  '\x10cc7'# -> unI64 68743
+  '\x10cc8'# -> unI64 68744
+  '\x10cc9'# -> unI64 68745
+  '\x10cca'# -> unI64 68746
+  '\x10ccb'# -> unI64 68747
+  '\x10ccc'# -> unI64 68748
+  '\x10ccd'# -> unI64 68749
+  '\x10cce'# -> unI64 68750
+  '\x10ccf'# -> unI64 68751
+  '\x10cd0'# -> unI64 68752
+  '\x10cd1'# -> unI64 68753
+  '\x10cd2'# -> unI64 68754
+  '\x10cd3'# -> unI64 68755
+  '\x10cd4'# -> unI64 68756
+  '\x10cd5'# -> unI64 68757
+  '\x10cd6'# -> unI64 68758
+  '\x10cd7'# -> unI64 68759
+  '\x10cd8'# -> unI64 68760
+  '\x10cd9'# -> unI64 68761
+  '\x10cda'# -> unI64 68762
+  '\x10cdb'# -> unI64 68763
+  '\x10cdc'# -> unI64 68764
+  '\x10cdd'# -> unI64 68765
+  '\x10cde'# -> unI64 68766
+  '\x10cdf'# -> unI64 68767
+  '\x10ce0'# -> unI64 68768
+  '\x10ce1'# -> unI64 68769
+  '\x10ce2'# -> unI64 68770
+  '\x10ce3'# -> unI64 68771
+  '\x10ce4'# -> unI64 68772
+  '\x10ce5'# -> unI64 68773
+  '\x10ce6'# -> unI64 68774
+  '\x10ce7'# -> unI64 68775
+  '\x10ce8'# -> unI64 68776
+  '\x10ce9'# -> unI64 68777
+  '\x10cea'# -> unI64 68778
+  '\x10ceb'# -> unI64 68779
+  '\x10cec'# -> unI64 68780
+  '\x10ced'# -> unI64 68781
+  '\x10cee'# -> unI64 68782
+  '\x10cef'# -> unI64 68783
+  '\x10cf0'# -> unI64 68784
+  '\x10cf1'# -> unI64 68785
+  '\x10cf2'# -> unI64 68786
+  '\x10d70'# -> unI64 68944
+  '\x10d71'# -> unI64 68945
+  '\x10d72'# -> unI64 68946
+  '\x10d73'# -> unI64 68947
+  '\x10d74'# -> unI64 68948
+  '\x10d75'# -> unI64 68949
+  '\x10d76'# -> unI64 68950
+  '\x10d77'# -> unI64 68951
+  '\x10d78'# -> unI64 68952
+  '\x10d79'# -> unI64 68953
+  '\x10d7a'# -> unI64 68954
+  '\x10d7b'# -> unI64 68955
+  '\x10d7c'# -> unI64 68956
+  '\x10d7d'# -> unI64 68957
+  '\x10d7e'# -> unI64 68958
+  '\x10d7f'# -> unI64 68959
+  '\x10d80'# -> unI64 68960
+  '\x10d81'# -> unI64 68961
+  '\x10d82'# -> unI64 68962
+  '\x10d83'# -> unI64 68963
+  '\x10d84'# -> unI64 68964
+  '\x10d85'# -> unI64 68965
+  '\x118c0'# -> unI64 71840
+  '\x118c1'# -> unI64 71841
+  '\x118c2'# -> unI64 71842
+  '\x118c3'# -> unI64 71843
+  '\x118c4'# -> unI64 71844
+  '\x118c5'# -> unI64 71845
+  '\x118c6'# -> unI64 71846
+  '\x118c7'# -> unI64 71847
+  '\x118c8'# -> unI64 71848
+  '\x118c9'# -> unI64 71849
+  '\x118ca'# -> unI64 71850
+  '\x118cb'# -> unI64 71851
+  '\x118cc'# -> unI64 71852
+  '\x118cd'# -> unI64 71853
+  '\x118ce'# -> unI64 71854
+  '\x118cf'# -> unI64 71855
+  '\x118d0'# -> unI64 71856
+  '\x118d1'# -> unI64 71857
+  '\x118d2'# -> unI64 71858
+  '\x118d3'# -> unI64 71859
+  '\x118d4'# -> unI64 71860
+  '\x118d5'# -> unI64 71861
+  '\x118d6'# -> unI64 71862
+  '\x118d7'# -> unI64 71863
+  '\x118d8'# -> unI64 71864
+  '\x118d9'# -> unI64 71865
+  '\x118da'# -> unI64 71866
+  '\x118db'# -> unI64 71867
+  '\x118dc'# -> unI64 71868
+  '\x118dd'# -> unI64 71869
+  '\x118de'# -> unI64 71870
+  '\x118df'# -> unI64 71871
+  '\x16e60'# -> unI64 93760
+  '\x16e61'# -> unI64 93761
+  '\x16e62'# -> unI64 93762
+  '\x16e63'# -> unI64 93763
+  '\x16e64'# -> unI64 93764
+  '\x16e65'# -> unI64 93765
+  '\x16e66'# -> unI64 93766
+  '\x16e67'# -> unI64 93767
+  '\x16e68'# -> unI64 93768
+  '\x16e69'# -> unI64 93769
+  '\x16e6a'# -> unI64 93770
+  '\x16e6b'# -> unI64 93771
+  '\x16e6c'# -> unI64 93772
+  '\x16e6d'# -> unI64 93773
+  '\x16e6e'# -> unI64 93774
+  '\x16e6f'# -> unI64 93775
+  '\x16e70'# -> unI64 93776
+  '\x16e71'# -> unI64 93777
+  '\x16e72'# -> unI64 93778
+  '\x16e73'# -> unI64 93779
+  '\x16e74'# -> unI64 93780
+  '\x16e75'# -> unI64 93781
+  '\x16e76'# -> unI64 93782
+  '\x16e77'# -> unI64 93783
+  '\x16e78'# -> unI64 93784
+  '\x16e79'# -> unI64 93785
+  '\x16e7a'# -> unI64 93786
+  '\x16e7b'# -> unI64 93787
+  '\x16e7c'# -> unI64 93788
+  '\x16e7d'# -> unI64 93789
+  '\x16e7e'# -> unI64 93790
+  '\x16e7f'# -> unI64 93791
+  '\x16ebb'# -> unI64 93856
+  '\x16ebc'# -> unI64 93857
+  '\x16ebd'# -> unI64 93858
+  '\x16ebe'# -> unI64 93859
+  '\x16ebf'# -> unI64 93860
+  '\x16ec0'# -> unI64 93861
+  '\x16ec1'# -> unI64 93862
+  '\x16ec2'# -> unI64 93863
+  '\x16ec3'# -> unI64 93864
+  '\x16ec4'# -> unI64 93865
+  '\x16ec5'# -> unI64 93866
+  '\x16ec6'# -> unI64 93867
+  '\x16ec7'# -> unI64 93868
+  '\x16ec8'# -> unI64 93869
+  '\x16ec9'# -> unI64 93870
+  '\x16eca'# -> unI64 93871
+  '\x16ecb'# -> unI64 93872
+  '\x16ecc'# -> unI64 93873
+  '\x16ecd'# -> unI64 93874
+  '\x16ece'# -> unI64 93875
+  '\x16ecf'# -> unI64 93876
+  '\x16ed0'# -> unI64 93877
+  '\x16ed1'# -> unI64 93878
+  '\x16ed2'# -> unI64 93879
+  '\x16ed3'# -> unI64 93880
+  '\x1e922'# -> unI64 125184
+  '\x1e923'# -> unI64 125185
+  '\x1e924'# -> unI64 125186
+  '\x1e925'# -> unI64 125187
+  '\x1e926'# -> unI64 125188
+  '\x1e927'# -> unI64 125189
+  '\x1e928'# -> unI64 125190
+  '\x1e929'# -> unI64 125191
+  '\x1e92a'# -> unI64 125192
+  '\x1e92b'# -> unI64 125193
+  '\x1e92c'# -> unI64 125194
+  '\x1e92d'# -> unI64 125195
+  '\x1e92e'# -> unI64 125196
+  '\x1e92f'# -> unI64 125197
+  '\x1e930'# -> unI64 125198
+  '\x1e931'# -> unI64 125199
+  '\x1e932'# -> unI64 125200
+  '\x1e933'# -> unI64 125201
+  '\x1e934'# -> unI64 125202
+  '\x1e935'# -> unI64 125203
+  '\x1e936'# -> unI64 125204
+  '\x1e937'# -> unI64 125205
+  '\x1e938'# -> unI64 125206
+  '\x1e939'# -> unI64 125207
+  '\x1e93a'# -> unI64 125208
+  '\x1e93b'# -> unI64 125209
+  '\x1e93c'# -> unI64 125210
+  '\x1e93d'# -> unI64 125211
+  '\x1e93e'# -> unI64 125212
+  '\x1e93f'# -> unI64 125213
+  '\x1e940'# -> unI64 125214
+  '\x1e941'# -> unI64 125215
+  '\x1e942'# -> unI64 125216
+  '\x1e943'# -> unI64 125217
+  _ -> unI64 0
+lowerMapping :: Char# -> _ {- unboxed Int64 -}
+{-# NOINLINE lowerMapping #-}
+lowerMapping = \case
+  -- LATIN CAPITAL LETTER I WITH DOT ABOVE
+  '\x0130'# -> unI64 1625292905
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f88'# -> unI64 8064
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f89'# -> unI64 8065
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f8a'# -> unI64 8066
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f8b'# -> unI64 8067
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f8c'# -> unI64 8068
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f8d'# -> unI64 8069
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8e'# -> unI64 8070
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8f'# -> unI64 8071
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f98'# -> unI64 8080
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f99'# -> unI64 8081
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f9a'# -> unI64 8082
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f9b'# -> unI64 8083
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f9c'# -> unI64 8084
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f9d'# -> unI64 8085
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9e'# -> unI64 8086
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9f'# -> unI64 8087
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+  '\x1fa8'# -> unI64 8096
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+  '\x1fa9'# -> unI64 8097
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1faa'# -> unI64 8098
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1fab'# -> unI64 8099
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1fac'# -> unI64 8100
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1fad'# -> unI64 8101
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1fae'# -> unI64 8102
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1faf'# -> unI64 8103
+  -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+  '\x1fbc'# -> unI64 8115
+  -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+  '\x1fcc'# -> unI64 8131
+  -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+  '\x1ffc'# -> unI64 8179
+  '\x0041'# -> unI64 97
+  '\x0042'# -> unI64 98
+  '\x0043'# -> unI64 99
+  '\x0044'# -> unI64 100
+  '\x0045'# -> unI64 101
+  '\x0046'# -> unI64 102
+  '\x0047'# -> unI64 103
+  '\x0048'# -> unI64 104
+  '\x0049'# -> unI64 105
+  '\x004a'# -> unI64 106
+  '\x004b'# -> unI64 107
+  '\x004c'# -> unI64 108
+  '\x004d'# -> unI64 109
+  '\x004e'# -> unI64 110
+  '\x004f'# -> unI64 111
+  '\x0050'# -> unI64 112
+  '\x0051'# -> unI64 113
+  '\x0052'# -> unI64 114
+  '\x0053'# -> unI64 115
+  '\x0054'# -> unI64 116
+  '\x0055'# -> unI64 117
+  '\x0056'# -> unI64 118
+  '\x0057'# -> unI64 119
+  '\x0058'# -> unI64 120
+  '\x0059'# -> unI64 121
+  '\x005a'# -> unI64 122
+  '\x00c0'# -> unI64 224
+  '\x00c1'# -> unI64 225
+  '\x00c2'# -> unI64 226
+  '\x00c3'# -> unI64 227
+  '\x00c4'# -> unI64 228
+  '\x00c5'# -> unI64 229
+  '\x00c6'# -> unI64 230
+  '\x00c7'# -> unI64 231
+  '\x00c8'# -> unI64 232
+  '\x00c9'# -> unI64 233
+  '\x00ca'# -> unI64 234
+  '\x00cb'# -> unI64 235
+  '\x00cc'# -> unI64 236
+  '\x00cd'# -> unI64 237
+  '\x00ce'# -> unI64 238
+  '\x00cf'# -> unI64 239
+  '\x00d0'# -> unI64 240
+  '\x00d1'# -> unI64 241
+  '\x00d2'# -> unI64 242
+  '\x00d3'# -> unI64 243
+  '\x00d4'# -> unI64 244
+  '\x00d5'# -> unI64 245
+  '\x00d6'# -> unI64 246
+  '\x00d8'# -> unI64 248
+  '\x00d9'# -> unI64 249
+  '\x00da'# -> unI64 250
+  '\x00db'# -> unI64 251
+  '\x00dc'# -> unI64 252
+  '\x00dd'# -> unI64 253
+  '\x00de'# -> unI64 254
+  '\x0100'# -> unI64 257
+  '\x0102'# -> unI64 259
+  '\x0104'# -> unI64 261
+  '\x0106'# -> unI64 263
+  '\x0108'# -> unI64 265
+  '\x010a'# -> unI64 267
+  '\x010c'# -> unI64 269
+  '\x010e'# -> unI64 271
+  '\x0110'# -> unI64 273
+  '\x0112'# -> unI64 275
+  '\x0114'# -> unI64 277
+  '\x0116'# -> unI64 279
+  '\x0118'# -> unI64 281
+  '\x011a'# -> unI64 283
+  '\x011c'# -> unI64 285
+  '\x011e'# -> unI64 287
+  '\x0120'# -> unI64 289
+  '\x0122'# -> unI64 291
+  '\x0124'# -> unI64 293
+  '\x0126'# -> unI64 295
+  '\x0128'# -> unI64 297
+  '\x012a'# -> unI64 299
+  '\x012c'# -> unI64 301
+  '\x012e'# -> unI64 303
+  '\x0132'# -> unI64 307
+  '\x0134'# -> unI64 309
+  '\x0136'# -> unI64 311
+  '\x0139'# -> unI64 314
+  '\x013b'# -> unI64 316
+  '\x013d'# -> unI64 318
+  '\x013f'# -> unI64 320
+  '\x0141'# -> unI64 322
+  '\x0143'# -> unI64 324
+  '\x0145'# -> unI64 326
+  '\x0147'# -> unI64 328
+  '\x014a'# -> unI64 331
+  '\x014c'# -> unI64 333
+  '\x014e'# -> unI64 335
+  '\x0150'# -> unI64 337
+  '\x0152'# -> unI64 339
+  '\x0154'# -> unI64 341
+  '\x0156'# -> unI64 343
+  '\x0158'# -> unI64 345
+  '\x015a'# -> unI64 347
+  '\x015c'# -> unI64 349
+  '\x015e'# -> unI64 351
+  '\x0160'# -> unI64 353
+  '\x0162'# -> unI64 355
+  '\x0164'# -> unI64 357
+  '\x0166'# -> unI64 359
+  '\x0168'# -> unI64 361
+  '\x016a'# -> unI64 363
+  '\x016c'# -> unI64 365
+  '\x016e'# -> unI64 367
+  '\x0170'# -> unI64 369
+  '\x0172'# -> unI64 371
+  '\x0174'# -> unI64 373
+  '\x0176'# -> unI64 375
+  '\x0178'# -> unI64 255
+  '\x0179'# -> unI64 378
+  '\x017b'# -> unI64 380
+  '\x017d'# -> unI64 382
+  '\x0181'# -> unI64 595
+  '\x0182'# -> unI64 387
+  '\x0184'# -> unI64 389
+  '\x0186'# -> unI64 596
+  '\x0187'# -> unI64 392
+  '\x0189'# -> unI64 598
+  '\x018a'# -> unI64 599
+  '\x018b'# -> unI64 396
+  '\x018e'# -> unI64 477
+  '\x018f'# -> unI64 601
+  '\x0190'# -> unI64 603
+  '\x0191'# -> unI64 402
+  '\x0193'# -> unI64 608
+  '\x0194'# -> unI64 611
+  '\x0196'# -> unI64 617
+  '\x0197'# -> unI64 616
+  '\x0198'# -> unI64 409
+  '\x019c'# -> unI64 623
+  '\x019d'# -> unI64 626
+  '\x019f'# -> unI64 629
+  '\x01a0'# -> unI64 417
+  '\x01a2'# -> unI64 419
+  '\x01a4'# -> unI64 421
+  '\x01a6'# -> unI64 640
+  '\x01a7'# -> unI64 424
+  '\x01a9'# -> unI64 643
+  '\x01ac'# -> unI64 429
+  '\x01ae'# -> unI64 648
+  '\x01af'# -> unI64 432
+  '\x01b1'# -> unI64 650
+  '\x01b2'# -> unI64 651
+  '\x01b3'# -> unI64 436
+  '\x01b5'# -> unI64 438
+  '\x01b7'# -> unI64 658
+  '\x01b8'# -> unI64 441
+  '\x01bc'# -> unI64 445
+  '\x01c4'# -> unI64 454
+  '\x01c5'# -> unI64 454
+  '\x01c7'# -> unI64 457
+  '\x01c8'# -> unI64 457
+  '\x01ca'# -> unI64 460
+  '\x01cb'# -> unI64 460
+  '\x01cd'# -> unI64 462
+  '\x01cf'# -> unI64 464
+  '\x01d1'# -> unI64 466
+  '\x01d3'# -> unI64 468
+  '\x01d5'# -> unI64 470
+  '\x01d7'# -> unI64 472
+  '\x01d9'# -> unI64 474
+  '\x01db'# -> unI64 476
+  '\x01de'# -> unI64 479
+  '\x01e0'# -> unI64 481
+  '\x01e2'# -> unI64 483
+  '\x01e4'# -> unI64 485
+  '\x01e6'# -> unI64 487
+  '\x01e8'# -> unI64 489
+  '\x01ea'# -> unI64 491
+  '\x01ec'# -> unI64 493
+  '\x01ee'# -> unI64 495
+  '\x01f1'# -> unI64 499
+  '\x01f2'# -> unI64 499
+  '\x01f4'# -> unI64 501
+  '\x01f6'# -> unI64 405
+  '\x01f7'# -> unI64 447
+  '\x01f8'# -> unI64 505
+  '\x01fa'# -> unI64 507
+  '\x01fc'# -> unI64 509
+  '\x01fe'# -> unI64 511
+  '\x0200'# -> unI64 513
+  '\x0202'# -> unI64 515
+  '\x0204'# -> unI64 517
+  '\x0206'# -> unI64 519
+  '\x0208'# -> unI64 521
+  '\x020a'# -> unI64 523
+  '\x020c'# -> unI64 525
+  '\x020e'# -> unI64 527
+  '\x0210'# -> unI64 529
+  '\x0212'# -> unI64 531
+  '\x0214'# -> unI64 533
+  '\x0216'# -> unI64 535
+  '\x0218'# -> unI64 537
+  '\x021a'# -> unI64 539
+  '\x021c'# -> unI64 541
+  '\x021e'# -> unI64 543
+  '\x0220'# -> unI64 414
+  '\x0222'# -> unI64 547
+  '\x0224'# -> unI64 549
+  '\x0226'# -> unI64 551
+  '\x0228'# -> unI64 553
+  '\x022a'# -> unI64 555
+  '\x022c'# -> unI64 557
+  '\x022e'# -> unI64 559
+  '\x0230'# -> unI64 561
+  '\x0232'# -> unI64 563
+  '\x023a'# -> unI64 11365
+  '\x023b'# -> unI64 572
+  '\x023d'# -> unI64 410
+  '\x023e'# -> unI64 11366
+  '\x0241'# -> unI64 578
+  '\x0243'# -> unI64 384
+  '\x0244'# -> unI64 649
+  '\x0245'# -> unI64 652
+  '\x0246'# -> unI64 583
+  '\x0248'# -> unI64 585
+  '\x024a'# -> unI64 587
+  '\x024c'# -> unI64 589
+  '\x024e'# -> unI64 591
+  '\x0370'# -> unI64 881
+  '\x0372'# -> unI64 883
+  '\x0376'# -> unI64 887
+  '\x037f'# -> unI64 1011
+  '\x0386'# -> unI64 940
+  '\x0388'# -> unI64 941
+  '\x0389'# -> unI64 942
+  '\x038a'# -> unI64 943
+  '\x038c'# -> unI64 972
+  '\x038e'# -> unI64 973
+  '\x038f'# -> unI64 974
+  '\x0391'# -> unI64 945
+  '\x0392'# -> unI64 946
+  '\x0393'# -> unI64 947
+  '\x0394'# -> unI64 948
+  '\x0395'# -> unI64 949
+  '\x0396'# -> unI64 950
+  '\x0397'# -> unI64 951
+  '\x0398'# -> unI64 952
+  '\x0399'# -> unI64 953
+  '\x039a'# -> unI64 954
+  '\x039b'# -> unI64 955
+  '\x039c'# -> unI64 956
+  '\x039d'# -> unI64 957
+  '\x039e'# -> unI64 958
+  '\x039f'# -> unI64 959
+  '\x03a0'# -> unI64 960
+  '\x03a1'# -> unI64 961
+  '\x03a3'# -> unI64 963
+  '\x03a4'# -> unI64 964
+  '\x03a5'# -> unI64 965
+  '\x03a6'# -> unI64 966
+  '\x03a7'# -> unI64 967
+  '\x03a8'# -> unI64 968
+  '\x03a9'# -> unI64 969
+  '\x03aa'# -> unI64 970
+  '\x03ab'# -> unI64 971
+  '\x03cf'# -> unI64 983
+  '\x03d8'# -> unI64 985
+  '\x03da'# -> unI64 987
+  '\x03dc'# -> unI64 989
+  '\x03de'# -> unI64 991
+  '\x03e0'# -> unI64 993
+  '\x03e2'# -> unI64 995
+  '\x03e4'# -> unI64 997
+  '\x03e6'# -> unI64 999
+  '\x03e8'# -> unI64 1001
+  '\x03ea'# -> unI64 1003
+  '\x03ec'# -> unI64 1005
+  '\x03ee'# -> unI64 1007
+  '\x03f4'# -> unI64 952
+  '\x03f7'# -> unI64 1016
+  '\x03f9'# -> unI64 1010
+  '\x03fa'# -> unI64 1019
+  '\x03fd'# -> unI64 891
+  '\x03fe'# -> unI64 892
+  '\x03ff'# -> unI64 893
+  '\x0400'# -> unI64 1104
+  '\x0401'# -> unI64 1105
+  '\x0402'# -> unI64 1106
+  '\x0403'# -> unI64 1107
+  '\x0404'# -> unI64 1108
+  '\x0405'# -> unI64 1109
+  '\x0406'# -> unI64 1110
+  '\x0407'# -> unI64 1111
+  '\x0408'# -> unI64 1112
+  '\x0409'# -> unI64 1113
+  '\x040a'# -> unI64 1114
+  '\x040b'# -> unI64 1115
+  '\x040c'# -> unI64 1116
+  '\x040d'# -> unI64 1117
+  '\x040e'# -> unI64 1118
+  '\x040f'# -> unI64 1119
+  '\x0410'# -> unI64 1072
+  '\x0411'# -> unI64 1073
+  '\x0412'# -> unI64 1074
+  '\x0413'# -> unI64 1075
+  '\x0414'# -> unI64 1076
+  '\x0415'# -> unI64 1077
+  '\x0416'# -> unI64 1078
+  '\x0417'# -> unI64 1079
+  '\x0418'# -> unI64 1080
+  '\x0419'# -> unI64 1081
+  '\x041a'# -> unI64 1082
+  '\x041b'# -> unI64 1083
+  '\x041c'# -> unI64 1084
+  '\x041d'# -> unI64 1085
+  '\x041e'# -> unI64 1086
+  '\x041f'# -> unI64 1087
+  '\x0420'# -> unI64 1088
+  '\x0421'# -> unI64 1089
+  '\x0422'# -> unI64 1090
+  '\x0423'# -> unI64 1091
+  '\x0424'# -> unI64 1092
+  '\x0425'# -> unI64 1093
+  '\x0426'# -> unI64 1094
+  '\x0427'# -> unI64 1095
+  '\x0428'# -> unI64 1096
+  '\x0429'# -> unI64 1097
+  '\x042a'# -> unI64 1098
+  '\x042b'# -> unI64 1099
+  '\x042c'# -> unI64 1100
+  '\x042d'# -> unI64 1101
+  '\x042e'# -> unI64 1102
+  '\x042f'# -> unI64 1103
+  '\x0460'# -> unI64 1121
+  '\x0462'# -> unI64 1123
+  '\x0464'# -> unI64 1125
+  '\x0466'# -> unI64 1127
+  '\x0468'# -> unI64 1129
+  '\x046a'# -> unI64 1131
+  '\x046c'# -> unI64 1133
+  '\x046e'# -> unI64 1135
+  '\x0470'# -> unI64 1137
+  '\x0472'# -> unI64 1139
+  '\x0474'# -> unI64 1141
+  '\x0476'# -> unI64 1143
+  '\x0478'# -> unI64 1145
+  '\x047a'# -> unI64 1147
+  '\x047c'# -> unI64 1149
+  '\x047e'# -> unI64 1151
+  '\x0480'# -> unI64 1153
+  '\x048a'# -> unI64 1163
+  '\x048c'# -> unI64 1165
+  '\x048e'# -> unI64 1167
+  '\x0490'# -> unI64 1169
+  '\x0492'# -> unI64 1171
+  '\x0494'# -> unI64 1173
+  '\x0496'# -> unI64 1175
+  '\x0498'# -> unI64 1177
+  '\x049a'# -> unI64 1179
+  '\x049c'# -> unI64 1181
+  '\x049e'# -> unI64 1183
+  '\x04a0'# -> unI64 1185
+  '\x04a2'# -> unI64 1187
+  '\x04a4'# -> unI64 1189
+  '\x04a6'# -> unI64 1191
+  '\x04a8'# -> unI64 1193
+  '\x04aa'# -> unI64 1195
+  '\x04ac'# -> unI64 1197
+  '\x04ae'# -> unI64 1199
+  '\x04b0'# -> unI64 1201
+  '\x04b2'# -> unI64 1203
+  '\x04b4'# -> unI64 1205
+  '\x04b6'# -> unI64 1207
+  '\x04b8'# -> unI64 1209
+  '\x04ba'# -> unI64 1211
+  '\x04bc'# -> unI64 1213
+  '\x04be'# -> unI64 1215
+  '\x04c0'# -> unI64 1231
+  '\x04c1'# -> unI64 1218
+  '\x04c3'# -> unI64 1220
+  '\x04c5'# -> unI64 1222
+  '\x04c7'# -> unI64 1224
+  '\x04c9'# -> unI64 1226
+  '\x04cb'# -> unI64 1228
+  '\x04cd'# -> unI64 1230
+  '\x04d0'# -> unI64 1233
+  '\x04d2'# -> unI64 1235
+  '\x04d4'# -> unI64 1237
+  '\x04d6'# -> unI64 1239
+  '\x04d8'# -> unI64 1241
+  '\x04da'# -> unI64 1243
+  '\x04dc'# -> unI64 1245
+  '\x04de'# -> unI64 1247
+  '\x04e0'# -> unI64 1249
+  '\x04e2'# -> unI64 1251
+  '\x04e4'# -> unI64 1253
+  '\x04e6'# -> unI64 1255
+  '\x04e8'# -> unI64 1257
+  '\x04ea'# -> unI64 1259
+  '\x04ec'# -> unI64 1261
+  '\x04ee'# -> unI64 1263
+  '\x04f0'# -> unI64 1265
+  '\x04f2'# -> unI64 1267
+  '\x04f4'# -> unI64 1269
+  '\x04f6'# -> unI64 1271
+  '\x04f8'# -> unI64 1273
+  '\x04fa'# -> unI64 1275
+  '\x04fc'# -> unI64 1277
+  '\x04fe'# -> unI64 1279
+  '\x0500'# -> unI64 1281
+  '\x0502'# -> unI64 1283
+  '\x0504'# -> unI64 1285
+  '\x0506'# -> unI64 1287
+  '\x0508'# -> unI64 1289
+  '\x050a'# -> unI64 1291
+  '\x050c'# -> unI64 1293
+  '\x050e'# -> unI64 1295
+  '\x0510'# -> unI64 1297
+  '\x0512'# -> unI64 1299
+  '\x0514'# -> unI64 1301
+  '\x0516'# -> unI64 1303
+  '\x0518'# -> unI64 1305
+  '\x051a'# -> unI64 1307
+  '\x051c'# -> unI64 1309
+  '\x051e'# -> unI64 1311
+  '\x0520'# -> unI64 1313
+  '\x0522'# -> unI64 1315
+  '\x0524'# -> unI64 1317
+  '\x0526'# -> unI64 1319
+  '\x0528'# -> unI64 1321
+  '\x052a'# -> unI64 1323
+  '\x052c'# -> unI64 1325
+  '\x052e'# -> unI64 1327
+  '\x0531'# -> unI64 1377
+  '\x0532'# -> unI64 1378
+  '\x0533'# -> unI64 1379
+  '\x0534'# -> unI64 1380
+  '\x0535'# -> unI64 1381
+  '\x0536'# -> unI64 1382
+  '\x0537'# -> unI64 1383
+  '\x0538'# -> unI64 1384
+  '\x0539'# -> unI64 1385
+  '\x053a'# -> unI64 1386
+  '\x053b'# -> unI64 1387
+  '\x053c'# -> unI64 1388
+  '\x053d'# -> unI64 1389
+  '\x053e'# -> unI64 1390
+  '\x053f'# -> unI64 1391
+  '\x0540'# -> unI64 1392
+  '\x0541'# -> unI64 1393
+  '\x0542'# -> unI64 1394
+  '\x0543'# -> unI64 1395
+  '\x0544'# -> unI64 1396
+  '\x0545'# -> unI64 1397
+  '\x0546'# -> unI64 1398
+  '\x0547'# -> unI64 1399
+  '\x0548'# -> unI64 1400
+  '\x0549'# -> unI64 1401
+  '\x054a'# -> unI64 1402
+  '\x054b'# -> unI64 1403
+  '\x054c'# -> unI64 1404
+  '\x054d'# -> unI64 1405
+  '\x054e'# -> unI64 1406
+  '\x054f'# -> unI64 1407
+  '\x0550'# -> unI64 1408
+  '\x0551'# -> unI64 1409
+  '\x0552'# -> unI64 1410
+  '\x0553'# -> unI64 1411
+  '\x0554'# -> unI64 1412
+  '\x0555'# -> unI64 1413
+  '\x0556'# -> unI64 1414
+  '\x10a0'# -> unI64 11520
+  '\x10a1'# -> unI64 11521
+  '\x10a2'# -> unI64 11522
+  '\x10a3'# -> unI64 11523
+  '\x10a4'# -> unI64 11524
+  '\x10a5'# -> unI64 11525
+  '\x10a6'# -> unI64 11526
+  '\x10a7'# -> unI64 11527
+  '\x10a8'# -> unI64 11528
+  '\x10a9'# -> unI64 11529
+  '\x10aa'# -> unI64 11530
+  '\x10ab'# -> unI64 11531
+  '\x10ac'# -> unI64 11532
+  '\x10ad'# -> unI64 11533
+  '\x10ae'# -> unI64 11534
+  '\x10af'# -> unI64 11535
+  '\x10b0'# -> unI64 11536
+  '\x10b1'# -> unI64 11537
+  '\x10b2'# -> unI64 11538
+  '\x10b3'# -> unI64 11539
+  '\x10b4'# -> unI64 11540
+  '\x10b5'# -> unI64 11541
+  '\x10b6'# -> unI64 11542
+  '\x10b7'# -> unI64 11543
+  '\x10b8'# -> unI64 11544
+  '\x10b9'# -> unI64 11545
+  '\x10ba'# -> unI64 11546
+  '\x10bb'# -> unI64 11547
+  '\x10bc'# -> unI64 11548
+  '\x10bd'# -> unI64 11549
+  '\x10be'# -> unI64 11550
+  '\x10bf'# -> unI64 11551
+  '\x10c0'# -> unI64 11552
+  '\x10c1'# -> unI64 11553
+  '\x10c2'# -> unI64 11554
+  '\x10c3'# -> unI64 11555
+  '\x10c4'# -> unI64 11556
+  '\x10c5'# -> unI64 11557
+  '\x10c7'# -> unI64 11559
+  '\x10cd'# -> unI64 11565
+  '\x13a0'# -> unI64 43888
+  '\x13a1'# -> unI64 43889
+  '\x13a2'# -> unI64 43890
+  '\x13a3'# -> unI64 43891
+  '\x13a4'# -> unI64 43892
+  '\x13a5'# -> unI64 43893
+  '\x13a6'# -> unI64 43894
+  '\x13a7'# -> unI64 43895
+  '\x13a8'# -> unI64 43896
+  '\x13a9'# -> unI64 43897
+  '\x13aa'# -> unI64 43898
+  '\x13ab'# -> unI64 43899
+  '\x13ac'# -> unI64 43900
+  '\x13ad'# -> unI64 43901
+  '\x13ae'# -> unI64 43902
+  '\x13af'# -> unI64 43903
+  '\x13b0'# -> unI64 43904
+  '\x13b1'# -> unI64 43905
+  '\x13b2'# -> unI64 43906
+  '\x13b3'# -> unI64 43907
+  '\x13b4'# -> unI64 43908
+  '\x13b5'# -> unI64 43909
+  '\x13b6'# -> unI64 43910
+  '\x13b7'# -> unI64 43911
+  '\x13b8'# -> unI64 43912
+  '\x13b9'# -> unI64 43913
+  '\x13ba'# -> unI64 43914
+  '\x13bb'# -> unI64 43915
+  '\x13bc'# -> unI64 43916
+  '\x13bd'# -> unI64 43917
+  '\x13be'# -> unI64 43918
+  '\x13bf'# -> unI64 43919
+  '\x13c0'# -> unI64 43920
+  '\x13c1'# -> unI64 43921
+  '\x13c2'# -> unI64 43922
+  '\x13c3'# -> unI64 43923
+  '\x13c4'# -> unI64 43924
+  '\x13c5'# -> unI64 43925
+  '\x13c6'# -> unI64 43926
+  '\x13c7'# -> unI64 43927
+  '\x13c8'# -> unI64 43928
+  '\x13c9'# -> unI64 43929
+  '\x13ca'# -> unI64 43930
+  '\x13cb'# -> unI64 43931
+  '\x13cc'# -> unI64 43932
+  '\x13cd'# -> unI64 43933
+  '\x13ce'# -> unI64 43934
+  '\x13cf'# -> unI64 43935
+  '\x13d0'# -> unI64 43936
+  '\x13d1'# -> unI64 43937
+  '\x13d2'# -> unI64 43938
+  '\x13d3'# -> unI64 43939
+  '\x13d4'# -> unI64 43940
+  '\x13d5'# -> unI64 43941
+  '\x13d6'# -> unI64 43942
+  '\x13d7'# -> unI64 43943
+  '\x13d8'# -> unI64 43944
+  '\x13d9'# -> unI64 43945
+  '\x13da'# -> unI64 43946
+  '\x13db'# -> unI64 43947
+  '\x13dc'# -> unI64 43948
+  '\x13dd'# -> unI64 43949
+  '\x13de'# -> unI64 43950
+  '\x13df'# -> unI64 43951
+  '\x13e0'# -> unI64 43952
+  '\x13e1'# -> unI64 43953
+  '\x13e2'# -> unI64 43954
+  '\x13e3'# -> unI64 43955
+  '\x13e4'# -> unI64 43956
+  '\x13e5'# -> unI64 43957
+  '\x13e6'# -> unI64 43958
+  '\x13e7'# -> unI64 43959
+  '\x13e8'# -> unI64 43960
+  '\x13e9'# -> unI64 43961
+  '\x13ea'# -> unI64 43962
+  '\x13eb'# -> unI64 43963
+  '\x13ec'# -> unI64 43964
+  '\x13ed'# -> unI64 43965
+  '\x13ee'# -> unI64 43966
+  '\x13ef'# -> unI64 43967
+  '\x13f0'# -> unI64 5112
+  '\x13f1'# -> unI64 5113
+  '\x13f2'# -> unI64 5114
+  '\x13f3'# -> unI64 5115
+  '\x13f4'# -> unI64 5116
+  '\x13f5'# -> unI64 5117
+  '\x1c89'# -> unI64 7306
+  '\x1c90'# -> unI64 4304
+  '\x1c91'# -> unI64 4305
+  '\x1c92'# -> unI64 4306
+  '\x1c93'# -> unI64 4307
+  '\x1c94'# -> unI64 4308
+  '\x1c95'# -> unI64 4309
+  '\x1c96'# -> unI64 4310
+  '\x1c97'# -> unI64 4311
+  '\x1c98'# -> unI64 4312
+  '\x1c99'# -> unI64 4313
+  '\x1c9a'# -> unI64 4314
+  '\x1c9b'# -> unI64 4315
+  '\x1c9c'# -> unI64 4316
+  '\x1c9d'# -> unI64 4317
+  '\x1c9e'# -> unI64 4318
+  '\x1c9f'# -> unI64 4319
+  '\x1ca0'# -> unI64 4320
+  '\x1ca1'# -> unI64 4321
+  '\x1ca2'# -> unI64 4322
+  '\x1ca3'# -> unI64 4323
+  '\x1ca4'# -> unI64 4324
+  '\x1ca5'# -> unI64 4325
+  '\x1ca6'# -> unI64 4326
+  '\x1ca7'# -> unI64 4327
+  '\x1ca8'# -> unI64 4328
+  '\x1ca9'# -> unI64 4329
+  '\x1caa'# -> unI64 4330
+  '\x1cab'# -> unI64 4331
+  '\x1cac'# -> unI64 4332
+  '\x1cad'# -> unI64 4333
+  '\x1cae'# -> unI64 4334
+  '\x1caf'# -> unI64 4335
+  '\x1cb0'# -> unI64 4336
+  '\x1cb1'# -> unI64 4337
+  '\x1cb2'# -> unI64 4338
+  '\x1cb3'# -> unI64 4339
+  '\x1cb4'# -> unI64 4340
+  '\x1cb5'# -> unI64 4341
+  '\x1cb6'# -> unI64 4342
+  '\x1cb7'# -> unI64 4343
+  '\x1cb8'# -> unI64 4344
+  '\x1cb9'# -> unI64 4345
+  '\x1cba'# -> unI64 4346
+  '\x1cbd'# -> unI64 4349
+  '\x1cbe'# -> unI64 4350
+  '\x1cbf'# -> unI64 4351
+  '\x1e00'# -> unI64 7681
+  '\x1e02'# -> unI64 7683
+  '\x1e04'# -> unI64 7685
+  '\x1e06'# -> unI64 7687
+  '\x1e08'# -> unI64 7689
+  '\x1e0a'# -> unI64 7691
+  '\x1e0c'# -> unI64 7693
+  '\x1e0e'# -> unI64 7695
+  '\x1e10'# -> unI64 7697
+  '\x1e12'# -> unI64 7699
+  '\x1e14'# -> unI64 7701
+  '\x1e16'# -> unI64 7703
+  '\x1e18'# -> unI64 7705
+  '\x1e1a'# -> unI64 7707
+  '\x1e1c'# -> unI64 7709
+  '\x1e1e'# -> unI64 7711
+  '\x1e20'# -> unI64 7713
+  '\x1e22'# -> unI64 7715
+  '\x1e24'# -> unI64 7717
+  '\x1e26'# -> unI64 7719
+  '\x1e28'# -> unI64 7721
+  '\x1e2a'# -> unI64 7723
+  '\x1e2c'# -> unI64 7725
+  '\x1e2e'# -> unI64 7727
+  '\x1e30'# -> unI64 7729
+  '\x1e32'# -> unI64 7731
+  '\x1e34'# -> unI64 7733
+  '\x1e36'# -> unI64 7735
+  '\x1e38'# -> unI64 7737
+  '\x1e3a'# -> unI64 7739
+  '\x1e3c'# -> unI64 7741
+  '\x1e3e'# -> unI64 7743
+  '\x1e40'# -> unI64 7745
+  '\x1e42'# -> unI64 7747
+  '\x1e44'# -> unI64 7749
+  '\x1e46'# -> unI64 7751
+  '\x1e48'# -> unI64 7753
+  '\x1e4a'# -> unI64 7755
+  '\x1e4c'# -> unI64 7757
+  '\x1e4e'# -> unI64 7759
+  '\x1e50'# -> unI64 7761
+  '\x1e52'# -> unI64 7763
+  '\x1e54'# -> unI64 7765
+  '\x1e56'# -> unI64 7767
+  '\x1e58'# -> unI64 7769
+  '\x1e5a'# -> unI64 7771
+  '\x1e5c'# -> unI64 7773
+  '\x1e5e'# -> unI64 7775
+  '\x1e60'# -> unI64 7777
+  '\x1e62'# -> unI64 7779
+  '\x1e64'# -> unI64 7781
+  '\x1e66'# -> unI64 7783
+  '\x1e68'# -> unI64 7785
+  '\x1e6a'# -> unI64 7787
+  '\x1e6c'# -> unI64 7789
+  '\x1e6e'# -> unI64 7791
+  '\x1e70'# -> unI64 7793
+  '\x1e72'# -> unI64 7795
+  '\x1e74'# -> unI64 7797
+  '\x1e76'# -> unI64 7799
+  '\x1e78'# -> unI64 7801
+  '\x1e7a'# -> unI64 7803
+  '\x1e7c'# -> unI64 7805
+  '\x1e7e'# -> unI64 7807
+  '\x1e80'# -> unI64 7809
+  '\x1e82'# -> unI64 7811
+  '\x1e84'# -> unI64 7813
+  '\x1e86'# -> unI64 7815
+  '\x1e88'# -> unI64 7817
+  '\x1e8a'# -> unI64 7819
+  '\x1e8c'# -> unI64 7821
+  '\x1e8e'# -> unI64 7823
+  '\x1e90'# -> unI64 7825
+  '\x1e92'# -> unI64 7827
+  '\x1e94'# -> unI64 7829
+  '\x1e9e'# -> unI64 223
+  '\x1ea0'# -> unI64 7841
+  '\x1ea2'# -> unI64 7843
+  '\x1ea4'# -> unI64 7845
+  '\x1ea6'# -> unI64 7847
+  '\x1ea8'# -> unI64 7849
+  '\x1eaa'# -> unI64 7851
+  '\x1eac'# -> unI64 7853
+  '\x1eae'# -> unI64 7855
+  '\x1eb0'# -> unI64 7857
+  '\x1eb2'# -> unI64 7859
+  '\x1eb4'# -> unI64 7861
+  '\x1eb6'# -> unI64 7863
+  '\x1eb8'# -> unI64 7865
+  '\x1eba'# -> unI64 7867
+  '\x1ebc'# -> unI64 7869
+  '\x1ebe'# -> unI64 7871
+  '\x1ec0'# -> unI64 7873
+  '\x1ec2'# -> unI64 7875
+  '\x1ec4'# -> unI64 7877
+  '\x1ec6'# -> unI64 7879
+  '\x1ec8'# -> unI64 7881
+  '\x1eca'# -> unI64 7883
+  '\x1ecc'# -> unI64 7885
+  '\x1ece'# -> unI64 7887
+  '\x1ed0'# -> unI64 7889
+  '\x1ed2'# -> unI64 7891
+  '\x1ed4'# -> unI64 7893
+  '\x1ed6'# -> unI64 7895
+  '\x1ed8'# -> unI64 7897
+  '\x1eda'# -> unI64 7899
+  '\x1edc'# -> unI64 7901
+  '\x1ede'# -> unI64 7903
+  '\x1ee0'# -> unI64 7905
+  '\x1ee2'# -> unI64 7907
+  '\x1ee4'# -> unI64 7909
+  '\x1ee6'# -> unI64 7911
+  '\x1ee8'# -> unI64 7913
+  '\x1eea'# -> unI64 7915
+  '\x1eec'# -> unI64 7917
+  '\x1eee'# -> unI64 7919
+  '\x1ef0'# -> unI64 7921
+  '\x1ef2'# -> unI64 7923
+  '\x1ef4'# -> unI64 7925
+  '\x1ef6'# -> unI64 7927
+  '\x1ef8'# -> unI64 7929
+  '\x1efa'# -> unI64 7931
+  '\x1efc'# -> unI64 7933
+  '\x1efe'# -> unI64 7935
+  '\x1f08'# -> unI64 7936
+  '\x1f09'# -> unI64 7937
+  '\x1f0a'# -> unI64 7938
+  '\x1f0b'# -> unI64 7939
+  '\x1f0c'# -> unI64 7940
+  '\x1f0d'# -> unI64 7941
+  '\x1f0e'# -> unI64 7942
+  '\x1f0f'# -> unI64 7943
+  '\x1f18'# -> unI64 7952
+  '\x1f19'# -> unI64 7953
+  '\x1f1a'# -> unI64 7954
+  '\x1f1b'# -> unI64 7955
+  '\x1f1c'# -> unI64 7956
+  '\x1f1d'# -> unI64 7957
+  '\x1f28'# -> unI64 7968
+  '\x1f29'# -> unI64 7969
+  '\x1f2a'# -> unI64 7970
+  '\x1f2b'# -> unI64 7971
+  '\x1f2c'# -> unI64 7972
+  '\x1f2d'# -> unI64 7973
+  '\x1f2e'# -> unI64 7974
+  '\x1f2f'# -> unI64 7975
+  '\x1f38'# -> unI64 7984
+  '\x1f39'# -> unI64 7985
+  '\x1f3a'# -> unI64 7986
+  '\x1f3b'# -> unI64 7987
+  '\x1f3c'# -> unI64 7988
+  '\x1f3d'# -> unI64 7989
+  '\x1f3e'# -> unI64 7990
+  '\x1f3f'# -> unI64 7991
+  '\x1f48'# -> unI64 8000
+  '\x1f49'# -> unI64 8001
+  '\x1f4a'# -> unI64 8002
+  '\x1f4b'# -> unI64 8003
+  '\x1f4c'# -> unI64 8004
+  '\x1f4d'# -> unI64 8005
+  '\x1f59'# -> unI64 8017
+  '\x1f5b'# -> unI64 8019
+  '\x1f5d'# -> unI64 8021
+  '\x1f5f'# -> unI64 8023
+  '\x1f68'# -> unI64 8032
+  '\x1f69'# -> unI64 8033
+  '\x1f6a'# -> unI64 8034
+  '\x1f6b'# -> unI64 8035
+  '\x1f6c'# -> unI64 8036
+  '\x1f6d'# -> unI64 8037
+  '\x1f6e'# -> unI64 8038
+  '\x1f6f'# -> unI64 8039
+  '\x1fb8'# -> unI64 8112
+  '\x1fb9'# -> unI64 8113
+  '\x1fba'# -> unI64 8048
+  '\x1fbb'# -> unI64 8049
+  '\x1fc8'# -> unI64 8050
+  '\x1fc9'# -> unI64 8051
+  '\x1fca'# -> unI64 8052
+  '\x1fcb'# -> unI64 8053
+  '\x1fd8'# -> unI64 8144
+  '\x1fd9'# -> unI64 8145
+  '\x1fda'# -> unI64 8054
+  '\x1fdb'# -> unI64 8055
+  '\x1fe8'# -> unI64 8160
+  '\x1fe9'# -> unI64 8161
+  '\x1fea'# -> unI64 8058
+  '\x1feb'# -> unI64 8059
+  '\x1fec'# -> unI64 8165
+  '\x1ff8'# -> unI64 8056
+  '\x1ff9'# -> unI64 8057
+  '\x1ffa'# -> unI64 8060
+  '\x1ffb'# -> unI64 8061
+  '\x2126'# -> unI64 969
+  '\x212a'# -> unI64 107
+  '\x212b'# -> unI64 229
+  '\x2132'# -> unI64 8526
+  '\x2160'# -> unI64 8560
+  '\x2161'# -> unI64 8561
+  '\x2162'# -> unI64 8562
+  '\x2163'# -> unI64 8563
+  '\x2164'# -> unI64 8564
+  '\x2165'# -> unI64 8565
+  '\x2166'# -> unI64 8566
+  '\x2167'# -> unI64 8567
+  '\x2168'# -> unI64 8568
+  '\x2169'# -> unI64 8569
+  '\x216a'# -> unI64 8570
+  '\x216b'# -> unI64 8571
+  '\x216c'# -> unI64 8572
+  '\x216d'# -> unI64 8573
+  '\x216e'# -> unI64 8574
+  '\x216f'# -> unI64 8575
+  '\x2183'# -> unI64 8580
+  '\x24b6'# -> unI64 9424
+  '\x24b7'# -> unI64 9425
+  '\x24b8'# -> unI64 9426
+  '\x24b9'# -> unI64 9427
+  '\x24ba'# -> unI64 9428
+  '\x24bb'# -> unI64 9429
+  '\x24bc'# -> unI64 9430
+  '\x24bd'# -> unI64 9431
+  '\x24be'# -> unI64 9432
+  '\x24bf'# -> unI64 9433
+  '\x24c0'# -> unI64 9434
+  '\x24c1'# -> unI64 9435
+  '\x24c2'# -> unI64 9436
+  '\x24c3'# -> unI64 9437
+  '\x24c4'# -> unI64 9438
+  '\x24c5'# -> unI64 9439
+  '\x24c6'# -> unI64 9440
+  '\x24c7'# -> unI64 9441
+  '\x24c8'# -> unI64 9442
+  '\x24c9'# -> unI64 9443
+  '\x24ca'# -> unI64 9444
+  '\x24cb'# -> unI64 9445
+  '\x24cc'# -> unI64 9446
+  '\x24cd'# -> unI64 9447
+  '\x24ce'# -> unI64 9448
+  '\x24cf'# -> unI64 9449
+  '\x2c00'# -> unI64 11312
+  '\x2c01'# -> unI64 11313
+  '\x2c02'# -> unI64 11314
+  '\x2c03'# -> unI64 11315
+  '\x2c04'# -> unI64 11316
+  '\x2c05'# -> unI64 11317
+  '\x2c06'# -> unI64 11318
+  '\x2c07'# -> unI64 11319
+  '\x2c08'# -> unI64 11320
+  '\x2c09'# -> unI64 11321
+  '\x2c0a'# -> unI64 11322
+  '\x2c0b'# -> unI64 11323
+  '\x2c0c'# -> unI64 11324
+  '\x2c0d'# -> unI64 11325
+  '\x2c0e'# -> unI64 11326
+  '\x2c0f'# -> unI64 11327
+  '\x2c10'# -> unI64 11328
+  '\x2c11'# -> unI64 11329
+  '\x2c12'# -> unI64 11330
+  '\x2c13'# -> unI64 11331
+  '\x2c14'# -> unI64 11332
+  '\x2c15'# -> unI64 11333
+  '\x2c16'# -> unI64 11334
+  '\x2c17'# -> unI64 11335
+  '\x2c18'# -> unI64 11336
+  '\x2c19'# -> unI64 11337
+  '\x2c1a'# -> unI64 11338
+  '\x2c1b'# -> unI64 11339
+  '\x2c1c'# -> unI64 11340
+  '\x2c1d'# -> unI64 11341
+  '\x2c1e'# -> unI64 11342
+  '\x2c1f'# -> unI64 11343
+  '\x2c20'# -> unI64 11344
+  '\x2c21'# -> unI64 11345
+  '\x2c22'# -> unI64 11346
+  '\x2c23'# -> unI64 11347
+  '\x2c24'# -> unI64 11348
+  '\x2c25'# -> unI64 11349
+  '\x2c26'# -> unI64 11350
+  '\x2c27'# -> unI64 11351
+  '\x2c28'# -> unI64 11352
+  '\x2c29'# -> unI64 11353
+  '\x2c2a'# -> unI64 11354
+  '\x2c2b'# -> unI64 11355
+  '\x2c2c'# -> unI64 11356
+  '\x2c2d'# -> unI64 11357
+  '\x2c2e'# -> unI64 11358
+  '\x2c2f'# -> unI64 11359
+  '\x2c60'# -> unI64 11361
+  '\x2c62'# -> unI64 619
+  '\x2c63'# -> unI64 7549
+  '\x2c64'# -> unI64 637
+  '\x2c67'# -> unI64 11368
+  '\x2c69'# -> unI64 11370
+  '\x2c6b'# -> unI64 11372
+  '\x2c6d'# -> unI64 593
+  '\x2c6e'# -> unI64 625
+  '\x2c6f'# -> unI64 592
+  '\x2c70'# -> unI64 594
+  '\x2c72'# -> unI64 11379
+  '\x2c75'# -> unI64 11382
+  '\x2c7e'# -> unI64 575
+  '\x2c7f'# -> unI64 576
+  '\x2c80'# -> unI64 11393
+  '\x2c82'# -> unI64 11395
+  '\x2c84'# -> unI64 11397
+  '\x2c86'# -> unI64 11399
+  '\x2c88'# -> unI64 11401
+  '\x2c8a'# -> unI64 11403
+  '\x2c8c'# -> unI64 11405
+  '\x2c8e'# -> unI64 11407
+  '\x2c90'# -> unI64 11409
+  '\x2c92'# -> unI64 11411
+  '\x2c94'# -> unI64 11413
+  '\x2c96'# -> unI64 11415
+  '\x2c98'# -> unI64 11417
+  '\x2c9a'# -> unI64 11419
+  '\x2c9c'# -> unI64 11421
+  '\x2c9e'# -> unI64 11423
+  '\x2ca0'# -> unI64 11425
+  '\x2ca2'# -> unI64 11427
+  '\x2ca4'# -> unI64 11429
+  '\x2ca6'# -> unI64 11431
+  '\x2ca8'# -> unI64 11433
+  '\x2caa'# -> unI64 11435
+  '\x2cac'# -> unI64 11437
+  '\x2cae'# -> unI64 11439
+  '\x2cb0'# -> unI64 11441
+  '\x2cb2'# -> unI64 11443
+  '\x2cb4'# -> unI64 11445
+  '\x2cb6'# -> unI64 11447
+  '\x2cb8'# -> unI64 11449
+  '\x2cba'# -> unI64 11451
+  '\x2cbc'# -> unI64 11453
+  '\x2cbe'# -> unI64 11455
+  '\x2cc0'# -> unI64 11457
+  '\x2cc2'# -> unI64 11459
+  '\x2cc4'# -> unI64 11461
+  '\x2cc6'# -> unI64 11463
+  '\x2cc8'# -> unI64 11465
+  '\x2cca'# -> unI64 11467
+  '\x2ccc'# -> unI64 11469
+  '\x2cce'# -> unI64 11471
+  '\x2cd0'# -> unI64 11473
+  '\x2cd2'# -> unI64 11475
+  '\x2cd4'# -> unI64 11477
+  '\x2cd6'# -> unI64 11479
+  '\x2cd8'# -> unI64 11481
+  '\x2cda'# -> unI64 11483
+  '\x2cdc'# -> unI64 11485
+  '\x2cde'# -> unI64 11487
+  '\x2ce0'# -> unI64 11489
+  '\x2ce2'# -> unI64 11491
+  '\x2ceb'# -> unI64 11500
+  '\x2ced'# -> unI64 11502
+  '\x2cf2'# -> unI64 11507
+  '\xa640'# -> unI64 42561
+  '\xa642'# -> unI64 42563
+  '\xa644'# -> unI64 42565
+  '\xa646'# -> unI64 42567
+  '\xa648'# -> unI64 42569
+  '\xa64a'# -> unI64 42571
+  '\xa64c'# -> unI64 42573
+  '\xa64e'# -> unI64 42575
+  '\xa650'# -> unI64 42577
+  '\xa652'# -> unI64 42579
+  '\xa654'# -> unI64 42581
+  '\xa656'# -> unI64 42583
+  '\xa658'# -> unI64 42585
+  '\xa65a'# -> unI64 42587
+  '\xa65c'# -> unI64 42589
+  '\xa65e'# -> unI64 42591
+  '\xa660'# -> unI64 42593
+  '\xa662'# -> unI64 42595
+  '\xa664'# -> unI64 42597
+  '\xa666'# -> unI64 42599
+  '\xa668'# -> unI64 42601
+  '\xa66a'# -> unI64 42603
+  '\xa66c'# -> unI64 42605
+  '\xa680'# -> unI64 42625
+  '\xa682'# -> unI64 42627
+  '\xa684'# -> unI64 42629
+  '\xa686'# -> unI64 42631
+  '\xa688'# -> unI64 42633
+  '\xa68a'# -> unI64 42635
+  '\xa68c'# -> unI64 42637
+  '\xa68e'# -> unI64 42639
+  '\xa690'# -> unI64 42641
+  '\xa692'# -> unI64 42643
+  '\xa694'# -> unI64 42645
+  '\xa696'# -> unI64 42647
+  '\xa698'# -> unI64 42649
+  '\xa69a'# -> unI64 42651
+  '\xa722'# -> unI64 42787
+  '\xa724'# -> unI64 42789
+  '\xa726'# -> unI64 42791
+  '\xa728'# -> unI64 42793
+  '\xa72a'# -> unI64 42795
+  '\xa72c'# -> unI64 42797
+  '\xa72e'# -> unI64 42799
+  '\xa732'# -> unI64 42803
+  '\xa734'# -> unI64 42805
+  '\xa736'# -> unI64 42807
+  '\xa738'# -> unI64 42809
+  '\xa73a'# -> unI64 42811
+  '\xa73c'# -> unI64 42813
+  '\xa73e'# -> unI64 42815
+  '\xa740'# -> unI64 42817
+  '\xa742'# -> unI64 42819
+  '\xa744'# -> unI64 42821
+  '\xa746'# -> unI64 42823
+  '\xa748'# -> unI64 42825
+  '\xa74a'# -> unI64 42827
+  '\xa74c'# -> unI64 42829
+  '\xa74e'# -> unI64 42831
+  '\xa750'# -> unI64 42833
+  '\xa752'# -> unI64 42835
+  '\xa754'# -> unI64 42837
+  '\xa756'# -> unI64 42839
+  '\xa758'# -> unI64 42841
+  '\xa75a'# -> unI64 42843
+  '\xa75c'# -> unI64 42845
+  '\xa75e'# -> unI64 42847
+  '\xa760'# -> unI64 42849
+  '\xa762'# -> unI64 42851
+  '\xa764'# -> unI64 42853
+  '\xa766'# -> unI64 42855
+  '\xa768'# -> unI64 42857
+  '\xa76a'# -> unI64 42859
+  '\xa76c'# -> unI64 42861
+  '\xa76e'# -> unI64 42863
+  '\xa779'# -> unI64 42874
+  '\xa77b'# -> unI64 42876
+  '\xa77d'# -> unI64 7545
+  '\xa77e'# -> unI64 42879
+  '\xa780'# -> unI64 42881
+  '\xa782'# -> unI64 42883
+  '\xa784'# -> unI64 42885
+  '\xa786'# -> unI64 42887
+  '\xa78b'# -> unI64 42892
+  '\xa78d'# -> unI64 613
+  '\xa790'# -> unI64 42897
+  '\xa792'# -> unI64 42899
+  '\xa796'# -> unI64 42903
+  '\xa798'# -> unI64 42905
+  '\xa79a'# -> unI64 42907
+  '\xa79c'# -> unI64 42909
+  '\xa79e'# -> unI64 42911
+  '\xa7a0'# -> unI64 42913
+  '\xa7a2'# -> unI64 42915
+  '\xa7a4'# -> unI64 42917
+  '\xa7a6'# -> unI64 42919
+  '\xa7a8'# -> unI64 42921
+  '\xa7aa'# -> unI64 614
+  '\xa7ab'# -> unI64 604
+  '\xa7ac'# -> unI64 609
+  '\xa7ad'# -> unI64 620
+  '\xa7ae'# -> unI64 618
+  '\xa7b0'# -> unI64 670
+  '\xa7b1'# -> unI64 647
+  '\xa7b2'# -> unI64 669
+  '\xa7b3'# -> unI64 43859
+  '\xa7b4'# -> unI64 42933
+  '\xa7b6'# -> unI64 42935
+  '\xa7b8'# -> unI64 42937
+  '\xa7ba'# -> unI64 42939
+  '\xa7bc'# -> unI64 42941
+  '\xa7be'# -> unI64 42943
+  '\xa7c0'# -> unI64 42945
+  '\xa7c2'# -> unI64 42947
+  '\xa7c4'# -> unI64 42900
+  '\xa7c5'# -> unI64 642
+  '\xa7c6'# -> unI64 7566
+  '\xa7c7'# -> unI64 42952
+  '\xa7c9'# -> unI64 42954
+  '\xa7cb'# -> unI64 612
+  '\xa7cc'# -> unI64 42957
+  '\xa7ce'# -> unI64 42959
+  '\xa7d0'# -> unI64 42961
+  '\xa7d2'# -> unI64 42963
+  '\xa7d4'# -> unI64 42965
+  '\xa7d6'# -> unI64 42967
+  '\xa7d8'# -> unI64 42969
+  '\xa7da'# -> unI64 42971
+  '\xa7dc'# -> unI64 411
+  '\xa7f5'# -> unI64 42998
+  '\xff21'# -> unI64 65345
+  '\xff22'# -> unI64 65346
+  '\xff23'# -> unI64 65347
+  '\xff24'# -> unI64 65348
+  '\xff25'# -> unI64 65349
+  '\xff26'# -> unI64 65350
+  '\xff27'# -> unI64 65351
+  '\xff28'# -> unI64 65352
+  '\xff29'# -> unI64 65353
+  '\xff2a'# -> unI64 65354
+  '\xff2b'# -> unI64 65355
+  '\xff2c'# -> unI64 65356
+  '\xff2d'# -> unI64 65357
+  '\xff2e'# -> unI64 65358
+  '\xff2f'# -> unI64 65359
+  '\xff30'# -> unI64 65360
+  '\xff31'# -> unI64 65361
+  '\xff32'# -> unI64 65362
+  '\xff33'# -> unI64 65363
+  '\xff34'# -> unI64 65364
+  '\xff35'# -> unI64 65365
+  '\xff36'# -> unI64 65366
+  '\xff37'# -> unI64 65367
+  '\xff38'# -> unI64 65368
+  '\xff39'# -> unI64 65369
+  '\xff3a'# -> unI64 65370
+  '\x10400'# -> unI64 66600
+  '\x10401'# -> unI64 66601
+  '\x10402'# -> unI64 66602
+  '\x10403'# -> unI64 66603
+  '\x10404'# -> unI64 66604
+  '\x10405'# -> unI64 66605
+  '\x10406'# -> unI64 66606
+  '\x10407'# -> unI64 66607
+  '\x10408'# -> unI64 66608
+  '\x10409'# -> unI64 66609
+  '\x1040a'# -> unI64 66610
+  '\x1040b'# -> unI64 66611
+  '\x1040c'# -> unI64 66612
+  '\x1040d'# -> unI64 66613
+  '\x1040e'# -> unI64 66614
+  '\x1040f'# -> unI64 66615
+  '\x10410'# -> unI64 66616
+  '\x10411'# -> unI64 66617
+  '\x10412'# -> unI64 66618
+  '\x10413'# -> unI64 66619
+  '\x10414'# -> unI64 66620
+  '\x10415'# -> unI64 66621
+  '\x10416'# -> unI64 66622
+  '\x10417'# -> unI64 66623
+  '\x10418'# -> unI64 66624
+  '\x10419'# -> unI64 66625
+  '\x1041a'# -> unI64 66626
+  '\x1041b'# -> unI64 66627
+  '\x1041c'# -> unI64 66628
+  '\x1041d'# -> unI64 66629
+  '\x1041e'# -> unI64 66630
+  '\x1041f'# -> unI64 66631
+  '\x10420'# -> unI64 66632
+  '\x10421'# -> unI64 66633
+  '\x10422'# -> unI64 66634
+  '\x10423'# -> unI64 66635
+  '\x10424'# -> unI64 66636
+  '\x10425'# -> unI64 66637
+  '\x10426'# -> unI64 66638
+  '\x10427'# -> unI64 66639
+  '\x104b0'# -> unI64 66776
+  '\x104b1'# -> unI64 66777
+  '\x104b2'# -> unI64 66778
+  '\x104b3'# -> unI64 66779
+  '\x104b4'# -> unI64 66780
+  '\x104b5'# -> unI64 66781
+  '\x104b6'# -> unI64 66782
+  '\x104b7'# -> unI64 66783
+  '\x104b8'# -> unI64 66784
+  '\x104b9'# -> unI64 66785
+  '\x104ba'# -> unI64 66786
+  '\x104bb'# -> unI64 66787
+  '\x104bc'# -> unI64 66788
+  '\x104bd'# -> unI64 66789
+  '\x104be'# -> unI64 66790
+  '\x104bf'# -> unI64 66791
+  '\x104c0'# -> unI64 66792
+  '\x104c1'# -> unI64 66793
+  '\x104c2'# -> unI64 66794
+  '\x104c3'# -> unI64 66795
+  '\x104c4'# -> unI64 66796
+  '\x104c5'# -> unI64 66797
+  '\x104c6'# -> unI64 66798
+  '\x104c7'# -> unI64 66799
+  '\x104c8'# -> unI64 66800
+  '\x104c9'# -> unI64 66801
+  '\x104ca'# -> unI64 66802
+  '\x104cb'# -> unI64 66803
+  '\x104cc'# -> unI64 66804
+  '\x104cd'# -> unI64 66805
+  '\x104ce'# -> unI64 66806
+  '\x104cf'# -> unI64 66807
+  '\x104d0'# -> unI64 66808
+  '\x104d1'# -> unI64 66809
+  '\x104d2'# -> unI64 66810
+  '\x104d3'# -> unI64 66811
+  '\x10570'# -> unI64 66967
+  '\x10571'# -> unI64 66968
+  '\x10572'# -> unI64 66969
+  '\x10573'# -> unI64 66970
+  '\x10574'# -> unI64 66971
+  '\x10575'# -> unI64 66972
+  '\x10576'# -> unI64 66973
+  '\x10577'# -> unI64 66974
+  '\x10578'# -> unI64 66975
+  '\x10579'# -> unI64 66976
+  '\x1057a'# -> unI64 66977
+  '\x1057c'# -> unI64 66979
+  '\x1057d'# -> unI64 66980
+  '\x1057e'# -> unI64 66981
+  '\x1057f'# -> unI64 66982
+  '\x10580'# -> unI64 66983
+  '\x10581'# -> unI64 66984
+  '\x10582'# -> unI64 66985
+  '\x10583'# -> unI64 66986
+  '\x10584'# -> unI64 66987
+  '\x10585'# -> unI64 66988
+  '\x10586'# -> unI64 66989
+  '\x10587'# -> unI64 66990
+  '\x10588'# -> unI64 66991
+  '\x10589'# -> unI64 66992
+  '\x1058a'# -> unI64 66993
+  '\x1058c'# -> unI64 66995
+  '\x1058d'# -> unI64 66996
+  '\x1058e'# -> unI64 66997
+  '\x1058f'# -> unI64 66998
+  '\x10590'# -> unI64 66999
+  '\x10591'# -> unI64 67000
+  '\x10592'# -> unI64 67001
+  '\x10594'# -> unI64 67003
+  '\x10595'# -> unI64 67004
+  '\x10c80'# -> unI64 68800
+  '\x10c81'# -> unI64 68801
+  '\x10c82'# -> unI64 68802
+  '\x10c83'# -> unI64 68803
+  '\x10c84'# -> unI64 68804
+  '\x10c85'# -> unI64 68805
+  '\x10c86'# -> unI64 68806
+  '\x10c87'# -> unI64 68807
+  '\x10c88'# -> unI64 68808
+  '\x10c89'# -> unI64 68809
+  '\x10c8a'# -> unI64 68810
+  '\x10c8b'# -> unI64 68811
+  '\x10c8c'# -> unI64 68812
+  '\x10c8d'# -> unI64 68813
+  '\x10c8e'# -> unI64 68814
+  '\x10c8f'# -> unI64 68815
+  '\x10c90'# -> unI64 68816
+  '\x10c91'# -> unI64 68817
+  '\x10c92'# -> unI64 68818
+  '\x10c93'# -> unI64 68819
+  '\x10c94'# -> unI64 68820
+  '\x10c95'# -> unI64 68821
+  '\x10c96'# -> unI64 68822
+  '\x10c97'# -> unI64 68823
+  '\x10c98'# -> unI64 68824
+  '\x10c99'# -> unI64 68825
+  '\x10c9a'# -> unI64 68826
+  '\x10c9b'# -> unI64 68827
+  '\x10c9c'# -> unI64 68828
+  '\x10c9d'# -> unI64 68829
+  '\x10c9e'# -> unI64 68830
+  '\x10c9f'# -> unI64 68831
+  '\x10ca0'# -> unI64 68832
+  '\x10ca1'# -> unI64 68833
+  '\x10ca2'# -> unI64 68834
+  '\x10ca3'# -> unI64 68835
+  '\x10ca4'# -> unI64 68836
+  '\x10ca5'# -> unI64 68837
+  '\x10ca6'# -> unI64 68838
+  '\x10ca7'# -> unI64 68839
+  '\x10ca8'# -> unI64 68840
+  '\x10ca9'# -> unI64 68841
+  '\x10caa'# -> unI64 68842
+  '\x10cab'# -> unI64 68843
+  '\x10cac'# -> unI64 68844
+  '\x10cad'# -> unI64 68845
+  '\x10cae'# -> unI64 68846
+  '\x10caf'# -> unI64 68847
+  '\x10cb0'# -> unI64 68848
+  '\x10cb1'# -> unI64 68849
+  '\x10cb2'# -> unI64 68850
+  '\x10d50'# -> unI64 68976
+  '\x10d51'# -> unI64 68977
+  '\x10d52'# -> unI64 68978
+  '\x10d53'# -> unI64 68979
+  '\x10d54'# -> unI64 68980
+  '\x10d55'# -> unI64 68981
+  '\x10d56'# -> unI64 68982
+  '\x10d57'# -> unI64 68983
+  '\x10d58'# -> unI64 68984
+  '\x10d59'# -> unI64 68985
+  '\x10d5a'# -> unI64 68986
+  '\x10d5b'# -> unI64 68987
+  '\x10d5c'# -> unI64 68988
+  '\x10d5d'# -> unI64 68989
+  '\x10d5e'# -> unI64 68990
+  '\x10d5f'# -> unI64 68991
+  '\x10d60'# -> unI64 68992
+  '\x10d61'# -> unI64 68993
+  '\x10d62'# -> unI64 68994
+  '\x10d63'# -> unI64 68995
+  '\x10d64'# -> unI64 68996
+  '\x10d65'# -> unI64 68997
+  '\x118a0'# -> unI64 71872
+  '\x118a1'# -> unI64 71873
+  '\x118a2'# -> unI64 71874
+  '\x118a3'# -> unI64 71875
+  '\x118a4'# -> unI64 71876
+  '\x118a5'# -> unI64 71877
+  '\x118a6'# -> unI64 71878
+  '\x118a7'# -> unI64 71879
+  '\x118a8'# -> unI64 71880
+  '\x118a9'# -> unI64 71881
+  '\x118aa'# -> unI64 71882
+  '\x118ab'# -> unI64 71883
+  '\x118ac'# -> unI64 71884
+  '\x118ad'# -> unI64 71885
+  '\x118ae'# -> unI64 71886
+  '\x118af'# -> unI64 71887
+  '\x118b0'# -> unI64 71888
+  '\x118b1'# -> unI64 71889
+  '\x118b2'# -> unI64 71890
+  '\x118b3'# -> unI64 71891
+  '\x118b4'# -> unI64 71892
+  '\x118b5'# -> unI64 71893
+  '\x118b6'# -> unI64 71894
+  '\x118b7'# -> unI64 71895
+  '\x118b8'# -> unI64 71896
+  '\x118b9'# -> unI64 71897
+  '\x118ba'# -> unI64 71898
+  '\x118bb'# -> unI64 71899
+  '\x118bc'# -> unI64 71900
+  '\x118bd'# -> unI64 71901
+  '\x118be'# -> unI64 71902
+  '\x118bf'# -> unI64 71903
+  '\x16e40'# -> unI64 93792
+  '\x16e41'# -> unI64 93793
+  '\x16e42'# -> unI64 93794
+  '\x16e43'# -> unI64 93795
+  '\x16e44'# -> unI64 93796
+  '\x16e45'# -> unI64 93797
+  '\x16e46'# -> unI64 93798
+  '\x16e47'# -> unI64 93799
+  '\x16e48'# -> unI64 93800
+  '\x16e49'# -> unI64 93801
+  '\x16e4a'# -> unI64 93802
+  '\x16e4b'# -> unI64 93803
+  '\x16e4c'# -> unI64 93804
+  '\x16e4d'# -> unI64 93805
+  '\x16e4e'# -> unI64 93806
+  '\x16e4f'# -> unI64 93807
+  '\x16e50'# -> unI64 93808
+  '\x16e51'# -> unI64 93809
+  '\x16e52'# -> unI64 93810
+  '\x16e53'# -> unI64 93811
+  '\x16e54'# -> unI64 93812
+  '\x16e55'# -> unI64 93813
+  '\x16e56'# -> unI64 93814
+  '\x16e57'# -> unI64 93815
+  '\x16e58'# -> unI64 93816
+  '\x16e59'# -> unI64 93817
+  '\x16e5a'# -> unI64 93818
+  '\x16e5b'# -> unI64 93819
+  '\x16e5c'# -> unI64 93820
+  '\x16e5d'# -> unI64 93821
+  '\x16e5e'# -> unI64 93822
+  '\x16e5f'# -> unI64 93823
+  '\x16ea0'# -> unI64 93883
+  '\x16ea1'# -> unI64 93884
+  '\x16ea2'# -> unI64 93885
+  '\x16ea3'# -> unI64 93886
+  '\x16ea4'# -> unI64 93887
+  '\x16ea5'# -> unI64 93888
+  '\x16ea6'# -> unI64 93889
+  '\x16ea7'# -> unI64 93890
+  '\x16ea8'# -> unI64 93891
+  '\x16ea9'# -> unI64 93892
+  '\x16eaa'# -> unI64 93893
+  '\x16eab'# -> unI64 93894
+  '\x16eac'# -> unI64 93895
+  '\x16ead'# -> unI64 93896
+  '\x16eae'# -> unI64 93897
+  '\x16eaf'# -> unI64 93898
+  '\x16eb0'# -> unI64 93899
+  '\x16eb1'# -> unI64 93900
+  '\x16eb2'# -> unI64 93901
+  '\x16eb3'# -> unI64 93902
+  '\x16eb4'# -> unI64 93903
+  '\x16eb5'# -> unI64 93904
+  '\x16eb6'# -> unI64 93905
+  '\x16eb7'# -> unI64 93906
+  '\x16eb8'# -> unI64 93907
+  '\x1e900'# -> unI64 125218
+  '\x1e901'# -> unI64 125219
+  '\x1e902'# -> unI64 125220
+  '\x1e903'# -> unI64 125221
+  '\x1e904'# -> unI64 125222
+  '\x1e905'# -> unI64 125223
+  '\x1e906'# -> unI64 125224
+  '\x1e907'# -> unI64 125225
+  '\x1e908'# -> unI64 125226
+  '\x1e909'# -> unI64 125227
+  '\x1e90a'# -> unI64 125228
+  '\x1e90b'# -> unI64 125229
+  '\x1e90c'# -> unI64 125230
+  '\x1e90d'# -> unI64 125231
+  '\x1e90e'# -> unI64 125232
+  '\x1e90f'# -> unI64 125233
+  '\x1e910'# -> unI64 125234
+  '\x1e911'# -> unI64 125235
+  '\x1e912'# -> unI64 125236
+  '\x1e913'# -> unI64 125237
+  '\x1e914'# -> unI64 125238
+  '\x1e915'# -> unI64 125239
+  '\x1e916'# -> unI64 125240
+  '\x1e917'# -> unI64 125241
+  '\x1e918'# -> unI64 125242
+  '\x1e919'# -> unI64 125243
+  '\x1e91a'# -> unI64 125244
+  '\x1e91b'# -> unI64 125245
+  '\x1e91c'# -> unI64 125246
+  '\x1e91d'# -> unI64 125247
+  '\x1e91e'# -> unI64 125248
+  '\x1e91f'# -> unI64 125249
+  '\x1e920'# -> unI64 125250
+  '\x1e921'# -> unI64 125251
+  _ -> unI64 0
+titleMapping :: Char# -> _ {- unboxed Int64 -}
+{-# NOINLINE titleMapping #-}
+titleMapping = \case
+  -- LATIN SMALL LETTER SHARP S
+  '\x00df'# -> unI64 241172563
+  -- LATIN SMALL LIGATURE FF
+  '\xfb00'# -> unI64 213909574
+  -- LATIN SMALL LIGATURE FI
+  '\xfb01'# -> unI64 220201030
+  -- LATIN SMALL LIGATURE FL
+  '\xfb02'# -> unI64 226492486
+  -- LATIN SMALL LIGATURE FFI
+  '\xfb03'# -> unI64 461795097575494
+  -- LATIN SMALL LIGATURE FFL
+  '\xfb04'# -> unI64 474989237108806
+  -- LATIN SMALL LIGATURE LONG S T
+  '\xfb05'# -> unI64 243269715
+  -- LATIN SMALL LIGATURE ST
+  '\xfb06'# -> unI64 243269715
+  -- ARMENIAN SMALL LIGATURE ECH YIWN
+  '\x0587'# -> unI64 2956985653
+  -- ARMENIAN SMALL LIGATURE MEN NOW
+  '\xfb13'# -> unI64 2931819844
+  -- ARMENIAN SMALL LIGATURE MEN ECH
+  '\xfb14'# -> unI64 2896168260
+  -- ARMENIAN SMALL LIGATURE MEN INI
+  '\xfb15'# -> unI64 2908751172
+  -- ARMENIAN SMALL LIGATURE VEW NOW
+  '\xfb16'# -> unI64 2931819854
+  -- ARMENIAN SMALL LIGATURE MEN XEH
+  '\xfb17'# -> unI64 2912945476
+  -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+  '\x0149'# -> unI64 163578556
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+  '\x0390'# -> unI64 3382099394429849
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+  '\x03b0'# -> unI64 3382099394429861
+  -- LATIN SMALL LETTER J WITH CARON
+  '\x01f0'# -> unI64 1635778634
+  -- LATIN SMALL LETTER H WITH LINE BELOW
+  '\x1e96'# -> unI64 1713373256
+  -- LATIN SMALL LETTER T WITH DIAERESIS
+  '\x1e97'# -> unI64 1627390036
+  -- LATIN SMALL LETTER W WITH RING ABOVE
+  '\x1e98'# -> unI64 1631584343
+  -- LATIN SMALL LETTER Y WITH RING ABOVE
+  '\x1e99'# -> unI64 1631584345
+  -- LATIN SMALL LETTER A WITH RIGHT HALF RING
+  '\x1e9a'# -> unI64 1472200769
+  -- GREEK SMALL LETTER UPSILON WITH PSILI
+  '\x1f50'# -> unI64 1650459557
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+  '\x1f52'# -> unI64 3377701370987429
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+  '\x1f54'# -> unI64 3382099417498533
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+  '\x1f56'# -> unI64 3667972440720293
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+  '\x1fb6'# -> unI64 1749025681
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI
+  '\x1fc6'# -> unI64 1749025687
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+  '\x1fd2'# -> unI64 3377701347918745
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+  '\x1fd3'# -> unI64 3382099394429849
+  -- GREEK SMALL LETTER IOTA WITH PERISPOMENI
+  '\x1fd6'# -> unI64 1749025689
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+  '\x1fd7'# -> unI64 3667972417651609
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+  '\x1fe2'# -> unI64 3377701347918757
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+  '\x1fe3'# -> unI64 3382099394429861
+  -- GREEK SMALL LETTER RHO WITH PSILI
+  '\x1fe4'# -> unI64 1650459553
+  -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+  '\x1fe6'# -> unI64 1749025701
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+  '\x1fe7'# -> unI64 3667972417651621
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+  '\x1ff6'# -> unI64 1749025705
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f80'# -> unI64 8072
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f81'# -> unI64 8073
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f82'# -> unI64 8074
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f83'# -> unI64 8075
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f84'# -> unI64 8076
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f85'# -> unI64 8077
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f86'# -> unI64 8078
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f87'# -> unI64 8079
+  -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f90'# -> unI64 8088
+  -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f91'# -> unI64 8089
+  -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f92'# -> unI64 8090
+  -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f93'# -> unI64 8091
+  -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f94'# -> unI64 8092
+  -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f95'# -> unI64 8093
+  -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f96'# -> unI64 8094
+  -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f97'# -> unI64 8095
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+  '\x1fa0'# -> unI64 8104
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+  '\x1fa1'# -> unI64 8105
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1fa2'# -> unI64 8106
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1fa3'# -> unI64 8107
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1fa4'# -> unI64 8108
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1fa5'# -> unI64 8109
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa6'# -> unI64 8110
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa7'# -> unI64 8111
+  -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+  '\x1fb3'# -> unI64 8124
+  -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+  '\x1fc3'# -> unI64 8140
+  -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+  '\x1ff3'# -> unI64 8188
+  -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fb2'# -> unI64 1755324346
+  -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fb4'# -> unI64 1755317126
+  -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fc2'# -> unI64 1755324362
+  -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fc4'# -> unI64 1755317129
+  -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+  '\x1ff2'# -> unI64 1755324410
+  -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+  '\x1ff4'# -> unI64 1755317135
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fb7'# -> unI64 3681166678819729
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fc7'# -> unI64 3681166678819735
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1ff7'# -> unI64 3681166678819753
+  '\x0061'# -> unI64 65
+  '\x0062'# -> unI64 66
+  '\x0063'# -> unI64 67
+  '\x0064'# -> unI64 68
+  '\x0065'# -> unI64 69
+  '\x0066'# -> unI64 70
+  '\x0067'# -> unI64 71
+  '\x0068'# -> unI64 72
+  '\x0069'# -> unI64 73
+  '\x006a'# -> unI64 74
+  '\x006b'# -> unI64 75
+  '\x006c'# -> unI64 76
+  '\x006d'# -> unI64 77
+  '\x006e'# -> unI64 78
+  '\x006f'# -> unI64 79
+  '\x0070'# -> unI64 80
+  '\x0071'# -> unI64 81
+  '\x0072'# -> unI64 82
+  '\x0073'# -> unI64 83
+  '\x0074'# -> unI64 84
+  '\x0075'# -> unI64 85
+  '\x0076'# -> unI64 86
+  '\x0077'# -> unI64 87
+  '\x0078'# -> unI64 88
+  '\x0079'# -> unI64 89
+  '\x007a'# -> unI64 90
+  '\x00b5'# -> unI64 924
+  '\x00e0'# -> unI64 192
+  '\x00e1'# -> unI64 193
+  '\x00e2'# -> unI64 194
+  '\x00e3'# -> unI64 195
+  '\x00e4'# -> unI64 196
+  '\x00e5'# -> unI64 197
+  '\x00e6'# -> unI64 198
+  '\x00e7'# -> unI64 199
+  '\x00e8'# -> unI64 200
+  '\x00e9'# -> unI64 201
+  '\x00ea'# -> unI64 202
+  '\x00eb'# -> unI64 203
+  '\x00ec'# -> unI64 204
+  '\x00ed'# -> unI64 205
+  '\x00ee'# -> unI64 206
+  '\x00ef'# -> unI64 207
+  '\x00f0'# -> unI64 208
+  '\x00f1'# -> unI64 209
+  '\x00f2'# -> unI64 210
+  '\x00f3'# -> unI64 211
+  '\x00f4'# -> unI64 212
+  '\x00f5'# -> unI64 213
+  '\x00f6'# -> unI64 214
+  '\x00f8'# -> unI64 216
+  '\x00f9'# -> unI64 217
+  '\x00fa'# -> unI64 218
+  '\x00fb'# -> unI64 219
+  '\x00fc'# -> unI64 220
+  '\x00fd'# -> unI64 221
+  '\x00fe'# -> unI64 222
+  '\x00ff'# -> unI64 376
+  '\x0101'# -> unI64 256
+  '\x0103'# -> unI64 258
+  '\x0105'# -> unI64 260
+  '\x0107'# -> unI64 262
+  '\x0109'# -> unI64 264
+  '\x010b'# -> unI64 266
+  '\x010d'# -> unI64 268
+  '\x010f'# -> unI64 270
+  '\x0111'# -> unI64 272
+  '\x0113'# -> unI64 274
+  '\x0115'# -> unI64 276
+  '\x0117'# -> unI64 278
+  '\x0119'# -> unI64 280
+  '\x011b'# -> unI64 282
+  '\x011d'# -> unI64 284
+  '\x011f'# -> unI64 286
+  '\x0121'# -> unI64 288
+  '\x0123'# -> unI64 290
+  '\x0125'# -> unI64 292
+  '\x0127'# -> unI64 294
+  '\x0129'# -> unI64 296
+  '\x012b'# -> unI64 298
+  '\x012d'# -> unI64 300
+  '\x012f'# -> unI64 302
+  '\x0131'# -> unI64 73
+  '\x0133'# -> unI64 306
+  '\x0135'# -> unI64 308
+  '\x0137'# -> unI64 310
+  '\x013a'# -> unI64 313
+  '\x013c'# -> unI64 315
+  '\x013e'# -> unI64 317
+  '\x0140'# -> unI64 319
+  '\x0142'# -> unI64 321
+  '\x0144'# -> unI64 323
+  '\x0146'# -> unI64 325
+  '\x0148'# -> unI64 327
+  '\x014b'# -> unI64 330
+  '\x014d'# -> unI64 332
+  '\x014f'# -> unI64 334
+  '\x0151'# -> unI64 336
+  '\x0153'# -> unI64 338
+  '\x0155'# -> unI64 340
+  '\x0157'# -> unI64 342
+  '\x0159'# -> unI64 344
+  '\x015b'# -> unI64 346
+  '\x015d'# -> unI64 348
+  '\x015f'# -> unI64 350
+  '\x0161'# -> unI64 352
+  '\x0163'# -> unI64 354
+  '\x0165'# -> unI64 356
+  '\x0167'# -> unI64 358
+  '\x0169'# -> unI64 360
+  '\x016b'# -> unI64 362
+  '\x016d'# -> unI64 364
+  '\x016f'# -> unI64 366
+  '\x0171'# -> unI64 368
+  '\x0173'# -> unI64 370
+  '\x0175'# -> unI64 372
+  '\x0177'# -> unI64 374
+  '\x017a'# -> unI64 377
+  '\x017c'# -> unI64 379
+  '\x017e'# -> unI64 381
+  '\x017f'# -> unI64 83
+  '\x0180'# -> unI64 579
+  '\x0183'# -> unI64 386
+  '\x0185'# -> unI64 388
+  '\x0188'# -> unI64 391
+  '\x018c'# -> unI64 395
+  '\x0192'# -> unI64 401
+  '\x0195'# -> unI64 502
+  '\x0199'# -> unI64 408
+  '\x019a'# -> unI64 573
+  '\x019b'# -> unI64 42972
+  '\x019e'# -> unI64 544
+  '\x01a1'# -> unI64 416
+  '\x01a3'# -> unI64 418
+  '\x01a5'# -> unI64 420
+  '\x01a8'# -> unI64 423
+  '\x01ad'# -> unI64 428
+  '\x01b0'# -> unI64 431
+  '\x01b4'# -> unI64 435
+  '\x01b6'# -> unI64 437
+  '\x01b9'# -> unI64 440
+  '\x01bd'# -> unI64 444
+  '\x01bf'# -> unI64 503
+  '\x01c4'# -> unI64 453
+  '\x01c6'# -> unI64 453
+  '\x01c7'# -> unI64 456
+  '\x01c9'# -> unI64 456
+  '\x01ca'# -> unI64 459
+  '\x01cc'# -> unI64 459
+  '\x01ce'# -> unI64 461
+  '\x01d0'# -> unI64 463
+  '\x01d2'# -> unI64 465
+  '\x01d4'# -> unI64 467
+  '\x01d6'# -> unI64 469
+  '\x01d8'# -> unI64 471
+  '\x01da'# -> unI64 473
+  '\x01dc'# -> unI64 475
+  '\x01dd'# -> unI64 398
+  '\x01df'# -> unI64 478
+  '\x01e1'# -> unI64 480
+  '\x01e3'# -> unI64 482
+  '\x01e5'# -> unI64 484
+  '\x01e7'# -> unI64 486
+  '\x01e9'# -> unI64 488
+  '\x01eb'# -> unI64 490
+  '\x01ed'# -> unI64 492
+  '\x01ef'# -> unI64 494
+  '\x01f1'# -> unI64 498
+  '\x01f3'# -> unI64 498
+  '\x01f5'# -> unI64 500
+  '\x01f9'# -> unI64 504
+  '\x01fb'# -> unI64 506
+  '\x01fd'# -> unI64 508
+  '\x01ff'# -> unI64 510
+  '\x0201'# -> unI64 512
+  '\x0203'# -> unI64 514
+  '\x0205'# -> unI64 516
+  '\x0207'# -> unI64 518
+  '\x0209'# -> unI64 520
+  '\x020b'# -> unI64 522
+  '\x020d'# -> unI64 524
+  '\x020f'# -> unI64 526
+  '\x0211'# -> unI64 528
+  '\x0213'# -> unI64 530
+  '\x0215'# -> unI64 532
+  '\x0217'# -> unI64 534
+  '\x0219'# -> unI64 536
+  '\x021b'# -> unI64 538
+  '\x021d'# -> unI64 540
+  '\x021f'# -> unI64 542
+  '\x0223'# -> unI64 546
+  '\x0225'# -> unI64 548
+  '\x0227'# -> unI64 550
+  '\x0229'# -> unI64 552
+  '\x022b'# -> unI64 554
+  '\x022d'# -> unI64 556
+  '\x022f'# -> unI64 558
+  '\x0231'# -> unI64 560
+  '\x0233'# -> unI64 562
+  '\x023c'# -> unI64 571
+  '\x023f'# -> unI64 11390
+  '\x0240'# -> unI64 11391
+  '\x0242'# -> unI64 577
+  '\x0247'# -> unI64 582
+  '\x0249'# -> unI64 584
+  '\x024b'# -> unI64 586
+  '\x024d'# -> unI64 588
+  '\x024f'# -> unI64 590
+  '\x0250'# -> unI64 11375
+  '\x0251'# -> unI64 11373
+  '\x0252'# -> unI64 11376
+  '\x0253'# -> unI64 385
+  '\x0254'# -> unI64 390
+  '\x0256'# -> unI64 393
+  '\x0257'# -> unI64 394
+  '\x0259'# -> unI64 399
+  '\x025b'# -> unI64 400
+  '\x025c'# -> unI64 42923
+  '\x0260'# -> unI64 403
+  '\x0261'# -> unI64 42924
+  '\x0263'# -> unI64 404
+  '\x0264'# -> unI64 42955
+  '\x0265'# -> unI64 42893
+  '\x0266'# -> unI64 42922
+  '\x0268'# -> unI64 407
+  '\x0269'# -> unI64 406
+  '\x026a'# -> unI64 42926
+  '\x026b'# -> unI64 11362
+  '\x026c'# -> unI64 42925
+  '\x026f'# -> unI64 412
+  '\x0271'# -> unI64 11374
+  '\x0272'# -> unI64 413
+  '\x0275'# -> unI64 415
+  '\x027d'# -> unI64 11364
+  '\x0280'# -> unI64 422
+  '\x0282'# -> unI64 42949
+  '\x0283'# -> unI64 425
+  '\x0287'# -> unI64 42929
+  '\x0288'# -> unI64 430
+  '\x0289'# -> unI64 580
+  '\x028a'# -> unI64 433
+  '\x028b'# -> unI64 434
+  '\x028c'# -> unI64 581
+  '\x0292'# -> unI64 439
+  '\x029d'# -> unI64 42930
+  '\x029e'# -> unI64 42928
+  '\x0345'# -> unI64 921
+  '\x0371'# -> unI64 880
+  '\x0373'# -> unI64 882
+  '\x0377'# -> unI64 886
+  '\x037b'# -> unI64 1021
+  '\x037c'# -> unI64 1022
+  '\x037d'# -> unI64 1023
+  '\x03ac'# -> unI64 902
+  '\x03ad'# -> unI64 904
+  '\x03ae'# -> unI64 905
+  '\x03af'# -> unI64 906
+  '\x03b1'# -> unI64 913
+  '\x03b2'# -> unI64 914
+  '\x03b3'# -> unI64 915
+  '\x03b4'# -> unI64 916
+  '\x03b5'# -> unI64 917
+  '\x03b6'# -> unI64 918
+  '\x03b7'# -> unI64 919
+  '\x03b8'# -> unI64 920
+  '\x03b9'# -> unI64 921
+  '\x03ba'# -> unI64 922
+  '\x03bb'# -> unI64 923
+  '\x03bc'# -> unI64 924
+  '\x03bd'# -> unI64 925
+  '\x03be'# -> unI64 926
+  '\x03bf'# -> unI64 927
+  '\x03c0'# -> unI64 928
+  '\x03c1'# -> unI64 929
+  '\x03c2'# -> unI64 931
+  '\x03c3'# -> unI64 931
+  '\x03c4'# -> unI64 932
+  '\x03c5'# -> unI64 933
+  '\x03c6'# -> unI64 934
+  '\x03c7'# -> unI64 935
+  '\x03c8'# -> unI64 936
+  '\x03c9'# -> unI64 937
+  '\x03ca'# -> unI64 938
+  '\x03cb'# -> unI64 939
+  '\x03cc'# -> unI64 908
+  '\x03cd'# -> unI64 910
+  '\x03ce'# -> unI64 911
+  '\x03d0'# -> unI64 914
+  '\x03d1'# -> unI64 920
+  '\x03d5'# -> unI64 934
+  '\x03d6'# -> unI64 928
+  '\x03d7'# -> unI64 975
+  '\x03d9'# -> unI64 984
+  '\x03db'# -> unI64 986
+  '\x03dd'# -> unI64 988
+  '\x03df'# -> unI64 990
+  '\x03e1'# -> unI64 992
+  '\x03e3'# -> unI64 994
+  '\x03e5'# -> unI64 996
+  '\x03e7'# -> unI64 998
+  '\x03e9'# -> unI64 1000
+  '\x03eb'# -> unI64 1002
+  '\x03ed'# -> unI64 1004
+  '\x03ef'# -> unI64 1006
+  '\x03f0'# -> unI64 922
+  '\x03f1'# -> unI64 929
+  '\x03f2'# -> unI64 1017
+  '\x03f3'# -> unI64 895
+  '\x03f5'# -> unI64 917
+  '\x03f8'# -> unI64 1015
+  '\x03fb'# -> unI64 1018
+  '\x0430'# -> unI64 1040
+  '\x0431'# -> unI64 1041
+  '\x0432'# -> unI64 1042
+  '\x0433'# -> unI64 1043
+  '\x0434'# -> unI64 1044
+  '\x0435'# -> unI64 1045
+  '\x0436'# -> unI64 1046
+  '\x0437'# -> unI64 1047
+  '\x0438'# -> unI64 1048
+  '\x0439'# -> unI64 1049
+  '\x043a'# -> unI64 1050
+  '\x043b'# -> unI64 1051
+  '\x043c'# -> unI64 1052
+  '\x043d'# -> unI64 1053
+  '\x043e'# -> unI64 1054
+  '\x043f'# -> unI64 1055
+  '\x0440'# -> unI64 1056
+  '\x0441'# -> unI64 1057
+  '\x0442'# -> unI64 1058
+  '\x0443'# -> unI64 1059
+  '\x0444'# -> unI64 1060
+  '\x0445'# -> unI64 1061
+  '\x0446'# -> unI64 1062
+  '\x0447'# -> unI64 1063
+  '\x0448'# -> unI64 1064
+  '\x0449'# -> unI64 1065
+  '\x044a'# -> unI64 1066
+  '\x044b'# -> unI64 1067
+  '\x044c'# -> unI64 1068
+  '\x044d'# -> unI64 1069
+  '\x044e'# -> unI64 1070
+  '\x044f'# -> unI64 1071
+  '\x0450'# -> unI64 1024
+  '\x0451'# -> unI64 1025
+  '\x0452'# -> unI64 1026
+  '\x0453'# -> unI64 1027
+  '\x0454'# -> unI64 1028
+  '\x0455'# -> unI64 1029
+  '\x0456'# -> unI64 1030
+  '\x0457'# -> unI64 1031
+  '\x0458'# -> unI64 1032
+  '\x0459'# -> unI64 1033
+  '\x045a'# -> unI64 1034
+  '\x045b'# -> unI64 1035
+  '\x045c'# -> unI64 1036
+  '\x045d'# -> unI64 1037
+  '\x045e'# -> unI64 1038
+  '\x045f'# -> unI64 1039
+  '\x0461'# -> unI64 1120
+  '\x0463'# -> unI64 1122
+  '\x0465'# -> unI64 1124
+  '\x0467'# -> unI64 1126
+  '\x0469'# -> unI64 1128
+  '\x046b'# -> unI64 1130
+  '\x046d'# -> unI64 1132
+  '\x046f'# -> unI64 1134
+  '\x0471'# -> unI64 1136
+  '\x0473'# -> unI64 1138
+  '\x0475'# -> unI64 1140
+  '\x0477'# -> unI64 1142
+  '\x0479'# -> unI64 1144
+  '\x047b'# -> unI64 1146
+  '\x047d'# -> unI64 1148
+  '\x047f'# -> unI64 1150
+  '\x0481'# -> unI64 1152
+  '\x048b'# -> unI64 1162
+  '\x048d'# -> unI64 1164
+  '\x048f'# -> unI64 1166
+  '\x0491'# -> unI64 1168
+  '\x0493'# -> unI64 1170
+  '\x0495'# -> unI64 1172
+  '\x0497'# -> unI64 1174
+  '\x0499'# -> unI64 1176
+  '\x049b'# -> unI64 1178
+  '\x049d'# -> unI64 1180
+  '\x049f'# -> unI64 1182
+  '\x04a1'# -> unI64 1184
+  '\x04a3'# -> unI64 1186
+  '\x04a5'# -> unI64 1188
+  '\x04a7'# -> unI64 1190
+  '\x04a9'# -> unI64 1192
+  '\x04ab'# -> unI64 1194
+  '\x04ad'# -> unI64 1196
+  '\x04af'# -> unI64 1198
+  '\x04b1'# -> unI64 1200
+  '\x04b3'# -> unI64 1202
+  '\x04b5'# -> unI64 1204
+  '\x04b7'# -> unI64 1206
+  '\x04b9'# -> unI64 1208
+  '\x04bb'# -> unI64 1210
+  '\x04bd'# -> unI64 1212
+  '\x04bf'# -> unI64 1214
+  '\x04c2'# -> unI64 1217
+  '\x04c4'# -> unI64 1219
+  '\x04c6'# -> unI64 1221
+  '\x04c8'# -> unI64 1223
+  '\x04ca'# -> unI64 1225
+  '\x04cc'# -> unI64 1227
+  '\x04ce'# -> unI64 1229
+  '\x04cf'# -> unI64 1216
+  '\x04d1'# -> unI64 1232
+  '\x04d3'# -> unI64 1234
+  '\x04d5'# -> unI64 1236
+  '\x04d7'# -> unI64 1238
+  '\x04d9'# -> unI64 1240
+  '\x04db'# -> unI64 1242
+  '\x04dd'# -> unI64 1244
+  '\x04df'# -> unI64 1246
+  '\x04e1'# -> unI64 1248
+  '\x04e3'# -> unI64 1250
+  '\x04e5'# -> unI64 1252
+  '\x04e7'# -> unI64 1254
+  '\x04e9'# -> unI64 1256
+  '\x04eb'# -> unI64 1258
+  '\x04ed'# -> unI64 1260
+  '\x04ef'# -> unI64 1262
+  '\x04f1'# -> unI64 1264
+  '\x04f3'# -> unI64 1266
+  '\x04f5'# -> unI64 1268
+  '\x04f7'# -> unI64 1270
+  '\x04f9'# -> unI64 1272
+  '\x04fb'# -> unI64 1274
+  '\x04fd'# -> unI64 1276
+  '\x04ff'# -> unI64 1278
+  '\x0501'# -> unI64 1280
+  '\x0503'# -> unI64 1282
+  '\x0505'# -> unI64 1284
+  '\x0507'# -> unI64 1286
+  '\x0509'# -> unI64 1288
+  '\x050b'# -> unI64 1290
+  '\x050d'# -> unI64 1292
+  '\x050f'# -> unI64 1294
+  '\x0511'# -> unI64 1296
+  '\x0513'# -> unI64 1298
+  '\x0515'# -> unI64 1300
+  '\x0517'# -> unI64 1302
+  '\x0519'# -> unI64 1304
+  '\x051b'# -> unI64 1306
+  '\x051d'# -> unI64 1308
+  '\x051f'# -> unI64 1310
+  '\x0521'# -> unI64 1312
+  '\x0523'# -> unI64 1314
+  '\x0525'# -> unI64 1316
+  '\x0527'# -> unI64 1318
+  '\x0529'# -> unI64 1320
+  '\x052b'# -> unI64 1322
+  '\x052d'# -> unI64 1324
+  '\x052f'# -> unI64 1326
+  '\x0561'# -> unI64 1329
+  '\x0562'# -> unI64 1330
+  '\x0563'# -> unI64 1331
+  '\x0564'# -> unI64 1332
+  '\x0565'# -> unI64 1333
+  '\x0566'# -> unI64 1334
+  '\x0567'# -> unI64 1335
+  '\x0568'# -> unI64 1336
+  '\x0569'# -> unI64 1337
+  '\x056a'# -> unI64 1338
+  '\x056b'# -> unI64 1339
+  '\x056c'# -> unI64 1340
+  '\x056d'# -> unI64 1341
+  '\x056e'# -> unI64 1342
+  '\x056f'# -> unI64 1343
+  '\x0570'# -> unI64 1344
+  '\x0571'# -> unI64 1345
+  '\x0572'# -> unI64 1346
+  '\x0573'# -> unI64 1347
+  '\x0574'# -> unI64 1348
+  '\x0575'# -> unI64 1349
+  '\x0576'# -> unI64 1350
+  '\x0577'# -> unI64 1351
+  '\x0578'# -> unI64 1352
+  '\x0579'# -> unI64 1353
+  '\x057a'# -> unI64 1354
+  '\x057b'# -> unI64 1355
+  '\x057c'# -> unI64 1356
+  '\x057d'# -> unI64 1357
+  '\x057e'# -> unI64 1358
+  '\x057f'# -> unI64 1359
+  '\x0580'# -> unI64 1360
+  '\x0581'# -> unI64 1361
+  '\x0582'# -> unI64 1362
+  '\x0583'# -> unI64 1363
+  '\x0584'# -> unI64 1364
+  '\x0585'# -> unI64 1365
+  '\x0586'# -> unI64 1366
+  '\x13f8'# -> unI64 5104
+  '\x13f9'# -> unI64 5105
+  '\x13fa'# -> unI64 5106
+  '\x13fb'# -> unI64 5107
+  '\x13fc'# -> unI64 5108
+  '\x13fd'# -> unI64 5109
+  '\x1c80'# -> unI64 1042
+  '\x1c81'# -> unI64 1044
+  '\x1c82'# -> unI64 1054
+  '\x1c83'# -> unI64 1057
+  '\x1c84'# -> unI64 1058
+  '\x1c85'# -> unI64 1058
+  '\x1c86'# -> unI64 1066
+  '\x1c87'# -> unI64 1122
+  '\x1c88'# -> unI64 42570
+  '\x1c8a'# -> unI64 7305
+  '\x1d79'# -> unI64 42877
+  '\x1d7d'# -> unI64 11363
+  '\x1d8e'# -> unI64 42950
+  '\x1e01'# -> unI64 7680
+  '\x1e03'# -> unI64 7682
+  '\x1e05'# -> unI64 7684
+  '\x1e07'# -> unI64 7686
+  '\x1e09'# -> unI64 7688
+  '\x1e0b'# -> unI64 7690
+  '\x1e0d'# -> unI64 7692
+  '\x1e0f'# -> unI64 7694
+  '\x1e11'# -> unI64 7696
+  '\x1e13'# -> unI64 7698
+  '\x1e15'# -> unI64 7700
+  '\x1e17'# -> unI64 7702
+  '\x1e19'# -> unI64 7704
+  '\x1e1b'# -> unI64 7706
+  '\x1e1d'# -> unI64 7708
+  '\x1e1f'# -> unI64 7710
+  '\x1e21'# -> unI64 7712
+  '\x1e23'# -> unI64 7714
+  '\x1e25'# -> unI64 7716
+  '\x1e27'# -> unI64 7718
+  '\x1e29'# -> unI64 7720
+  '\x1e2b'# -> unI64 7722
+  '\x1e2d'# -> unI64 7724
+  '\x1e2f'# -> unI64 7726
+  '\x1e31'# -> unI64 7728
+  '\x1e33'# -> unI64 7730
+  '\x1e35'# -> unI64 7732
+  '\x1e37'# -> unI64 7734
+  '\x1e39'# -> unI64 7736
+  '\x1e3b'# -> unI64 7738
+  '\x1e3d'# -> unI64 7740
+  '\x1e3f'# -> unI64 7742
+  '\x1e41'# -> unI64 7744
+  '\x1e43'# -> unI64 7746
+  '\x1e45'# -> unI64 7748
+  '\x1e47'# -> unI64 7750
+  '\x1e49'# -> unI64 7752
+  '\x1e4b'# -> unI64 7754
+  '\x1e4d'# -> unI64 7756
+  '\x1e4f'# -> unI64 7758
+  '\x1e51'# -> unI64 7760
+  '\x1e53'# -> unI64 7762
+  '\x1e55'# -> unI64 7764
+  '\x1e57'# -> unI64 7766
+  '\x1e59'# -> unI64 7768
+  '\x1e5b'# -> unI64 7770
+  '\x1e5d'# -> unI64 7772
+  '\x1e5f'# -> unI64 7774
+  '\x1e61'# -> unI64 7776
+  '\x1e63'# -> unI64 7778
+  '\x1e65'# -> unI64 7780
+  '\x1e67'# -> unI64 7782
+  '\x1e69'# -> unI64 7784
+  '\x1e6b'# -> unI64 7786
+  '\x1e6d'# -> unI64 7788
+  '\x1e6f'# -> unI64 7790
+  '\x1e71'# -> unI64 7792
+  '\x1e73'# -> unI64 7794
+  '\x1e75'# -> unI64 7796
+  '\x1e77'# -> unI64 7798
+  '\x1e79'# -> unI64 7800
+  '\x1e7b'# -> unI64 7802
+  '\x1e7d'# -> unI64 7804
+  '\x1e7f'# -> unI64 7806
+  '\x1e81'# -> unI64 7808
+  '\x1e83'# -> unI64 7810
+  '\x1e85'# -> unI64 7812
+  '\x1e87'# -> unI64 7814
+  '\x1e89'# -> unI64 7816
+  '\x1e8b'# -> unI64 7818
+  '\x1e8d'# -> unI64 7820
+  '\x1e8f'# -> unI64 7822
+  '\x1e91'# -> unI64 7824
+  '\x1e93'# -> unI64 7826
+  '\x1e95'# -> unI64 7828
+  '\x1e9b'# -> unI64 7776
+  '\x1ea1'# -> unI64 7840
+  '\x1ea3'# -> unI64 7842
+  '\x1ea5'# -> unI64 7844
+  '\x1ea7'# -> unI64 7846
+  '\x1ea9'# -> unI64 7848
+  '\x1eab'# -> unI64 7850
+  '\x1ead'# -> unI64 7852
+  '\x1eaf'# -> unI64 7854
+  '\x1eb1'# -> unI64 7856
+  '\x1eb3'# -> unI64 7858
+  '\x1eb5'# -> unI64 7860
+  '\x1eb7'# -> unI64 7862
+  '\x1eb9'# -> unI64 7864
+  '\x1ebb'# -> unI64 7866
+  '\x1ebd'# -> unI64 7868
+  '\x1ebf'# -> unI64 7870
+  '\x1ec1'# -> unI64 7872
+  '\x1ec3'# -> unI64 7874
+  '\x1ec5'# -> unI64 7876
+  '\x1ec7'# -> unI64 7878
+  '\x1ec9'# -> unI64 7880
+  '\x1ecb'# -> unI64 7882
+  '\x1ecd'# -> unI64 7884
+  '\x1ecf'# -> unI64 7886
+  '\x1ed1'# -> unI64 7888
+  '\x1ed3'# -> unI64 7890
+  '\x1ed5'# -> unI64 7892
+  '\x1ed7'# -> unI64 7894
+  '\x1ed9'# -> unI64 7896
+  '\x1edb'# -> unI64 7898
+  '\x1edd'# -> unI64 7900
+  '\x1edf'# -> unI64 7902
+  '\x1ee1'# -> unI64 7904
+  '\x1ee3'# -> unI64 7906
+  '\x1ee5'# -> unI64 7908
+  '\x1ee7'# -> unI64 7910
+  '\x1ee9'# -> unI64 7912
+  '\x1eeb'# -> unI64 7914
+  '\x1eed'# -> unI64 7916
+  '\x1eef'# -> unI64 7918
+  '\x1ef1'# -> unI64 7920
+  '\x1ef3'# -> unI64 7922
+  '\x1ef5'# -> unI64 7924
+  '\x1ef7'# -> unI64 7926
+  '\x1ef9'# -> unI64 7928
+  '\x1efb'# -> unI64 7930
+  '\x1efd'# -> unI64 7932
+  '\x1eff'# -> unI64 7934
+  '\x1f00'# -> unI64 7944
+  '\x1f01'# -> unI64 7945
+  '\x1f02'# -> unI64 7946
+  '\x1f03'# -> unI64 7947
+  '\x1f04'# -> unI64 7948
+  '\x1f05'# -> unI64 7949
+  '\x1f06'# -> unI64 7950
+  '\x1f07'# -> unI64 7951
+  '\x1f10'# -> unI64 7960
+  '\x1f11'# -> unI64 7961
+  '\x1f12'# -> unI64 7962
+  '\x1f13'# -> unI64 7963
+  '\x1f14'# -> unI64 7964
+  '\x1f15'# -> unI64 7965
+  '\x1f20'# -> unI64 7976
+  '\x1f21'# -> unI64 7977
+  '\x1f22'# -> unI64 7978
+  '\x1f23'# -> unI64 7979
+  '\x1f24'# -> unI64 7980
+  '\x1f25'# -> unI64 7981
+  '\x1f26'# -> unI64 7982
+  '\x1f27'# -> unI64 7983
+  '\x1f30'# -> unI64 7992
+  '\x1f31'# -> unI64 7993
+  '\x1f32'# -> unI64 7994
+  '\x1f33'# -> unI64 7995
+  '\x1f34'# -> unI64 7996
+  '\x1f35'# -> unI64 7997
+  '\x1f36'# -> unI64 7998
+  '\x1f37'# -> unI64 7999
+  '\x1f40'# -> unI64 8008
+  '\x1f41'# -> unI64 8009
+  '\x1f42'# -> unI64 8010
+  '\x1f43'# -> unI64 8011
+  '\x1f44'# -> unI64 8012
+  '\x1f45'# -> unI64 8013
+  '\x1f51'# -> unI64 8025
+  '\x1f53'# -> unI64 8027
+  '\x1f55'# -> unI64 8029
+  '\x1f57'# -> unI64 8031
+  '\x1f60'# -> unI64 8040
+  '\x1f61'# -> unI64 8041
+  '\x1f62'# -> unI64 8042
+  '\x1f63'# -> unI64 8043
+  '\x1f64'# -> unI64 8044
+  '\x1f65'# -> unI64 8045
+  '\x1f66'# -> unI64 8046
+  '\x1f67'# -> unI64 8047
+  '\x1f70'# -> unI64 8122
+  '\x1f71'# -> unI64 8123
+  '\x1f72'# -> unI64 8136
+  '\x1f73'# -> unI64 8137
+  '\x1f74'# -> unI64 8138
+  '\x1f75'# -> unI64 8139
+  '\x1f76'# -> unI64 8154
+  '\x1f77'# -> unI64 8155
+  '\x1f78'# -> unI64 8184
+  '\x1f79'# -> unI64 8185
+  '\x1f7a'# -> unI64 8170
+  '\x1f7b'# -> unI64 8171
+  '\x1f7c'# -> unI64 8186
+  '\x1f7d'# -> unI64 8187
+  '\x1fb0'# -> unI64 8120
+  '\x1fb1'# -> unI64 8121
+  '\x1fbe'# -> unI64 921
+  '\x1fd0'# -> unI64 8152
+  '\x1fd1'# -> unI64 8153
+  '\x1fe0'# -> unI64 8168
+  '\x1fe1'# -> unI64 8169
+  '\x1fe5'# -> unI64 8172
+  '\x214e'# -> unI64 8498
+  '\x2170'# -> unI64 8544
+  '\x2171'# -> unI64 8545
+  '\x2172'# -> unI64 8546
+  '\x2173'# -> unI64 8547
+  '\x2174'# -> unI64 8548
+  '\x2175'# -> unI64 8549
+  '\x2176'# -> unI64 8550
+  '\x2177'# -> unI64 8551
+  '\x2178'# -> unI64 8552
+  '\x2179'# -> unI64 8553
+  '\x217a'# -> unI64 8554
+  '\x217b'# -> unI64 8555
+  '\x217c'# -> unI64 8556
+  '\x217d'# -> unI64 8557
+  '\x217e'# -> unI64 8558
+  '\x217f'# -> unI64 8559
+  '\x2184'# -> unI64 8579
+  '\x24d0'# -> unI64 9398
+  '\x24d1'# -> unI64 9399
+  '\x24d2'# -> unI64 9400
+  '\x24d3'# -> unI64 9401
+  '\x24d4'# -> unI64 9402
+  '\x24d5'# -> unI64 9403
+  '\x24d6'# -> unI64 9404
+  '\x24d7'# -> unI64 9405
+  '\x24d8'# -> unI64 9406
+  '\x24d9'# -> unI64 9407
+  '\x24da'# -> unI64 9408
+  '\x24db'# -> unI64 9409
+  '\x24dc'# -> unI64 9410
+  '\x24dd'# -> unI64 9411
+  '\x24de'# -> unI64 9412
+  '\x24df'# -> unI64 9413
+  '\x24e0'# -> unI64 9414
+  '\x24e1'# -> unI64 9415
+  '\x24e2'# -> unI64 9416
+  '\x24e3'# -> unI64 9417
+  '\x24e4'# -> unI64 9418
+  '\x24e5'# -> unI64 9419
+  '\x24e6'# -> unI64 9420
+  '\x24e7'# -> unI64 9421
+  '\x24e8'# -> unI64 9422
+  '\x24e9'# -> unI64 9423
+  '\x2c30'# -> unI64 11264
+  '\x2c31'# -> unI64 11265
+  '\x2c32'# -> unI64 11266
+  '\x2c33'# -> unI64 11267
+  '\x2c34'# -> unI64 11268
+  '\x2c35'# -> unI64 11269
+  '\x2c36'# -> unI64 11270
+  '\x2c37'# -> unI64 11271
+  '\x2c38'# -> unI64 11272
+  '\x2c39'# -> unI64 11273
+  '\x2c3a'# -> unI64 11274
+  '\x2c3b'# -> unI64 11275
+  '\x2c3c'# -> unI64 11276
+  '\x2c3d'# -> unI64 11277
+  '\x2c3e'# -> unI64 11278
+  '\x2c3f'# -> unI64 11279
+  '\x2c40'# -> unI64 11280
+  '\x2c41'# -> unI64 11281
+  '\x2c42'# -> unI64 11282
+  '\x2c43'# -> unI64 11283
+  '\x2c44'# -> unI64 11284
+  '\x2c45'# -> unI64 11285
+  '\x2c46'# -> unI64 11286
+  '\x2c47'# -> unI64 11287
+  '\x2c48'# -> unI64 11288
+  '\x2c49'# -> unI64 11289
+  '\x2c4a'# -> unI64 11290
+  '\x2c4b'# -> unI64 11291
+  '\x2c4c'# -> unI64 11292
+  '\x2c4d'# -> unI64 11293
+  '\x2c4e'# -> unI64 11294
+  '\x2c4f'# -> unI64 11295
+  '\x2c50'# -> unI64 11296
+  '\x2c51'# -> unI64 11297
+  '\x2c52'# -> unI64 11298
+  '\x2c53'# -> unI64 11299
+  '\x2c54'# -> unI64 11300
+  '\x2c55'# -> unI64 11301
+  '\x2c56'# -> unI64 11302
+  '\x2c57'# -> unI64 11303
+  '\x2c58'# -> unI64 11304
+  '\x2c59'# -> unI64 11305
+  '\x2c5a'# -> unI64 11306
+  '\x2c5b'# -> unI64 11307
+  '\x2c5c'# -> unI64 11308
+  '\x2c5d'# -> unI64 11309
+  '\x2c5e'# -> unI64 11310
+  '\x2c5f'# -> unI64 11311
+  '\x2c61'# -> unI64 11360
+  '\x2c65'# -> unI64 570
+  '\x2c66'# -> unI64 574
+  '\x2c68'# -> unI64 11367
+  '\x2c6a'# -> unI64 11369
+  '\x2c6c'# -> unI64 11371
+  '\x2c73'# -> unI64 11378
+  '\x2c76'# -> unI64 11381
+  '\x2c81'# -> unI64 11392
+  '\x2c83'# -> unI64 11394
+  '\x2c85'# -> unI64 11396
+  '\x2c87'# -> unI64 11398
+  '\x2c89'# -> unI64 11400
+  '\x2c8b'# -> unI64 11402
+  '\x2c8d'# -> unI64 11404
+  '\x2c8f'# -> unI64 11406
+  '\x2c91'# -> unI64 11408
+  '\x2c93'# -> unI64 11410
+  '\x2c95'# -> unI64 11412
+  '\x2c97'# -> unI64 11414
+  '\x2c99'# -> unI64 11416
+  '\x2c9b'# -> unI64 11418
+  '\x2c9d'# -> unI64 11420
+  '\x2c9f'# -> unI64 11422
+  '\x2ca1'# -> unI64 11424
+  '\x2ca3'# -> unI64 11426
+  '\x2ca5'# -> unI64 11428
+  '\x2ca7'# -> unI64 11430
+  '\x2ca9'# -> unI64 11432
+  '\x2cab'# -> unI64 11434
+  '\x2cad'# -> unI64 11436
+  '\x2caf'# -> unI64 11438
+  '\x2cb1'# -> unI64 11440
+  '\x2cb3'# -> unI64 11442
+  '\x2cb5'# -> unI64 11444
+  '\x2cb7'# -> unI64 11446
+  '\x2cb9'# -> unI64 11448
+  '\x2cbb'# -> unI64 11450
+  '\x2cbd'# -> unI64 11452
+  '\x2cbf'# -> unI64 11454
+  '\x2cc1'# -> unI64 11456
+  '\x2cc3'# -> unI64 11458
+  '\x2cc5'# -> unI64 11460
+  '\x2cc7'# -> unI64 11462
+  '\x2cc9'# -> unI64 11464
+  '\x2ccb'# -> unI64 11466
+  '\x2ccd'# -> unI64 11468
+  '\x2ccf'# -> unI64 11470
+  '\x2cd1'# -> unI64 11472
+  '\x2cd3'# -> unI64 11474
+  '\x2cd5'# -> unI64 11476
+  '\x2cd7'# -> unI64 11478
+  '\x2cd9'# -> unI64 11480
+  '\x2cdb'# -> unI64 11482
+  '\x2cdd'# -> unI64 11484
+  '\x2cdf'# -> unI64 11486
+  '\x2ce1'# -> unI64 11488
+  '\x2ce3'# -> unI64 11490
+  '\x2cec'# -> unI64 11499
+  '\x2cee'# -> unI64 11501
+  '\x2cf3'# -> unI64 11506
+  '\x2d00'# -> unI64 4256
+  '\x2d01'# -> unI64 4257
+  '\x2d02'# -> unI64 4258
+  '\x2d03'# -> unI64 4259
+  '\x2d04'# -> unI64 4260
+  '\x2d05'# -> unI64 4261
+  '\x2d06'# -> unI64 4262
+  '\x2d07'# -> unI64 4263
+  '\x2d08'# -> unI64 4264
+  '\x2d09'# -> unI64 4265
+  '\x2d0a'# -> unI64 4266
+  '\x2d0b'# -> unI64 4267
+  '\x2d0c'# -> unI64 4268
+  '\x2d0d'# -> unI64 4269
+  '\x2d0e'# -> unI64 4270
+  '\x2d0f'# -> unI64 4271
+  '\x2d10'# -> unI64 4272
+  '\x2d11'# -> unI64 4273
+  '\x2d12'# -> unI64 4274
+  '\x2d13'# -> unI64 4275
+  '\x2d14'# -> unI64 4276
+  '\x2d15'# -> unI64 4277
+  '\x2d16'# -> unI64 4278
+  '\x2d17'# -> unI64 4279
+  '\x2d18'# -> unI64 4280
+  '\x2d19'# -> unI64 4281
+  '\x2d1a'# -> unI64 4282
+  '\x2d1b'# -> unI64 4283
+  '\x2d1c'# -> unI64 4284
+  '\x2d1d'# -> unI64 4285
+  '\x2d1e'# -> unI64 4286
+  '\x2d1f'# -> unI64 4287
+  '\x2d20'# -> unI64 4288
+  '\x2d21'# -> unI64 4289
+  '\x2d22'# -> unI64 4290
+  '\x2d23'# -> unI64 4291
+  '\x2d24'# -> unI64 4292
+  '\x2d25'# -> unI64 4293
+  '\x2d27'# -> unI64 4295
+  '\x2d2d'# -> unI64 4301
+  '\xa641'# -> unI64 42560
+  '\xa643'# -> unI64 42562
+  '\xa645'# -> unI64 42564
+  '\xa647'# -> unI64 42566
+  '\xa649'# -> unI64 42568
+  '\xa64b'# -> unI64 42570
+  '\xa64d'# -> unI64 42572
+  '\xa64f'# -> unI64 42574
+  '\xa651'# -> unI64 42576
+  '\xa653'# -> unI64 42578
+  '\xa655'# -> unI64 42580
+  '\xa657'# -> unI64 42582
+  '\xa659'# -> unI64 42584
+  '\xa65b'# -> unI64 42586
+  '\xa65d'# -> unI64 42588
+  '\xa65f'# -> unI64 42590
+  '\xa661'# -> unI64 42592
+  '\xa663'# -> unI64 42594
+  '\xa665'# -> unI64 42596
+  '\xa667'# -> unI64 42598
+  '\xa669'# -> unI64 42600
+  '\xa66b'# -> unI64 42602
+  '\xa66d'# -> unI64 42604
+  '\xa681'# -> unI64 42624
+  '\xa683'# -> unI64 42626
+  '\xa685'# -> unI64 42628
+  '\xa687'# -> unI64 42630
+  '\xa689'# -> unI64 42632
+  '\xa68b'# -> unI64 42634
+  '\xa68d'# -> unI64 42636
+  '\xa68f'# -> unI64 42638
+  '\xa691'# -> unI64 42640
+  '\xa693'# -> unI64 42642
+  '\xa695'# -> unI64 42644
+  '\xa697'# -> unI64 42646
+  '\xa699'# -> unI64 42648
+  '\xa69b'# -> unI64 42650
+  '\xa723'# -> unI64 42786
+  '\xa725'# -> unI64 42788
+  '\xa727'# -> unI64 42790
+  '\xa729'# -> unI64 42792
+  '\xa72b'# -> unI64 42794
+  '\xa72d'# -> unI64 42796
+  '\xa72f'# -> unI64 42798
+  '\xa733'# -> unI64 42802
+  '\xa735'# -> unI64 42804
+  '\xa737'# -> unI64 42806
+  '\xa739'# -> unI64 42808
+  '\xa73b'# -> unI64 42810
+  '\xa73d'# -> unI64 42812
+  '\xa73f'# -> unI64 42814
+  '\xa741'# -> unI64 42816
+  '\xa743'# -> unI64 42818
+  '\xa745'# -> unI64 42820
+  '\xa747'# -> unI64 42822
+  '\xa749'# -> unI64 42824
+  '\xa74b'# -> unI64 42826
+  '\xa74d'# -> unI64 42828
+  '\xa74f'# -> unI64 42830
+  '\xa751'# -> unI64 42832
+  '\xa753'# -> unI64 42834
+  '\xa755'# -> unI64 42836
+  '\xa757'# -> unI64 42838
+  '\xa759'# -> unI64 42840
+  '\xa75b'# -> unI64 42842
+  '\xa75d'# -> unI64 42844
+  '\xa75f'# -> unI64 42846
+  '\xa761'# -> unI64 42848
+  '\xa763'# -> unI64 42850
+  '\xa765'# -> unI64 42852
+  '\xa767'# -> unI64 42854
+  '\xa769'# -> unI64 42856
+  '\xa76b'# -> unI64 42858
+  '\xa76d'# -> unI64 42860
+  '\xa76f'# -> unI64 42862
+  '\xa77a'# -> unI64 42873
+  '\xa77c'# -> unI64 42875
+  '\xa77f'# -> unI64 42878
+  '\xa781'# -> unI64 42880
+  '\xa783'# -> unI64 42882
+  '\xa785'# -> unI64 42884
+  '\xa787'# -> unI64 42886
+  '\xa78c'# -> unI64 42891
+  '\xa791'# -> unI64 42896
+  '\xa793'# -> unI64 42898
+  '\xa794'# -> unI64 42948
+  '\xa797'# -> unI64 42902
+  '\xa799'# -> unI64 42904
+  '\xa79b'# -> unI64 42906
+  '\xa79d'# -> unI64 42908
+  '\xa79f'# -> unI64 42910
+  '\xa7a1'# -> unI64 42912
+  '\xa7a3'# -> unI64 42914
+  '\xa7a5'# -> unI64 42916
+  '\xa7a7'# -> unI64 42918
+  '\xa7a9'# -> unI64 42920
+  '\xa7b5'# -> unI64 42932
+  '\xa7b7'# -> unI64 42934
+  '\xa7b9'# -> unI64 42936
+  '\xa7bb'# -> unI64 42938
+  '\xa7bd'# -> unI64 42940
+  '\xa7bf'# -> unI64 42942
+  '\xa7c1'# -> unI64 42944
+  '\xa7c3'# -> unI64 42946
+  '\xa7c8'# -> unI64 42951
+  '\xa7ca'# -> unI64 42953
+  '\xa7cd'# -> unI64 42956
+  '\xa7cf'# -> unI64 42958
+  '\xa7d1'# -> unI64 42960
+  '\xa7d3'# -> unI64 42962
+  '\xa7d5'# -> unI64 42964
+  '\xa7d7'# -> unI64 42966
+  '\xa7d9'# -> unI64 42968
+  '\xa7db'# -> unI64 42970
+  '\xa7f6'# -> unI64 42997
+  '\xab53'# -> unI64 42931
+  '\xab70'# -> unI64 5024
+  '\xab71'# -> unI64 5025
+  '\xab72'# -> unI64 5026
+  '\xab73'# -> unI64 5027
+  '\xab74'# -> unI64 5028
+  '\xab75'# -> unI64 5029
+  '\xab76'# -> unI64 5030
+  '\xab77'# -> unI64 5031
+  '\xab78'# -> unI64 5032
+  '\xab79'# -> unI64 5033
+  '\xab7a'# -> unI64 5034
+  '\xab7b'# -> unI64 5035
+  '\xab7c'# -> unI64 5036
+  '\xab7d'# -> unI64 5037
+  '\xab7e'# -> unI64 5038
+  '\xab7f'# -> unI64 5039
+  '\xab80'# -> unI64 5040
+  '\xab81'# -> unI64 5041
+  '\xab82'# -> unI64 5042
+  '\xab83'# -> unI64 5043
+  '\xab84'# -> unI64 5044
+  '\xab85'# -> unI64 5045
+  '\xab86'# -> unI64 5046
+  '\xab87'# -> unI64 5047
+  '\xab88'# -> unI64 5048
+  '\xab89'# -> unI64 5049
+  '\xab8a'# -> unI64 5050
+  '\xab8b'# -> unI64 5051
+  '\xab8c'# -> unI64 5052
+  '\xab8d'# -> unI64 5053
+  '\xab8e'# -> unI64 5054
+  '\xab8f'# -> unI64 5055
+  '\xab90'# -> unI64 5056
+  '\xab91'# -> unI64 5057
+  '\xab92'# -> unI64 5058
+  '\xab93'# -> unI64 5059
+  '\xab94'# -> unI64 5060
+  '\xab95'# -> unI64 5061
+  '\xab96'# -> unI64 5062
+  '\xab97'# -> unI64 5063
+  '\xab98'# -> unI64 5064
+  '\xab99'# -> unI64 5065
+  '\xab9a'# -> unI64 5066
+  '\xab9b'# -> unI64 5067
+  '\xab9c'# -> unI64 5068
+  '\xab9d'# -> unI64 5069
+  '\xab9e'# -> unI64 5070
+  '\xab9f'# -> unI64 5071
+  '\xaba0'# -> unI64 5072
+  '\xaba1'# -> unI64 5073
+  '\xaba2'# -> unI64 5074
+  '\xaba3'# -> unI64 5075
+  '\xaba4'# -> unI64 5076
+  '\xaba5'# -> unI64 5077
+  '\xaba6'# -> unI64 5078
+  '\xaba7'# -> unI64 5079
+  '\xaba8'# -> unI64 5080
+  '\xaba9'# -> unI64 5081
+  '\xabaa'# -> unI64 5082
+  '\xabab'# -> unI64 5083
+  '\xabac'# -> unI64 5084
+  '\xabad'# -> unI64 5085
+  '\xabae'# -> unI64 5086
+  '\xabaf'# -> unI64 5087
+  '\xabb0'# -> unI64 5088
+  '\xabb1'# -> unI64 5089
+  '\xabb2'# -> unI64 5090
+  '\xabb3'# -> unI64 5091
+  '\xabb4'# -> unI64 5092
+  '\xabb5'# -> unI64 5093
+  '\xabb6'# -> unI64 5094
+  '\xabb7'# -> unI64 5095
+  '\xabb8'# -> unI64 5096
+  '\xabb9'# -> unI64 5097
+  '\xabba'# -> unI64 5098
+  '\xabbb'# -> unI64 5099
+  '\xabbc'# -> unI64 5100
+  '\xabbd'# -> unI64 5101
+  '\xabbe'# -> unI64 5102
+  '\xabbf'# -> unI64 5103
+  '\xff41'# -> unI64 65313
+  '\xff42'# -> unI64 65314
+  '\xff43'# -> unI64 65315
+  '\xff44'# -> unI64 65316
+  '\xff45'# -> unI64 65317
+  '\xff46'# -> unI64 65318
+  '\xff47'# -> unI64 65319
+  '\xff48'# -> unI64 65320
+  '\xff49'# -> unI64 65321
+  '\xff4a'# -> unI64 65322
+  '\xff4b'# -> unI64 65323
+  '\xff4c'# -> unI64 65324
+  '\xff4d'# -> unI64 65325
+  '\xff4e'# -> unI64 65326
+  '\xff4f'# -> unI64 65327
+  '\xff50'# -> unI64 65328
+  '\xff51'# -> unI64 65329
+  '\xff52'# -> unI64 65330
+  '\xff53'# -> unI64 65331
+  '\xff54'# -> unI64 65332
+  '\xff55'# -> unI64 65333
+  '\xff56'# -> unI64 65334
+  '\xff57'# -> unI64 65335
+  '\xff58'# -> unI64 65336
+  '\xff59'# -> unI64 65337
+  '\xff5a'# -> unI64 65338
+  '\x10428'# -> unI64 66560
+  '\x10429'# -> unI64 66561
+  '\x1042a'# -> unI64 66562
+  '\x1042b'# -> unI64 66563
+  '\x1042c'# -> unI64 66564
+  '\x1042d'# -> unI64 66565
+  '\x1042e'# -> unI64 66566
+  '\x1042f'# -> unI64 66567
+  '\x10430'# -> unI64 66568
+  '\x10431'# -> unI64 66569
+  '\x10432'# -> unI64 66570
+  '\x10433'# -> unI64 66571
+  '\x10434'# -> unI64 66572
+  '\x10435'# -> unI64 66573
+  '\x10436'# -> unI64 66574
+  '\x10437'# -> unI64 66575
+  '\x10438'# -> unI64 66576
+  '\x10439'# -> unI64 66577
+  '\x1043a'# -> unI64 66578
+  '\x1043b'# -> unI64 66579
+  '\x1043c'# -> unI64 66580
+  '\x1043d'# -> unI64 66581
+  '\x1043e'# -> unI64 66582
+  '\x1043f'# -> unI64 66583
+  '\x10440'# -> unI64 66584
+  '\x10441'# -> unI64 66585
+  '\x10442'# -> unI64 66586
+  '\x10443'# -> unI64 66587
+  '\x10444'# -> unI64 66588
+  '\x10445'# -> unI64 66589
+  '\x10446'# -> unI64 66590
+  '\x10447'# -> unI64 66591
+  '\x10448'# -> unI64 66592
+  '\x10449'# -> unI64 66593
+  '\x1044a'# -> unI64 66594
+  '\x1044b'# -> unI64 66595
+  '\x1044c'# -> unI64 66596
+  '\x1044d'# -> unI64 66597
+  '\x1044e'# -> unI64 66598
+  '\x1044f'# -> unI64 66599
+  '\x104d8'# -> unI64 66736
+  '\x104d9'# -> unI64 66737
+  '\x104da'# -> unI64 66738
+  '\x104db'# -> unI64 66739
+  '\x104dc'# -> unI64 66740
+  '\x104dd'# -> unI64 66741
+  '\x104de'# -> unI64 66742
+  '\x104df'# -> unI64 66743
+  '\x104e0'# -> unI64 66744
+  '\x104e1'# -> unI64 66745
+  '\x104e2'# -> unI64 66746
+  '\x104e3'# -> unI64 66747
+  '\x104e4'# -> unI64 66748
+  '\x104e5'# -> unI64 66749
+  '\x104e6'# -> unI64 66750
+  '\x104e7'# -> unI64 66751
+  '\x104e8'# -> unI64 66752
+  '\x104e9'# -> unI64 66753
+  '\x104ea'# -> unI64 66754
+  '\x104eb'# -> unI64 66755
+  '\x104ec'# -> unI64 66756
+  '\x104ed'# -> unI64 66757
+  '\x104ee'# -> unI64 66758
+  '\x104ef'# -> unI64 66759
+  '\x104f0'# -> unI64 66760
+  '\x104f1'# -> unI64 66761
+  '\x104f2'# -> unI64 66762
+  '\x104f3'# -> unI64 66763
+  '\x104f4'# -> unI64 66764
+  '\x104f5'# -> unI64 66765
+  '\x104f6'# -> unI64 66766
+  '\x104f7'# -> unI64 66767
+  '\x104f8'# -> unI64 66768
+  '\x104f9'# -> unI64 66769
+  '\x104fa'# -> unI64 66770
+  '\x104fb'# -> unI64 66771
+  '\x10597'# -> unI64 66928
+  '\x10598'# -> unI64 66929
+  '\x10599'# -> unI64 66930
+  '\x1059a'# -> unI64 66931
+  '\x1059b'# -> unI64 66932
+  '\x1059c'# -> unI64 66933
+  '\x1059d'# -> unI64 66934
+  '\x1059e'# -> unI64 66935
+  '\x1059f'# -> unI64 66936
+  '\x105a0'# -> unI64 66937
+  '\x105a1'# -> unI64 66938
+  '\x105a3'# -> unI64 66940
+  '\x105a4'# -> unI64 66941
+  '\x105a5'# -> unI64 66942
+  '\x105a6'# -> unI64 66943
+  '\x105a7'# -> unI64 66944
+  '\x105a8'# -> unI64 66945
+  '\x105a9'# -> unI64 66946
+  '\x105aa'# -> unI64 66947
+  '\x105ab'# -> unI64 66948
+  '\x105ac'# -> unI64 66949
+  '\x105ad'# -> unI64 66950
+  '\x105ae'# -> unI64 66951
+  '\x105af'# -> unI64 66952
+  '\x105b0'# -> unI64 66953
+  '\x105b1'# -> unI64 66954
+  '\x105b3'# -> unI64 66956
+  '\x105b4'# -> unI64 66957
+  '\x105b5'# -> unI64 66958
+  '\x105b6'# -> unI64 66959
+  '\x105b7'# -> unI64 66960
+  '\x105b8'# -> unI64 66961
+  '\x105b9'# -> unI64 66962
+  '\x105bb'# -> unI64 66964
+  '\x105bc'# -> unI64 66965
+  '\x10cc0'# -> unI64 68736
+  '\x10cc1'# -> unI64 68737
+  '\x10cc2'# -> unI64 68738
+  '\x10cc3'# -> unI64 68739
+  '\x10cc4'# -> unI64 68740
+  '\x10cc5'# -> unI64 68741
+  '\x10cc6'# -> unI64 68742
+  '\x10cc7'# -> unI64 68743
+  '\x10cc8'# -> unI64 68744
+  '\x10cc9'# -> unI64 68745
+  '\x10cca'# -> unI64 68746
+  '\x10ccb'# -> unI64 68747
+  '\x10ccc'# -> unI64 68748
+  '\x10ccd'# -> unI64 68749
+  '\x10cce'# -> unI64 68750
+  '\x10ccf'# -> unI64 68751
+  '\x10cd0'# -> unI64 68752
+  '\x10cd1'# -> unI64 68753
+  '\x10cd2'# -> unI64 68754
+  '\x10cd3'# -> unI64 68755
+  '\x10cd4'# -> unI64 68756
+  '\x10cd5'# -> unI64 68757
+  '\x10cd6'# -> unI64 68758
+  '\x10cd7'# -> unI64 68759
+  '\x10cd8'# -> unI64 68760
+  '\x10cd9'# -> unI64 68761
+  '\x10cda'# -> unI64 68762
+  '\x10cdb'# -> unI64 68763
+  '\x10cdc'# -> unI64 68764
+  '\x10cdd'# -> unI64 68765
+  '\x10cde'# -> unI64 68766
+  '\x10cdf'# -> unI64 68767
+  '\x10ce0'# -> unI64 68768
+  '\x10ce1'# -> unI64 68769
+  '\x10ce2'# -> unI64 68770
+  '\x10ce3'# -> unI64 68771
+  '\x10ce4'# -> unI64 68772
+  '\x10ce5'# -> unI64 68773
+  '\x10ce6'# -> unI64 68774
+  '\x10ce7'# -> unI64 68775
+  '\x10ce8'# -> unI64 68776
+  '\x10ce9'# -> unI64 68777
+  '\x10cea'# -> unI64 68778
+  '\x10ceb'# -> unI64 68779
+  '\x10cec'# -> unI64 68780
+  '\x10ced'# -> unI64 68781
+  '\x10cee'# -> unI64 68782
+  '\x10cef'# -> unI64 68783
+  '\x10cf0'# -> unI64 68784
+  '\x10cf1'# -> unI64 68785
+  '\x10cf2'# -> unI64 68786
+  '\x10d70'# -> unI64 68944
+  '\x10d71'# -> unI64 68945
+  '\x10d72'# -> unI64 68946
+  '\x10d73'# -> unI64 68947
+  '\x10d74'# -> unI64 68948
+  '\x10d75'# -> unI64 68949
+  '\x10d76'# -> unI64 68950
+  '\x10d77'# -> unI64 68951
+  '\x10d78'# -> unI64 68952
+  '\x10d79'# -> unI64 68953
+  '\x10d7a'# -> unI64 68954
+  '\x10d7b'# -> unI64 68955
+  '\x10d7c'# -> unI64 68956
+  '\x10d7d'# -> unI64 68957
+  '\x10d7e'# -> unI64 68958
+  '\x10d7f'# -> unI64 68959
+  '\x10d80'# -> unI64 68960
+  '\x10d81'# -> unI64 68961
+  '\x10d82'# -> unI64 68962
+  '\x10d83'# -> unI64 68963
+  '\x10d84'# -> unI64 68964
+  '\x10d85'# -> unI64 68965
+  '\x118c0'# -> unI64 71840
+  '\x118c1'# -> unI64 71841
+  '\x118c2'# -> unI64 71842
+  '\x118c3'# -> unI64 71843
+  '\x118c4'# -> unI64 71844
+  '\x118c5'# -> unI64 71845
+  '\x118c6'# -> unI64 71846
+  '\x118c7'# -> unI64 71847
+  '\x118c8'# -> unI64 71848
+  '\x118c9'# -> unI64 71849
+  '\x118ca'# -> unI64 71850
+  '\x118cb'# -> unI64 71851
+  '\x118cc'# -> unI64 71852
+  '\x118cd'# -> unI64 71853
+  '\x118ce'# -> unI64 71854
+  '\x118cf'# -> unI64 71855
+  '\x118d0'# -> unI64 71856
+  '\x118d1'# -> unI64 71857
+  '\x118d2'# -> unI64 71858
+  '\x118d3'# -> unI64 71859
+  '\x118d4'# -> unI64 71860
+  '\x118d5'# -> unI64 71861
+  '\x118d6'# -> unI64 71862
+  '\x118d7'# -> unI64 71863
+  '\x118d8'# -> unI64 71864
+  '\x118d9'# -> unI64 71865
+  '\x118da'# -> unI64 71866
+  '\x118db'# -> unI64 71867
+  '\x118dc'# -> unI64 71868
+  '\x118dd'# -> unI64 71869
+  '\x118de'# -> unI64 71870
+  '\x118df'# -> unI64 71871
+  '\x16e60'# -> unI64 93760
+  '\x16e61'# -> unI64 93761
+  '\x16e62'# -> unI64 93762
+  '\x16e63'# -> unI64 93763
+  '\x16e64'# -> unI64 93764
+  '\x16e65'# -> unI64 93765
+  '\x16e66'# -> unI64 93766
+  '\x16e67'# -> unI64 93767
+  '\x16e68'# -> unI64 93768
+  '\x16e69'# -> unI64 93769
+  '\x16e6a'# -> unI64 93770
+  '\x16e6b'# -> unI64 93771
+  '\x16e6c'# -> unI64 93772
+  '\x16e6d'# -> unI64 93773
+  '\x16e6e'# -> unI64 93774
+  '\x16e6f'# -> unI64 93775
+  '\x16e70'# -> unI64 93776
+  '\x16e71'# -> unI64 93777
+  '\x16e72'# -> unI64 93778
+  '\x16e73'# -> unI64 93779
+  '\x16e74'# -> unI64 93780
+  '\x16e75'# -> unI64 93781
+  '\x16e76'# -> unI64 93782
+  '\x16e77'# -> unI64 93783
+  '\x16e78'# -> unI64 93784
+  '\x16e79'# -> unI64 93785
+  '\x16e7a'# -> unI64 93786
+  '\x16e7b'# -> unI64 93787
+  '\x16e7c'# -> unI64 93788
+  '\x16e7d'# -> unI64 93789
+  '\x16e7e'# -> unI64 93790
+  '\x16e7f'# -> unI64 93791
+  '\x16ebb'# -> unI64 93856
+  '\x16ebc'# -> unI64 93857
+  '\x16ebd'# -> unI64 93858
+  '\x16ebe'# -> unI64 93859
+  '\x16ebf'# -> unI64 93860
+  '\x16ec0'# -> unI64 93861
+  '\x16ec1'# -> unI64 93862
+  '\x16ec2'# -> unI64 93863
+  '\x16ec3'# -> unI64 93864
+  '\x16ec4'# -> unI64 93865
+  '\x16ec5'# -> unI64 93866
+  '\x16ec6'# -> unI64 93867
+  '\x16ec7'# -> unI64 93868
+  '\x16ec8'# -> unI64 93869
+  '\x16ec9'# -> unI64 93870
+  '\x16eca'# -> unI64 93871
+  '\x16ecb'# -> unI64 93872
+  '\x16ecc'# -> unI64 93873
+  '\x16ecd'# -> unI64 93874
+  '\x16ece'# -> unI64 93875
+  '\x16ecf'# -> unI64 93876
+  '\x16ed0'# -> unI64 93877
+  '\x16ed1'# -> unI64 93878
+  '\x16ed2'# -> unI64 93879
+  '\x16ed3'# -> unI64 93880
+  '\x1e922'# -> unI64 125184
+  '\x1e923'# -> unI64 125185
+  '\x1e924'# -> unI64 125186
+  '\x1e925'# -> unI64 125187
+  '\x1e926'# -> unI64 125188
+  '\x1e927'# -> unI64 125189
+  '\x1e928'# -> unI64 125190
+  '\x1e929'# -> unI64 125191
+  '\x1e92a'# -> unI64 125192
+  '\x1e92b'# -> unI64 125193
+  '\x1e92c'# -> unI64 125194
+  '\x1e92d'# -> unI64 125195
+  '\x1e92e'# -> unI64 125196
+  '\x1e92f'# -> unI64 125197
+  '\x1e930'# -> unI64 125198
+  '\x1e931'# -> unI64 125199
+  '\x1e932'# -> unI64 125200
+  '\x1e933'# -> unI64 125201
+  '\x1e934'# -> unI64 125202
+  '\x1e935'# -> unI64 125203
+  '\x1e936'# -> unI64 125204
+  '\x1e937'# -> unI64 125205
+  '\x1e938'# -> unI64 125206
+  '\x1e939'# -> unI64 125207
+  '\x1e93a'# -> unI64 125208
+  '\x1e93b'# -> unI64 125209
+  '\x1e93c'# -> unI64 125210
+  '\x1e93d'# -> unI64 125211
+  '\x1e93e'# -> unI64 125212
+  '\x1e93f'# -> unI64 125213
+  '\x1e940'# -> unI64 125214
+  '\x1e941'# -> unI64 125215
+  '\x1e942'# -> unI64 125216
+  '\x1e943'# -> unI64 125217
+  _ -> unI64 0
+foldMapping :: Char# -> _ {- unboxed Int64 -}
+{-# NOINLINE foldMapping #-}
+foldMapping = \case
+  -- LATIN CAPITAL LETTER A
+  '\x0041'# -> unI64 97
+  -- LATIN CAPITAL LETTER B
+  '\x0042'# -> unI64 98
+  -- LATIN CAPITAL LETTER C
+  '\x0043'# -> unI64 99
+  -- LATIN CAPITAL LETTER D
+  '\x0044'# -> unI64 100
+  -- LATIN CAPITAL LETTER E
+  '\x0045'# -> unI64 101
+  -- LATIN CAPITAL LETTER F
+  '\x0046'# -> unI64 102
+  -- LATIN CAPITAL LETTER G
+  '\x0047'# -> unI64 103
+  -- LATIN CAPITAL LETTER H
+  '\x0048'# -> unI64 104
+  -- LATIN CAPITAL LETTER I
+  '\x0049'# -> unI64 105
+  -- LATIN CAPITAL LETTER J
+  '\x004a'# -> unI64 106
+  -- LATIN CAPITAL LETTER K
+  '\x004b'# -> unI64 107
+  -- LATIN CAPITAL LETTER L
+  '\x004c'# -> unI64 108
+  -- LATIN CAPITAL LETTER M
+  '\x004d'# -> unI64 109
+  -- LATIN CAPITAL LETTER N
+  '\x004e'# -> unI64 110
+  -- LATIN CAPITAL LETTER O
+  '\x004f'# -> unI64 111
+  -- LATIN CAPITAL LETTER P
+  '\x0050'# -> unI64 112
+  -- LATIN CAPITAL LETTER Q
+  '\x0051'# -> unI64 113
+  -- LATIN CAPITAL LETTER R
+  '\x0052'# -> unI64 114
+  -- LATIN CAPITAL LETTER S
+  '\x0053'# -> unI64 115
+  -- LATIN CAPITAL LETTER T
+  '\x0054'# -> unI64 116
+  -- LATIN CAPITAL LETTER U
+  '\x0055'# -> unI64 117
+  -- LATIN CAPITAL LETTER V
+  '\x0056'# -> unI64 118
+  -- LATIN CAPITAL LETTER W
+  '\x0057'# -> unI64 119
+  -- LATIN CAPITAL LETTER X
+  '\x0058'# -> unI64 120
+  -- LATIN CAPITAL LETTER Y
+  '\x0059'# -> unI64 121
+  -- LATIN CAPITAL LETTER Z
+  '\x005a'# -> unI64 122
+  -- MICRO SIGN
+  '\x00b5'# -> unI64 956
+  -- LATIN CAPITAL LETTER A WITH GRAVE
+  '\x00c0'# -> unI64 224
+  -- LATIN CAPITAL LETTER A WITH ACUTE
+  '\x00c1'# -> unI64 225
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+  '\x00c2'# -> unI64 226
+  -- LATIN CAPITAL LETTER A WITH TILDE
+  '\x00c3'# -> unI64 227
+  -- LATIN CAPITAL LETTER A WITH DIAERESIS
+  '\x00c4'# -> unI64 228
+  -- LATIN CAPITAL LETTER A WITH RING ABOVE
+  '\x00c5'# -> unI64 229
+  -- LATIN CAPITAL LETTER AE
+  '\x00c6'# -> unI64 230
+  -- LATIN CAPITAL LETTER C WITH CEDILLA
+  '\x00c7'# -> unI64 231
+  -- LATIN CAPITAL LETTER E WITH GRAVE
+  '\x00c8'# -> unI64 232
+  -- LATIN CAPITAL LETTER E WITH ACUTE
+  '\x00c9'# -> unI64 233
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+  '\x00ca'# -> unI64 234
+  -- LATIN CAPITAL LETTER E WITH DIAERESIS
+  '\x00cb'# -> unI64 235
+  -- LATIN CAPITAL LETTER I WITH GRAVE
+  '\x00cc'# -> unI64 236
+  -- LATIN CAPITAL LETTER I WITH ACUTE
+  '\x00cd'# -> unI64 237
+  -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+  '\x00ce'# -> unI64 238
+  -- LATIN CAPITAL LETTER I WITH DIAERESIS
+  '\x00cf'# -> unI64 239
+  -- LATIN CAPITAL LETTER ETH
+  '\x00d0'# -> unI64 240
+  -- LATIN CAPITAL LETTER N WITH TILDE
+  '\x00d1'# -> unI64 241
+  -- LATIN CAPITAL LETTER O WITH GRAVE
+  '\x00d2'# -> unI64 242
+  -- LATIN CAPITAL LETTER O WITH ACUTE
+  '\x00d3'# -> unI64 243
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+  '\x00d4'# -> unI64 244
+  -- LATIN CAPITAL LETTER O WITH TILDE
+  '\x00d5'# -> unI64 245
+  -- LATIN CAPITAL LETTER O WITH DIAERESIS
+  '\x00d6'# -> unI64 246
+  -- LATIN CAPITAL LETTER O WITH STROKE
+  '\x00d8'# -> unI64 248
+  -- LATIN CAPITAL LETTER U WITH GRAVE
+  '\x00d9'# -> unI64 249
+  -- LATIN CAPITAL LETTER U WITH ACUTE
+  '\x00da'# -> unI64 250
+  -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+  '\x00db'# -> unI64 251
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS
+  '\x00dc'# -> unI64 252
+  -- LATIN CAPITAL LETTER Y WITH ACUTE
+  '\x00dd'# -> unI64 253
+  -- LATIN CAPITAL LETTER THORN
+  '\x00de'# -> unI64 254
+  -- LATIN SMALL LETTER SHARP S
+  '\x00df'# -> unI64 241172595
+  -- LATIN CAPITAL LETTER A WITH MACRON
+  '\x0100'# -> unI64 257
+  -- LATIN CAPITAL LETTER A WITH BREVE
+  '\x0102'# -> unI64 259
+  -- LATIN CAPITAL LETTER A WITH OGONEK
+  '\x0104'# -> unI64 261
+  -- LATIN CAPITAL LETTER C WITH ACUTE
+  '\x0106'# -> unI64 263
+  -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+  '\x0108'# -> unI64 265
+  -- LATIN CAPITAL LETTER C WITH DOT ABOVE
+  '\x010a'# -> unI64 267
+  -- LATIN CAPITAL LETTER C WITH CARON
+  '\x010c'# -> unI64 269
+  -- LATIN CAPITAL LETTER D WITH CARON
+  '\x010e'# -> unI64 271
+  -- LATIN CAPITAL LETTER D WITH STROKE
+  '\x0110'# -> unI64 273
+  -- LATIN CAPITAL LETTER E WITH MACRON
+  '\x0112'# -> unI64 275
+  -- LATIN CAPITAL LETTER E WITH BREVE
+  '\x0114'# -> unI64 277
+  -- LATIN CAPITAL LETTER E WITH DOT ABOVE
+  '\x0116'# -> unI64 279
+  -- LATIN CAPITAL LETTER E WITH OGONEK
+  '\x0118'# -> unI64 281
+  -- LATIN CAPITAL LETTER E WITH CARON
+  '\x011a'# -> unI64 283
+  -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+  '\x011c'# -> unI64 285
+  -- LATIN CAPITAL LETTER G WITH BREVE
+  '\x011e'# -> unI64 287
+  -- LATIN CAPITAL LETTER G WITH DOT ABOVE
+  '\x0120'# -> unI64 289
+  -- LATIN CAPITAL LETTER G WITH CEDILLA
+  '\x0122'# -> unI64 291
+  -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+  '\x0124'# -> unI64 293
+  -- LATIN CAPITAL LETTER H WITH STROKE
+  '\x0126'# -> unI64 295
+  -- LATIN CAPITAL LETTER I WITH TILDE
+  '\x0128'# -> unI64 297
+  -- LATIN CAPITAL LETTER I WITH MACRON
+  '\x012a'# -> unI64 299
+  -- LATIN CAPITAL LETTER I WITH BREVE
+  '\x012c'# -> unI64 301
+  -- LATIN CAPITAL LETTER I WITH OGONEK
+  '\x012e'# -> unI64 303
+  -- LATIN CAPITAL LETTER I WITH DOT ABOVE
+  '\x0130'# -> unI64 1625292905
+  -- LATIN CAPITAL LIGATURE IJ
+  '\x0132'# -> unI64 307
+  -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+  '\x0134'# -> unI64 309
+  -- LATIN CAPITAL LETTER K WITH CEDILLA
+  '\x0136'# -> unI64 311
+  -- LATIN CAPITAL LETTER L WITH ACUTE
+  '\x0139'# -> unI64 314
+  -- LATIN CAPITAL LETTER L WITH CEDILLA
+  '\x013b'# -> unI64 316
+  -- LATIN CAPITAL LETTER L WITH CARON
+  '\x013d'# -> unI64 318
+  -- LATIN CAPITAL LETTER L WITH MIDDLE DOT
+  '\x013f'# -> unI64 320
+  -- LATIN CAPITAL LETTER L WITH STROKE
+  '\x0141'# -> unI64 322
+  -- LATIN CAPITAL LETTER N WITH ACUTE
+  '\x0143'# -> unI64 324
+  -- LATIN CAPITAL LETTER N WITH CEDILLA
+  '\x0145'# -> unI64 326
+  -- LATIN CAPITAL LETTER N WITH CARON
+  '\x0147'# -> unI64 328
+  -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+  '\x0149'# -> unI64 230687420
+  -- LATIN CAPITAL LETTER ENG
+  '\x014a'# -> unI64 331
+  -- LATIN CAPITAL LETTER O WITH MACRON
+  '\x014c'# -> unI64 333
+  -- LATIN CAPITAL LETTER O WITH BREVE
+  '\x014e'# -> unI64 335
+  -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+  '\x0150'# -> unI64 337
+  -- LATIN CAPITAL LIGATURE OE
+  '\x0152'# -> unI64 339
+  -- LATIN CAPITAL LETTER R WITH ACUTE
+  '\x0154'# -> unI64 341
+  -- LATIN CAPITAL LETTER R WITH CEDILLA
+  '\x0156'# -> unI64 343
+  -- LATIN CAPITAL LETTER R WITH CARON
+  '\x0158'# -> unI64 345
+  -- LATIN CAPITAL LETTER S WITH ACUTE
+  '\x015a'# -> unI64 347
+  -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+  '\x015c'# -> unI64 349
+  -- LATIN CAPITAL LETTER S WITH CEDILLA
+  '\x015e'# -> unI64 351
+  -- LATIN CAPITAL LETTER S WITH CARON
+  '\x0160'# -> unI64 353
+  -- LATIN CAPITAL LETTER T WITH CEDILLA
+  '\x0162'# -> unI64 355
+  -- LATIN CAPITAL LETTER T WITH CARON
+  '\x0164'# -> unI64 357
+  -- LATIN CAPITAL LETTER T WITH STROKE
+  '\x0166'# -> unI64 359
+  -- LATIN CAPITAL LETTER U WITH TILDE
+  '\x0168'# -> unI64 361
+  -- LATIN CAPITAL LETTER U WITH MACRON
+  '\x016a'# -> unI64 363
+  -- LATIN CAPITAL LETTER U WITH BREVE
+  '\x016c'# -> unI64 365
+  -- LATIN CAPITAL LETTER U WITH RING ABOVE
+  '\x016e'# -> unI64 367
+  -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+  '\x0170'# -> unI64 369
+  -- LATIN CAPITAL LETTER U WITH OGONEK
+  '\x0172'# -> unI64 371
+  -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+  '\x0174'# -> unI64 373
+  -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+  '\x0176'# -> unI64 375
+  -- LATIN CAPITAL LETTER Y WITH DIAERESIS
+  '\x0178'# -> unI64 255
+  -- LATIN CAPITAL LETTER Z WITH ACUTE
+  '\x0179'# -> unI64 378
+  -- LATIN CAPITAL LETTER Z WITH DOT ABOVE
+  '\x017b'# -> unI64 380
+  -- LATIN CAPITAL LETTER Z WITH CARON
+  '\x017d'# -> unI64 382
+  -- LATIN SMALL LETTER LONG S
+  '\x017f'# -> unI64 115
+  -- LATIN CAPITAL LETTER B WITH HOOK
+  '\x0181'# -> unI64 595
+  -- LATIN CAPITAL LETTER B WITH TOPBAR
+  '\x0182'# -> unI64 387
+  -- LATIN CAPITAL LETTER TONE SIX
+  '\x0184'# -> unI64 389
+  -- LATIN CAPITAL LETTER OPEN O
+  '\x0186'# -> unI64 596
+  -- LATIN CAPITAL LETTER C WITH HOOK
+  '\x0187'# -> unI64 392
+  -- LATIN CAPITAL LETTER AFRICAN D
+  '\x0189'# -> unI64 598
+  -- LATIN CAPITAL LETTER D WITH HOOK
+  '\x018a'# -> unI64 599
+  -- LATIN CAPITAL LETTER D WITH TOPBAR
+  '\x018b'# -> unI64 396
+  -- LATIN CAPITAL LETTER REVERSED E
+  '\x018e'# -> unI64 477
+  -- LATIN CAPITAL LETTER SCHWA
+  '\x018f'# -> unI64 601
+  -- LATIN CAPITAL LETTER OPEN E
+  '\x0190'# -> unI64 603
+  -- LATIN CAPITAL LETTER F WITH HOOK
+  '\x0191'# -> unI64 402
+  -- LATIN CAPITAL LETTER G WITH HOOK
+  '\x0193'# -> unI64 608
+  -- LATIN CAPITAL LETTER GAMMA
+  '\x0194'# -> unI64 611
+  -- LATIN CAPITAL LETTER IOTA
+  '\x0196'# -> unI64 617
+  -- LATIN CAPITAL LETTER I WITH STROKE
+  '\x0197'# -> unI64 616
+  -- LATIN CAPITAL LETTER K WITH HOOK
+  '\x0198'# -> unI64 409
+  -- LATIN CAPITAL LETTER TURNED M
+  '\x019c'# -> unI64 623
+  -- LATIN CAPITAL LETTER N WITH LEFT HOOK
+  '\x019d'# -> unI64 626
+  -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+  '\x019f'# -> unI64 629
+  -- LATIN CAPITAL LETTER O WITH HORN
+  '\x01a0'# -> unI64 417
+  -- LATIN CAPITAL LETTER OI
+  '\x01a2'# -> unI64 419
+  -- LATIN CAPITAL LETTER P WITH HOOK
+  '\x01a4'# -> unI64 421
+  -- LATIN LETTER YR
+  '\x01a6'# -> unI64 640
+  -- LATIN CAPITAL LETTER TONE TWO
+  '\x01a7'# -> unI64 424
+  -- LATIN CAPITAL LETTER ESH
+  '\x01a9'# -> unI64 643
+  -- LATIN CAPITAL LETTER T WITH HOOK
+  '\x01ac'# -> unI64 429
+  -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+  '\x01ae'# -> unI64 648
+  -- LATIN CAPITAL LETTER U WITH HORN
+  '\x01af'# -> unI64 432
+  -- LATIN CAPITAL LETTER UPSILON
+  '\x01b1'# -> unI64 650
+  -- LATIN CAPITAL LETTER V WITH HOOK
+  '\x01b2'# -> unI64 651
+  -- LATIN CAPITAL LETTER Y WITH HOOK
+  '\x01b3'# -> unI64 436
+  -- LATIN CAPITAL LETTER Z WITH STROKE
+  '\x01b5'# -> unI64 438
+  -- LATIN CAPITAL LETTER EZH
+  '\x01b7'# -> unI64 658
+  -- LATIN CAPITAL LETTER EZH REVERSED
+  '\x01b8'# -> unI64 441
+  -- LATIN CAPITAL LETTER TONE FIVE
+  '\x01bc'# -> unI64 445
+  -- LATIN CAPITAL LETTER DZ WITH CARON
+  '\x01c4'# -> unI64 454
+  -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+  '\x01c5'# -> unI64 454
+  -- LATIN CAPITAL LETTER LJ
+  '\x01c7'# -> unI64 457
+  -- LATIN CAPITAL LETTER L WITH SMALL LETTER J
+  '\x01c8'# -> unI64 457
+  -- LATIN CAPITAL LETTER NJ
+  '\x01ca'# -> unI64 460
+  -- LATIN CAPITAL LETTER N WITH SMALL LETTER J
+  '\x01cb'# -> unI64 460
+  -- LATIN CAPITAL LETTER A WITH CARON
+  '\x01cd'# -> unI64 462
+  -- LATIN CAPITAL LETTER I WITH CARON
+  '\x01cf'# -> unI64 464
+  -- LATIN CAPITAL LETTER O WITH CARON
+  '\x01d1'# -> unI64 466
+  -- LATIN CAPITAL LETTER U WITH CARON
+  '\x01d3'# -> unI64 468
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+  '\x01d5'# -> unI64 470
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+  '\x01d7'# -> unI64 472
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+  '\x01d9'# -> unI64 474
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+  '\x01db'# -> unI64 476
+  -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+  '\x01de'# -> unI64 479
+  -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+  '\x01e0'# -> unI64 481
+  -- LATIN CAPITAL LETTER AE WITH MACRON
+  '\x01e2'# -> unI64 483
+  -- LATIN CAPITAL LETTER G WITH STROKE
+  '\x01e4'# -> unI64 485
+  -- LATIN CAPITAL LETTER G WITH CARON
+  '\x01e6'# -> unI64 487
+  -- LATIN CAPITAL LETTER K WITH CARON
+  '\x01e8'# -> unI64 489
+  -- LATIN CAPITAL LETTER O WITH OGONEK
+  '\x01ea'# -> unI64 491
+  -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+  '\x01ec'# -> unI64 493
+  -- LATIN CAPITAL LETTER EZH WITH CARON
+  '\x01ee'# -> unI64 495
+  -- LATIN SMALL LETTER J WITH CARON
+  '\x01f0'# -> unI64 1635778666
+  -- LATIN CAPITAL LETTER DZ
+  '\x01f1'# -> unI64 499
+  -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+  '\x01f2'# -> unI64 499
+  -- LATIN CAPITAL LETTER G WITH ACUTE
+  '\x01f4'# -> unI64 501
+  -- LATIN CAPITAL LETTER HWAIR
+  '\x01f6'# -> unI64 405
+  -- LATIN CAPITAL LETTER WYNN
+  '\x01f7'# -> unI64 447
+  -- LATIN CAPITAL LETTER N WITH GRAVE
+  '\x01f8'# -> unI64 505
+  -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+  '\x01fa'# -> unI64 507
+  -- LATIN CAPITAL LETTER AE WITH ACUTE
+  '\x01fc'# -> unI64 509
+  -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+  '\x01fe'# -> unI64 511
+  -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+  '\x0200'# -> unI64 513
+  -- LATIN CAPITAL LETTER A WITH INVERTED BREVE
+  '\x0202'# -> unI64 515
+  -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+  '\x0204'# -> unI64 517
+  -- LATIN CAPITAL LETTER E WITH INVERTED BREVE
+  '\x0206'# -> unI64 519
+  -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+  '\x0208'# -> unI64 521
+  -- LATIN CAPITAL LETTER I WITH INVERTED BREVE
+  '\x020a'# -> unI64 523
+  -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+  '\x020c'# -> unI64 525
+  -- LATIN CAPITAL LETTER O WITH INVERTED BREVE
+  '\x020e'# -> unI64 527
+  -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+  '\x0210'# -> unI64 529
+  -- LATIN CAPITAL LETTER R WITH INVERTED BREVE
+  '\x0212'# -> unI64 531
+  -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+  '\x0214'# -> unI64 533
+  -- LATIN CAPITAL LETTER U WITH INVERTED BREVE
+  '\x0216'# -> unI64 535
+  -- LATIN CAPITAL LETTER S WITH COMMA BELOW
+  '\x0218'# -> unI64 537
+  -- LATIN CAPITAL LETTER T WITH COMMA BELOW
+  '\x021a'# -> unI64 539
+  -- LATIN CAPITAL LETTER YOGH
+  '\x021c'# -> unI64 541
+  -- LATIN CAPITAL LETTER H WITH CARON
+  '\x021e'# -> unI64 543
+  -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+  '\x0220'# -> unI64 414
+  -- LATIN CAPITAL LETTER OU
+  '\x0222'# -> unI64 547
+  -- LATIN CAPITAL LETTER Z WITH HOOK
+  '\x0224'# -> unI64 549
+  -- LATIN CAPITAL LETTER A WITH DOT ABOVE
+  '\x0226'# -> unI64 551
+  -- LATIN CAPITAL LETTER E WITH CEDILLA
+  '\x0228'# -> unI64 553
+  -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+  '\x022a'# -> unI64 555
+  -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+  '\x022c'# -> unI64 557
+  -- LATIN CAPITAL LETTER O WITH DOT ABOVE
+  '\x022e'# -> unI64 559
+  -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+  '\x0230'# -> unI64 561
+  -- LATIN CAPITAL LETTER Y WITH MACRON
+  '\x0232'# -> unI64 563
+  -- LATIN CAPITAL LETTER A WITH STROKE
+  '\x023a'# -> unI64 11365
+  -- LATIN CAPITAL LETTER C WITH STROKE
+  '\x023b'# -> unI64 572
+  -- LATIN CAPITAL LETTER L WITH BAR
+  '\x023d'# -> unI64 410
+  -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+  '\x023e'# -> unI64 11366
+  -- LATIN CAPITAL LETTER GLOTTAL STOP
+  '\x0241'# -> unI64 578
+  -- LATIN CAPITAL LETTER B WITH STROKE
+  '\x0243'# -> unI64 384
+  -- LATIN CAPITAL LETTER U BAR
+  '\x0244'# -> unI64 649
+  -- LATIN CAPITAL LETTER TURNED V
+  '\x0245'# -> unI64 652
+  -- LATIN CAPITAL LETTER E WITH STROKE
+  '\x0246'# -> unI64 583
+  -- LATIN CAPITAL LETTER J WITH STROKE
+  '\x0248'# -> unI64 585
+  -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+  '\x024a'# -> unI64 587
+  -- LATIN CAPITAL LETTER R WITH STROKE
+  '\x024c'# -> unI64 589
+  -- LATIN CAPITAL LETTER Y WITH STROKE
+  '\x024e'# -> unI64 591
+  -- COMBINING GREEK YPOGEGRAMMENI
+  '\x0345'# -> unI64 953
+  -- GREEK CAPITAL LETTER HETA
+  '\x0370'# -> unI64 881
+  -- GREEK CAPITAL LETTER ARCHAIC SAMPI
+  '\x0372'# -> unI64 883
+  -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+  '\x0376'# -> unI64 887
+  -- GREEK CAPITAL LETTER YOT
+  '\x037f'# -> unI64 1011
+  -- GREEK CAPITAL LETTER ALPHA WITH TONOS
+  '\x0386'# -> unI64 940
+  -- GREEK CAPITAL LETTER EPSILON WITH TONOS
+  '\x0388'# -> unI64 941
+  -- GREEK CAPITAL LETTER ETA WITH TONOS
+  '\x0389'# -> unI64 942
+  -- GREEK CAPITAL LETTER IOTA WITH TONOS
+  '\x038a'# -> unI64 943
+  -- GREEK CAPITAL LETTER OMICRON WITH TONOS
+  '\x038c'# -> unI64 972
+  -- GREEK CAPITAL LETTER UPSILON WITH TONOS
+  '\x038e'# -> unI64 973
+  -- GREEK CAPITAL LETTER OMEGA WITH TONOS
+  '\x038f'# -> unI64 974
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+  '\x0390'# -> unI64 3382099394429881
+  -- GREEK CAPITAL LETTER ALPHA
+  '\x0391'# -> unI64 945
+  -- GREEK CAPITAL LETTER BETA
+  '\x0392'# -> unI64 946
+  -- GREEK CAPITAL LETTER GAMMA
+  '\x0393'# -> unI64 947
+  -- GREEK CAPITAL LETTER DELTA
+  '\x0394'# -> unI64 948
+  -- GREEK CAPITAL LETTER EPSILON
+  '\x0395'# -> unI64 949
+  -- GREEK CAPITAL LETTER ZETA
+  '\x0396'# -> unI64 950
+  -- GREEK CAPITAL LETTER ETA
+  '\x0397'# -> unI64 951
+  -- GREEK CAPITAL LETTER THETA
+  '\x0398'# -> unI64 952
+  -- GREEK CAPITAL LETTER IOTA
+  '\x0399'# -> unI64 953
+  -- GREEK CAPITAL LETTER KAPPA
+  '\x039a'# -> unI64 954
+  -- GREEK CAPITAL LETTER LAMDA
+  '\x039b'# -> unI64 955
+  -- GREEK CAPITAL LETTER MU
+  '\x039c'# -> unI64 956
+  -- GREEK CAPITAL LETTER NU
+  '\x039d'# -> unI64 957
+  -- GREEK CAPITAL LETTER XI
+  '\x039e'# -> unI64 958
+  -- GREEK CAPITAL LETTER OMICRON
+  '\x039f'# -> unI64 959
+  -- GREEK CAPITAL LETTER PI
+  '\x03a0'# -> unI64 960
+  -- GREEK CAPITAL LETTER RHO
+  '\x03a1'# -> unI64 961
+  -- GREEK CAPITAL LETTER SIGMA
+  '\x03a3'# -> unI64 963
+  -- GREEK CAPITAL LETTER TAU
+  '\x03a4'# -> unI64 964
+  -- GREEK CAPITAL LETTER UPSILON
+  '\x03a5'# -> unI64 965
+  -- GREEK CAPITAL LETTER PHI
+  '\x03a6'# -> unI64 966
+  -- GREEK CAPITAL LETTER CHI
+  '\x03a7'# -> unI64 967
+  -- GREEK CAPITAL LETTER PSI
+  '\x03a8'# -> unI64 968
+  -- GREEK CAPITAL LETTER OMEGA
+  '\x03a9'# -> unI64 969
+  -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+  '\x03aa'# -> unI64 970
+  -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+  '\x03ab'# -> unI64 971
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+  '\x03b0'# -> unI64 3382099394429893
+  -- GREEK SMALL LETTER FINAL SIGMA
+  '\x03c2'# -> unI64 963
+  -- GREEK CAPITAL KAI SYMBOL
+  '\x03cf'# -> unI64 983
+  -- GREEK BETA SYMBOL
+  '\x03d0'# -> unI64 946
+  -- GREEK THETA SYMBOL
+  '\x03d1'# -> unI64 952
+  -- GREEK PHI SYMBOL
+  '\x03d5'# -> unI64 966
+  -- GREEK PI SYMBOL
+  '\x03d6'# -> unI64 960
+  -- GREEK LETTER ARCHAIC KOPPA
+  '\x03d8'# -> unI64 985
+  -- GREEK LETTER STIGMA
+  '\x03da'# -> unI64 987
+  -- GREEK LETTER DIGAMMA
+  '\x03dc'# -> unI64 989
+  -- GREEK LETTER KOPPA
+  '\x03de'# -> unI64 991
+  -- GREEK LETTER SAMPI
+  '\x03e0'# -> unI64 993
+  -- COPTIC CAPITAL LETTER SHEI
+  '\x03e2'# -> unI64 995
+  -- COPTIC CAPITAL LETTER FEI
+  '\x03e4'# -> unI64 997
+  -- COPTIC CAPITAL LETTER KHEI
+  '\x03e6'# -> unI64 999
+  -- COPTIC CAPITAL LETTER HORI
+  '\x03e8'# -> unI64 1001
+  -- COPTIC CAPITAL LETTER GANGIA
+  '\x03ea'# -> unI64 1003
+  -- COPTIC CAPITAL LETTER SHIMA
+  '\x03ec'# -> unI64 1005
+  -- COPTIC CAPITAL LETTER DEI
+  '\x03ee'# -> unI64 1007
+  -- GREEK KAPPA SYMBOL
+  '\x03f0'# -> unI64 954
+  -- GREEK RHO SYMBOL
+  '\x03f1'# -> unI64 961
+  -- GREEK CAPITAL THETA SYMBOL
+  '\x03f4'# -> unI64 952
+  -- GREEK LUNATE EPSILON SYMBOL
+  '\x03f5'# -> unI64 949
+  -- GREEK CAPITAL LETTER SHO
+  '\x03f7'# -> unI64 1016
+  -- GREEK CAPITAL LUNATE SIGMA SYMBOL
+  '\x03f9'# -> unI64 1010
+  -- GREEK CAPITAL LETTER SAN
+  '\x03fa'# -> unI64 1019
+  -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+  '\x03fd'# -> unI64 891
+  -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+  '\x03fe'# -> unI64 892
+  -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+  '\x03ff'# -> unI64 893
+  -- CYRILLIC CAPITAL LETTER IE WITH GRAVE
+  '\x0400'# -> unI64 1104
+  -- CYRILLIC CAPITAL LETTER IO
+  '\x0401'# -> unI64 1105
+  -- CYRILLIC CAPITAL LETTER DJE
+  '\x0402'# -> unI64 1106
+  -- CYRILLIC CAPITAL LETTER GJE
+  '\x0403'# -> unI64 1107
+  -- CYRILLIC CAPITAL LETTER UKRAINIAN IE
+  '\x0404'# -> unI64 1108
+  -- CYRILLIC CAPITAL LETTER DZE
+  '\x0405'# -> unI64 1109
+  -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+  '\x0406'# -> unI64 1110
+  -- CYRILLIC CAPITAL LETTER YI
+  '\x0407'# -> unI64 1111
+  -- CYRILLIC CAPITAL LETTER JE
+  '\x0408'# -> unI64 1112
+  -- CYRILLIC CAPITAL LETTER LJE
+  '\x0409'# -> unI64 1113
+  -- CYRILLIC CAPITAL LETTER NJE
+  '\x040a'# -> unI64 1114
+  -- CYRILLIC CAPITAL LETTER TSHE
+  '\x040b'# -> unI64 1115
+  -- CYRILLIC CAPITAL LETTER KJE
+  '\x040c'# -> unI64 1116
+  -- CYRILLIC CAPITAL LETTER I WITH GRAVE
+  '\x040d'# -> unI64 1117
+  -- CYRILLIC CAPITAL LETTER SHORT U
+  '\x040e'# -> unI64 1118
+  -- CYRILLIC CAPITAL LETTER DZHE
+  '\x040f'# -> unI64 1119
+  -- CYRILLIC CAPITAL LETTER A
+  '\x0410'# -> unI64 1072
+  -- CYRILLIC CAPITAL LETTER BE
+  '\x0411'# -> unI64 1073
+  -- CYRILLIC CAPITAL LETTER VE
+  '\x0412'# -> unI64 1074
+  -- CYRILLIC CAPITAL LETTER GHE
+  '\x0413'# -> unI64 1075
+  -- CYRILLIC CAPITAL LETTER DE
+  '\x0414'# -> unI64 1076
+  -- CYRILLIC CAPITAL LETTER IE
+  '\x0415'# -> unI64 1077
+  -- CYRILLIC CAPITAL LETTER ZHE
+  '\x0416'# -> unI64 1078
+  -- CYRILLIC CAPITAL LETTER ZE
+  '\x0417'# -> unI64 1079
+  -- CYRILLIC CAPITAL LETTER I
+  '\x0418'# -> unI64 1080
+  -- CYRILLIC CAPITAL LETTER SHORT I
+  '\x0419'# -> unI64 1081
+  -- CYRILLIC CAPITAL LETTER KA
+  '\x041a'# -> unI64 1082
+  -- CYRILLIC CAPITAL LETTER EL
+  '\x041b'# -> unI64 1083
+  -- CYRILLIC CAPITAL LETTER EM
+  '\x041c'# -> unI64 1084
+  -- CYRILLIC CAPITAL LETTER EN
+  '\x041d'# -> unI64 1085
+  -- CYRILLIC CAPITAL LETTER O
+  '\x041e'# -> unI64 1086
+  -- CYRILLIC CAPITAL LETTER PE
+  '\x041f'# -> unI64 1087
+  -- CYRILLIC CAPITAL LETTER ER
+  '\x0420'# -> unI64 1088
+  -- CYRILLIC CAPITAL LETTER ES
+  '\x0421'# -> unI64 1089
+  -- CYRILLIC CAPITAL LETTER TE
+  '\x0422'# -> unI64 1090
+  -- CYRILLIC CAPITAL LETTER U
+  '\x0423'# -> unI64 1091
+  -- CYRILLIC CAPITAL LETTER EF
+  '\x0424'# -> unI64 1092
+  -- CYRILLIC CAPITAL LETTER HA
+  '\x0425'# -> unI64 1093
+  -- CYRILLIC CAPITAL LETTER TSE
+  '\x0426'# -> unI64 1094
+  -- CYRILLIC CAPITAL LETTER CHE
+  '\x0427'# -> unI64 1095
+  -- CYRILLIC CAPITAL LETTER SHA
+  '\x0428'# -> unI64 1096
+  -- CYRILLIC CAPITAL LETTER SHCHA
+  '\x0429'# -> unI64 1097
+  -- CYRILLIC CAPITAL LETTER HARD SIGN
+  '\x042a'# -> unI64 1098
+  -- CYRILLIC CAPITAL LETTER YERU
+  '\x042b'# -> unI64 1099
+  -- CYRILLIC CAPITAL LETTER SOFT SIGN
+  '\x042c'# -> unI64 1100
+  -- CYRILLIC CAPITAL LETTER E
+  '\x042d'# -> unI64 1101
+  -- CYRILLIC CAPITAL LETTER YU
+  '\x042e'# -> unI64 1102
+  -- CYRILLIC CAPITAL LETTER YA
+  '\x042f'# -> unI64 1103
+  -- CYRILLIC CAPITAL LETTER OMEGA
+  '\x0460'# -> unI64 1121
+  -- CYRILLIC CAPITAL LETTER YAT
+  '\x0462'# -> unI64 1123
+  -- CYRILLIC CAPITAL LETTER IOTIFIED E
+  '\x0464'# -> unI64 1125
+  -- CYRILLIC CAPITAL LETTER LITTLE YUS
+  '\x0466'# -> unI64 1127
+  -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+  '\x0468'# -> unI64 1129
+  -- CYRILLIC CAPITAL LETTER BIG YUS
+  '\x046a'# -> unI64 1131
+  -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+  '\x046c'# -> unI64 1133
+  -- CYRILLIC CAPITAL LETTER KSI
+  '\x046e'# -> unI64 1135
+  -- CYRILLIC CAPITAL LETTER PSI
+  '\x0470'# -> unI64 1137
+  -- CYRILLIC CAPITAL LETTER FITA
+  '\x0472'# -> unI64 1139
+  -- CYRILLIC CAPITAL LETTER IZHITSA
+  '\x0474'# -> unI64 1141
+  -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+  '\x0476'# -> unI64 1143
+  -- CYRILLIC CAPITAL LETTER UK
+  '\x0478'# -> unI64 1145
+  -- CYRILLIC CAPITAL LETTER ROUND OMEGA
+  '\x047a'# -> unI64 1147
+  -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+  '\x047c'# -> unI64 1149
+  -- CYRILLIC CAPITAL LETTER OT
+  '\x047e'# -> unI64 1151
+  -- CYRILLIC CAPITAL LETTER KOPPA
+  '\x0480'# -> unI64 1153
+  -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+  '\x048a'# -> unI64 1163
+  -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+  '\x048c'# -> unI64 1165
+  -- CYRILLIC CAPITAL LETTER ER WITH TICK
+  '\x048e'# -> unI64 1167
+  -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+  '\x0490'# -> unI64 1169
+  -- CYRILLIC CAPITAL LETTER GHE WITH STROKE
+  '\x0492'# -> unI64 1171
+  -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+  '\x0494'# -> unI64 1173
+  -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+  '\x0496'# -> unI64 1175
+  -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+  '\x0498'# -> unI64 1177
+  -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+  '\x049a'# -> unI64 1179
+  -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+  '\x049c'# -> unI64 1181
+  -- CYRILLIC CAPITAL LETTER KA WITH STROKE
+  '\x049e'# -> unI64 1183
+  -- CYRILLIC CAPITAL LETTER BASHKIR KA
+  '\x04a0'# -> unI64 1185
+  -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+  '\x04a2'# -> unI64 1187
+  -- CYRILLIC CAPITAL LIGATURE EN GHE
+  '\x04a4'# -> unI64 1189
+  -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+  '\x04a6'# -> unI64 1191
+  -- CYRILLIC CAPITAL LETTER ABKHASIAN HA
+  '\x04a8'# -> unI64 1193
+  -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+  '\x04aa'# -> unI64 1195
+  -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+  '\x04ac'# -> unI64 1197
+  -- CYRILLIC CAPITAL LETTER STRAIGHT U
+  '\x04ae'# -> unI64 1199
+  -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+  '\x04b0'# -> unI64 1201
+  -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+  '\x04b2'# -> unI64 1203
+  -- CYRILLIC CAPITAL LIGATURE TE TSE
+  '\x04b4'# -> unI64 1205
+  -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+  '\x04b6'# -> unI64 1207
+  -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+  '\x04b8'# -> unI64 1209
+  -- CYRILLIC CAPITAL LETTER SHHA
+  '\x04ba'# -> unI64 1211
+  -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+  '\x04bc'# -> unI64 1213
+  -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+  '\x04be'# -> unI64 1215
+  -- CYRILLIC LETTER PALOCHKA
+  '\x04c0'# -> unI64 1231
+  -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+  '\x04c1'# -> unI64 1218
+  -- CYRILLIC CAPITAL LETTER KA WITH HOOK
+  '\x04c3'# -> unI64 1220
+  -- CYRILLIC CAPITAL LETTER EL WITH TAIL
+  '\x04c5'# -> unI64 1222
+  -- CYRILLIC CAPITAL LETTER EN WITH HOOK
+  '\x04c7'# -> unI64 1224
+  -- CYRILLIC CAPITAL LETTER EN WITH TAIL
+  '\x04c9'# -> unI64 1226
+  -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+  '\x04cb'# -> unI64 1228
+  -- CYRILLIC CAPITAL LETTER EM WITH TAIL
+  '\x04cd'# -> unI64 1230
+  -- CYRILLIC CAPITAL LETTER A WITH BREVE
+  '\x04d0'# -> unI64 1233
+  -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+  '\x04d2'# -> unI64 1235
+  -- CYRILLIC CAPITAL LIGATURE A IE
+  '\x04d4'# -> unI64 1237
+  -- CYRILLIC CAPITAL LETTER IE WITH BREVE
+  '\x04d6'# -> unI64 1239
+  -- CYRILLIC CAPITAL LETTER SCHWA
+  '\x04d8'# -> unI64 1241
+  -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+  '\x04da'# -> unI64 1243
+  -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+  '\x04dc'# -> unI64 1245
+  -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+  '\x04de'# -> unI64 1247
+  -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+  '\x04e0'# -> unI64 1249
+  -- CYRILLIC CAPITAL LETTER I WITH MACRON
+  '\x04e2'# -> unI64 1251
+  -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+  '\x04e4'# -> unI64 1253
+  -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+  '\x04e6'# -> unI64 1255
+  -- CYRILLIC CAPITAL LETTER BARRED O
+  '\x04e8'# -> unI64 1257
+  -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+  '\x04ea'# -> unI64 1259
+  -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+  '\x04ec'# -> unI64 1261
+  -- CYRILLIC CAPITAL LETTER U WITH MACRON
+  '\x04ee'# -> unI64 1263
+  -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+  '\x04f0'# -> unI64 1265
+  -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+  '\x04f2'# -> unI64 1267
+  -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+  '\x04f4'# -> unI64 1269
+  -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+  '\x04f6'# -> unI64 1271
+  -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+  '\x04f8'# -> unI64 1273
+  -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+  '\x04fa'# -> unI64 1275
+  -- CYRILLIC CAPITAL LETTER HA WITH HOOK
+  '\x04fc'# -> unI64 1277
+  -- CYRILLIC CAPITAL LETTER HA WITH STROKE
+  '\x04fe'# -> unI64 1279
+  -- CYRILLIC CAPITAL LETTER KOMI DE
+  '\x0500'# -> unI64 1281
+  -- CYRILLIC CAPITAL LETTER KOMI DJE
+  '\x0502'# -> unI64 1283
+  -- CYRILLIC CAPITAL LETTER KOMI ZJE
+  '\x0504'# -> unI64 1285
+  -- CYRILLIC CAPITAL LETTER KOMI DZJE
+  '\x0506'# -> unI64 1287
+  -- CYRILLIC CAPITAL LETTER KOMI LJE
+  '\x0508'# -> unI64 1289
+  -- CYRILLIC CAPITAL LETTER KOMI NJE
+  '\x050a'# -> unI64 1291
+  -- CYRILLIC CAPITAL LETTER KOMI SJE
+  '\x050c'# -> unI64 1293
+  -- CYRILLIC CAPITAL LETTER KOMI TJE
+  '\x050e'# -> unI64 1295
+  -- CYRILLIC CAPITAL LETTER REVERSED ZE
+  '\x0510'# -> unI64 1297
+  -- CYRILLIC CAPITAL LETTER EL WITH HOOK
+  '\x0512'# -> unI64 1299
+  -- CYRILLIC CAPITAL LETTER LHA
+  '\x0514'# -> unI64 1301
+  -- CYRILLIC CAPITAL LETTER RHA
+  '\x0516'# -> unI64 1303
+  -- CYRILLIC CAPITAL LETTER YAE
+  '\x0518'# -> unI64 1305
+  -- CYRILLIC CAPITAL LETTER QA
+  '\x051a'# -> unI64 1307
+  -- CYRILLIC CAPITAL LETTER WE
+  '\x051c'# -> unI64 1309
+  -- CYRILLIC CAPITAL LETTER ALEUT KA
+  '\x051e'# -> unI64 1311
+  -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+  '\x0520'# -> unI64 1313
+  -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+  '\x0522'# -> unI64 1315
+  -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+  '\x0524'# -> unI64 1317
+  -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+  '\x0526'# -> unI64 1319
+  -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+  '\x0528'# -> unI64 1321
+  -- CYRILLIC CAPITAL LETTER DZZHE
+  '\x052a'# -> unI64 1323
+  -- CYRILLIC CAPITAL LETTER DCHE
+  '\x052c'# -> unI64 1325
+  -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+  '\x052e'# -> unI64 1327
+  -- ARMENIAN CAPITAL LETTER AYB
+  '\x0531'# -> unI64 1377
+  -- ARMENIAN CAPITAL LETTER BEN
+  '\x0532'# -> unI64 1378
+  -- ARMENIAN CAPITAL LETTER GIM
+  '\x0533'# -> unI64 1379
+  -- ARMENIAN CAPITAL LETTER DA
+  '\x0534'# -> unI64 1380
+  -- ARMENIAN CAPITAL LETTER ECH
+  '\x0535'# -> unI64 1381
+  -- ARMENIAN CAPITAL LETTER ZA
+  '\x0536'# -> unI64 1382
+  -- ARMENIAN CAPITAL LETTER EH
+  '\x0537'# -> unI64 1383
+  -- ARMENIAN CAPITAL LETTER ET
+  '\x0538'# -> unI64 1384
+  -- ARMENIAN CAPITAL LETTER TO
+  '\x0539'# -> unI64 1385
+  -- ARMENIAN CAPITAL LETTER ZHE
+  '\x053a'# -> unI64 1386
+  -- ARMENIAN CAPITAL LETTER INI
+  '\x053b'# -> unI64 1387
+  -- ARMENIAN CAPITAL LETTER LIWN
+  '\x053c'# -> unI64 1388
+  -- ARMENIAN CAPITAL LETTER XEH
+  '\x053d'# -> unI64 1389
+  -- ARMENIAN CAPITAL LETTER CA
+  '\x053e'# -> unI64 1390
+  -- ARMENIAN CAPITAL LETTER KEN
+  '\x053f'# -> unI64 1391
+  -- ARMENIAN CAPITAL LETTER HO
+  '\x0540'# -> unI64 1392
+  -- ARMENIAN CAPITAL LETTER JA
+  '\x0541'# -> unI64 1393
+  -- ARMENIAN CAPITAL LETTER GHAD
+  '\x0542'# -> unI64 1394
+  -- ARMENIAN CAPITAL LETTER CHEH
+  '\x0543'# -> unI64 1395
+  -- ARMENIAN CAPITAL LETTER MEN
+  '\x0544'# -> unI64 1396
+  -- ARMENIAN CAPITAL LETTER YI
+  '\x0545'# -> unI64 1397
+  -- ARMENIAN CAPITAL LETTER NOW
+  '\x0546'# -> unI64 1398
+  -- ARMENIAN CAPITAL LETTER SHA
+  '\x0547'# -> unI64 1399
+  -- ARMENIAN CAPITAL LETTER VO
+  '\x0548'# -> unI64 1400
+  -- ARMENIAN CAPITAL LETTER CHA
+  '\x0549'# -> unI64 1401
+  -- ARMENIAN CAPITAL LETTER PEH
+  '\x054a'# -> unI64 1402
+  -- ARMENIAN CAPITAL LETTER JHEH
+  '\x054b'# -> unI64 1403
+  -- ARMENIAN CAPITAL LETTER RA
+  '\x054c'# -> unI64 1404
+  -- ARMENIAN CAPITAL LETTER SEH
+  '\x054d'# -> unI64 1405
+  -- ARMENIAN CAPITAL LETTER VEW
+  '\x054e'# -> unI64 1406
+  -- ARMENIAN CAPITAL LETTER TIWN
+  '\x054f'# -> unI64 1407
+  -- ARMENIAN CAPITAL LETTER REH
+  '\x0550'# -> unI64 1408
+  -- ARMENIAN CAPITAL LETTER CO
+  '\x0551'# -> unI64 1409
+  -- ARMENIAN CAPITAL LETTER YIWN
+  '\x0552'# -> unI64 1410
+  -- ARMENIAN CAPITAL LETTER PIWR
+  '\x0553'# -> unI64 1411
+  -- ARMENIAN CAPITAL LETTER KEH
+  '\x0554'# -> unI64 1412
+  -- ARMENIAN CAPITAL LETTER OH
+  '\x0555'# -> unI64 1413
+  -- ARMENIAN CAPITAL LETTER FEH
+  '\x0556'# -> unI64 1414
+  -- ARMENIAN SMALL LIGATURE ECH YIWN
+  '\x0587'# -> unI64 2956985701
+  -- GEORGIAN CAPITAL LETTER AN
+  '\x10a0'# -> unI64 11520
+  -- GEORGIAN CAPITAL LETTER BAN
+  '\x10a1'# -> unI64 11521
+  -- GEORGIAN CAPITAL LETTER GAN
+  '\x10a2'# -> unI64 11522
+  -- GEORGIAN CAPITAL LETTER DON
+  '\x10a3'# -> unI64 11523
+  -- GEORGIAN CAPITAL LETTER EN
+  '\x10a4'# -> unI64 11524
+  -- GEORGIAN CAPITAL LETTER VIN
+  '\x10a5'# -> unI64 11525
+  -- GEORGIAN CAPITAL LETTER ZEN
+  '\x10a6'# -> unI64 11526
+  -- GEORGIAN CAPITAL LETTER TAN
+  '\x10a7'# -> unI64 11527
+  -- GEORGIAN CAPITAL LETTER IN
+  '\x10a8'# -> unI64 11528
+  -- GEORGIAN CAPITAL LETTER KAN
+  '\x10a9'# -> unI64 11529
+  -- GEORGIAN CAPITAL LETTER LAS
+  '\x10aa'# -> unI64 11530
+  -- GEORGIAN CAPITAL LETTER MAN
+  '\x10ab'# -> unI64 11531
+  -- GEORGIAN CAPITAL LETTER NAR
+  '\x10ac'# -> unI64 11532
+  -- GEORGIAN CAPITAL LETTER ON
+  '\x10ad'# -> unI64 11533
+  -- GEORGIAN CAPITAL LETTER PAR
+  '\x10ae'# -> unI64 11534
+  -- GEORGIAN CAPITAL LETTER ZHAR
+  '\x10af'# -> unI64 11535
+  -- GEORGIAN CAPITAL LETTER RAE
+  '\x10b0'# -> unI64 11536
+  -- GEORGIAN CAPITAL LETTER SAN
+  '\x10b1'# -> unI64 11537
+  -- GEORGIAN CAPITAL LETTER TAR
+  '\x10b2'# -> unI64 11538
+  -- GEORGIAN CAPITAL LETTER UN
+  '\x10b3'# -> unI64 11539
+  -- GEORGIAN CAPITAL LETTER PHAR
+  '\x10b4'# -> unI64 11540
+  -- GEORGIAN CAPITAL LETTER KHAR
+  '\x10b5'# -> unI64 11541
+  -- GEORGIAN CAPITAL LETTER GHAN
+  '\x10b6'# -> unI64 11542
+  -- GEORGIAN CAPITAL LETTER QAR
+  '\x10b7'# -> unI64 11543
+  -- GEORGIAN CAPITAL LETTER SHIN
+  '\x10b8'# -> unI64 11544
+  -- GEORGIAN CAPITAL LETTER CHIN
+  '\x10b9'# -> unI64 11545
+  -- GEORGIAN CAPITAL LETTER CAN
+  '\x10ba'# -> unI64 11546
+  -- GEORGIAN CAPITAL LETTER JIL
+  '\x10bb'# -> unI64 11547
+  -- GEORGIAN CAPITAL LETTER CIL
+  '\x10bc'# -> unI64 11548
+  -- GEORGIAN CAPITAL LETTER CHAR
+  '\x10bd'# -> unI64 11549
+  -- GEORGIAN CAPITAL LETTER XAN
+  '\x10be'# -> unI64 11550
+  -- GEORGIAN CAPITAL LETTER JHAN
+  '\x10bf'# -> unI64 11551
+  -- GEORGIAN CAPITAL LETTER HAE
+  '\x10c0'# -> unI64 11552
+  -- GEORGIAN CAPITAL LETTER HE
+  '\x10c1'# -> unI64 11553
+  -- GEORGIAN CAPITAL LETTER HIE
+  '\x10c2'# -> unI64 11554
+  -- GEORGIAN CAPITAL LETTER WE
+  '\x10c3'# -> unI64 11555
+  -- GEORGIAN CAPITAL LETTER HAR
+  '\x10c4'# -> unI64 11556
+  -- GEORGIAN CAPITAL LETTER HOE
+  '\x10c5'# -> unI64 11557
+  -- GEORGIAN CAPITAL LETTER YN
+  '\x10c7'# -> unI64 11559
+  -- GEORGIAN CAPITAL LETTER AEN
+  '\x10cd'# -> unI64 11565
+  -- CHEROKEE SMALL LETTER YE
+  '\x13f8'# -> unI64 5104
+  -- CHEROKEE SMALL LETTER YI
+  '\x13f9'# -> unI64 5105
+  -- CHEROKEE SMALL LETTER YO
+  '\x13fa'# -> unI64 5106
+  -- CHEROKEE SMALL LETTER YU
+  '\x13fb'# -> unI64 5107
+  -- CHEROKEE SMALL LETTER YV
+  '\x13fc'# -> unI64 5108
+  -- CHEROKEE SMALL LETTER MV
+  '\x13fd'# -> unI64 5109
+  -- CYRILLIC SMALL LETTER ROUNDED VE
+  '\x1c80'# -> unI64 1074
+  -- CYRILLIC SMALL LETTER LONG-LEGGED DE
+  '\x1c81'# -> unI64 1076
+  -- CYRILLIC SMALL LETTER NARROW O
+  '\x1c82'# -> unI64 1086
+  -- CYRILLIC SMALL LETTER WIDE ES
+  '\x1c83'# -> unI64 1089
+  -- CYRILLIC SMALL LETTER TALL TE
+  '\x1c84'# -> unI64 1090
+  -- CYRILLIC SMALL LETTER THREE-LEGGED TE
+  '\x1c85'# -> unI64 1090
+  -- CYRILLIC SMALL LETTER TALL HARD SIGN
+  '\x1c86'# -> unI64 1098
+  -- CYRILLIC SMALL LETTER TALL YAT
+  '\x1c87'# -> unI64 1123
+  -- CYRILLIC SMALL LETTER UNBLENDED UK
+  '\x1c88'# -> unI64 42571
+  -- CYRILLIC CAPITAL LETTER TJE
+  '\x1c89'# -> unI64 7306
+  -- GEORGIAN MTAVRULI CAPITAL LETTER AN
+  '\x1c90'# -> unI64 4304
+  -- GEORGIAN MTAVRULI CAPITAL LETTER BAN
+  '\x1c91'# -> unI64 4305
+  -- GEORGIAN MTAVRULI CAPITAL LETTER GAN
+  '\x1c92'# -> unI64 4306
+  -- GEORGIAN MTAVRULI CAPITAL LETTER DON
+  '\x1c93'# -> unI64 4307
+  -- GEORGIAN MTAVRULI CAPITAL LETTER EN
+  '\x1c94'# -> unI64 4308
+  -- GEORGIAN MTAVRULI CAPITAL LETTER VIN
+  '\x1c95'# -> unI64 4309
+  -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN
+  '\x1c96'# -> unI64 4310
+  -- GEORGIAN MTAVRULI CAPITAL LETTER TAN
+  '\x1c97'# -> unI64 4311
+  -- GEORGIAN MTAVRULI CAPITAL LETTER IN
+  '\x1c98'# -> unI64 4312
+  -- GEORGIAN MTAVRULI CAPITAL LETTER KAN
+  '\x1c99'# -> unI64 4313
+  -- GEORGIAN MTAVRULI CAPITAL LETTER LAS
+  '\x1c9a'# -> unI64 4314
+  -- GEORGIAN MTAVRULI CAPITAL LETTER MAN
+  '\x1c9b'# -> unI64 4315
+  -- GEORGIAN MTAVRULI CAPITAL LETTER NAR
+  '\x1c9c'# -> unI64 4316
+  -- GEORGIAN MTAVRULI CAPITAL LETTER ON
+  '\x1c9d'# -> unI64 4317
+  -- GEORGIAN MTAVRULI CAPITAL LETTER PAR
+  '\x1c9e'# -> unI64 4318
+  -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR
+  '\x1c9f'# -> unI64 4319
+  -- GEORGIAN MTAVRULI CAPITAL LETTER RAE
+  '\x1ca0'# -> unI64 4320
+  -- GEORGIAN MTAVRULI CAPITAL LETTER SAN
+  '\x1ca1'# -> unI64 4321
+  -- GEORGIAN MTAVRULI CAPITAL LETTER TAR
+  '\x1ca2'# -> unI64 4322
+  -- GEORGIAN MTAVRULI CAPITAL LETTER UN
+  '\x1ca3'# -> unI64 4323
+  -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR
+  '\x1ca4'# -> unI64 4324
+  -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR
+  '\x1ca5'# -> unI64 4325
+  -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN
+  '\x1ca6'# -> unI64 4326
+  -- GEORGIAN MTAVRULI CAPITAL LETTER QAR
+  '\x1ca7'# -> unI64 4327
+  -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN
+  '\x1ca8'# -> unI64 4328
+  -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN
+  '\x1ca9'# -> unI64 4329
+  -- GEORGIAN MTAVRULI CAPITAL LETTER CAN
+  '\x1caa'# -> unI64 4330
+  -- GEORGIAN MTAVRULI CAPITAL LETTER JIL
+  '\x1cab'# -> unI64 4331
+  -- GEORGIAN MTAVRULI CAPITAL LETTER CIL
+  '\x1cac'# -> unI64 4332
+  -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR
+  '\x1cad'# -> unI64 4333
+  -- GEORGIAN MTAVRULI CAPITAL LETTER XAN
+  '\x1cae'# -> unI64 4334
+  -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN
+  '\x1caf'# -> unI64 4335
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HAE
+  '\x1cb0'# -> unI64 4336
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HE
+  '\x1cb1'# -> unI64 4337
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HIE
+  '\x1cb2'# -> unI64 4338
+  -- GEORGIAN MTAVRULI CAPITAL LETTER WE
+  '\x1cb3'# -> unI64 4339
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HAR
+  '\x1cb4'# -> unI64 4340
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HOE
+  '\x1cb5'# -> unI64 4341
+  -- GEORGIAN MTAVRULI CAPITAL LETTER FI
+  '\x1cb6'# -> unI64 4342
+  -- GEORGIAN MTAVRULI CAPITAL LETTER YN
+  '\x1cb7'# -> unI64 4343
+  -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI
+  '\x1cb8'# -> unI64 4344
+  -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN
+  '\x1cb9'# -> unI64 4345
+  -- GEORGIAN MTAVRULI CAPITAL LETTER AIN
+  '\x1cba'# -> unI64 4346
+  -- GEORGIAN MTAVRULI CAPITAL LETTER AEN
+  '\x1cbd'# -> unI64 4349
+  -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN
+  '\x1cbe'# -> unI64 4350
+  -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
+  '\x1cbf'# -> unI64 4351
+  -- LATIN CAPITAL LETTER A WITH RING BELOW
+  '\x1e00'# -> unI64 7681
+  -- LATIN CAPITAL LETTER B WITH DOT ABOVE
+  '\x1e02'# -> unI64 7683
+  -- LATIN CAPITAL LETTER B WITH DOT BELOW
+  '\x1e04'# -> unI64 7685
+  -- LATIN CAPITAL LETTER B WITH LINE BELOW
+  '\x1e06'# -> unI64 7687
+  -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+  '\x1e08'# -> unI64 7689
+  -- LATIN CAPITAL LETTER D WITH DOT ABOVE
+  '\x1e0a'# -> unI64 7691
+  -- LATIN CAPITAL LETTER D WITH DOT BELOW
+  '\x1e0c'# -> unI64 7693
+  -- LATIN CAPITAL LETTER D WITH LINE BELOW
+  '\x1e0e'# -> unI64 7695
+  -- LATIN CAPITAL LETTER D WITH CEDILLA
+  '\x1e10'# -> unI64 7697
+  -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+  '\x1e12'# -> unI64 7699
+  -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+  '\x1e14'# -> unI64 7701
+  -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+  '\x1e16'# -> unI64 7703
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+  '\x1e18'# -> unI64 7705
+  -- LATIN CAPITAL LETTER E WITH TILDE BELOW
+  '\x1e1a'# -> unI64 7707
+  -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+  '\x1e1c'# -> unI64 7709
+  -- LATIN CAPITAL LETTER F WITH DOT ABOVE
+  '\x1e1e'# -> unI64 7711
+  -- LATIN CAPITAL LETTER G WITH MACRON
+  '\x1e20'# -> unI64 7713
+  -- LATIN CAPITAL LETTER H WITH DOT ABOVE
+  '\x1e22'# -> unI64 7715
+  -- LATIN CAPITAL LETTER H WITH DOT BELOW
+  '\x1e24'# -> unI64 7717
+  -- LATIN CAPITAL LETTER H WITH DIAERESIS
+  '\x1e26'# -> unI64 7719
+  -- LATIN CAPITAL LETTER H WITH CEDILLA
+  '\x1e28'# -> unI64 7721
+  -- LATIN CAPITAL LETTER H WITH BREVE BELOW
+  '\x1e2a'# -> unI64 7723
+  -- LATIN CAPITAL LETTER I WITH TILDE BELOW
+  '\x1e2c'# -> unI64 7725
+  -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+  '\x1e2e'# -> unI64 7727
+  -- LATIN CAPITAL LETTER K WITH ACUTE
+  '\x1e30'# -> unI64 7729
+  -- LATIN CAPITAL LETTER K WITH DOT BELOW
+  '\x1e32'# -> unI64 7731
+  -- LATIN CAPITAL LETTER K WITH LINE BELOW
+  '\x1e34'# -> unI64 7733
+  -- LATIN CAPITAL LETTER L WITH DOT BELOW
+  '\x1e36'# -> unI64 7735
+  -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+  '\x1e38'# -> unI64 7737
+  -- LATIN CAPITAL LETTER L WITH LINE BELOW
+  '\x1e3a'# -> unI64 7739
+  -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+  '\x1e3c'# -> unI64 7741
+  -- LATIN CAPITAL LETTER M WITH ACUTE
+  '\x1e3e'# -> unI64 7743
+  -- LATIN CAPITAL LETTER M WITH DOT ABOVE
+  '\x1e40'# -> unI64 7745
+  -- LATIN CAPITAL LETTER M WITH DOT BELOW
+  '\x1e42'# -> unI64 7747
+  -- LATIN CAPITAL LETTER N WITH DOT ABOVE
+  '\x1e44'# -> unI64 7749
+  -- LATIN CAPITAL LETTER N WITH DOT BELOW
+  '\x1e46'# -> unI64 7751
+  -- LATIN CAPITAL LETTER N WITH LINE BELOW
+  '\x1e48'# -> unI64 7753
+  -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+  '\x1e4a'# -> unI64 7755
+  -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+  '\x1e4c'# -> unI64 7757
+  -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+  '\x1e4e'# -> unI64 7759
+  -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+  '\x1e50'# -> unI64 7761
+  -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+  '\x1e52'# -> unI64 7763
+  -- LATIN CAPITAL LETTER P WITH ACUTE
+  '\x1e54'# -> unI64 7765
+  -- LATIN CAPITAL LETTER P WITH DOT ABOVE
+  '\x1e56'# -> unI64 7767
+  -- LATIN CAPITAL LETTER R WITH DOT ABOVE
+  '\x1e58'# -> unI64 7769
+  -- LATIN CAPITAL LETTER R WITH DOT BELOW
+  '\x1e5a'# -> unI64 7771
+  -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+  '\x1e5c'# -> unI64 7773
+  -- LATIN CAPITAL LETTER R WITH LINE BELOW
+  '\x1e5e'# -> unI64 7775
+  -- LATIN CAPITAL LETTER S WITH DOT ABOVE
+  '\x1e60'# -> unI64 7777
+  -- LATIN CAPITAL LETTER S WITH DOT BELOW
+  '\x1e62'# -> unI64 7779
+  -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+  '\x1e64'# -> unI64 7781
+  -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+  '\x1e66'# -> unI64 7783
+  -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+  '\x1e68'# -> unI64 7785
+  -- LATIN CAPITAL LETTER T WITH DOT ABOVE
+  '\x1e6a'# -> unI64 7787
+  -- LATIN CAPITAL LETTER T WITH DOT BELOW
+  '\x1e6c'# -> unI64 7789
+  -- LATIN CAPITAL LETTER T WITH LINE BELOW
+  '\x1e6e'# -> unI64 7791
+  -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+  '\x1e70'# -> unI64 7793
+  -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+  '\x1e72'# -> unI64 7795
+  -- LATIN CAPITAL LETTER U WITH TILDE BELOW
+  '\x1e74'# -> unI64 7797
+  -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+  '\x1e76'# -> unI64 7799
+  -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+  '\x1e78'# -> unI64 7801
+  -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+  '\x1e7a'# -> unI64 7803
+  -- LATIN CAPITAL LETTER V WITH TILDE
+  '\x1e7c'# -> unI64 7805
+  -- LATIN CAPITAL LETTER V WITH DOT BELOW
+  '\x1e7e'# -> unI64 7807
+  -- LATIN CAPITAL LETTER W WITH GRAVE
+  '\x1e80'# -> unI64 7809
+  -- LATIN CAPITAL LETTER W WITH ACUTE
+  '\x1e82'# -> unI64 7811
+  -- LATIN CAPITAL LETTER W WITH DIAERESIS
+  '\x1e84'# -> unI64 7813
+  -- LATIN CAPITAL LETTER W WITH DOT ABOVE
+  '\x1e86'# -> unI64 7815
+  -- LATIN CAPITAL LETTER W WITH DOT BELOW
+  '\x1e88'# -> unI64 7817
+  -- LATIN CAPITAL LETTER X WITH DOT ABOVE
+  '\x1e8a'# -> unI64 7819
+  -- LATIN CAPITAL LETTER X WITH DIAERESIS
+  '\x1e8c'# -> unI64 7821
+  -- LATIN CAPITAL LETTER Y WITH DOT ABOVE
+  '\x1e8e'# -> unI64 7823
+  -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+  '\x1e90'# -> unI64 7825
+  -- LATIN CAPITAL LETTER Z WITH DOT BELOW
+  '\x1e92'# -> unI64 7827
+  -- LATIN CAPITAL LETTER Z WITH LINE BELOW
+  '\x1e94'# -> unI64 7829
+  -- LATIN SMALL LETTER H WITH LINE BELOW
+  '\x1e96'# -> unI64 1713373288
+  -- LATIN SMALL LETTER T WITH DIAERESIS
+  '\x1e97'# -> unI64 1627390068
+  -- LATIN SMALL LETTER W WITH RING ABOVE
+  '\x1e98'# -> unI64 1631584375
+  -- LATIN SMALL LETTER Y WITH RING ABOVE
+  '\x1e99'# -> unI64 1631584377
+  -- LATIN SMALL LETTER A WITH RIGHT HALF RING
+  '\x1e9a'# -> unI64 1472200801
+  -- LATIN SMALL LETTER LONG S WITH DOT ABOVE
+  '\x1e9b'# -> unI64 7777
+  -- LATIN CAPITAL LETTER SHARP S
+  '\x1e9e'# -> unI64 241172595
+  -- LATIN CAPITAL LETTER A WITH DOT BELOW
+  '\x1ea0'# -> unI64 7841
+  -- LATIN CAPITAL LETTER A WITH HOOK ABOVE
+  '\x1ea2'# -> unI64 7843
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+  '\x1ea4'# -> unI64 7845
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+  '\x1ea6'# -> unI64 7847
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+  '\x1ea8'# -> unI64 7849
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+  '\x1eaa'# -> unI64 7851
+  -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+  '\x1eac'# -> unI64 7853
+  -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+  '\x1eae'# -> unI64 7855
+  -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+  '\x1eb0'# -> unI64 7857
+  -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+  '\x1eb2'# -> unI64 7859
+  -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+  '\x1eb4'# -> unI64 7861
+  -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+  '\x1eb6'# -> unI64 7863
+  -- LATIN CAPITAL LETTER E WITH DOT BELOW
+  '\x1eb8'# -> unI64 7865
+  -- LATIN CAPITAL LETTER E WITH HOOK ABOVE
+  '\x1eba'# -> unI64 7867
+  -- LATIN CAPITAL LETTER E WITH TILDE
+  '\x1ebc'# -> unI64 7869
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+  '\x1ebe'# -> unI64 7871
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+  '\x1ec0'# -> unI64 7873
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+  '\x1ec2'# -> unI64 7875
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+  '\x1ec4'# -> unI64 7877
+  -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+  '\x1ec6'# -> unI64 7879
+  -- LATIN CAPITAL LETTER I WITH HOOK ABOVE
+  '\x1ec8'# -> unI64 7881
+  -- LATIN CAPITAL LETTER I WITH DOT BELOW
+  '\x1eca'# -> unI64 7883
+  -- LATIN CAPITAL LETTER O WITH DOT BELOW
+  '\x1ecc'# -> unI64 7885
+  -- LATIN CAPITAL LETTER O WITH HOOK ABOVE
+  '\x1ece'# -> unI64 7887
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+  '\x1ed0'# -> unI64 7889
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+  '\x1ed2'# -> unI64 7891
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+  '\x1ed4'# -> unI64 7893
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+  '\x1ed6'# -> unI64 7895
+  -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+  '\x1ed8'# -> unI64 7897
+  -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+  '\x1eda'# -> unI64 7899
+  -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+  '\x1edc'# -> unI64 7901
+  -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+  '\x1ede'# -> unI64 7903
+  -- LATIN CAPITAL LETTER O WITH HORN AND TILDE
+  '\x1ee0'# -> unI64 7905
+  -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+  '\x1ee2'# -> unI64 7907
+  -- LATIN CAPITAL LETTER U WITH DOT BELOW
+  '\x1ee4'# -> unI64 7909
+  -- LATIN CAPITAL LETTER U WITH HOOK ABOVE
+  '\x1ee6'# -> unI64 7911
+  -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+  '\x1ee8'# -> unI64 7913
+  -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+  '\x1eea'# -> unI64 7915
+  -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+  '\x1eec'# -> unI64 7917
+  -- LATIN CAPITAL LETTER U WITH HORN AND TILDE
+  '\x1eee'# -> unI64 7919
+  -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+  '\x1ef0'# -> unI64 7921
+  -- LATIN CAPITAL LETTER Y WITH GRAVE
+  '\x1ef2'# -> unI64 7923
+  -- LATIN CAPITAL LETTER Y WITH DOT BELOW
+  '\x1ef4'# -> unI64 7925
+  -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+  '\x1ef6'# -> unI64 7927
+  -- LATIN CAPITAL LETTER Y WITH TILDE
+  '\x1ef8'# -> unI64 7929
+  -- LATIN CAPITAL LETTER MIDDLE-WELSH LL
+  '\x1efa'# -> unI64 7931
+  -- LATIN CAPITAL LETTER MIDDLE-WELSH V
+  '\x1efc'# -> unI64 7933
+  -- LATIN CAPITAL LETTER Y WITH LOOP
+  '\x1efe'# -> unI64 7935
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI
+  '\x1f08'# -> unI64 7936
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA
+  '\x1f09'# -> unI64 7937
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+  '\x1f0a'# -> unI64 7938
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+  '\x1f0b'# -> unI64 7939
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+  '\x1f0c'# -> unI64 7940
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+  '\x1f0d'# -> unI64 7941
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+  '\x1f0e'# -> unI64 7942
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+  '\x1f0f'# -> unI64 7943
+  -- GREEK CAPITAL LETTER EPSILON WITH PSILI
+  '\x1f18'# -> unI64 7952
+  -- GREEK CAPITAL LETTER EPSILON WITH DASIA
+  '\x1f19'# -> unI64 7953
+  -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+  '\x1f1a'# -> unI64 7954
+  -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+  '\x1f1b'# -> unI64 7955
+  -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+  '\x1f1c'# -> unI64 7956
+  -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+  '\x1f1d'# -> unI64 7957
+  -- GREEK CAPITAL LETTER ETA WITH PSILI
+  '\x1f28'# -> unI64 7968
+  -- GREEK CAPITAL LETTER ETA WITH DASIA
+  '\x1f29'# -> unI64 7969
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+  '\x1f2a'# -> unI64 7970
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+  '\x1f2b'# -> unI64 7971
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+  '\x1f2c'# -> unI64 7972
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+  '\x1f2d'# -> unI64 7973
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+  '\x1f2e'# -> unI64 7974
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+  '\x1f2f'# -> unI64 7975
+  -- GREEK CAPITAL LETTER IOTA WITH PSILI
+  '\x1f38'# -> unI64 7984
+  -- GREEK CAPITAL LETTER IOTA WITH DASIA
+  '\x1f39'# -> unI64 7985
+  -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+  '\x1f3a'# -> unI64 7986
+  -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+  '\x1f3b'# -> unI64 7987
+  -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+  '\x1f3c'# -> unI64 7988
+  -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+  '\x1f3d'# -> unI64 7989
+  -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+  '\x1f3e'# -> unI64 7990
+  -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+  '\x1f3f'# -> unI64 7991
+  -- GREEK CAPITAL LETTER OMICRON WITH PSILI
+  '\x1f48'# -> unI64 8000
+  -- GREEK CAPITAL LETTER OMICRON WITH DASIA
+  '\x1f49'# -> unI64 8001
+  -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+  '\x1f4a'# -> unI64 8002
+  -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+  '\x1f4b'# -> unI64 8003
+  -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+  '\x1f4c'# -> unI64 8004
+  -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+  '\x1f4d'# -> unI64 8005
+  -- GREEK SMALL LETTER UPSILON WITH PSILI
+  '\x1f50'# -> unI64 1650459589
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+  '\x1f52'# -> unI64 3377701370987461
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+  '\x1f54'# -> unI64 3382099417498565
+  -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+  '\x1f56'# -> unI64 3667972440720325
+  -- GREEK CAPITAL LETTER UPSILON WITH DASIA
+  '\x1f59'# -> unI64 8017
+  -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+  '\x1f5b'# -> unI64 8019
+  -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+  '\x1f5d'# -> unI64 8021
+  -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+  '\x1f5f'# -> unI64 8023
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI
+  '\x1f68'# -> unI64 8032
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA
+  '\x1f69'# -> unI64 8033
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+  '\x1f6a'# -> unI64 8034
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+  '\x1f6b'# -> unI64 8035
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+  '\x1f6c'# -> unI64 8036
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+  '\x1f6d'# -> unI64 8037
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+  '\x1f6e'# -> unI64 8038
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+  '\x1f6f'# -> unI64 8039
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f80'# -> unI64 1998593792
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f81'# -> unI64 1998593793
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f82'# -> unI64 1998593794
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f83'# -> unI64 1998593795
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f84'# -> unI64 1998593796
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f85'# -> unI64 1998593797
+  -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f86'# -> unI64 1998593798
+  -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f87'# -> unI64 1998593799
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f88'# -> unI64 1998593792
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f89'# -> unI64 1998593793
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f8a'# -> unI64 1998593794
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f8b'# -> unI64 1998593795
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f8c'# -> unI64 1998593796
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f8d'# -> unI64 1998593797
+  -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8e'# -> unI64 1998593798
+  -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f8f'# -> unI64 1998593799
+  -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+  '\x1f90'# -> unI64 1998593824
+  -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+  '\x1f91'# -> unI64 1998593825
+  -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1f92'# -> unI64 1998593826
+  -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1f93'# -> unI64 1998593827
+  -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1f94'# -> unI64 1998593828
+  -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1f95'# -> unI64 1998593829
+  -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f96'# -> unI64 1998593830
+  -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1f97'# -> unI64 1998593831
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+  '\x1f98'# -> unI64 1998593824
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+  '\x1f99'# -> unI64 1998593825
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1f9a'# -> unI64 1998593826
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1f9b'# -> unI64 1998593827
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1f9c'# -> unI64 1998593828
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1f9d'# -> unI64 1998593829
+  -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9e'# -> unI64 1998593830
+  -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1f9f'# -> unI64 1998593831
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+  '\x1fa0'# -> unI64 1998593888
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+  '\x1fa1'# -> unI64 1998593889
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+  '\x1fa2'# -> unI64 1998593890
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+  '\x1fa3'# -> unI64 1998593891
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+  '\x1fa4'# -> unI64 1998593892
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+  '\x1fa5'# -> unI64 1998593893
+  -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa6'# -> unI64 1998593894
+  -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fa7'# -> unI64 1998593895
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+  '\x1fa8'# -> unI64 1998593888
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+  '\x1fa9'# -> unI64 1998593889
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+  '\x1faa'# -> unI64 1998593890
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+  '\x1fab'# -> unI64 1998593891
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+  '\x1fac'# -> unI64 1998593892
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+  '\x1fad'# -> unI64 1998593893
+  -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1fae'# -> unI64 1998593894
+  -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+  '\x1faf'# -> unI64 1998593895
+  -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fb2'# -> unI64 1998593904
+  -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+  '\x1fb3'# -> unI64 1998586801
+  -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fb4'# -> unI64 1998586796
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+  '\x1fb6'# -> unI64 1749025713
+  -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fb7'# -> unI64 4191340074107825
+  -- GREEK CAPITAL LETTER ALPHA WITH VRACHY
+  '\x1fb8'# -> unI64 8112
+  -- GREEK CAPITAL LETTER ALPHA WITH MACRON
+  '\x1fb9'# -> unI64 8113
+  -- GREEK CAPITAL LETTER ALPHA WITH VARIA
+  '\x1fba'# -> unI64 8048
+  -- GREEK CAPITAL LETTER ALPHA WITH OXIA
+  '\x1fbb'# -> unI64 8049
+  -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+  '\x1fbc'# -> unI64 1998586801
+  -- GREEK PROSGEGRAMMENI
+  '\x1fbe'# -> unI64 953
+  -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+  '\x1fc2'# -> unI64 1998593908
+  -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+  '\x1fc3'# -> unI64 1998586807
+  -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+  '\x1fc4'# -> unI64 1998586798
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI
+  '\x1fc6'# -> unI64 1749025719
+  -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1fc7'# -> unI64 4191340074107831
+  -- GREEK CAPITAL LETTER EPSILON WITH VARIA
+  '\x1fc8'# -> unI64 8050
+  -- GREEK CAPITAL LETTER EPSILON WITH OXIA
+  '\x1fc9'# -> unI64 8051
+  -- GREEK CAPITAL LETTER ETA WITH VARIA
+  '\x1fca'# -> unI64 8052
+  -- GREEK CAPITAL LETTER ETA WITH OXIA
+  '\x1fcb'# -> unI64 8053
+  -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+  '\x1fcc'# -> unI64 1998586807
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+  '\x1fd2'# -> unI64 3377701347918777
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+  '\x1fd3'# -> unI64 3382099394429881
+  -- GREEK SMALL LETTER IOTA WITH PERISPOMENI
+  '\x1fd6'# -> unI64 1749025721
+  -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+  '\x1fd7'# -> unI64 3667972417651641
+  -- GREEK CAPITAL LETTER IOTA WITH VRACHY
+  '\x1fd8'# -> unI64 8144
+  -- GREEK CAPITAL LETTER IOTA WITH MACRON
+  '\x1fd9'# -> unI64 8145
+  -- GREEK CAPITAL LETTER IOTA WITH VARIA
+  '\x1fda'# -> unI64 8054
+  -- GREEK CAPITAL LETTER IOTA WITH OXIA
+  '\x1fdb'# -> unI64 8055
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+  '\x1fe2'# -> unI64 3377701347918789
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+  '\x1fe3'# -> unI64 3382099394429893
+  -- GREEK SMALL LETTER RHO WITH PSILI
+  '\x1fe4'# -> unI64 1650459585
+  -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+  '\x1fe6'# -> unI64 1749025733
+  -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+  '\x1fe7'# -> unI64 3667972417651653
+  -- GREEK CAPITAL LETTER UPSILON WITH VRACHY
+  '\x1fe8'# -> unI64 8160
+  -- GREEK CAPITAL LETTER UPSILON WITH MACRON
+  '\x1fe9'# -> unI64 8161
+  -- GREEK CAPITAL LETTER UPSILON WITH VARIA
+  '\x1fea'# -> unI64 8058
+  -- GREEK CAPITAL LETTER UPSILON WITH OXIA
+  '\x1feb'# -> unI64 8059
+  -- GREEK CAPITAL LETTER RHO WITH DASIA
+  '\x1fec'# -> unI64 8165
+  -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+  '\x1ff2'# -> unI64 1998593916
+  -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+  '\x1ff3'# -> unI64 1998586825
+  -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+  '\x1ff4'# -> unI64 1998586830
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+  '\x1ff6'# -> unI64 1749025737
+  -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+  '\x1ff7'# -> unI64 4191340074107849
+  -- GREEK CAPITAL LETTER OMICRON WITH VARIA
+  '\x1ff8'# -> unI64 8056
+  -- GREEK CAPITAL LETTER OMICRON WITH OXIA
+  '\x1ff9'# -> unI64 8057
+  -- GREEK CAPITAL LETTER OMEGA WITH VARIA
+  '\x1ffa'# -> unI64 8060
+  -- GREEK CAPITAL LETTER OMEGA WITH OXIA
+  '\x1ffb'# -> unI64 8061
+  -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+  '\x1ffc'# -> unI64 1998586825
+  -- OHM SIGN
+  '\x2126'# -> unI64 969
+  -- KELVIN SIGN
+  '\x212a'# -> unI64 107
+  -- ANGSTROM SIGN
+  '\x212b'# -> unI64 229
+  -- TURNED CAPITAL F
+  '\x2132'# -> unI64 8526
+  -- ROMAN NUMERAL ONE
+  '\x2160'# -> unI64 8560
+  -- ROMAN NUMERAL TWO
+  '\x2161'# -> unI64 8561
+  -- ROMAN NUMERAL THREE
+  '\x2162'# -> unI64 8562
+  -- ROMAN NUMERAL FOUR
+  '\x2163'# -> unI64 8563
+  -- ROMAN NUMERAL FIVE
+  '\x2164'# -> unI64 8564
+  -- ROMAN NUMERAL SIX
+  '\x2165'# -> unI64 8565
+  -- ROMAN NUMERAL SEVEN
+  '\x2166'# -> unI64 8566
+  -- ROMAN NUMERAL EIGHT
+  '\x2167'# -> unI64 8567
+  -- ROMAN NUMERAL NINE
+  '\x2168'# -> unI64 8568
+  -- ROMAN NUMERAL TEN
+  '\x2169'# -> unI64 8569
+  -- ROMAN NUMERAL ELEVEN
+  '\x216a'# -> unI64 8570
+  -- ROMAN NUMERAL TWELVE
+  '\x216b'# -> unI64 8571
+  -- ROMAN NUMERAL FIFTY
+  '\x216c'# -> unI64 8572
+  -- ROMAN NUMERAL ONE HUNDRED
+  '\x216d'# -> unI64 8573
+  -- ROMAN NUMERAL FIVE HUNDRED
+  '\x216e'# -> unI64 8574
+  -- ROMAN NUMERAL ONE THOUSAND
+  '\x216f'# -> unI64 8575
+  -- ROMAN NUMERAL REVERSED ONE HUNDRED
+  '\x2183'# -> unI64 8580
+  -- CIRCLED LATIN CAPITAL LETTER A
+  '\x24b6'# -> unI64 9424
+  -- CIRCLED LATIN CAPITAL LETTER B
+  '\x24b7'# -> unI64 9425
+  -- CIRCLED LATIN CAPITAL LETTER C
+  '\x24b8'# -> unI64 9426
+  -- CIRCLED LATIN CAPITAL LETTER D
+  '\x24b9'# -> unI64 9427
+  -- CIRCLED LATIN CAPITAL LETTER E
+  '\x24ba'# -> unI64 9428
+  -- CIRCLED LATIN CAPITAL LETTER F
+  '\x24bb'# -> unI64 9429
+  -- CIRCLED LATIN CAPITAL LETTER G
+  '\x24bc'# -> unI64 9430
+  -- CIRCLED LATIN CAPITAL LETTER H
+  '\x24bd'# -> unI64 9431
+  -- CIRCLED LATIN CAPITAL LETTER I
+  '\x24be'# -> unI64 9432
+  -- CIRCLED LATIN CAPITAL LETTER J
+  '\x24bf'# -> unI64 9433
+  -- CIRCLED LATIN CAPITAL LETTER K
+  '\x24c0'# -> unI64 9434
+  -- CIRCLED LATIN CAPITAL LETTER L
+  '\x24c1'# -> unI64 9435
+  -- CIRCLED LATIN CAPITAL LETTER M
+  '\x24c2'# -> unI64 9436
+  -- CIRCLED LATIN CAPITAL LETTER N
+  '\x24c3'# -> unI64 9437
+  -- CIRCLED LATIN CAPITAL LETTER O
+  '\x24c4'# -> unI64 9438
+  -- CIRCLED LATIN CAPITAL LETTER P
+  '\x24c5'# -> unI64 9439
+  -- CIRCLED LATIN CAPITAL LETTER Q
+  '\x24c6'# -> unI64 9440
+  -- CIRCLED LATIN CAPITAL LETTER R
+  '\x24c7'# -> unI64 9441
+  -- CIRCLED LATIN CAPITAL LETTER S
+  '\x24c8'# -> unI64 9442
+  -- CIRCLED LATIN CAPITAL LETTER T
+  '\x24c9'# -> unI64 9443
+  -- CIRCLED LATIN CAPITAL LETTER U
+  '\x24ca'# -> unI64 9444
+  -- CIRCLED LATIN CAPITAL LETTER V
+  '\x24cb'# -> unI64 9445
+  -- CIRCLED LATIN CAPITAL LETTER W
+  '\x24cc'# -> unI64 9446
+  -- CIRCLED LATIN CAPITAL LETTER X
+  '\x24cd'# -> unI64 9447
+  -- CIRCLED LATIN CAPITAL LETTER Y
+  '\x24ce'# -> unI64 9448
+  -- CIRCLED LATIN CAPITAL LETTER Z
+  '\x24cf'# -> unI64 9449
+  -- GLAGOLITIC CAPITAL LETTER AZU
+  '\x2c00'# -> unI64 11312
+  -- GLAGOLITIC CAPITAL LETTER BUKY
+  '\x2c01'# -> unI64 11313
+  -- GLAGOLITIC CAPITAL LETTER VEDE
+  '\x2c02'# -> unI64 11314
+  -- GLAGOLITIC CAPITAL LETTER GLAGOLI
+  '\x2c03'# -> unI64 11315
+  -- GLAGOLITIC CAPITAL LETTER DOBRO
+  '\x2c04'# -> unI64 11316
+  -- GLAGOLITIC CAPITAL LETTER YESTU
+  '\x2c05'# -> unI64 11317
+  -- GLAGOLITIC CAPITAL LETTER ZHIVETE
+  '\x2c06'# -> unI64 11318
+  -- GLAGOLITIC CAPITAL LETTER DZELO
+  '\x2c07'# -> unI64 11319
+  -- GLAGOLITIC CAPITAL LETTER ZEMLJA
+  '\x2c08'# -> unI64 11320
+  -- GLAGOLITIC CAPITAL LETTER IZHE
+  '\x2c09'# -> unI64 11321
+  -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+  '\x2c0a'# -> unI64 11322
+  -- GLAGOLITIC CAPITAL LETTER I
+  '\x2c0b'# -> unI64 11323
+  -- GLAGOLITIC CAPITAL LETTER DJERVI
+  '\x2c0c'# -> unI64 11324
+  -- GLAGOLITIC CAPITAL LETTER KAKO
+  '\x2c0d'# -> unI64 11325
+  -- GLAGOLITIC CAPITAL LETTER LJUDIJE
+  '\x2c0e'# -> unI64 11326
+  -- GLAGOLITIC CAPITAL LETTER MYSLITE
+  '\x2c0f'# -> unI64 11327
+  -- GLAGOLITIC CAPITAL LETTER NASHI
+  '\x2c10'# -> unI64 11328
+  -- GLAGOLITIC CAPITAL LETTER ONU
+  '\x2c11'# -> unI64 11329
+  -- GLAGOLITIC CAPITAL LETTER POKOJI
+  '\x2c12'# -> unI64 11330
+  -- GLAGOLITIC CAPITAL LETTER RITSI
+  '\x2c13'# -> unI64 11331
+  -- GLAGOLITIC CAPITAL LETTER SLOVO
+  '\x2c14'# -> unI64 11332
+  -- GLAGOLITIC CAPITAL LETTER TVRIDO
+  '\x2c15'# -> unI64 11333
+  -- GLAGOLITIC CAPITAL LETTER UKU
+  '\x2c16'# -> unI64 11334
+  -- GLAGOLITIC CAPITAL LETTER FRITU
+  '\x2c17'# -> unI64 11335
+  -- GLAGOLITIC CAPITAL LETTER HERU
+  '\x2c18'# -> unI64 11336
+  -- GLAGOLITIC CAPITAL LETTER OTU
+  '\x2c19'# -> unI64 11337
+  -- GLAGOLITIC CAPITAL LETTER PE
+  '\x2c1a'# -> unI64 11338
+  -- GLAGOLITIC CAPITAL LETTER SHTA
+  '\x2c1b'# -> unI64 11339
+  -- GLAGOLITIC CAPITAL LETTER TSI
+  '\x2c1c'# -> unI64 11340
+  -- GLAGOLITIC CAPITAL LETTER CHRIVI
+  '\x2c1d'# -> unI64 11341
+  -- GLAGOLITIC CAPITAL LETTER SHA
+  '\x2c1e'# -> unI64 11342
+  -- GLAGOLITIC CAPITAL LETTER YERU
+  '\x2c1f'# -> unI64 11343
+  -- GLAGOLITIC CAPITAL LETTER YERI
+  '\x2c20'# -> unI64 11344
+  -- GLAGOLITIC CAPITAL LETTER YATI
+  '\x2c21'# -> unI64 11345
+  -- GLAGOLITIC CAPITAL LETTER SPIDERY HA
+  '\x2c22'# -> unI64 11346
+  -- GLAGOLITIC CAPITAL LETTER YU
+  '\x2c23'# -> unI64 11347
+  -- GLAGOLITIC CAPITAL LETTER SMALL YUS
+  '\x2c24'# -> unI64 11348
+  -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+  '\x2c25'# -> unI64 11349
+  -- GLAGOLITIC CAPITAL LETTER YO
+  '\x2c26'# -> unI64 11350
+  -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+  '\x2c27'# -> unI64 11351
+  -- GLAGOLITIC CAPITAL LETTER BIG YUS
+  '\x2c28'# -> unI64 11352
+  -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+  '\x2c29'# -> unI64 11353
+  -- GLAGOLITIC CAPITAL LETTER FITA
+  '\x2c2a'# -> unI64 11354
+  -- GLAGOLITIC CAPITAL LETTER IZHITSA
+  '\x2c2b'# -> unI64 11355
+  -- GLAGOLITIC CAPITAL LETTER SHTAPIC
+  '\x2c2c'# -> unI64 11356
+  -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+  '\x2c2d'# -> unI64 11357
+  -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+  '\x2c2e'# -> unI64 11358
+  -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI
+  '\x2c2f'# -> unI64 11359
+  -- LATIN CAPITAL LETTER L WITH DOUBLE BAR
+  '\x2c60'# -> unI64 11361
+  -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+  '\x2c62'# -> unI64 619
+  -- LATIN CAPITAL LETTER P WITH STROKE
+  '\x2c63'# -> unI64 7549
+  -- LATIN CAPITAL LETTER R WITH TAIL
+  '\x2c64'# -> unI64 637
+  -- LATIN CAPITAL LETTER H WITH DESCENDER
+  '\x2c67'# -> unI64 11368
+  -- LATIN CAPITAL LETTER K WITH DESCENDER
+  '\x2c69'# -> unI64 11370
+  -- LATIN CAPITAL LETTER Z WITH DESCENDER
+  '\x2c6b'# -> unI64 11372
+  -- LATIN CAPITAL LETTER ALPHA
+  '\x2c6d'# -> unI64 593
+  -- LATIN CAPITAL LETTER M WITH HOOK
+  '\x2c6e'# -> unI64 625
+  -- LATIN CAPITAL LETTER TURNED A
+  '\x2c6f'# -> unI64 592
+  -- LATIN CAPITAL LETTER TURNED ALPHA
+  '\x2c70'# -> unI64 594
+  -- LATIN CAPITAL LETTER W WITH HOOK
+  '\x2c72'# -> unI64 11379
+  -- LATIN CAPITAL LETTER HALF H
+  '\x2c75'# -> unI64 11382
+  -- LATIN CAPITAL LETTER S WITH SWASH TAIL
+  '\x2c7e'# -> unI64 575
+  -- LATIN CAPITAL LETTER Z WITH SWASH TAIL
+  '\x2c7f'# -> unI64 576
+  -- COPTIC CAPITAL LETTER ALFA
+  '\x2c80'# -> unI64 11393
+  -- COPTIC CAPITAL LETTER VIDA
+  '\x2c82'# -> unI64 11395
+  -- COPTIC CAPITAL LETTER GAMMA
+  '\x2c84'# -> unI64 11397
+  -- COPTIC CAPITAL LETTER DALDA
+  '\x2c86'# -> unI64 11399
+  -- COPTIC CAPITAL LETTER EIE
+  '\x2c88'# -> unI64 11401
+  -- COPTIC CAPITAL LETTER SOU
+  '\x2c8a'# -> unI64 11403
+  -- COPTIC CAPITAL LETTER ZATA
+  '\x2c8c'# -> unI64 11405
+  -- COPTIC CAPITAL LETTER HATE
+  '\x2c8e'# -> unI64 11407
+  -- COPTIC CAPITAL LETTER THETHE
+  '\x2c90'# -> unI64 11409
+  -- COPTIC CAPITAL LETTER IAUDA
+  '\x2c92'# -> unI64 11411
+  -- COPTIC CAPITAL LETTER KAPA
+  '\x2c94'# -> unI64 11413
+  -- COPTIC CAPITAL LETTER LAULA
+  '\x2c96'# -> unI64 11415
+  -- COPTIC CAPITAL LETTER MI
+  '\x2c98'# -> unI64 11417
+  -- COPTIC CAPITAL LETTER NI
+  '\x2c9a'# -> unI64 11419
+  -- COPTIC CAPITAL LETTER KSI
+  '\x2c9c'# -> unI64 11421
+  -- COPTIC CAPITAL LETTER O
+  '\x2c9e'# -> unI64 11423
+  -- COPTIC CAPITAL LETTER PI
+  '\x2ca0'# -> unI64 11425
+  -- COPTIC CAPITAL LETTER RO
+  '\x2ca2'# -> unI64 11427
+  -- COPTIC CAPITAL LETTER SIMA
+  '\x2ca4'# -> unI64 11429
+  -- COPTIC CAPITAL LETTER TAU
+  '\x2ca6'# -> unI64 11431
+  -- COPTIC CAPITAL LETTER UA
+  '\x2ca8'# -> unI64 11433
+  -- COPTIC CAPITAL LETTER FI
+  '\x2caa'# -> unI64 11435
+  -- COPTIC CAPITAL LETTER KHI
+  '\x2cac'# -> unI64 11437
+  -- COPTIC CAPITAL LETTER PSI
+  '\x2cae'# -> unI64 11439
+  -- COPTIC CAPITAL LETTER OOU
+  '\x2cb0'# -> unI64 11441
+  -- COPTIC CAPITAL LETTER DIALECT-P ALEF
+  '\x2cb2'# -> unI64 11443
+  -- COPTIC CAPITAL LETTER OLD COPTIC AIN
+  '\x2cb4'# -> unI64 11445
+  -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
+  '\x2cb6'# -> unI64 11447
+  -- COPTIC CAPITAL LETTER DIALECT-P KAPA
+  '\x2cb8'# -> unI64 11449
+  -- COPTIC CAPITAL LETTER DIALECT-P NI
+  '\x2cba'# -> unI64 11451
+  -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
+  '\x2cbc'# -> unI64 11453
+  -- COPTIC CAPITAL LETTER OLD COPTIC OOU
+  '\x2cbe'# -> unI64 11455
+  -- COPTIC CAPITAL LETTER SAMPI
+  '\x2cc0'# -> unI64 11457
+  -- COPTIC CAPITAL LETTER CROSSED SHEI
+  '\x2cc2'# -> unI64 11459
+  -- COPTIC CAPITAL LETTER OLD COPTIC SHEI
+  '\x2cc4'# -> unI64 11461
+  -- COPTIC CAPITAL LETTER OLD COPTIC ESH
+  '\x2cc6'# -> unI64 11463
+  -- COPTIC CAPITAL LETTER AKHMIMIC KHEI
+  '\x2cc8'# -> unI64 11465
+  -- COPTIC CAPITAL LETTER DIALECT-P HORI
+  '\x2cca'# -> unI64 11467
+  -- COPTIC CAPITAL LETTER OLD COPTIC HORI
+  '\x2ccc'# -> unI64 11469
+  -- COPTIC CAPITAL LETTER OLD COPTIC HA
+  '\x2cce'# -> unI64 11471
+  -- COPTIC CAPITAL LETTER L-SHAPED HA
+  '\x2cd0'# -> unI64 11473
+  -- COPTIC CAPITAL LETTER OLD COPTIC HEI
+  '\x2cd2'# -> unI64 11475
+  -- COPTIC CAPITAL LETTER OLD COPTIC HAT
+  '\x2cd4'# -> unI64 11477
+  -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA
+  '\x2cd6'# -> unI64 11479
+  -- COPTIC CAPITAL LETTER OLD COPTIC DJA
+  '\x2cd8'# -> unI64 11481
+  -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA
+  '\x2cda'# -> unI64 11483
+  -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
+  '\x2cdc'# -> unI64 11485
+  -- COPTIC CAPITAL LETTER OLD NUBIAN NGI
+  '\x2cde'# -> unI64 11487
+  -- COPTIC CAPITAL LETTER OLD NUBIAN NYI
+  '\x2ce0'# -> unI64 11489
+  -- COPTIC CAPITAL LETTER OLD NUBIAN WAU
+  '\x2ce2'# -> unI64 11491
+  -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
+  '\x2ceb'# -> unI64 11500
+  -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
+  '\x2ced'# -> unI64 11502
+  -- COPTIC CAPITAL LETTER BOHAIRIC KHEI
+  '\x2cf2'# -> unI64 11507
+  -- CYRILLIC CAPITAL LETTER ZEMLYA
+  '\xa640'# -> unI64 42561
+  -- CYRILLIC CAPITAL LETTER DZELO
+  '\xa642'# -> unI64 42563
+  -- CYRILLIC CAPITAL LETTER REVERSED DZE
+  '\xa644'# -> unI64 42565
+  -- CYRILLIC CAPITAL LETTER IOTA
+  '\xa646'# -> unI64 42567
+  -- CYRILLIC CAPITAL LETTER DJERV
+  '\xa648'# -> unI64 42569
+  -- CYRILLIC CAPITAL LETTER MONOGRAPH UK
+  '\xa64a'# -> unI64 42571
+  -- CYRILLIC CAPITAL LETTER BROAD OMEGA
+  '\xa64c'# -> unI64 42573
+  -- CYRILLIC CAPITAL LETTER NEUTRAL YER
+  '\xa64e'# -> unI64 42575
+  -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER
+  '\xa650'# -> unI64 42577
+  -- CYRILLIC CAPITAL LETTER IOTIFIED YAT
+  '\xa652'# -> unI64 42579
+  -- CYRILLIC CAPITAL LETTER REVERSED YU
+  '\xa654'# -> unI64 42581
+  -- CYRILLIC CAPITAL LETTER IOTIFIED A
+  '\xa656'# -> unI64 42583
+  -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
+  '\xa658'# -> unI64 42585
+  -- CYRILLIC CAPITAL LETTER BLENDED YUS
+  '\xa65a'# -> unI64 42587
+  -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
+  '\xa65c'# -> unI64 42589
+  -- CYRILLIC CAPITAL LETTER YN
+  '\xa65e'# -> unI64 42591
+  -- CYRILLIC CAPITAL LETTER REVERSED TSE
+  '\xa660'# -> unI64 42593
+  -- CYRILLIC CAPITAL LETTER SOFT DE
+  '\xa662'# -> unI64 42595
+  -- CYRILLIC CAPITAL LETTER SOFT EL
+  '\xa664'# -> unI64 42597
+  -- CYRILLIC CAPITAL LETTER SOFT EM
+  '\xa666'# -> unI64 42599
+  -- CYRILLIC CAPITAL LETTER MONOCULAR O
+  '\xa668'# -> unI64 42601
+  -- CYRILLIC CAPITAL LETTER BINOCULAR O
+  '\xa66a'# -> unI64 42603
+  -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
+  '\xa66c'# -> unI64 42605
+  -- CYRILLIC CAPITAL LETTER DWE
+  '\xa680'# -> unI64 42625
+  -- CYRILLIC CAPITAL LETTER DZWE
+  '\xa682'# -> unI64 42627
+  -- CYRILLIC CAPITAL LETTER ZHWE
+  '\xa684'# -> unI64 42629
+  -- CYRILLIC CAPITAL LETTER CCHE
+  '\xa686'# -> unI64 42631
+  -- CYRILLIC CAPITAL LETTER DZZE
+  '\xa688'# -> unI64 42633
+  -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
+  '\xa68a'# -> unI64 42635
+  -- CYRILLIC CAPITAL LETTER TWE
+  '\xa68c'# -> unI64 42637
+  -- CYRILLIC CAPITAL LETTER TSWE
+  '\xa68e'# -> unI64 42639
+  -- CYRILLIC CAPITAL LETTER TSSE
+  '\xa690'# -> unI64 42641
+  -- CYRILLIC CAPITAL LETTER TCHE
+  '\xa692'# -> unI64 42643
+  -- CYRILLIC CAPITAL LETTER HWE
+  '\xa694'# -> unI64 42645
+  -- CYRILLIC CAPITAL LETTER SHWE
+  '\xa696'# -> unI64 42647
+  -- CYRILLIC CAPITAL LETTER DOUBLE O
+  '\xa698'# -> unI64 42649
+  -- CYRILLIC CAPITAL LETTER CROSSED O
+  '\xa69a'# -> unI64 42651
+  -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
+  '\xa722'# -> unI64 42787
+  -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
+  '\xa724'# -> unI64 42789
+  -- LATIN CAPITAL LETTER HENG
+  '\xa726'# -> unI64 42791
+  -- LATIN CAPITAL LETTER TZ
+  '\xa728'# -> unI64 42793
+  -- LATIN CAPITAL LETTER TRESILLO
+  '\xa72a'# -> unI64 42795
+  -- LATIN CAPITAL LETTER CUATRILLO
+  '\xa72c'# -> unI64 42797
+  -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA
+  '\xa72e'# -> unI64 42799
+  -- LATIN CAPITAL LETTER AA
+  '\xa732'# -> unI64 42803
+  -- LATIN CAPITAL LETTER AO
+  '\xa734'# -> unI64 42805
+  -- LATIN CAPITAL LETTER AU
+  '\xa736'# -> unI64 42807
+  -- LATIN CAPITAL LETTER AV
+  '\xa738'# -> unI64 42809
+  -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
+  '\xa73a'# -> unI64 42811
+  -- LATIN CAPITAL LETTER AY
+  '\xa73c'# -> unI64 42813
+  -- LATIN CAPITAL LETTER REVERSED C WITH DOT
+  '\xa73e'# -> unI64 42815
+  -- LATIN CAPITAL LETTER K WITH STROKE
+  '\xa740'# -> unI64 42817
+  -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
+  '\xa742'# -> unI64 42819
+  -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
+  '\xa744'# -> unI64 42821
+  -- LATIN CAPITAL LETTER BROKEN L
+  '\xa746'# -> unI64 42823
+  -- LATIN CAPITAL LETTER L WITH HIGH STROKE
+  '\xa748'# -> unI64 42825
+  -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
+  '\xa74a'# -> unI64 42827
+  -- LATIN CAPITAL LETTER O WITH LOOP
+  '\xa74c'# -> unI64 42829
+  -- LATIN CAPITAL LETTER OO
+  '\xa74e'# -> unI64 42831
+  -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
+  '\xa750'# -> unI64 42833
+  -- LATIN CAPITAL LETTER P WITH FLOURISH
+  '\xa752'# -> unI64 42835
+  -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
+  '\xa754'# -> unI64 42837
+  -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
+  '\xa756'# -> unI64 42839
+  -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
+  '\xa758'# -> unI64 42841
+  -- LATIN CAPITAL LETTER R ROTUNDA
+  '\xa75a'# -> unI64 42843
+  -- LATIN CAPITAL LETTER RUM ROTUNDA
+  '\xa75c'# -> unI64 42845
+  -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
+  '\xa75e'# -> unI64 42847
+  -- LATIN CAPITAL LETTER VY
+  '\xa760'# -> unI64 42849
+  -- LATIN CAPITAL LETTER VISIGOTHIC Z
+  '\xa762'# -> unI64 42851
+  -- LATIN CAPITAL LETTER THORN WITH STROKE
+  '\xa764'# -> unI64 42853
+  -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
+  '\xa766'# -> unI64 42855
+  -- LATIN CAPITAL LETTER VEND
+  '\xa768'# -> unI64 42857
+  -- LATIN CAPITAL LETTER ET
+  '\xa76a'# -> unI64 42859
+  -- LATIN CAPITAL LETTER IS
+  '\xa76c'# -> unI64 42861
+  -- LATIN CAPITAL LETTER CON
+  '\xa76e'# -> unI64 42863
+  -- LATIN CAPITAL LETTER INSULAR D
+  '\xa779'# -> unI64 42874
+  -- LATIN CAPITAL LETTER INSULAR F
+  '\xa77b'# -> unI64 42876
+  -- LATIN CAPITAL LETTER INSULAR G
+  '\xa77d'# -> unI64 7545
+  -- LATIN CAPITAL LETTER TURNED INSULAR G
+  '\xa77e'# -> unI64 42879
+  -- LATIN CAPITAL LETTER TURNED L
+  '\xa780'# -> unI64 42881
+  -- LATIN CAPITAL LETTER INSULAR R
+  '\xa782'# -> unI64 42883
+  -- LATIN CAPITAL LETTER INSULAR S
+  '\xa784'# -> unI64 42885
+  -- LATIN CAPITAL LETTER INSULAR T
+  '\xa786'# -> unI64 42887
+  -- LATIN CAPITAL LETTER SALTILLO
+  '\xa78b'# -> unI64 42892
+  -- LATIN CAPITAL LETTER TURNED H
+  '\xa78d'# -> unI64 613
+  -- LATIN CAPITAL LETTER N WITH DESCENDER
+  '\xa790'# -> unI64 42897
+  -- LATIN CAPITAL LETTER C WITH BAR
+  '\xa792'# -> unI64 42899
+  -- LATIN CAPITAL LETTER B WITH FLOURISH
+  '\xa796'# -> unI64 42903
+  -- LATIN CAPITAL LETTER F WITH STROKE
+  '\xa798'# -> unI64 42905
+  -- LATIN CAPITAL LETTER VOLAPUK AE
+  '\xa79a'# -> unI64 42907
+  -- LATIN CAPITAL LETTER VOLAPUK OE
+  '\xa79c'# -> unI64 42909
+  -- LATIN CAPITAL LETTER VOLAPUK UE
+  '\xa79e'# -> unI64 42911
+  -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
+  '\xa7a0'# -> unI64 42913
+  -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
+  '\xa7a2'# -> unI64 42915
+  -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
+  '\xa7a4'# -> unI64 42917
+  -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
+  '\xa7a6'# -> unI64 42919
+  -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
+  '\xa7a8'# -> unI64 42921
+  -- LATIN CAPITAL LETTER H WITH HOOK
+  '\xa7aa'# -> unI64 614
+  -- LATIN CAPITAL LETTER REVERSED OPEN E
+  '\xa7ab'# -> unI64 604
+  -- LATIN CAPITAL LETTER SCRIPT G
+  '\xa7ac'# -> unI64 609
+  -- LATIN CAPITAL LETTER L WITH BELT
+  '\xa7ad'# -> unI64 620
+  -- LATIN CAPITAL LETTER SMALL CAPITAL I
+  '\xa7ae'# -> unI64 618
+  -- LATIN CAPITAL LETTER TURNED K
+  '\xa7b0'# -> unI64 670
+  -- LATIN CAPITAL LETTER TURNED T
+  '\xa7b1'# -> unI64 647
+  -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL
+  '\xa7b2'# -> unI64 669
+  -- LATIN CAPITAL LETTER CHI
+  '\xa7b3'# -> unI64 43859
+  -- LATIN CAPITAL LETTER BETA
+  '\xa7b4'# -> unI64 42933
+  -- LATIN CAPITAL LETTER OMEGA
+  '\xa7b6'# -> unI64 42935
+  -- LATIN CAPITAL LETTER U WITH STROKE
+  '\xa7b8'# -> unI64 42937
+  -- LATIN CAPITAL LETTER GLOTTAL A
+  '\xa7ba'# -> unI64 42939
+  -- LATIN CAPITAL LETTER GLOTTAL I
+  '\xa7bc'# -> unI64 42941
+  -- LATIN CAPITAL LETTER GLOTTAL U
+  '\xa7be'# -> unI64 42943
+  -- LATIN CAPITAL LETTER OLD POLISH O
+  '\xa7c0'# -> unI64 42945
+  -- LATIN CAPITAL LETTER ANGLICANA W
+  '\xa7c2'# -> unI64 42947
+  -- LATIN CAPITAL LETTER C WITH PALATAL HOOK
+  '\xa7c4'# -> unI64 42900
+  -- LATIN CAPITAL LETTER S WITH HOOK
+  '\xa7c5'# -> unI64 642
+  -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK
+  '\xa7c6'# -> unI64 7566
+  -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
+  '\xa7c7'# -> unI64 42952
+  -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+  '\xa7c9'# -> unI64 42954
+  -- LATIN CAPITAL LETTER RAMS HORN
+  '\xa7cb'# -> unI64 612
+  -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
+  '\xa7cc'# -> unI64 42957
+  -- LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE
+  '\xa7ce'# -> unI64 42959
+  -- LATIN CAPITAL LETTER CLOSED INSULAR G
+  '\xa7d0'# -> unI64 42961
+  -- LATIN CAPITAL LETTER DOUBLE THORN
+  '\xa7d2'# -> unI64 42963
+  -- LATIN CAPITAL LETTER DOUBLE WYNN
+  '\xa7d4'# -> unI64 42965
+  -- LATIN CAPITAL LETTER MIDDLE SCOTS S
+  '\xa7d6'# -> unI64 42967
+  -- LATIN CAPITAL LETTER SIGMOID S
+  '\xa7d8'# -> unI64 42969
+  -- LATIN CAPITAL LETTER LAMBDA
+  '\xa7da'# -> unI64 42971
+  -- LATIN CAPITAL LETTER LAMBDA WITH STROKE
+  '\xa7dc'# -> unI64 411
+  -- LATIN CAPITAL LETTER REVERSED HALF H
+  '\xa7f5'# -> unI64 42998
+  -- CHEROKEE SMALL LETTER A
+  '\xab70'# -> unI64 5024
+  -- CHEROKEE SMALL LETTER E
+  '\xab71'# -> unI64 5025
+  -- CHEROKEE SMALL LETTER I
+  '\xab72'# -> unI64 5026
+  -- CHEROKEE SMALL LETTER O
+  '\xab73'# -> unI64 5027
+  -- CHEROKEE SMALL LETTER U
+  '\xab74'# -> unI64 5028
+  -- CHEROKEE SMALL LETTER V
+  '\xab75'# -> unI64 5029
+  -- CHEROKEE SMALL LETTER GA
+  '\xab76'# -> unI64 5030
+  -- CHEROKEE SMALL LETTER KA
+  '\xab77'# -> unI64 5031
+  -- CHEROKEE SMALL LETTER GE
+  '\xab78'# -> unI64 5032
+  -- CHEROKEE SMALL LETTER GI
+  '\xab79'# -> unI64 5033
+  -- CHEROKEE SMALL LETTER GO
+  '\xab7a'# -> unI64 5034
+  -- CHEROKEE SMALL LETTER GU
+  '\xab7b'# -> unI64 5035
+  -- CHEROKEE SMALL LETTER GV
+  '\xab7c'# -> unI64 5036
+  -- CHEROKEE SMALL LETTER HA
+  '\xab7d'# -> unI64 5037
+  -- CHEROKEE SMALL LETTER HE
+  '\xab7e'# -> unI64 5038
+  -- CHEROKEE SMALL LETTER HI
+  '\xab7f'# -> unI64 5039
+  -- CHEROKEE SMALL LETTER HO
+  '\xab80'# -> unI64 5040
+  -- CHEROKEE SMALL LETTER HU
+  '\xab81'# -> unI64 5041
+  -- CHEROKEE SMALL LETTER HV
+  '\xab82'# -> unI64 5042
+  -- CHEROKEE SMALL LETTER LA
+  '\xab83'# -> unI64 5043
+  -- CHEROKEE SMALL LETTER LE
+  '\xab84'# -> unI64 5044
+  -- CHEROKEE SMALL LETTER LI
+  '\xab85'# -> unI64 5045
+  -- CHEROKEE SMALL LETTER LO
+  '\xab86'# -> unI64 5046
+  -- CHEROKEE SMALL LETTER LU
+  '\xab87'# -> unI64 5047
+  -- CHEROKEE SMALL LETTER LV
+  '\xab88'# -> unI64 5048
+  -- CHEROKEE SMALL LETTER MA
+  '\xab89'# -> unI64 5049
+  -- CHEROKEE SMALL LETTER ME
+  '\xab8a'# -> unI64 5050
+  -- CHEROKEE SMALL LETTER MI
+  '\xab8b'# -> unI64 5051
+  -- CHEROKEE SMALL LETTER MO
+  '\xab8c'# -> unI64 5052
+  -- CHEROKEE SMALL LETTER MU
+  '\xab8d'# -> unI64 5053
+  -- CHEROKEE SMALL LETTER NA
+  '\xab8e'# -> unI64 5054
+  -- CHEROKEE SMALL LETTER HNA
+  '\xab8f'# -> unI64 5055
+  -- CHEROKEE SMALL LETTER NAH
+  '\xab90'# -> unI64 5056
+  -- CHEROKEE SMALL LETTER NE
+  '\xab91'# -> unI64 5057
+  -- CHEROKEE SMALL LETTER NI
+  '\xab92'# -> unI64 5058
+  -- CHEROKEE SMALL LETTER NO
+  '\xab93'# -> unI64 5059
+  -- CHEROKEE SMALL LETTER NU
+  '\xab94'# -> unI64 5060
+  -- CHEROKEE SMALL LETTER NV
+  '\xab95'# -> unI64 5061
+  -- CHEROKEE SMALL LETTER QUA
+  '\xab96'# -> unI64 5062
+  -- CHEROKEE SMALL LETTER QUE
+  '\xab97'# -> unI64 5063
+  -- CHEROKEE SMALL LETTER QUI
+  '\xab98'# -> unI64 5064
+  -- CHEROKEE SMALL LETTER QUO
+  '\xab99'# -> unI64 5065
+  -- CHEROKEE SMALL LETTER QUU
+  '\xab9a'# -> unI64 5066
+  -- CHEROKEE SMALL LETTER QUV
+  '\xab9b'# -> unI64 5067
+  -- CHEROKEE SMALL LETTER SA
+  '\xab9c'# -> unI64 5068
+  -- CHEROKEE SMALL LETTER S
+  '\xab9d'# -> unI64 5069
+  -- CHEROKEE SMALL LETTER SE
+  '\xab9e'# -> unI64 5070
+  -- CHEROKEE SMALL LETTER SI
+  '\xab9f'# -> unI64 5071
+  -- CHEROKEE SMALL LETTER SO
+  '\xaba0'# -> unI64 5072
+  -- CHEROKEE SMALL LETTER SU
+  '\xaba1'# -> unI64 5073
+  -- CHEROKEE SMALL LETTER SV
+  '\xaba2'# -> unI64 5074
+  -- CHEROKEE SMALL LETTER DA
+  '\xaba3'# -> unI64 5075
+  -- CHEROKEE SMALL LETTER TA
+  '\xaba4'# -> unI64 5076
+  -- CHEROKEE SMALL LETTER DE
+  '\xaba5'# -> unI64 5077
+  -- CHEROKEE SMALL LETTER TE
+  '\xaba6'# -> unI64 5078
+  -- CHEROKEE SMALL LETTER DI
+  '\xaba7'# -> unI64 5079
+  -- CHEROKEE SMALL LETTER TI
+  '\xaba8'# -> unI64 5080
+  -- CHEROKEE SMALL LETTER DO
+  '\xaba9'# -> unI64 5081
+  -- CHEROKEE SMALL LETTER DU
+  '\xabaa'# -> unI64 5082
+  -- CHEROKEE SMALL LETTER DV
+  '\xabab'# -> unI64 5083
+  -- CHEROKEE SMALL LETTER DLA
+  '\xabac'# -> unI64 5084
+  -- CHEROKEE SMALL LETTER TLA
+  '\xabad'# -> unI64 5085
+  -- CHEROKEE SMALL LETTER TLE
+  '\xabae'# -> unI64 5086
+  -- CHEROKEE SMALL LETTER TLI
+  '\xabaf'# -> unI64 5087
+  -- CHEROKEE SMALL LETTER TLO
+  '\xabb0'# -> unI64 5088
+  -- CHEROKEE SMALL LETTER TLU
+  '\xabb1'# -> unI64 5089
+  -- CHEROKEE SMALL LETTER TLV
+  '\xabb2'# -> unI64 5090
+  -- CHEROKEE SMALL LETTER TSA
+  '\xabb3'# -> unI64 5091
+  -- CHEROKEE SMALL LETTER TSE
+  '\xabb4'# -> unI64 5092
+  -- CHEROKEE SMALL LETTER TSI
+  '\xabb5'# -> unI64 5093
+  -- CHEROKEE SMALL LETTER TSO
+  '\xabb6'# -> unI64 5094
+  -- CHEROKEE SMALL LETTER TSU
+  '\xabb7'# -> unI64 5095
+  -- CHEROKEE SMALL LETTER TSV
+  '\xabb8'# -> unI64 5096
+  -- CHEROKEE SMALL LETTER WA
+  '\xabb9'# -> unI64 5097
+  -- CHEROKEE SMALL LETTER WE
+  '\xabba'# -> unI64 5098
+  -- CHEROKEE SMALL LETTER WI
+  '\xabbb'# -> unI64 5099
+  -- CHEROKEE SMALL LETTER WO
+  '\xabbc'# -> unI64 5100
+  -- CHEROKEE SMALL LETTER WU
+  '\xabbd'# -> unI64 5101
+  -- CHEROKEE SMALL LETTER WV
+  '\xabbe'# -> unI64 5102
+  -- CHEROKEE SMALL LETTER YA
+  '\xabbf'# -> unI64 5103
+  -- LATIN SMALL LIGATURE FF
+  '\xfb00'# -> unI64 213909606
+  -- LATIN SMALL LIGATURE FI
+  '\xfb01'# -> unI64 220201062
+  -- LATIN SMALL LIGATURE FL
+  '\xfb02'# -> unI64 226492518
+  -- LATIN SMALL LIGATURE FFI
+  '\xfb03'# -> unI64 461795097575526
+  -- LATIN SMALL LIGATURE FFL
+  '\xfb04'# -> unI64 474989237108838
+  -- LATIN SMALL LIGATURE LONG S T
+  '\xfb05'# -> unI64 243269747
+  -- LATIN SMALL LIGATURE ST
+  '\xfb06'# -> unI64 243269747
+  -- ARMENIAN SMALL LIGATURE MEN NOW
+  '\xfb13'# -> unI64 2931819892
+  -- ARMENIAN SMALL LIGATURE MEN ECH
+  '\xfb14'# -> unI64 2896168308
+  -- ARMENIAN SMALL LIGATURE MEN INI
+  '\xfb15'# -> unI64 2908751220
+  -- ARMENIAN SMALL LIGATURE VEW NOW
+  '\xfb16'# -> unI64 2931819902
+  -- ARMENIAN SMALL LIGATURE MEN XEH
+  '\xfb17'# -> unI64 2912945524
+  -- FULLWIDTH LATIN CAPITAL LETTER A
+  '\xff21'# -> unI64 65345
+  -- FULLWIDTH LATIN CAPITAL LETTER B
+  '\xff22'# -> unI64 65346
+  -- FULLWIDTH LATIN CAPITAL LETTER C
+  '\xff23'# -> unI64 65347
+  -- FULLWIDTH LATIN CAPITAL LETTER D
+  '\xff24'# -> unI64 65348
+  -- FULLWIDTH LATIN CAPITAL LETTER E
+  '\xff25'# -> unI64 65349
+  -- FULLWIDTH LATIN CAPITAL LETTER F
+  '\xff26'# -> unI64 65350
+  -- FULLWIDTH LATIN CAPITAL LETTER G
+  '\xff27'# -> unI64 65351
+  -- FULLWIDTH LATIN CAPITAL LETTER H
+  '\xff28'# -> unI64 65352
+  -- FULLWIDTH LATIN CAPITAL LETTER I
+  '\xff29'# -> unI64 65353
+  -- FULLWIDTH LATIN CAPITAL LETTER J
+  '\xff2a'# -> unI64 65354
+  -- FULLWIDTH LATIN CAPITAL LETTER K
+  '\xff2b'# -> unI64 65355
+  -- FULLWIDTH LATIN CAPITAL LETTER L
+  '\xff2c'# -> unI64 65356
+  -- FULLWIDTH LATIN CAPITAL LETTER M
+  '\xff2d'# -> unI64 65357
+  -- FULLWIDTH LATIN CAPITAL LETTER N
+  '\xff2e'# -> unI64 65358
+  -- FULLWIDTH LATIN CAPITAL LETTER O
+  '\xff2f'# -> unI64 65359
+  -- FULLWIDTH LATIN CAPITAL LETTER P
+  '\xff30'# -> unI64 65360
+  -- FULLWIDTH LATIN CAPITAL LETTER Q
+  '\xff31'# -> unI64 65361
+  -- FULLWIDTH LATIN CAPITAL LETTER R
+  '\xff32'# -> unI64 65362
+  -- FULLWIDTH LATIN CAPITAL LETTER S
+  '\xff33'# -> unI64 65363
+  -- FULLWIDTH LATIN CAPITAL LETTER T
+  '\xff34'# -> unI64 65364
+  -- FULLWIDTH LATIN CAPITAL LETTER U
+  '\xff35'# -> unI64 65365
+  -- FULLWIDTH LATIN CAPITAL LETTER V
+  '\xff36'# -> unI64 65366
+  -- FULLWIDTH LATIN CAPITAL LETTER W
+  '\xff37'# -> unI64 65367
+  -- FULLWIDTH LATIN CAPITAL LETTER X
+  '\xff38'# -> unI64 65368
+  -- FULLWIDTH LATIN CAPITAL LETTER Y
+  '\xff39'# -> unI64 65369
+  -- FULLWIDTH LATIN CAPITAL LETTER Z
+  '\xff3a'# -> unI64 65370
+  -- DESERET CAPITAL LETTER LONG I
+  '\x10400'# -> unI64 66600
+  -- DESERET CAPITAL LETTER LONG E
+  '\x10401'# -> unI64 66601
+  -- DESERET CAPITAL LETTER LONG A
+  '\x10402'# -> unI64 66602
+  -- DESERET CAPITAL LETTER LONG AH
+  '\x10403'# -> unI64 66603
+  -- DESERET CAPITAL LETTER LONG O
+  '\x10404'# -> unI64 66604
+  -- DESERET CAPITAL LETTER LONG OO
+  '\x10405'# -> unI64 66605
+  -- DESERET CAPITAL LETTER SHORT I
+  '\x10406'# -> unI64 66606
+  -- DESERET CAPITAL LETTER SHORT E
+  '\x10407'# -> unI64 66607
+  -- DESERET CAPITAL LETTER SHORT A
+  '\x10408'# -> unI64 66608
+  -- DESERET CAPITAL LETTER SHORT AH
+  '\x10409'# -> unI64 66609
+  -- DESERET CAPITAL LETTER SHORT O
+  '\x1040a'# -> unI64 66610
+  -- DESERET CAPITAL LETTER SHORT OO
+  '\x1040b'# -> unI64 66611
+  -- DESERET CAPITAL LETTER AY
+  '\x1040c'# -> unI64 66612
+  -- DESERET CAPITAL LETTER OW
+  '\x1040d'# -> unI64 66613
+  -- DESERET CAPITAL LETTER WU
+  '\x1040e'# -> unI64 66614
+  -- DESERET CAPITAL LETTER YEE
+  '\x1040f'# -> unI64 66615
+  -- DESERET CAPITAL LETTER H
+  '\x10410'# -> unI64 66616
+  -- DESERET CAPITAL LETTER PEE
+  '\x10411'# -> unI64 66617
+  -- DESERET CAPITAL LETTER BEE
+  '\x10412'# -> unI64 66618
+  -- DESERET CAPITAL LETTER TEE
+  '\x10413'# -> unI64 66619
+  -- DESERET CAPITAL LETTER DEE
+  '\x10414'# -> unI64 66620
+  -- DESERET CAPITAL LETTER CHEE
+  '\x10415'# -> unI64 66621
+  -- DESERET CAPITAL LETTER JEE
+  '\x10416'# -> unI64 66622
+  -- DESERET CAPITAL LETTER KAY
+  '\x10417'# -> unI64 66623
+  -- DESERET CAPITAL LETTER GAY
+  '\x10418'# -> unI64 66624
+  -- DESERET CAPITAL LETTER EF
+  '\x10419'# -> unI64 66625
+  -- DESERET CAPITAL LETTER VEE
+  '\x1041a'# -> unI64 66626
+  -- DESERET CAPITAL LETTER ETH
+  '\x1041b'# -> unI64 66627
+  -- DESERET CAPITAL LETTER THEE
+  '\x1041c'# -> unI64 66628
+  -- DESERET CAPITAL LETTER ES
+  '\x1041d'# -> unI64 66629
+  -- DESERET CAPITAL LETTER ZEE
+  '\x1041e'# -> unI64 66630
+  -- DESERET CAPITAL LETTER ESH
+  '\x1041f'# -> unI64 66631
+  -- DESERET CAPITAL LETTER ZHEE
+  '\x10420'# -> unI64 66632
+  -- DESERET CAPITAL LETTER ER
+  '\x10421'# -> unI64 66633
+  -- DESERET CAPITAL LETTER EL
+  '\x10422'# -> unI64 66634
+  -- DESERET CAPITAL LETTER EM
+  '\x10423'# -> unI64 66635
+  -- DESERET CAPITAL LETTER EN
+  '\x10424'# -> unI64 66636
+  -- DESERET CAPITAL LETTER ENG
+  '\x10425'# -> unI64 66637
+  -- DESERET CAPITAL LETTER OI
+  '\x10426'# -> unI64 66638
+  -- DESERET CAPITAL LETTER EW
+  '\x10427'# -> unI64 66639
+  -- OSAGE CAPITAL LETTER A
+  '\x104b0'# -> unI64 66776
+  -- OSAGE CAPITAL LETTER AI
+  '\x104b1'# -> unI64 66777
+  -- OSAGE CAPITAL LETTER AIN
+  '\x104b2'# -> unI64 66778
+  -- OSAGE CAPITAL LETTER AH
+  '\x104b3'# -> unI64 66779
+  -- OSAGE CAPITAL LETTER BRA
+  '\x104b4'# -> unI64 66780
+  -- OSAGE CAPITAL LETTER CHA
+  '\x104b5'# -> unI64 66781
+  -- OSAGE CAPITAL LETTER EHCHA
+  '\x104b6'# -> unI64 66782
+  -- OSAGE CAPITAL LETTER E
+  '\x104b7'# -> unI64 66783
+  -- OSAGE CAPITAL LETTER EIN
+  '\x104b8'# -> unI64 66784
+  -- OSAGE CAPITAL LETTER HA
+  '\x104b9'# -> unI64 66785
+  -- OSAGE CAPITAL LETTER HYA
+  '\x104ba'# -> unI64 66786
+  -- OSAGE CAPITAL LETTER I
+  '\x104bb'# -> unI64 66787
+  -- OSAGE CAPITAL LETTER KA
+  '\x104bc'# -> unI64 66788
+  -- OSAGE CAPITAL LETTER EHKA
+  '\x104bd'# -> unI64 66789
+  -- OSAGE CAPITAL LETTER KYA
+  '\x104be'# -> unI64 66790
+  -- OSAGE CAPITAL LETTER LA
+  '\x104bf'# -> unI64 66791
+  -- OSAGE CAPITAL LETTER MA
+  '\x104c0'# -> unI64 66792
+  -- OSAGE CAPITAL LETTER NA
+  '\x104c1'# -> unI64 66793
+  -- OSAGE CAPITAL LETTER O
+  '\x104c2'# -> unI64 66794
+  -- OSAGE CAPITAL LETTER OIN
+  '\x104c3'# -> unI64 66795
+  -- OSAGE CAPITAL LETTER PA
+  '\x104c4'# -> unI64 66796
+  -- OSAGE CAPITAL LETTER EHPA
+  '\x104c5'# -> unI64 66797
+  -- OSAGE CAPITAL LETTER SA
+  '\x104c6'# -> unI64 66798
+  -- OSAGE CAPITAL LETTER SHA
+  '\x104c7'# -> unI64 66799
+  -- OSAGE CAPITAL LETTER TA
+  '\x104c8'# -> unI64 66800
+  -- OSAGE CAPITAL LETTER EHTA
+  '\x104c9'# -> unI64 66801
+  -- OSAGE CAPITAL LETTER TSA
+  '\x104ca'# -> unI64 66802
+  -- OSAGE CAPITAL LETTER EHTSA
+  '\x104cb'# -> unI64 66803
+  -- OSAGE CAPITAL LETTER TSHA
+  '\x104cc'# -> unI64 66804
+  -- OSAGE CAPITAL LETTER DHA
+  '\x104cd'# -> unI64 66805
+  -- OSAGE CAPITAL LETTER U
+  '\x104ce'# -> unI64 66806
+  -- OSAGE CAPITAL LETTER WA
+  '\x104cf'# -> unI64 66807
+  -- OSAGE CAPITAL LETTER KHA
+  '\x104d0'# -> unI64 66808
+  -- OSAGE CAPITAL LETTER GHA
+  '\x104d1'# -> unI64 66809
+  -- OSAGE CAPITAL LETTER ZA
+  '\x104d2'# -> unI64 66810
+  -- OSAGE CAPITAL LETTER ZHA
+  '\x104d3'# -> unI64 66811
+  -- VITHKUQI CAPITAL LETTER A
+  '\x10570'# -> unI64 66967
+  -- VITHKUQI CAPITAL LETTER BBE
+  '\x10571'# -> unI64 66968
+  -- VITHKUQI CAPITAL LETTER BE
+  '\x10572'# -> unI64 66969
+  -- VITHKUQI CAPITAL LETTER CE
+  '\x10573'# -> unI64 66970
+  -- VITHKUQI CAPITAL LETTER CHE
+  '\x10574'# -> unI64 66971
+  -- VITHKUQI CAPITAL LETTER DE
+  '\x10575'# -> unI64 66972
+  -- VITHKUQI CAPITAL LETTER DHE
+  '\x10576'# -> unI64 66973
+  -- VITHKUQI CAPITAL LETTER EI
+  '\x10577'# -> unI64 66974
+  -- VITHKUQI CAPITAL LETTER E
+  '\x10578'# -> unI64 66975
+  -- VITHKUQI CAPITAL LETTER FE
+  '\x10579'# -> unI64 66976
+  -- VITHKUQI CAPITAL LETTER GA
+  '\x1057a'# -> unI64 66977
+  -- VITHKUQI CAPITAL LETTER HA
+  '\x1057c'# -> unI64 66979
+  -- VITHKUQI CAPITAL LETTER HHA
+  '\x1057d'# -> unI64 66980
+  -- VITHKUQI CAPITAL LETTER I
+  '\x1057e'# -> unI64 66981
+  -- VITHKUQI CAPITAL LETTER IJE
+  '\x1057f'# -> unI64 66982
+  -- VITHKUQI CAPITAL LETTER JE
+  '\x10580'# -> unI64 66983
+  -- VITHKUQI CAPITAL LETTER KA
+  '\x10581'# -> unI64 66984
+  -- VITHKUQI CAPITAL LETTER LA
+  '\x10582'# -> unI64 66985
+  -- VITHKUQI CAPITAL LETTER LLA
+  '\x10583'# -> unI64 66986
+  -- VITHKUQI CAPITAL LETTER ME
+  '\x10584'# -> unI64 66987
+  -- VITHKUQI CAPITAL LETTER NE
+  '\x10585'# -> unI64 66988
+  -- VITHKUQI CAPITAL LETTER NJE
+  '\x10586'# -> unI64 66989
+  -- VITHKUQI CAPITAL LETTER O
+  '\x10587'# -> unI64 66990
+  -- VITHKUQI CAPITAL LETTER PE
+  '\x10588'# -> unI64 66991
+  -- VITHKUQI CAPITAL LETTER QA
+  '\x10589'# -> unI64 66992
+  -- VITHKUQI CAPITAL LETTER RE
+  '\x1058a'# -> unI64 66993
+  -- VITHKUQI CAPITAL LETTER SE
+  '\x1058c'# -> unI64 66995
+  -- VITHKUQI CAPITAL LETTER SHE
+  '\x1058d'# -> unI64 66996
+  -- VITHKUQI CAPITAL LETTER TE
+  '\x1058e'# -> unI64 66997
+  -- VITHKUQI CAPITAL LETTER THE
+  '\x1058f'# -> unI64 66998
+  -- VITHKUQI CAPITAL LETTER U
+  '\x10590'# -> unI64 66999
+  -- VITHKUQI CAPITAL LETTER VE
+  '\x10591'# -> unI64 67000
+  -- VITHKUQI CAPITAL LETTER XE
+  '\x10592'# -> unI64 67001
+  -- VITHKUQI CAPITAL LETTER Y
+  '\x10594'# -> unI64 67003
+  -- VITHKUQI CAPITAL LETTER ZE
+  '\x10595'# -> unI64 67004
+  -- OLD HUNGARIAN CAPITAL LETTER A
+  '\x10c80'# -> unI64 68800
+  -- OLD HUNGARIAN CAPITAL LETTER AA
+  '\x10c81'# -> unI64 68801
+  -- OLD HUNGARIAN CAPITAL LETTER EB
+  '\x10c82'# -> unI64 68802
+  -- OLD HUNGARIAN CAPITAL LETTER AMB
+  '\x10c83'# -> unI64 68803
+  -- OLD HUNGARIAN CAPITAL LETTER EC
+  '\x10c84'# -> unI64 68804
+  -- OLD HUNGARIAN CAPITAL LETTER ENC
+  '\x10c85'# -> unI64 68805
+  -- OLD HUNGARIAN CAPITAL LETTER ECS
+  '\x10c86'# -> unI64 68806
+  -- OLD HUNGARIAN CAPITAL LETTER ED
+  '\x10c87'# -> unI64 68807
+  -- OLD HUNGARIAN CAPITAL LETTER AND
+  '\x10c88'# -> unI64 68808
+  -- OLD HUNGARIAN CAPITAL LETTER E
+  '\x10c89'# -> unI64 68809
+  -- OLD HUNGARIAN CAPITAL LETTER CLOSE E
+  '\x10c8a'# -> unI64 68810
+  -- OLD HUNGARIAN CAPITAL LETTER EE
+  '\x10c8b'# -> unI64 68811
+  -- OLD HUNGARIAN CAPITAL LETTER EF
+  '\x10c8c'# -> unI64 68812
+  -- OLD HUNGARIAN CAPITAL LETTER EG
+  '\x10c8d'# -> unI64 68813
+  -- OLD HUNGARIAN CAPITAL LETTER EGY
+  '\x10c8e'# -> unI64 68814
+  -- OLD HUNGARIAN CAPITAL LETTER EH
+  '\x10c8f'# -> unI64 68815
+  -- OLD HUNGARIAN CAPITAL LETTER I
+  '\x10c90'# -> unI64 68816
+  -- OLD HUNGARIAN CAPITAL LETTER II
+  '\x10c91'# -> unI64 68817
+  -- OLD HUNGARIAN CAPITAL LETTER EJ
+  '\x10c92'# -> unI64 68818
+  -- OLD HUNGARIAN CAPITAL LETTER EK
+  '\x10c93'# -> unI64 68819
+  -- OLD HUNGARIAN CAPITAL LETTER AK
+  '\x10c94'# -> unI64 68820
+  -- OLD HUNGARIAN CAPITAL LETTER UNK
+  '\x10c95'# -> unI64 68821
+  -- OLD HUNGARIAN CAPITAL LETTER EL
+  '\x10c96'# -> unI64 68822
+  -- OLD HUNGARIAN CAPITAL LETTER ELY
+  '\x10c97'# -> unI64 68823
+  -- OLD HUNGARIAN CAPITAL LETTER EM
+  '\x10c98'# -> unI64 68824
+  -- OLD HUNGARIAN CAPITAL LETTER EN
+  '\x10c99'# -> unI64 68825
+  -- OLD HUNGARIAN CAPITAL LETTER ENY
+  '\x10c9a'# -> unI64 68826
+  -- OLD HUNGARIAN CAPITAL LETTER O
+  '\x10c9b'# -> unI64 68827
+  -- OLD HUNGARIAN CAPITAL LETTER OO
+  '\x10c9c'# -> unI64 68828
+  -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
+  '\x10c9d'# -> unI64 68829
+  -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
+  '\x10c9e'# -> unI64 68830
+  -- OLD HUNGARIAN CAPITAL LETTER OEE
+  '\x10c9f'# -> unI64 68831
+  -- OLD HUNGARIAN CAPITAL LETTER EP
+  '\x10ca0'# -> unI64 68832
+  -- OLD HUNGARIAN CAPITAL LETTER EMP
+  '\x10ca1'# -> unI64 68833
+  -- OLD HUNGARIAN CAPITAL LETTER ER
+  '\x10ca2'# -> unI64 68834
+  -- OLD HUNGARIAN CAPITAL LETTER SHORT ER
+  '\x10ca3'# -> unI64 68835
+  -- OLD HUNGARIAN CAPITAL LETTER ES
+  '\x10ca4'# -> unI64 68836
+  -- OLD HUNGARIAN CAPITAL LETTER ESZ
+  '\x10ca5'# -> unI64 68837
+  -- OLD HUNGARIAN CAPITAL LETTER ET
+  '\x10ca6'# -> unI64 68838
+  -- OLD HUNGARIAN CAPITAL LETTER ENT
+  '\x10ca7'# -> unI64 68839
+  -- OLD HUNGARIAN CAPITAL LETTER ETY
+  '\x10ca8'# -> unI64 68840
+  -- OLD HUNGARIAN CAPITAL LETTER ECH
+  '\x10ca9'# -> unI64 68841
+  -- OLD HUNGARIAN CAPITAL LETTER U
+  '\x10caa'# -> unI64 68842
+  -- OLD HUNGARIAN CAPITAL LETTER UU
+  '\x10cab'# -> unI64 68843
+  -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
+  '\x10cac'# -> unI64 68844
+  -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
+  '\x10cad'# -> unI64 68845
+  -- OLD HUNGARIAN CAPITAL LETTER EV
+  '\x10cae'# -> unI64 68846
+  -- OLD HUNGARIAN CAPITAL LETTER EZ
+  '\x10caf'# -> unI64 68847
+  -- OLD HUNGARIAN CAPITAL LETTER EZS
+  '\x10cb0'# -> unI64 68848
+  -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
+  '\x10cb1'# -> unI64 68849
+  -- OLD HUNGARIAN CAPITAL LETTER US
+  '\x10cb2'# -> unI64 68850
+  -- GARAY CAPITAL LETTER A
+  '\x10d50'# -> unI64 68976
+  -- GARAY CAPITAL LETTER CA
+  '\x10d51'# -> unI64 68977
+  -- GARAY CAPITAL LETTER MA
+  '\x10d52'# -> unI64 68978
+  -- GARAY CAPITAL LETTER KA
+  '\x10d53'# -> unI64 68979
+  -- GARAY CAPITAL LETTER BA
+  '\x10d54'# -> unI64 68980
+  -- GARAY CAPITAL LETTER JA
+  '\x10d55'# -> unI64 68981
+  -- GARAY CAPITAL LETTER SA
+  '\x10d56'# -> unI64 68982
+  -- GARAY CAPITAL LETTER WA
+  '\x10d57'# -> unI64 68983
+  -- GARAY CAPITAL LETTER LA
+  '\x10d58'# -> unI64 68984
+  -- GARAY CAPITAL LETTER GA
+  '\x10d59'# -> unI64 68985
+  -- GARAY CAPITAL LETTER DA
+  '\x10d5a'# -> unI64 68986
+  -- GARAY CAPITAL LETTER XA
+  '\x10d5b'# -> unI64 68987
+  -- GARAY CAPITAL LETTER YA
+  '\x10d5c'# -> unI64 68988
+  -- GARAY CAPITAL LETTER TA
+  '\x10d5d'# -> unI64 68989
+  -- GARAY CAPITAL LETTER RA
+  '\x10d5e'# -> unI64 68990
+  -- GARAY CAPITAL LETTER NYA
+  '\x10d5f'# -> unI64 68991
+  -- GARAY CAPITAL LETTER FA
+  '\x10d60'# -> unI64 68992
+  -- GARAY CAPITAL LETTER NA
+  '\x10d61'# -> unI64 68993
+  -- GARAY CAPITAL LETTER PA
+  '\x10d62'# -> unI64 68994
+  -- GARAY CAPITAL LETTER HA
+  '\x10d63'# -> unI64 68995
+  -- GARAY CAPITAL LETTER OLD KA
+  '\x10d64'# -> unI64 68996
+  -- GARAY CAPITAL LETTER OLD NA
+  '\x10d65'# -> unI64 68997
+  -- WARANG CITI CAPITAL LETTER NGAA
+  '\x118a0'# -> unI64 71872
+  -- WARANG CITI CAPITAL LETTER A
+  '\x118a1'# -> unI64 71873
+  -- WARANG CITI CAPITAL LETTER WI
+  '\x118a2'# -> unI64 71874
+  -- WARANG CITI CAPITAL LETTER YU
+  '\x118a3'# -> unI64 71875
+  -- WARANG CITI CAPITAL LETTER YA
+  '\x118a4'# -> unI64 71876
+  -- WARANG CITI CAPITAL LETTER YO
+  '\x118a5'# -> unI64 71877
+  -- WARANG CITI CAPITAL LETTER II
+  '\x118a6'# -> unI64 71878
+  -- WARANG CITI CAPITAL LETTER UU
+  '\x118a7'# -> unI64 71879
+  -- WARANG CITI CAPITAL LETTER E
+  '\x118a8'# -> unI64 71880
+  -- WARANG CITI CAPITAL LETTER O
+  '\x118a9'# -> unI64 71881
+  -- WARANG CITI CAPITAL LETTER ANG
+  '\x118aa'# -> unI64 71882
+  -- WARANG CITI CAPITAL LETTER GA
+  '\x118ab'# -> unI64 71883
+  -- WARANG CITI CAPITAL LETTER KO
+  '\x118ac'# -> unI64 71884
+  -- WARANG CITI CAPITAL LETTER ENY
+  '\x118ad'# -> unI64 71885
+  -- WARANG CITI CAPITAL LETTER YUJ
+  '\x118ae'# -> unI64 71886
+  -- WARANG CITI CAPITAL LETTER UC
+  '\x118af'# -> unI64 71887
+  -- WARANG CITI CAPITAL LETTER ENN
+  '\x118b0'# -> unI64 71888
+  -- WARANG CITI CAPITAL LETTER ODD
+  '\x118b1'# -> unI64 71889
+  -- WARANG CITI CAPITAL LETTER TTE
+  '\x118b2'# -> unI64 71890
+  -- WARANG CITI CAPITAL LETTER NUNG
+  '\x118b3'# -> unI64 71891
+  -- WARANG CITI CAPITAL LETTER DA
+  '\x118b4'# -> unI64 71892
+  -- WARANG CITI CAPITAL LETTER AT
+  '\x118b5'# -> unI64 71893
+  -- WARANG CITI CAPITAL LETTER AM
+  '\x118b6'# -> unI64 71894
+  -- WARANG CITI CAPITAL LETTER BU
+  '\x118b7'# -> unI64 71895
+  -- WARANG CITI CAPITAL LETTER PU
+  '\x118b8'# -> unI64 71896
+  -- WARANG CITI CAPITAL LETTER HIYO
+  '\x118b9'# -> unI64 71897
+  -- WARANG CITI CAPITAL LETTER HOLO
+  '\x118ba'# -> unI64 71898
+  -- WARANG CITI CAPITAL LETTER HORR
+  '\x118bb'# -> unI64 71899
+  -- WARANG CITI CAPITAL LETTER HAR
+  '\x118bc'# -> unI64 71900
+  -- WARANG CITI CAPITAL LETTER SSUU
+  '\x118bd'# -> unI64 71901
+  -- WARANG CITI CAPITAL LETTER SII
+  '\x118be'# -> unI64 71902
+  -- WARANG CITI CAPITAL LETTER VIYO
+  '\x118bf'# -> unI64 71903
+  -- MEDEFAIDRIN CAPITAL LETTER M
+  '\x16e40'# -> unI64 93792
+  -- MEDEFAIDRIN CAPITAL LETTER S
+  '\x16e41'# -> unI64 93793
+  -- MEDEFAIDRIN CAPITAL LETTER V
+  '\x16e42'# -> unI64 93794
+  -- MEDEFAIDRIN CAPITAL LETTER W
+  '\x16e43'# -> unI64 93795
+  -- MEDEFAIDRIN CAPITAL LETTER ATIU
+  '\x16e44'# -> unI64 93796
+  -- MEDEFAIDRIN CAPITAL LETTER Z
+  '\x16e45'# -> unI64 93797
+  -- MEDEFAIDRIN CAPITAL LETTER KP
+  '\x16e46'# -> unI64 93798
+  -- MEDEFAIDRIN CAPITAL LETTER P
+  '\x16e47'# -> unI64 93799
+  -- MEDEFAIDRIN CAPITAL LETTER T
+  '\x16e48'# -> unI64 93800
+  -- MEDEFAIDRIN CAPITAL LETTER G
+  '\x16e49'# -> unI64 93801
+  -- MEDEFAIDRIN CAPITAL LETTER F
+  '\x16e4a'# -> unI64 93802
+  -- MEDEFAIDRIN CAPITAL LETTER I
+  '\x16e4b'# -> unI64 93803
+  -- MEDEFAIDRIN CAPITAL LETTER K
+  '\x16e4c'# -> unI64 93804
+  -- MEDEFAIDRIN CAPITAL LETTER A
+  '\x16e4d'# -> unI64 93805
+  -- MEDEFAIDRIN CAPITAL LETTER J
+  '\x16e4e'# -> unI64 93806
+  -- MEDEFAIDRIN CAPITAL LETTER E
+  '\x16e4f'# -> unI64 93807
+  -- MEDEFAIDRIN CAPITAL LETTER B
+  '\x16e50'# -> unI64 93808
+  -- MEDEFAIDRIN CAPITAL LETTER C
+  '\x16e51'# -> unI64 93809
+  -- MEDEFAIDRIN CAPITAL LETTER U
+  '\x16e52'# -> unI64 93810
+  -- MEDEFAIDRIN CAPITAL LETTER YU
+  '\x16e53'# -> unI64 93811
+  -- MEDEFAIDRIN CAPITAL LETTER L
+  '\x16e54'# -> unI64 93812
+  -- MEDEFAIDRIN CAPITAL LETTER Q
+  '\x16e55'# -> unI64 93813
+  -- MEDEFAIDRIN CAPITAL LETTER HP
+  '\x16e56'# -> unI64 93814
+  -- MEDEFAIDRIN CAPITAL LETTER NY
+  '\x16e57'# -> unI64 93815
+  -- MEDEFAIDRIN CAPITAL LETTER X
+  '\x16e58'# -> unI64 93816
+  -- MEDEFAIDRIN CAPITAL LETTER D
+  '\x16e59'# -> unI64 93817
+  -- MEDEFAIDRIN CAPITAL LETTER OE
+  '\x16e5a'# -> unI64 93818
+  -- MEDEFAIDRIN CAPITAL LETTER N
+  '\x16e5b'# -> unI64 93819
+  -- MEDEFAIDRIN CAPITAL LETTER R
+  '\x16e5c'# -> unI64 93820
+  -- MEDEFAIDRIN CAPITAL LETTER O
+  '\x16e5d'# -> unI64 93821
+  -- MEDEFAIDRIN CAPITAL LETTER AI
+  '\x16e5e'# -> unI64 93822
+  -- MEDEFAIDRIN CAPITAL LETTER Y
+  '\x16e5f'# -> unI64 93823
+  -- BERIA ERFE CAPITAL LETTER ARKAB
+  '\x16ea0'# -> unI64 93883
+  -- BERIA ERFE CAPITAL LETTER BASIGNA
+  '\x16ea1'# -> unI64 93884
+  -- BERIA ERFE CAPITAL LETTER DARBAI
+  '\x16ea2'# -> unI64 93885
+  -- BERIA ERFE CAPITAL LETTER EH
+  '\x16ea3'# -> unI64 93886
+  -- BERIA ERFE CAPITAL LETTER FITKO
+  '\x16ea4'# -> unI64 93887
+  -- BERIA ERFE CAPITAL LETTER GOWAY
+  '\x16ea5'# -> unI64 93888
+  -- BERIA ERFE CAPITAL LETTER HIRDEABO
+  '\x16ea6'# -> unI64 93889
+  -- BERIA ERFE CAPITAL LETTER I
+  '\x16ea7'# -> unI64 93890
+  -- BERIA ERFE CAPITAL LETTER DJAI
+  '\x16ea8'# -> unI64 93891
+  -- BERIA ERFE CAPITAL LETTER KOBO
+  '\x16ea9'# -> unI64 93892
+  -- BERIA ERFE CAPITAL LETTER LAKKO
+  '\x16eaa'# -> unI64 93893
+  -- BERIA ERFE CAPITAL LETTER MERI
+  '\x16eab'# -> unI64 93894
+  -- BERIA ERFE CAPITAL LETTER NINI
+  '\x16eac'# -> unI64 93895
+  -- BERIA ERFE CAPITAL LETTER GNA
+  '\x16ead'# -> unI64 93896
+  -- BERIA ERFE CAPITAL LETTER NGAY
+  '\x16eae'# -> unI64 93897
+  -- BERIA ERFE CAPITAL LETTER OI
+  '\x16eaf'# -> unI64 93898
+  -- BERIA ERFE CAPITAL LETTER PI
+  '\x16eb0'# -> unI64 93899
+  -- BERIA ERFE CAPITAL LETTER ERIGO
+  '\x16eb1'# -> unI64 93900
+  -- BERIA ERFE CAPITAL LETTER ERIGO TAMURA
+  '\x16eb2'# -> unI64 93901
+  -- BERIA ERFE CAPITAL LETTER SERI
+  '\x16eb3'# -> unI64 93902
+  -- BERIA ERFE CAPITAL LETTER SHEP
+  '\x16eb4'# -> unI64 93903
+  -- BERIA ERFE CAPITAL LETTER TATASOUE
+  '\x16eb5'# -> unI64 93904
+  -- BERIA ERFE CAPITAL LETTER UI
+  '\x16eb6'# -> unI64 93905
+  -- BERIA ERFE CAPITAL LETTER WASSE
+  '\x16eb7'# -> unI64 93906
+  -- BERIA ERFE CAPITAL LETTER AY
+  '\x16eb8'# -> unI64 93907
+  -- ADLAM CAPITAL LETTER ALIF
+  '\x1e900'# -> unI64 125218
+  -- ADLAM CAPITAL LETTER DAALI
+  '\x1e901'# -> unI64 125219
+  -- ADLAM CAPITAL LETTER LAAM
+  '\x1e902'# -> unI64 125220
+  -- ADLAM CAPITAL LETTER MIIM
+  '\x1e903'# -> unI64 125221
+  -- ADLAM CAPITAL LETTER BA
+  '\x1e904'# -> unI64 125222
+  -- ADLAM CAPITAL LETTER SINNYIIYHE
+  '\x1e905'# -> unI64 125223
+  -- ADLAM CAPITAL LETTER PE
+  '\x1e906'# -> unI64 125224
+  -- ADLAM CAPITAL LETTER BHE
+  '\x1e907'# -> unI64 125225
+  -- ADLAM CAPITAL LETTER RA
+  '\x1e908'# -> unI64 125226
+  -- ADLAM CAPITAL LETTER E
+  '\x1e909'# -> unI64 125227
+  -- ADLAM CAPITAL LETTER FA
+  '\x1e90a'# -> unI64 125228
+  -- ADLAM CAPITAL LETTER I
+  '\x1e90b'# -> unI64 125229
+  -- ADLAM CAPITAL LETTER O
+  '\x1e90c'# -> unI64 125230
+  -- ADLAM CAPITAL LETTER DHA
+  '\x1e90d'# -> unI64 125231
+  -- ADLAM CAPITAL LETTER YHE
+  '\x1e90e'# -> unI64 125232
+  -- ADLAM CAPITAL LETTER WAW
+  '\x1e90f'# -> unI64 125233
+  -- ADLAM CAPITAL LETTER NUN
+  '\x1e910'# -> unI64 125234
+  -- ADLAM CAPITAL LETTER KAF
+  '\x1e911'# -> unI64 125235
+  -- ADLAM CAPITAL LETTER YA
+  '\x1e912'# -> unI64 125236
+  -- ADLAM CAPITAL LETTER U
+  '\x1e913'# -> unI64 125237
+  -- ADLAM CAPITAL LETTER JIIM
+  '\x1e914'# -> unI64 125238
+  -- ADLAM CAPITAL LETTER CHI
+  '\x1e915'# -> unI64 125239
+  -- ADLAM CAPITAL LETTER HA
+  '\x1e916'# -> unI64 125240
+  -- ADLAM CAPITAL LETTER QAAF
+  '\x1e917'# -> unI64 125241
+  -- ADLAM CAPITAL LETTER GA
+  '\x1e918'# -> unI64 125242
+  -- ADLAM CAPITAL LETTER NYA
+  '\x1e919'# -> unI64 125243
+  -- ADLAM CAPITAL LETTER TU
+  '\x1e91a'# -> unI64 125244
+  -- ADLAM CAPITAL LETTER NHA
+  '\x1e91b'# -> unI64 125245
+  -- ADLAM CAPITAL LETTER VA
+  '\x1e91c'# -> unI64 125246
+  -- ADLAM CAPITAL LETTER KHA
+  '\x1e91d'# -> unI64 125247
+  -- ADLAM CAPITAL LETTER GBE
+  '\x1e91e'# -> unI64 125248
+  -- ADLAM CAPITAL LETTER ZAL
+  '\x1e91f'# -> unI64 125249
+  -- ADLAM CAPITAL LETTER KPO
+  '\x1e920'# -> unI64 125250
+  -- ADLAM CAPITAL LETTER SHA
+  '\x1e921'# -> unI64 125251
+  _ -> unI64 0
+ src/Data/Text/Internal/Fusion/Common.hs view
@@ -0,0 +1,1213 @@+{-# LANGUAGE BangPatterns, MagicHash, RankNTypes, PartialTypeSignatures #-}+{-# OPTIONS_GHC -Wno-partial-type-signatures #-}+-- |+-- Module      : Data.Text.Internal.Fusion.Common+-- Copyright   : (c) Bryan O'Sullivan 2009, 2012+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- This module provides a common stream fusion interface for text.+-- The stream interface allows us to write text pipelines which+-- do not allocate intermediate text values. For example, we could+-- guarantee no intermediate text is allocated by writing the following:+--+-- @+--   getNucleotides :: 'Data.Text.Internal.Text' -> 'Data.Text.Internal.Text'+--   getNucleotides =+--         'Data.Text.Internal.Fusion.unstream'+--       . 'filter' isNucleotide+--       . 'toLower'+--       . 'Data.Text.Internal.Fusion.stream'+--     where+--       isNucleotide chr =+--         chr == \'a\' ||+--         chr == \'c\' ||+--         chr == \'t\' ||+--         chr == \'g\'+-- @++module Data.Text.Internal.Fusion.Common+    (+    -- * Creation and elimination+      singleton+    , streamList+    , unstreamList+    , streamCString#++    -- * Basic interface+    , cons+    , snoc+    , append+    , head+    , uncons+    , last+    , tail+    , init+    , null+    , lengthI+    , compareLengthI+    , isSingleton++    -- * Transformations+    , map+    , intercalate+    , intersperse++    -- ** Case conversion+    -- $case+    , toCaseFold+    , toLower+    , toTitle+    , toUpper++    -- ** Justification+    , justifyLeftI++    -- * Folds+    , foldl+    , foldl'+    , foldl1+    , foldl1'+    , foldr+    , foldr1+    , foldlM'++    -- ** Special folds+    , concat+    , concatMap+    , any+    , all+    , maximum+    , minimum++    -- * Construction+    -- ** Scans+    , scanl++    -- ** Generation and unfolding+    , replicateCharI+    , replicateI+    , unfoldr+    , unfoldrNI++    -- * Substrings+    -- ** Breaking strings+    , take+    , drop+    , takeWhile+    , dropWhile++    -- * Predicates+    , isPrefixOf++    -- * Searching+    , elem+    , filter++    -- * Indexing+    , findBy+    , indexI+    , findIndexI+    , countCharI++    -- * Zipping and unzipping+    , zipWith+    ) where++import Prelude (Bool(..), Char, Eq, (==), Int, Integral, Maybe(..),+                Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++),+                (&&), fromIntegral, otherwise)+import qualified Data.List as L hiding (head, tail)+import qualified Prelude as P+import Data.Bits (shiftL, shiftR, (.&.))+import Data.Char (isLetter, isSpace)+import GHC.Int (Int64(..))+import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader)+import Data.Text.Internal.Fusion.Types+import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping,+                                     upperMapping)+import Data.Text.Internal.Fusion.Size+import GHC.Exts (Char(..), Char#, chr#)+import GHC.Prim (Addr#, indexWord8OffAddr#)+import GHC.Stack (HasCallStack)+import GHC.Types (Int(..))+import Data.Text.Internal.Unsafe.Char (unsafeChr8)+import GHC.Word++-- | /O(1)/ Convert a character into a 'Stream'+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'singleton' = 'Data.Text.singleton'@+singleton :: Char -> Stream Char+singleton c = Stream next False (codePointsSize 1)+    where next False = Yield c True+          next True  = Done+{-# INLINE [0] singleton #-}++-- | /O(n)/ Convert a list into a 'Stream'.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'streamList' = 'Data.Text.pack'@+streamList :: [a] -> Stream a+{-# INLINE [0] streamList #-}+streamList s  = Stream next s unknownSize+    where next []       = Done+          next (x:xs)   = Yield x xs++-- | /O(n)/ Convert a 'Stream' into a list.+--+-- __Properties__+--+-- @'unstreamList' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.unpack'@+unstreamList :: Stream a -> [a]+unstreamList (Stream next s0 _len) = unfold s0+    where unfold !s = case next s of+                        Done       -> []+                        Skip s'    -> unfold s'+                        Yield x s' -> x : unfold s'+{-# INLINE [0] unstreamList #-}++{-# RULES "STREAM streamList/unstreamList fusion" forall s. streamList (unstreamList s) = s #-}++-- | Stream the UTF-8-like packed encoding used by GHC to represent+-- constant strings in generated code.+--+-- This encoding uses the byte sequence "\xc0\x80" to represent NUL,+-- and the string is NUL-terminated.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.Fusion.unstream' . 'streamCString#' addr# = 'Data.Text.Show.unpackCString#' addr#@+streamCString# :: Addr# -> Stream Char+streamCString# addr = Stream step 0 unknownSize+  where+    step !i+        | b == 0    = Done+        | otherwise = Yield chr (i + l)+      where b = at# i+            l = utf8LengthByLeader b+            next n = at# (i+n)+            chr = case l of+              1 -> unsafeChr8 b+              2 -> chr2 b (next 1)+              3 -> chr3 b (next 1) (next 2)+              _ -> chr4 b (next 1) (next 2) (next 3)+    at# (I# i#) = W8# (indexWord8OffAddr# addr i#)+{-# INLINE [0] streamCString# #-}++-- ----------------------------------------------------------------------------+-- * Basic stream functions++data C s = C0 !s+         | C1 !s++-- | /O(n)/ Adds a character to the front of a Stream Char.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.Fusion.unstream' . 'cons' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.cons' c @+cons :: Char -> Stream Char -> Stream Char+cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1)+    where+      next (C1 s) = Yield w (C0 s)+      next (C0 s) = case next0 s of+                          Done -> Done+                          Skip s' -> Skip (C0 s')+                          Yield x s' -> Yield x (C0 s')+{-# INLINE [0] cons #-}++data Snoc a = N+            | J !a++-- | /O(n)/ Adds a character to the end of a stream.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.Fusion.unstream' . 'snoc' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.snoc' c @+snoc :: Stream Char -> Char -> Stream Char+snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1)+  where+    next (J xs) = case next0 xs of+      Done        -> Yield w N+      Skip xs'    -> Skip    (J xs')+      Yield x xs' -> Yield x (J xs')+    next N = Done+{-# INLINE [0] snoc #-}++data E l r = L !l+           | R !r++-- | /O(n)/ Appends one Stream to the other.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.Fusion.unstream' ('append' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.append' t1 t2@+append :: Stream Char -> Stream Char -> Stream Char+append (Stream next0 s01 len1) (Stream next1 s02 len2) =+    Stream next (L s01) (len1 + len2)+    where+      next (L s1) = case next0 s1 of+                         Done        -> Skip    (R s02)+                         Skip s1'    -> Skip    (L s1')+                         Yield x s1' -> Yield x (L s1')+      next (R s2) = case next1 s2 of+                          Done        -> Done+                          Skip s2'    -> Skip    (R s2')+                          Yield x s2' -> Yield x (R s2')+{-# INLINE [0] append #-}++-- | /O(1)/ Returns the first character of a 'Stream' 'Char', which must be non-empty.+-- This is a partial function, consider using 'uncons'.+--+-- __Properties__+--+-- @ 'head' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.head' @+head :: HasCallStack => Stream Char -> Char+head (Stream next s0 _len) = loop_head s0+    where+      loop_head !s = case next s of+                      Yield x _ -> x+                      Skip s'   -> loop_head s'+                      Done      -> head_empty+{-# INLINE [0] head #-}++head_empty :: HasCallStack => a+head_empty = streamError "head" "Empty stream"+{-# NOINLINE head_empty #-}++-- | /O(1)/ Returns the first character and remainder of a 'Stream'+-- 'Char', or 'Nothing' if empty.+--+-- __Properties__+--+-- @ 'Data.Functor.fmap' 'Data.Tuple.fst' . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.fst' . 'Data.Text.uncons' @+--+-- @ 'Data.Functor.fmap' ('Data.Text.Internal.Fusion.unstream' . 'Data.Tuple.snd') . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.snd' . 'Data.Text.uncons' @+uncons :: Stream Char -> Maybe (Char, Stream Char)+uncons (Stream next s0 len) = loop_uncons s0+    where+      loop_uncons !s = case next s of+                         Yield x s1 -> Just (x, Stream next s1 (len - codePointsSize 1))+                         Skip s'    -> loop_uncons s'+                         Done       -> Nothing+{-# INLINE [0] uncons #-}++-- | /O(n)/ Returns the last character of a 'Stream' 'Char', which must+-- be non-empty.+--+-- __Properties__+--+-- @ 'last' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.last' @+last :: HasCallStack => Stream Char -> Char+last (Stream next s0 _len) = loop0_last s0+    where+      loop0_last !s = case next s of+                        Done       -> emptyError "last"+                        Skip s'    -> loop0_last  s'+                        Yield x s' -> loop_last x s'+      loop_last !x !s = case next s of+                         Done        -> x+                         Skip s'     -> loop_last x  s'+                         Yield x' s' -> loop_last x' s'+{-# INLINE[0] last #-}++-- | /O(1)/ Returns all characters after the head of a 'Stream' 'Char', which must+-- be non-empty. This is a partial function, consider using 'uncons'.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'tail' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.tail' @+tail :: HasCallStack => Stream Char -> Stream Char+tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1)+    where+      next (C0 s) = case next0 s of+                      Done       -> emptyError "tail"+                      Skip s'    -> Skip (C0 s')+                      Yield _ s' -> Skip (C1 s')+      next (C1 s) = case next0 s of+                      Done       -> Done+                      Skip s'    -> Skip    (C1 s')+                      Yield x s' -> Yield x (C1 s')+{-# INLINE [0] tail #-}++data Init s = Init0 !s+            | Init1 {-# UNPACK #-} !Char !s++-- | /O(1)/ Returns all but the last character of a 'Stream' 'Char', which+-- must be non-empty.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'init' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.init' @+init :: HasCallStack => Stream Char -> Stream Char+init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1)+    where+      next (Init0 s) = case next0 s of+                         Done       -> emptyError "init"+                         Skip s'    -> Skip (Init0 s')+                         Yield x s' -> Skip (Init1 x s')+      next (Init1 x s)  = case next0 s of+                            Done        -> Done+                            Skip s'     -> Skip    (Init1 x s')+                            Yield x' s' -> Yield x (Init1 x' s')+{-# INLINE [0] init #-}++-- | /O(1)/ Tests whether a 'Stream' 'Char' is empty or not.+--+-- __Properties__+--+-- @ 'null' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.null' @+null :: Stream Char -> Bool+null (Stream next s0 _len) = loop_null s0+    where+      loop_null !s = case next s of+                       Done      -> True+                       Yield _ _ -> False+                       Skip s'   -> loop_null s'+{-# INLINE[0] null #-}++-- | /O(n)/ Returns the number of characters in a string.+lengthI :: Integral a => Stream Char -> a+lengthI (Stream next s0 _len) = loop_length 0 s0+    where+      loop_length !z s  = case next s of+                           Done       -> z+                           Skip    s' -> loop_length z s'+                           Yield _ s' -> loop_length (z + 1) s'+{-# INLINE[0] lengthI #-}++-- | /O(n)/ Compares the count of characters in a string to a number.+--+-- This function gives the same answer as comparing against the result+-- of 'lengthI', but can short circuit if the count of characters is+-- greater than the number or if the stream can't possibly be as long+-- as the number supplied, and hence be more efficient.+compareLengthI :: Integral a => Stream Char -> a -> Ordering+compareLengthI (Stream next s0 len) n+    -- Note that @len@ tracks code units whereas we want to compare the length+    -- in code points. Specifically, a stream with hint @len@ may consist of+    -- anywhere from @len/2@ to @len@ code points.+  | n < 0 = GT+  | Just r <- compareSize len n' = r+  | otherwise = loop_cmp 0 s0+    where+      n' = codePointsSize $ fromIntegral n+      loop_cmp !z s  = case next s of+                         Done       -> compare z n+                         Skip    s' -> loop_cmp z s'+                         Yield _ s' | z > n     -> GT+                                    | otherwise -> loop_cmp (z + 1) s'+{-# INLINE[0] compareLengthI #-}++-- | /O(n)/ Indicate whether a string contains exactly one element.+--+-- __Properties__+--+-- @ 'isSingleton' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.isSingleton' @+isSingleton :: Stream Char -> Bool+isSingleton (Stream next s0 _len) = loop 0 s0+    where+      loop !z s  = case next s of+                     Done            -> z == (1::Int)+                     Skip    s'      -> loop z s'+                     Yield _ s'+                         | z >= 1    -> False+                         | otherwise -> loop (z+1) s'+{-# INLINE[0] isSingleton #-}++-- ----------------------------------------------------------------------------+-- * Stream transformations++-- | /O(n)/ 'map' @f @xs is the 'Stream' 'Char' obtained by applying @f@+-- to each element of @xs@.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'map' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.map' f @+map :: (Char -> Char) -> Stream Char -> Stream Char+map f (Stream next0 s0 len) = Stream next s0 len+    where+      next !s = case next0 s of+                  Done       -> Done+                  Skip s'    -> Skip s'+                  Yield x s' -> Yield (f x) s'+{-# INLINE [0] map #-}++{-#+  RULES "STREAM map/map fusion" forall f g s.+     map f (map g s) = map (\x -> f (g x)) s+ #-}++data I s = I1 !s+         | I2 !s {-# UNPACK #-} !Char+         | I3 !s++-- | /O(n)/ Take a character and place it between each of the+-- characters of a 'Stream Char'.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'intersperse' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.intersperse' c @+intersperse :: Char -> Stream Char -> Stream Char+intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize)+    where+      next (I1 s) = case next0 s of+        Done       -> Done+        Skip s'    -> Skip (I1 s')+        Yield x s' -> Skip (I2 s' x)+      next (I2 s x)  = Yield x (I3 s)+      next (I3 s) = case next0 s of+        Done       -> Done+        Skip s'    -> Skip    (I3 s')+        Yield x s' -> Yield c (I2 s' x)+{-# INLINE [0] intersperse #-}++-- ----------------------------------------------------------------------------+-- ** Case conversions (folds)++-- $case+--+-- With Unicode text, it is incorrect to use combinators like @map+-- toUpper@ to case convert each character of a string individually.+-- Instead, use the whole-string case conversion functions from this+-- module.  For correctness in different writing systems, these+-- functions may map one input character to two or three output+-- characters.++-- | Map a 'Stream' through the given case-mapping function.+caseConvert :: (Char# -> _ {- unboxed Int64 -})+            -> Stream Char -> Stream Char+caseConvert remap (Stream next0 s0 len) =+    Stream next (CC s0 0) (len `unionSize` (3*len))+  where+    next (CC s 0) =+        case next0 s of+          Done       -> Done+          Skip s'    -> Skip (CC s' 0)+          Yield c@(C# c#) s' -> case I64# (remap c#) of+            0 -> Yield c (CC s' 0)+            ab -> let (a, b) = chopOffChar ab in+              Yield a (CC s' b)+    next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b)++chopOffChar :: Int64 -> (Char, Int64)+chopOffChar ab = (chr a, ab `shiftR` 21)+  where+    chr (I# n) = C# (chr# n)+    mask = (1 `shiftL` 21) - 1+    a = fromIntegral $ ab .&. mask++-- | /O(n)/ Convert a string to folded case.  This function is mainly+-- useful for performing caseless (or case insensitive) string+-- comparisons.+--+-- A string @x@ is a caseless match for a string @y@ if and only if:+--+-- @'toCaseFold' x == 'toCaseFold' y@+--+-- The result string may be longer than the input string, and may+-- differ from applying 'toLower' to the input string.  For instance,+-- the Armenian small ligature men now (U+FB13) is case folded to the+-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is+-- case folded to the Greek small letter letter mu (U+03BC) instead of+-- itself.+toCaseFold :: Stream Char -> Stream Char+toCaseFold = caseConvert foldMapping+{-# INLINE [0] toCaseFold #-}++-- | /O(n)/ Convert a string to upper case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the German eszett (U+00DF) maps to the two-letter+-- sequence SS.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'toUpper' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toUpper' @+toUpper :: Stream Char -> Stream Char+toUpper = caseConvert upperMapping+{-# INLINE [0] toUpper #-}++-- | /O(n)/ Convert a string to lower case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the Latin capital letter I with dot above (U+0130)+-- maps to the sequence Latin small letter i (U+0069) followed by+-- combining dot above (U+0307).+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'toLower' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toLower' @+toLower :: Stream Char -> Stream Char+toLower = caseConvert lowerMapping+{-# INLINE [0] toLower #-}++-- | /O(n)/ Convert a string to title case, using simple case+-- conversion.+--+-- The first letter (as determined by 'Data.Char.isLetter')+-- of the input is converted to title case, as is+-- every subsequent letter that immediately follows a non-letter.+-- Every letter that immediately follows another letter is converted+-- to lower case.+--+-- The result string may be longer than the input string. For example,+-- the Latin small ligature &#xfb02; (U+FB02) is converted to the+-- sequence Latin capital letter F (U+0046) followed by Latin small+-- letter l (U+006C).+--+-- This function is not idempotent.+-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).+-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input+-- is converted to title case, becoming two letters.+-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter+-- and as such is recognised as a letter by 'Data.Char.isLetter',+-- so 'T.toTitle' @"ʼN"@ = @"'n"@.+--+-- /Note/: this function does not take language or culture specific+-- rules into account. For instance, in English, different style+-- guides disagree on whether the book name \"The Hill of the Red+-- Fox\" is correctly title cased&#x2014;but this function will+-- capitalize /every/ word.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'toTitle' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toTitle' @+toTitle :: Stream Char -> Stream Char+toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) 0) (len + unknownSize)+  where+    next (CC (letter :*: s) 0) =+      case next0 s of+        Done            -> Done+        Skip s'         -> Skip (CC (letter :*: s') 0)+        Yield c@(C# c#) s'+          | nonSpace, letter -> case I64# (lowerMapping c#) of+            0 -> Yield c (CC (nonSpace :*: s') 0)+            ab -> let (a, b) = chopOffChar ab in+              Yield a (CC (nonSpace :*: s') b)+          | nonSpace    ->  case I64# (titleMapping c#) of+            0 -> Yield c (CC (letter' :*: s') 0)+            ab -> let (a, b) = chopOffChar ab in+              Yield a (CC (letter' :*: s') b)+          | otherwise   -> Yield c (CC (letter' :*: s') 0)+          where nonSpace = P.not (isSpace c)+                letter'  = isLetter c+    next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b)+{-# INLINE [0] toTitle #-}++data Justify i s = Just1 !i !s+                 | Just2 !i !s++justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char+justifyLeftI k c (Stream next0 s0 len) =+    Stream next (Just1 0 s0) (larger (fromIntegral k * charSize c + len) len)+  where+    next (Just1 n s) =+        case next0 s of+          Done       -> next (Just2 n s)+          Skip s'    -> Skip (Just1 n s')+          Yield x s' -> Yield x (Just1 (n+1) s')+    next (Just2 n s)+        | n < k       = Yield c (Just2 (n+1) s)+        | otherwise   = Done+    {-# INLINE next #-}+{-# INLINE [0] justifyLeftI #-}++-- ----------------------------------------------------------------------------+-- * Reducing Streams (folds)++-- | foldl, applied to a binary operator, a starting value (typically the+-- left-identity of the operator), and a 'Stream', reduces the 'Stream' using the+-- binary operator, from left to right.+--+-- __Properties__+--+-- @ 'foldl' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl' f z0 @+foldl :: (b -> Char -> b) -> b -> Stream Char -> b+foldl f z0 (Stream next s0 _len) = loop_foldl z0 s0+    where+      loop_foldl z !s = case next s of+                          Done -> z+                          Skip s' -> loop_foldl z s'+                          Yield x s' -> loop_foldl (f z x) s'+{-# INLINE [0] foldl #-}++-- | A strict version of foldl.+--+-- __Properties__+--+-- @ 'foldl'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl'' f z0 @+foldl' :: (b -> Char -> b) -> b -> Stream Char -> b+foldl' f z0 (Stream next s0 _len) = loop_foldl' z0 s0+    where+      loop_foldl' !z !s = case next s of+                            Done -> z+                            Skip s' -> loop_foldl' z s'+                            Yield x s' -> loop_foldl' (f z x) s'+{-# INLINE [0] foldl' #-}++-- | foldl1 is a variant of foldl that has no starting value argument,+-- and thus must be applied to non-empty Streams.+--+-- __Properties__+--+-- @ 'foldl1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1' f @+foldl1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char+foldl1 f (Stream next s0 _len) = loop0_foldl1 s0+    where+      loop0_foldl1 !s = case next s of+                          Skip s' -> loop0_foldl1 s'+                          Yield x s' -> loop_foldl1 x s'+                          Done -> emptyError "foldl1"+      loop_foldl1 z !s = case next s of+                           Done -> z+                           Skip s' -> loop_foldl1 z s'+                           Yield x s' -> loop_foldl1 (f z x) s'+{-# INLINE [0] foldl1 #-}++-- | A strict version of foldl1.+--+-- __Properties__+--+-- @ 'foldl1'' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1'' f @+foldl1' :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char+foldl1' f (Stream next s0 _len) = loop0_foldl1' s0+    where+      loop0_foldl1' !s = case next s of+                           Skip s' -> loop0_foldl1' s'+                           Yield x s' -> loop_foldl1' x s'+                           Done -> emptyError "foldl1"+      loop_foldl1' !z !s = case next s of+                             Done -> z+                             Skip s' -> loop_foldl1' z s'+                             Yield x s' -> loop_foldl1' (f z x) s'+{-# INLINE [0] foldl1' #-}++-- | A monadic version of 'foldl'.+--+-- __Properties__+--+-- @ 'foldlM'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldlM'' f z0 @+foldlM' :: P.Monad m => (b -> Char -> m b) -> b -> Stream Char -> m b+foldlM' f z0 (Stream next s0 _len) = loop_foldlM' z0 s0+    where+      loop_foldlM' !z !s = case next s of+                            Done -> P.pure z+                            Skip s' -> loop_foldlM' z s'+                            Yield x s' -> f z x P.>>= \z' -> loop_foldlM' z' s'+{-# INLINE [0] foldlM' #-}++-- | 'foldr', applied to a binary operator, a starting value (typically the+-- right-identity of the operator), and a stream, reduces the stream using the+-- binary operator, from right to left.+--+-- __Properties__+--+-- @ 'foldr' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr' f z0 @+foldr :: (Char -> b -> b) -> b -> Stream Char -> b+foldr f z (Stream next s0 _len) = loop_foldr s0+    where+      loop_foldr !s = case next s of+                        Done -> z+                        Skip s' -> loop_foldr s'+                        Yield x s' -> f x (loop_foldr s')+{-# INLINE [0] foldr #-}++-- | foldr1 is a variant of 'foldr' that has no starting value argument,+-- and thus must be applied to non-empty streams.+--+-- __Properties__+--+-- @ 'foldr1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr1' f @+foldr1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char+foldr1 f (Stream next s0 _len) = loop0_foldr1 s0+  where+    loop0_foldr1 !s = case next s of+      Done       -> emptyError "foldr1"+      Skip    s' -> loop0_foldr1  s'+      Yield x s' -> loop_foldr1 x s'++    loop_foldr1 x !s = case next s of+      Done        -> x+      Skip     s' -> loop_foldr1 x s'+      Yield x' s' -> f x (loop_foldr1 x' s')+{-# INLINE [0] foldr1 #-}++-- | intercalate str strs inserts the stream str in between the streams strs and+-- concatenates the result.+--+-- __Properties__+--+-- @ 'intercalate' s = 'concat' . 'L.intersperse' s @+intercalate :: Stream Char -> [Stream Char] -> Stream Char+intercalate s = concat . (L.intersperse s)+{-# INLINE [0] intercalate #-}++-- ----------------------------------------------------------------------------+-- ** Special folds++-- | /O(n)/ Concatenate a list of streams.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'concat' . 'Data.Functor.fmap' 'Data.Text.Internal.Fusion.stream'  = 'Data.Text.concat'@+concat :: [Stream Char] -> Stream Char+concat = L.foldr append empty+{-# INLINE [0] concat #-}++-- | Map a function over a stream that results in a stream and concatenate the+-- results.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'concatMap' ('Data.Text.Fusion.stream' . f) . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.concatMap' f@+concatMap :: (Char -> Stream Char) -> Stream Char -> Stream Char+concatMap f = foldr (append . f) empty+{-# INLINE [0] concatMap #-}++-- | /O(n)/ any @p @xs determines if any character in the stream+-- @xs@ satisfies the predicate @p@.+--+-- __Properties__+--+-- @'any' f . 'Data.Text.Fusion.stream' = 'Data.Text.any' f@+any :: (Char -> Bool) -> Stream Char -> Bool+any p (Stream next0 s0 _len) = loop_any s0+    where+      loop_any !s = case next0 s of+                      Done                   -> False+                      Skip s'                -> loop_any s'+                      Yield x s' | p x       -> True+                                 | otherwise -> loop_any s'+{-# INLINE [0] any #-}++-- | /O(n)/ all @p @xs determines if all characters in the 'Text'+-- @xs@ satisfy the predicate @p@.+--+-- __Properties__+--+-- @'all' f . 'Data.Text.Fusion.stream' = 'Data.Text.all' f@+all :: (Char -> Bool) -> Stream Char -> Bool+all p (Stream next0 s0 _len) = loop_all s0+    where+      loop_all !s = case next0 s of+                      Done                   -> True+                      Skip s'                -> loop_all s'+                      Yield x s' | p x       -> loop_all s'+                                 | otherwise -> False+{-# INLINE [0] all #-}++-- | /O(n)/ maximum returns the maximum value from a stream, which must be+-- non-empty.+--+-- __Properties__+--+-- @'maximum' . 'Data.Text.Fusion.stream' = 'Data.Text.maximum'@+maximum :: HasCallStack => Stream Char -> Char+maximum (Stream next0 s0 _len) = loop0_maximum s0+    where+      loop0_maximum !s   = case next0 s of+                             Done       -> emptyError "maximum"+                             Skip s'    -> loop0_maximum s'+                             Yield x s' -> loop_maximum x s'+      loop_maximum !z !s = case next0 s of+                             Done            -> z+                             Skip s'         -> loop_maximum z s'+                             Yield x s'+                                 | x > z     -> loop_maximum x s'+                                 | otherwise -> loop_maximum z s'+{-# INLINE [0] maximum #-}++-- | /O(n)/ minimum returns the minimum value from a 'Text', which must be+-- non-empty.+--+-- __Properties__+--+-- @'minimum' . 'Data.Text.Fusion.stream' = 'Data.Text.minimum'@+minimum :: HasCallStack => Stream Char -> Char+minimum (Stream next0 s0 _len) = loop0_minimum s0+    where+      loop0_minimum !s   = case next0 s of+                             Done       -> emptyError "minimum"+                             Skip s'    -> loop0_minimum s'+                             Yield x s' -> loop_minimum x s'+      loop_minimum !z !s = case next0 s of+                             Done            -> z+                             Skip s'         -> loop_minimum z s'+                             Yield x s'+                                 | x < z     -> loop_minimum x s'+                                 | otherwise -> loop_minimum z s'+{-# INLINE [0] minimum #-}++-- -----------------------------------------------------------------------------+-- * Building streams+--+-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a stream of+-- successive reduced values from the left. Conceptually, if we+-- write the input stream as a list then we have:+--+-- > scanl f z [x1, x2, ...] == [z, z 'f' x1, (z 'f' x1) 'f' x2, ...]+--+-- __Properties__+--+-- @'head' ('scanl' f z xs) = z@+--+-- @'last' ('scanl' f z xs) = 'foldl' f z xs@+scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char+scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low+  where+    {-# INLINE next #-}+    next (Scan1 z s) = Yield z (Scan2 z s)+    next (Scan2 z s) = case next0 s of+                         Yield x s' -> let !x' = f z x+                                       in Yield x' (Scan2 x' s')+                         Skip s'    -> Skip (Scan2 z s')+                         Done       -> Done+{-# INLINE [0] scanl #-}++-- -----------------------------------------------------------------------------+-- ** Generating and unfolding streams++-- | /O(n)/ 'replicateCharI' @n@ @c@ is a 'Stream' 'Char' of length @n@ with @c@ the+-- value of every element.+replicateCharI :: Integral a => a -> Char -> Stream Char+replicateCharI !n !c+    | n < 0     = empty+    | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low+  where+    next !i | i >= n    = Done+            | otherwise = Yield c (i + 1)+{-# INLINE [0] replicateCharI #-}++data RI s = RI !s {-# UNPACK #-} !Int64+++-- | /O(n*m)/ 'replicateI' @n@ @t@ is a 'Stream' 'Char' consisting of the input+-- @t@ repeated @n@ times.+replicateI :: Int64 -> Stream Char -> Stream Char+replicateI n (Stream next0 s0 len) =+    Stream next (RI s0 0) (int64ToSize (max 0 n) * len)+  where+    next (RI s k)+        | k >= n = Done+        | otherwise = case next0 s of+                        Done       -> Skip    (RI s0 (k+1))+                        Skip s'    -> Skip    (RI s' k)+                        Yield x s' -> Yield x (RI s' k)+{-# INLINE [0] replicateI #-}++-- | /O(n)/, where @n@ is the length of the result. The unfoldr function+-- is analogous to the List 'unfoldr'. unfoldr builds a stream+-- from a seed value. The function takes the element and returns+-- Nothing if it is done producing the stream or returns Just+-- (a,b), in which case, a is the next Char in the string, and b is+-- the seed value for further production.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'unfoldr' f z = 'Data.Text.unfoldr' f z@+unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char+unfoldr f s0 = Stream next s0 unknownSize+    where+      {-# INLINE next #-}+      next !s = case f s of+                 Nothing      -> Done+                 Just (w, s') -> Yield w s'+{-# INLINE [0] unfoldr #-}++-- | /O(n)/ Like 'unfoldr', 'unfoldrNI' builds a stream from a seed+-- value. However, the length of the result is limited by the+-- first argument to 'unfoldrNI'. This function is more efficient than+-- 'unfoldr' when the length of the result is known.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' ('unfoldrNI' n f z) = 'Data.Text.unfoldrN' n f z@+unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char+unfoldrNI n f s0 | n <  0    = empty+                 | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2))+    where+      {-# INLINE next #-}+      next (z :*: s) = case f s of+          Nothing                  -> Done+          Just (w, s') | z >= n    -> Done+                       | otherwise -> Yield w ((z + 1) :*: s')+{-# INLINE unfoldrNI #-}++-------------------------------------------------------------------------------+--  * Substreams++-- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the+-- stream of length @n@, or the stream itself if @n@ is greater than the+-- length of the stream.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'take' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.take' n@+take :: Integral a => a -> Stream Char -> Stream Char+take n0 (Stream next0 s0 len) =+    Stream next (n0' :*: s0) (smaller len (codePointsSize $ fromIntegral n0'))+    where+      n0' = max n0 0++      {-# INLINE next #-}+      next (n :*: s) | n <= 0    = Done+                     | otherwise = case next0 s of+                                     Done -> Done+                                     Skip s' -> Skip (n :*: s')+                                     Yield x s' -> Yield x ((n-1) :*: s')+{-# INLINE [0] take #-}++data Drop a s = NS !s+              | JS !a !s++-- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the+-- stream after the first @n@ characters, or the empty stream if @n@+-- is greater than the length of the stream.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'drop' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.drop' n@+drop :: Integral a => a -> Stream Char -> Stream Char+drop n0 (Stream next0 s0 len) =+    Stream next (JS n0' s0) (len - codePointsSize (fromIntegral n0'))+  where+    n0' = max n0 0++    {-# INLINE next #-}+    next (JS n s)+      | n <= 0    = Skip (NS s)+      | otherwise = case next0 s of+          Done       -> Done+          Skip    s' -> Skip (JS n    s')+          Yield _ s' -> Skip (JS (n-1) s')+    next (NS s) = case next0 s of+      Done       -> Done+      Skip    s' -> Skip    (NS s')+      Yield x s' -> Yield x (NS s')+{-# INLINE [0] drop #-}++-- | 'takeWhile', applied to a predicate @p@ and a stream, returns the+-- longest prefix (possibly empty) of elements that satisfy @p@.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'takeWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.takeWhile' p@+takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char+takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize)+    where+      {-# INLINE next #-}+      next !s = case next0 s of+                  Done    -> Done+                  Skip s' -> Skip s'+                  Yield x s' | p x       -> Yield x s'+                             | otherwise -> Done+{-# INLINE [0] takeWhile #-}++-- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@.+--+-- __Properties__+--+-- @'Data.Text.Internal.Fusion.unstream' . 'dropWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.dropWhile' p@+dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char+dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize)+    where+    {-# INLINE next #-}+    next (L s)  = case next0 s of+      Done                   -> Done+      Skip    s'             -> Skip    (L s')+      Yield x s' | p x       -> Skip    (L s')+                 | otherwise -> Yield x (R s')+    next (R s) = case next0 s of+      Done       -> Done+      Skip    s' -> Skip    (R s')+      Yield x s' -> Yield x (R s')+{-# INLINE [0] dropWhile #-}++-- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns+-- 'True' if and only if the first is a prefix of the second.+--+-- __Properties__+--+-- @ 'isPrefixOf' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2) = 'Data.Text.isPrefixOf' t1 t2@+isPrefixOf :: (Eq a) => Stream a -> Stream a -> Bool+isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)+    where+      loop Done      _ = True+      loop _    Done = False+      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')+      loop (Skip s1')     x2             = loop (next1 s1') x2+      loop x1             (Skip s2')     = loop x1          (next2 s2')+      loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&+                                           loop (next1 s1') (next2 s2')+{-# INLINE [0] isPrefixOf #-}++-- ----------------------------------------------------------------------------+-- * Searching++-------------------------------------------------------------------------------+-- ** Searching by equality++-- | /O(n)/ 'elem' is the stream membership predicate.+--+-- __Properties__+--+-- @ 'elem' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.elem' c@+elem :: Char -> Stream Char -> Bool+elem w (Stream next s0 _len) = loop_elem s0+    where+      loop_elem !s = case next s of+                       Done -> False+                       Skip s' -> loop_elem s'+                       Yield x s' | x == w -> True+                                  | otherwise -> loop_elem s'+{-# INLINE [0] elem #-}++-------------------------------------------------------------------------------+-- ** Searching with a predicate++-- | /O(n)/ The 'findBy' function takes a predicate and a stream,+-- and returns the first element in matching the predicate, or 'Nothing'+-- if there is no such element.+--+-- __Properties__+--+-- @ 'findBy' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.find' p@+findBy :: (Char -> Bool) -> Stream Char -> Maybe Char+findBy p (Stream next s0 _len) = loop_find s0+    where+      loop_find !s = case next s of+                       Done -> Nothing+                       Skip s' -> loop_find s'+                       Yield x s' | p x -> Just x+                                  | otherwise -> loop_find s'+{-# INLINE [0] findBy #-}++-- | /O(n)/ Stream index (subscript) operator, starting from 0.+--+-- __Properties__+--+-- @ 'indexI' ('Data.Text.Internal.Fusion.stream' t) n = 'Data.Text.index' t n@+indexI :: (HasCallStack, Integral a) => Stream Char -> a -> Char+indexI (Stream next s0 _len) n0+  | n0 < 0    = streamError "index" "Negative index"+  | otherwise = loop_index n0 s0+  where+    loop_index !n !s = case next s of+      Done                   -> streamError "index" "Index too large"+      Skip    s'             -> loop_index  n    s'+      Yield x s' | n == 0    -> x+                 | otherwise -> loop_index (n-1) s'+{-# INLINE [0] indexI #-}++-- | /O(n)/ 'filter', applied to a predicate and a stream,+-- returns a stream containing those characters that satisfy the+-- predicate.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.unstream' . 'filter' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.filter' p @+filter :: (Char -> Bool) -> Stream Char -> Stream Char+filter p (Stream next0 s0 len) =+    Stream next s0 (len - unknownSize) -- HINT maybe too high+  where+    next !s = case next0 s of+                Done                   -> Done+                Skip    s'             -> Skip    s'+                Yield x s' | p x       -> Yield x s'+                           | otherwise -> Skip    s'+{-# INLINE [0] filter #-}++{-# RULES+  "STREAM filter/filter fusion" forall p q s.+  filter p (filter q s) = filter (\x -> q x && p x) s+  #-}++-- | The 'findIndexI' function takes a predicate and a stream and+-- returns the index of the first element in the stream satisfying the+-- predicate.+--+-- __Properties__+--+-- @'findIndexI' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.findIndex' p @+findIndexI :: Integral a => (Char -> Bool) -> Stream Char -> Maybe a+findIndexI p s = case findIndicesI p s of+                  (i:_) -> Just i+                  _     -> Nothing+{-# INLINE [0] findIndexI #-}++-- | The 'findIndicesI' function takes a predicate and a stream and+-- returns all indices of the elements in the stream satisfying the+-- predicate.+findIndicesI :: Integral a => (Char -> Bool) -> Stream Char -> [a]+findIndicesI p (Stream next s0 _len) = loop_findIndex 0 s0+  where+    loop_findIndex !i !s = case next s of+      Done                   -> []+      Skip    s'             -> loop_findIndex i     s' -- hmm. not caught by QC+      Yield x s' | p x       -> i : loop_findIndex (i+1) s'+                 | otherwise -> loop_findIndex (i+1) s'+{-# INLINE [0] findIndicesI #-}++-------------------------------------------------------------------------------+-- * Zipping++-- | Strict triple.+data Zip a b m = Z1 !a !b+               | Z2 !a !b !m++-- | zipWith generalises 'zip' by zipping with the function given as+-- the first argument, instead of a tupling function.+--+-- __Properties__+--+-- @ 'Data.Text.Internal.Fusion.unstream' ('zipWith' f ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.zipWith' f t1 t2@+zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b+zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) =+    Stream next (Z1 sa0 sb0) (smaller len1 len2)+    where+      next (Z1 sa sb) = case next0 sa of+                          Done -> Done+                          Skip sa' -> Skip (Z1 sa' sb)+                          Yield a sa' -> Skip (Z2 sa' sb a)++      next (Z2 sa' sb a) = case next1 sb of+                             Done -> Done+                             Skip sb' -> Skip (Z2 sa' sb' a)+                             Yield b sb' -> Yield (f a b) (Z1 sa' sb')+{-# INLINE [0] zipWith #-}++-- | /O(n)/ The 'countCharI' function returns the number of times the+-- query element appears in the given stream.+--+-- __Properties__+--+-- @'countCharI' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.countChar' c @+countCharI :: Integral a => Char -> Stream Char -> a+countCharI a (Stream next s0 _len) = loop 0 s0+  where+    loop !i !s = case next s of+      Done                   -> i+      Skip    s'             -> loop i s'+      Yield x s' | a == x    -> loop (i+1) s'+                 | otherwise -> loop i s'+{-# INLINE [0] countCharI #-}++streamError :: HasCallStack => String -> String -> a+streamError func msg = P.error $ "Data.Text.Internal.Fusion.Common." ++ func ++ ": " ++ msg++emptyError :: HasCallStack => String -> a+emptyError func = internalError func "Empty input"++internalError :: HasCallStack => String -> a+internalError func = streamError func "Internal error"++int64ToSize :: Int64 -> Size+int64ToSize = fromIntegral
+ src/Data/Text/Internal/Fusion/Size.hs view
@@ -0,0 +1,185 @@+{-# LANGUAGE CPP #-}+{-# OPTIONS_GHC -fno-warn-missing-methods #-}+-- |+-- Module      : Data.Text.Internal.Fusion.Internal+-- Copyright   : (c) Roman Leshchinskiy 2008,+--               (c) Bryan O'Sullivan 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Size hints.++module Data.Text.Internal.Fusion.Size+    (+      Size+      -- * Sizes+    , exactSize+    , maxSize+    , betweenSize+    , unknownSize+    , unionSize+    , charSize+    , codePointsSize+      -- * Querying sizes+    , exactly+    , smaller+    , larger+    , upperBound+    , lowerBound+    , compareSize+    , isEmpty+    ) where++import Data.Text.Internal.Encoding.Utf8 (utf8Length)+import Data.Text.Internal (mul)+#if defined(ASSERTS)+import Control.Exception (assert)+#endif++-- | A size in UTF-8 code units (which is bytes).+data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size.+          | Unknown                                         -- ^ Unknown size.+            deriving (Eq, Show)++exactly :: Size -> Maybe Int+exactly (Between na nb) | na == nb = Just na+exactly _ = Nothing+{-# INLINE exactly #-}++-- | The 'Size' of the given code point.+charSize :: Char -> Size+charSize = exactSize . utf8Length++-- | The 'Size' of @n@ code points.+codePointsSize :: Int -> Size+codePointsSize n =+#if defined(ASSERTS)+    assert (n >= 0)+#endif+    Between n (4*n)+{-# INLINE codePointsSize #-}++exactSize :: Int -> Size+exactSize n =+#if defined(ASSERTS)+    assert (n >= 0)+#endif+    Between n n+{-# INLINE exactSize #-}++maxSize :: Int -> Size+maxSize n =+#if defined(ASSERTS)+    assert (n >= 0)+#endif+    Between 0 n+{-# INLINE maxSize #-}++betweenSize :: Int -> Int -> Size+betweenSize m n =+#if defined(ASSERTS)+    assert (m >= 0)+    assert (n >= m)+#endif+    Between m n+{-# INLINE betweenSize #-}++unionSize :: Size -> Size -> Size+unionSize (Between a b) (Between c d) = Between (min a c) (max b d)+unionSize _ _ = Unknown++unknownSize :: Size+unknownSize = Unknown+{-# INLINE unknownSize #-}++instance Num Size where+    (+) = addSize+    (-) = subtractSize+    (*) = mulSize++    fromInteger = f where f = exactSize . fromInteger+                          {-# INLINE f #-}++add :: Int -> Int -> Int+add m n | mn >=   0 = mn+        | otherwise = overflowError+  where mn = m + n+{-# INLINE add #-}++addSize :: Size -> Size -> Size+addSize (Between ma mb) (Between na nb) = Between (add ma na) (add mb nb)+addSize _               _               = Unknown+{-# INLINE addSize #-}++subtractSize :: Size -> Size -> Size+subtractSize (Between ma mb) (Between na nb) = Between (max (ma-nb) 0) (max (mb-na) 0)+subtractSize a@(Between 0 _) Unknown         = a+subtractSize (Between _ mb)  Unknown         = Between 0 mb+subtractSize _               _               = Unknown+{-# INLINE subtractSize #-}++mulSize :: Size -> Size -> Size+mulSize (Between ma mb) (Between na nb) = Between (mul ma na) (mul mb nb)+mulSize _               _               = Unknown+{-# INLINE mulSize #-}++-- | Minimum of two size hints.+smaller :: Size -> Size -> Size+smaller a@(Between ma mb) b@(Between na nb)+    | mb <= na  = a+    | nb <= ma  = b+    | otherwise = Between (ma `min` na) (mb `min` nb)+smaller a@(Between 0 _) Unknown         = a+smaller (Between _ mb)  Unknown         = Between 0 mb+smaller Unknown         b@(Between 0 _) = b+smaller Unknown         (Between _ nb)  = Between 0 nb+smaller Unknown         Unknown         = Unknown+{-# INLINE smaller #-}++-- | Maximum of two size hints.+larger :: Size -> Size -> Size+larger a@(Between ma mb) b@(Between na nb)+    | ma >= nb  = a+    | na >= mb  = b+    | otherwise = Between (ma `max` na) (mb `max` nb)+larger _ _ = Unknown+{-# INLINE larger #-}++-- | Compute the maximum size from a size hint, if possible.+upperBound :: Int -> Size -> Int+upperBound _ (Between _ n) = n+upperBound k _             = k+{-# INLINE upperBound #-}++-- | Compute the minimum size from a size hint, if possible.+lowerBound :: Int -> Size -> Int+lowerBound _ (Between n _) = n+lowerBound k _             = k+{-# INLINE lowerBound #-}++-- | Determine the ordering relationship between two 'Size's, or 'Nothing' in+-- the indeterminate case.+compareSize :: Size -> Size -> Maybe Ordering+compareSize (Between ma mb) (Between na nb)+  | mb < na            = Just LT+  | ma > nb            = Just GT+  | ma == mb+  , ma == na+  , ma == nb           = Just EQ+compareSize _ _        = Nothing+++isEmpty :: Size -> Bool+isEmpty (Between _ n) = n <= 0+isEmpty _             = False+{-# INLINE isEmpty #-}++overflowError :: Int+overflowError = error "Data.Text.Internal.Fusion.Size: size overflow"
+ src/Data/Text/Internal/Fusion/Types.hs view
@@ -0,0 +1,125 @@+{-# LANGUAGE BangPatterns, ExistentialQuantification #-}+-- |+-- Module      : Data.Text.Internal.Fusion.Types+-- Copyright   : (c) Tom Harper 2008-2009,+--               (c) Bryan O'Sullivan 2009,+--               (c) Duncan Coutts 2009,+--               (c) Jasper Van der Jeugt 2011+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Core stream fusion functionality for text.++module Data.Text.Internal.Fusion.Types+    (+      CC(..)+    , PairS(..)+    , Scan(..)+    , RS(..)+    , Step(..)+    , Stream(..)+    , empty+    ) where++import Data.Text.Internal.Fusion.Size+import Data.Int (Int64)+import Data.Word (Word8)++-- | Specialised tuple for case conversion.+data CC s = CC !s {-# UNPACK #-} !Int64++-- | Restreaming state.+data RS s+    = RS0 !s+    | RS1 !s {-# UNPACK #-} !Word8+    | RS2 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8+    | RS3 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8++-- | Strict pair.+data PairS a b = !a :*: !b+                 -- deriving (Eq, Ord, Show)+infixl 2 :*:++-- | An intermediate result in a scan.+data Scan s = Scan1 {-# UNPACK #-} !Char !s+            | Scan2 {-# UNPACK #-} !Char !s++-- | Intermediate result in a processing pipeline.+data Step s a = Done+              | Skip !s+              | Yield !a !s++{-+instance (Show a) => Show (Step s a)+    where show Done        = "Done"+          show (Skip _)    = "Skip"+          show (Yield x _) = "Yield " ++ show x+-}++instance (Eq a) => Eq (Stream a) where+    (==) = eq++instance (Ord a) => Ord (Stream a) where+    compare = cmp++-- The length hint in a Stream has two roles.  If its value is zero,+-- we trust it, and treat the stream as empty.  Otherwise, we treat it+-- as a hint: it should usually be accurate, so we use it when+-- unstreaming to decide what size array to allocate.  However, the+-- unstreaming functions must be able to cope with the hint being too+-- small or too large.+--+-- The size hint tries to track the UTF-8 code units in a stream,+-- but often counts the number of code points instead.  It can easily+-- undercount if, for instance, a transformed stream contains astral+-- plane code points (those above 0x10000).++-- | A co-recursive type yielding a single element at a time depending+-- on the internal state it carries.+data Stream a =+    forall s. Stream+    (s -> Step s a)             -- stepper function+    !s                          -- current state+    !Size                       -- size hint in code units++-- | /O(n)/ Determines if two streams are equal.+eq :: (Eq a) => Stream a -> Stream a -> Bool+eq (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)+    where+      loop Done Done                     = True+      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')+      loop (Skip s1')     x2             = loop (next1 s1') x2+      loop x1             (Skip s2')     = loop x1          (next2 s2')+      loop Done _                        = False+      loop _    Done                     = False+      loop (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&+                                           loop (next1 s1') (next2 s2')+{-# INLINE [0] eq #-}++cmp :: (Ord a) => Stream a -> Stream a -> Ordering+cmp (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2)+    where+      loop Done Done                     = EQ+      loop (Skip s1')     (Skip s2')     = loop (next1 s1') (next2 s2')+      loop (Skip s1')     x2             = loop (next1 s1') x2+      loop x1             (Skip s2')     = loop x1          (next2 s2')+      loop Done _                        = LT+      loop _    Done                     = GT+      loop (Yield x1 s1') (Yield x2 s2') =+          case compare x1 x2 of+            EQ    -> loop (next1 s1') (next2 s2')+            other -> other+{-# INLINE [0] cmp #-}++-- | The empty stream.+empty :: Stream a+empty = Stream next () 0+    where next _ = Done+{-# INLINE [0] empty #-}
+ src/Data/Text/Internal/IO.hs view
@@ -0,0 +1,308 @@+{-# LANGUAGE BangPatterns, RecordWildCards #-}
+{-# LANGUAGE MagicHash #-}
+-- |
+-- Module      : Data.Text.Internal.IO
+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,
+--               (c) 2009 Simon Marlow
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- /Warning/: this is an internal module, and does not have a stable
+-- API or name. Functions in this module may not check or enforce
+-- preconditions expected by public modules. Use at your own risk!
+--
+-- Low-level support for text I\/O.
+
+module Data.Text.Internal.IO
+    (
+      hGetLineWith
+    , readChunk
+    , hPutStream
+    , hPutStr
+    , hPutStrLn
+    ) where
+
+import qualified Control.Exception as E
+import qualified Data.ByteString as B
+import Data.ByteString.Builder (hPutBuilder, charUtf8)
+import Data.IORef (readIORef, writeIORef)
+import Data.Text (Text)
+import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder)
+import Data.Text.Internal.Fusion (stream, streamLn, unstream)
+import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))
+import Data.Text.Internal.Fusion.Size (exactSize, maxSize)
+import Data.Text.Unsafe (inlinePerformIO)
+import Foreign.Storable (peekElemOff)
+import GHC.Exts (reallyUnsafePtrEquality#, isTrue#)
+import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer,
+                      bufferAdjustL, bufferElems, charSize, emptyBuffer,
+                      isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer,
+                      writeCharBuf)
+import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_,
+                                wantWritableHandle)
+import GHC.IO.Handle.Text (commitBuffer')
+import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..))
+import System.IO (Handle, hPutChar, utf8)
+import System.IO.Error (isEOFError)
+import qualified Data.Text as T
+
+-- | Read a single line of input from a handle, constructing a list of
+-- decoded chunks as we go.  When we're done, transform them into the
+-- destination type.
+hGetLineWith :: ([Text] -> t) -> Handle -> IO t
+hGetLineWith f h = wantReadableHandle_ "hGetLine" h go
+  where
+    go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh []
+
+hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text]
+hGetLineLoop hh@Handle__{..} = go where
+ go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do
+  let findEOL raw r | r == w    = return (False, w)
+                    | otherwise = do
+        (c,r') <- readCharBuf raw r
+        if c == '\n'
+          then return (True, r)
+          else findEOL raw r'
+  (eol, off) <- findEOL raw0 r0
+  (t,r') <- if haInputNL == CRLF
+            then unpack_nl raw0 r0 off
+            else do t <- unpack raw0 r0 off
+                    return (t,off)
+  if eol
+    then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf)
+            return $ reverse (t:ts)
+    else do
+      let buf1 = bufferAdjustL r' buf
+      maybe_buf <- maybeFillReadBuffer hh buf1
+      case maybe_buf of
+         -- Nothing indicates we caught an EOF, and we may have a
+         -- partial line to return.
+         Nothing -> do
+              -- we reached EOF.  There might be a lone \r left
+              -- in the buffer, so check for that and
+              -- append it to the line if necessary.
+              let pre | isEmptyBuffer buf1 = T.empty
+                      | otherwise          = T.singleton '\r'
+              writeIORef haCharBuffer buf1{ bufL=0, bufR=0 }
+              let str = reverse . filter (not . T.null) $ pre:t:ts
+              if null str
+                then ioe_EOF
+                else return str
+         Just new_buf -> go (t:ts) new_buf
+
+-- This function is lifted almost verbatim from GHC.IO.Handle.Text.
+maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer)
+maybeFillReadBuffer handle_ buf
+  = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e ->
+      if isEOFError e
+      then return Nothing
+      else ioError e
+
+unpack :: RawCharBuffer -> Int -> Int -> IO Text
+unpack !buf !r !w
+ | charSize /= 4 = sizeError "unpack"
+ | r >= w        = return T.empty
+ | otherwise     = withRawBuffer buf go
+ where
+  go pbuf = return $! unstream (Stream next r (exactSize (w-r)))
+   where
+    next !i | i >= w    = Done
+            | otherwise = Yield (ix i) (i+1)
+    ix i = inlinePerformIO $ peekElemOff pbuf i
+
+-- Variant of 'unpack' with CRLF decoding. If there is a trailing '\r', leave it in the buffer.
+unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int)
+unpack_nl !buf !r !w
+ | charSize /= 4 = sizeError "unpack_nl"
+ | r >= w        = return (T.empty, 0)
+ | otherwise     = withRawBuffer buf $ go
+ where
+  go pbuf = do
+    let !t = unstream (Stream next r (maxSize (w-r)))
+        w' = w - 1
+    return $ if ix w' == '\r'
+             then (t,w')
+             else (t,w)
+   where
+    next !i | i >= w = Done
+            | c == '\r' = let i' = i + 1
+                          in if i' < w
+                             then if ix i' == '\n'
+                                  then Yield '\n' (i+2)
+                                  else Yield '\r' i'
+                             else Done
+            | otherwise = Yield c (i+1)
+            where c = ix i
+    ix i = inlinePerformIO $ peekElemOff pbuf i
+
+-- This function is completely lifted from GHC.IO.Handle.Text.
+getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer
+getSomeCharacters handle_@Handle__{..} buf@Buffer{..} =
+  case bufferElems buf of
+    -- buffer empty: read some more
+    0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf
+
+    -- if the buffer has a single '\r' in it and we're doing newline
+    -- translation: read some more
+    1 | haInputNL == CRLF -> do
+      (c,_) <- readCharBuf bufRaw bufL
+      if c == '\r'
+         then do -- shuffle the '\r' to the beginning.  This is only safe
+                 -- if we're about to call readTextDevice, otherwise it
+                 -- would mess up flushCharBuffer.
+                 -- See [note Buffer Flushing], GHC.IO.Handle.Types
+                 _ <- writeCharBuf bufRaw 0 '\r'
+                 let buf' = buf{ bufL=0, bufR=1 }
+                 readTextDevice handle_ buf'
+         else do
+                 return buf
+
+    -- buffer has some chars in it already: just return it
+    _otherwise -> {-# SCC "otherwise" #-} return buf
+
+-- | Read a single chunk of strict text from a buffer. Used by both
+-- the strict and lazy implementations of hGetContents.
+readChunk :: Handle__ -> CharBuffer -> IO Text
+readChunk hh@Handle__{..} buf = do
+  buf'@Buffer{..} <- getSomeCharacters hh buf
+  (t,r) <- if haInputNL == CRLF
+           then unpack_nl bufRaw bufL bufR
+           else do t <- unpack bufRaw bufL bufR
+                   return (t,bufR)
+  writeIORef haCharBuffer (bufferAdjustL r buf')
+  return t
+
+-- | Print a @Stream Char@.
+hPutStream :: Handle -> Stream Char -> IO ()
+hPutStream h str = hPutStreamOrUtf8 h str Nothing
+
+-- | Write a string to a handle.
+hPutStr :: Handle -> Text -> IO ()
+hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8)
+  where
+    putUtf8 = B.hPutStr h (encodeUtf8 t)
+
+-- | Write a string to a handle, followed by a newline.
+hPutStrLn :: Handle -> Text -> IO ()
+hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8)
+  where
+    -- Not using B.hPutStrLn because it's not necessarily atomic:
+    -- https://github.com/haskell/bytestring/issues/200
+    putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n')
+
+-- | 'hPutStream' with an optional special case when the output encoding is
+-- UTF-8 and without newline conversion.
+hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO ()
+-- This function is modified from GHC.IO.Handle.Text.
+hPutStreamOrUtf8 h str mPutUtf8 = do
+  (buffer_mode, nl, isUtf8) <-
+       wantWritableHandle "hPutStr" h $ \h_ -> do
+                     bmode <- getSpareBuffer h_
+                     return (bmode, haOutputNL h_, eqUTF8 h_)
+  case buffer_mode of
+     _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8
+     (NoBuffering, _)        -> hPutChars h str
+     (LineBuffering, buf)    -> writeLines h nl buf str
+     (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str
+
+  where
+  -- If the encoding is UTF-8, it's most likely pointer-equal to
+  -- 'System.IO.utf8', letting us avoid a String comparison.
+  -- If it is somehow UTF-8 but not pointer-equal to 'utf8',
+  -- we will just take a slower branch, but the result is still correct.
+  eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec
+{-# INLINE hPutStreamOrUtf8 #-}
+
+hPutChars :: Handle -> Stream Char -> IO ()
+hPutChars h (Stream next0 s0 _len) = loop s0
+  where
+    loop !s = case next0 s of
+                Done       -> return ()
+                Skip s'    -> loop s'
+                Yield x s' -> hPutChar h x >> loop s'
+
+-- The following functions are largely lifted from GHC.IO.Handle.Text,
+-- but adapted to a coinductive stream of data instead of an inductive
+-- list.
+--
+-- We have several variations of more or less the same code for
+-- performance reasons.  Splitting the original buffered write
+-- function into line- and block-oriented versions gave us a 2.1x
+-- performance improvement.  Lifting out the raw/cooked newline
+-- handling gave a few more percent on top.
+
+writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO ()
+writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0
+ where
+  outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)
+   where
+    inner !s !n =
+      case next0 s of
+        Done -> commit n False{-no flush-} True{-release-} >> return ()
+        Skip s' -> inner s' n
+        Yield x s'
+          | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s
+          | x == '\n'    -> do
+                   n' <- if nl == CRLF
+                         then do n1 <- writeCharBuf' raw len n '\r'
+                                 writeCharBuf' raw len n1 '\n'
+                         else writeCharBuf' raw len n x
+                   commit n' True{-needs flush-} False >>= outer s'
+          | otherwise    -> writeCharBuf' raw len n x >>= inner s'
+    commit = commitBuffer h raw len
+
+writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO ()
+writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0
+ where
+  outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)
+   where
+    inner !s !n =
+      case next0 s of
+        Done -> commit n False{-no flush-} True{-release-} >> return ()
+        Skip s' -> inner s' n
+        Yield x s'
+          -- Leave room for two characters for CRLF decoding
+          | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s
+          | x == '\n' && isCRLF -> do
+              n1 <- writeCharBuf' raw len n '\r'
+              writeCharBuf' raw len n1 '\n' >>= inner s'
+          | otherwise -> writeCharBuf' raw len n x >>= inner s'
+    commit = commitBuffer h raw len
+
+-- | Only modifies the raw buffer and not the buffer attributes
+writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int
+writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $
+  writeCharBuf bufRaw n c
+
+-- This function is completely lifted from GHC.IO.Handle.Text.
+getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer)
+getSpareBuffer Handle__{haCharBuffer=ref,
+                        haBuffers=spare_ref,
+                        haBufferMode=mode}
+ = do
+   case mode of
+     NoBuffering -> return (mode, error "no buffer!")
+     _ -> do
+          bufs <- readIORef spare_ref
+          buf  <- readIORef ref
+          case bufs of
+            BufferListCons b rest -> do
+                writeIORef spare_ref rest
+                return ( mode, emptyBuffer b (bufSize buf) WriteBuffer)
+            BufferListNil -> do
+                new_buf <- newCharBuffer (bufSize buf) WriteBuffer
+                return (mode, new_buf)
+
+
+-- This function is modified from GHC.Internal.IO.Handle.Text.
+commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool
+             -> IO CharBuffer
+commitBuffer hdl !raw !sz !count flush release =
+  wantWritableHandle "commitAndReleaseBuffer" hdl $
+     commitBuffer' raw sz count flush release
+{-# INLINE commitBuffer #-}
+
+sizeError :: String -> a
+sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"
+ src/Data/Text/Internal/IsAscii.hs view
@@ -0,0 +1,94 @@+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE UnliftedFFITypes #-}+#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'isAscii', using efficient C routines by default.+--+-- Similarly implements asciiPrefixLength, used internally in Data.Text.Encoding.+module Data.Text.Internal.IsAscii where++#if defined(PURE_HASKELL)+import Prelude hiding (all)+import qualified Data.Char as Char+import qualified Data.ByteString as BS+import Data.Text.Unsafe (iter, Iter(..))+#else+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Data.Word (Word8)+import Foreign.C.Types+import Foreign.Ptr (Ptr, plusPtr)+import GHC.Base (ByteArray#)+import Prelude (Bool(..), Int, (==), ($), IO, (<$>))+import qualified Data.Text.Array as A+#endif+import Data.ByteString (ByteString)+import Data.Text.Internal (Text(..))+import qualified Prelude as P++-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only+--   U+0000 through U+007F).+--+-- This is a more efficient version of @'all' 'Data.Char.isAscii'@.+--+-- >>> isAscii ""+-- True+--+-- >>> isAscii "abc\NUL"+-- True+--+-- >>> isAscii "abcd€"+-- False+--+-- prop> isAscii t == all (< '\x80') t+--+-- @since 2.0.2+isAscii :: Text -> Bool+#if defined(PURE_HASKELL)+isAscii = all Char.isAscii++-- | (Re)implemented to avoid circular dependency on Data.Text.+all :: (Char -> Bool) -> Text -> Bool+all p t@(Text _ _ len) = go 0+  where+    go i | i >= len = True+         | otherwise =+             let !(Iter c j) = iter t i+             in p c && go (i+j)+#else+cSizeToInt :: CSize -> Int+cSizeToInt = P.fromIntegral+{-# INLINE cSizeToInt #-}++intToCSize :: Int -> CSize+intToCSize = P.fromIntegral++isAscii (Text (A.ByteArray arr) off len) =+    cSizeToInt (c_is_ascii_offset arr (intToCSize off) (intToCSize len)) == len+#endif+{-# INLINE isAscii #-}++-- | Length of the longest ASCII prefix.+asciiPrefixLength :: ByteString -> Int+#if defined(PURE_HASKELL)+asciiPrefixLength = BS.length P.. BS.takeWhile (P.< 0x80)+#else+asciiPrefixLength bs = unsafeDupablePerformIO $ withBS bs $ \ fp len ->+  unsafeWithForeignPtr fp $ \src -> do+    P.fromIntegral <$> c_is_ascii src (src `plusPtr` len)+#endif+{-# INLINE asciiPrefixLength #-}++#if !defined(PURE_HASKELL)+foreign import ccall unsafe "_hs_text_is_ascii_offset" c_is_ascii_offset+    :: ByteArray# -> CSize -> CSize -> CSize++foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii+    :: Ptr Word8 -> Ptr Word8 -> IO CSize+#endif
+ src/Data/Text/Internal/Lazy.hs view
@@ -0,0 +1,151 @@+{-# LANGUAGE BangPatterns #-}
+{-# OPTIONS_HADDOCK not-home #-}
+
+-- |
+-- Module      : Data.Text.Internal.Lazy
+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- /Warning/: this is an internal module, and does not have a stable
+-- API or name. Functions in this module may not check or enforce
+-- preconditions expected by public modules. Use at your own risk!
+--
+-- A module containing private 'Text' internals. This exposes the
+-- 'Text' representation and low level construction functions.
+-- Modules which extend the 'Text' system may need to use this module.
+
+module Data.Text.Internal.Lazy
+    (
+      Text(..)
+    , LazyText
+    , chunk
+    , empty
+    , foldrChunks
+    , foldlChunks
+    -- * Data type invariant and abstraction functions
+
+    -- $invariant
+    , strictInvariant
+    , lazyInvariant
+    , showStructure
+
+    -- * Chunk allocation sizes
+    , defaultChunkSize
+    , smallChunkSize
+    , chunkOverhead
+
+    , equal
+    ) where
+
+import Data.Bits (shiftL)
+import Data.Text ()
+import Foreign.Storable (sizeOf)
+import qualified Data.Text.Array as A
+import qualified Data.Text.Internal as T
+import qualified Data.Text as T
+
+data Text = Empty
+          -- ^ Empty text.
+          --
+          -- @since 2.1.2
+          | Chunk {-# UNPACK #-} !T.Text Text
+          -- ^ Chunks must be non-empty, this invariant is not checked.
+
+-- | Type synonym for the lazy flavour of 'Text'.
+--
+-- @since 2.1.1
+type LazyText = Text
+
+-- $invariant
+--
+-- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or
+-- consists of non-null 'T.Text's.  All functions must preserve this,
+-- and the QC properties must check this.
+
+-- | Check the invariant strictly.
+strictInvariant :: Text -> Bool
+strictInvariant Empty = True
+strictInvariant x@(Chunk (T.Text _ _ len) cs)
+    | len > 0   = strictInvariant cs
+    | otherwise = error $ "Data.Text.Lazy: invariant violation: "
+                  ++ showStructure x
+
+-- | Check the invariant lazily.
+lazyInvariant :: Text -> Text
+lazyInvariant Empty = Empty
+lazyInvariant x@(Chunk c@(T.Text _ _ len) cs)
+    | len > 0   = Chunk c (lazyInvariant cs)
+    | otherwise = error $ "Data.Text.Lazy: invariant violation: "
+                  ++ showStructure x
+
+-- | Display the internal structure of a lazy 'Text'.
+showStructure :: Text -> String
+showStructure Empty           = "Empty"
+showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"
+showStructure (Chunk t ts)    =
+    "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"
+
+-- | Smart constructor for 'Chunk'. Guarantees the data type invariant.
+chunk :: T.Text -> Text -> Text
+{-# INLINE [0] chunk #-}
+chunk t ts | T.null t = ts
+           | otherwise = Chunk t ts
+
+{-# RULES
+"TEXT chunk/text" forall arr off len.
+    chunk (T.text arr off len) = chunk (T.Text arr off len)
+"TEXT chunk/empty" forall ts.
+    chunk T.empty ts = ts
+#-}
+
+-- | Smart constructor for 'Empty'.
+empty :: Text
+{-# INLINE [0] empty #-}
+empty = Empty
+
+-- | Consume the chunks of a lazy 'Text' with a natural right fold.
+foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a
+foldrChunks f z = go
+  where go Empty        = z
+        go (Chunk c cs) = f c (go cs)
+{-# INLINE foldrChunks #-}
+
+-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,
+-- accumulating left fold.
+foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a
+foldlChunks f z = go z
+  where go !a Empty        = a
+        go !a (Chunk c cs) = go (f a c) cs
+{-# INLINE foldlChunks #-}
+
+-- | Currently set to 16 KiB, less the memory management overhead.
+defaultChunkSize :: Int
+defaultChunkSize = 16384 - chunkOverhead
+{-# INLINE defaultChunkSize #-}
+
+-- | Currently set to 128 bytes, less the memory management overhead.
+smallChunkSize :: Int
+smallChunkSize = 128 - chunkOverhead
+{-# INLINE smallChunkSize #-}
+
+-- | The memory management overhead. Currently this is tuned for GHC only.
+chunkOverhead :: Int
+chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1
+{-# INLINE chunkOverhead #-}
+
+equal :: Text -> Text -> Bool
+equal Empty Empty = True
+equal Empty _     = False
+equal _ Empty     = False
+equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =
+    case compare lenA lenB of
+      LT -> A.equal arrA offA arrB offB lenA &&
+            as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs
+      EQ -> A.equal arrA offA arrB offB lenA &&
+            as `equal` bs
+      GT -> A.equal arrA offA arrB offB lenB &&
+            Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs
+ src/Data/Text/Internal/Lazy/Encoding/Fusion.hs view
@@ -0,0 +1,332 @@+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-}++-- |+-- Module      : Data.Text.Lazy.Encoding.Fusion+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Fusible 'Stream'-oriented functions for converting between lazy+-- 'Text' and several common encodings.++module Data.Text.Internal.Lazy.Encoding.Fusion+    (+    -- * Streaming+    --  streamASCII+      streamUtf8+    , streamUtf16LE+    , streamUtf16BE+    , streamUtf32LE+    , streamUtf32BE++    -- * Unstreaming+    , unstream++    , module Data.Text.Internal.Encoding.Fusion.Common+    ) where++import Data.Bits (shiftL)+import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize)+import qualified Data.ByteString as B+import qualified Data.ByteString.Unsafe as B+import Data.Text.Internal.ByteStringCompat+import Data.Text.Internal.Encoding.Fusion.Common+import Data.Text.Encoding.Error+import Data.Text.Internal.Fusion (Step(..), Stream(..))+import Data.Text.Internal.Fusion.Size+import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Word (Word8, Word16, Word32)+import qualified Data.Text.Internal.Encoding.Utf8 as U8+import qualified Data.Text.Internal.Encoding.Utf16 as U16+import qualified Data.Text.Internal.Encoding.Utf32 as U32+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Foreign.ForeignPtr (ForeignPtr)+import Foreign.Marshal.Utils (copyBytes)+import Foreign.Storable (pokeByteOff)+import Data.ByteString.Internal (mallocByteString)+#if defined(ASSERTS)+import Control.Exception (assert)+#endif++data S = S0+       | S1 {-# UNPACK #-} !Word8+       | S2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8+       | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8+       | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8++data T = T !ByteString !S {-# UNPACK #-} !Int++-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using+-- UTF-8 encoding.+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char+streamUtf8 onErr bs0 = Stream next (T bs0 S0 0) unknownSize+  where+    next (T bs@(Chunk ps _) S0 i)+      | i < len && U8.validate1 a =+          Yield (unsafeChr8 a)    (T bs S0 (i+1))+      | i + 1 < len && U8.validate2 a b =+          Yield (U8.chr2 a b)     (T bs S0 (i+2))+      | i + 2 < len && U8.validate3 a b c =+          Yield (U8.chr3 a b c)   (T bs S0 (i+3))+      | i + 3 < len && U8.validate4 a b c d =+          Yield (U8.chr4 a b c d) (T bs S0 (i+4))+      where len = B.length ps+            a = B.unsafeIndex ps i+            b = B.unsafeIndex ps (i+1)+            c = B.unsafeIndex ps (i+2)+            d = B.unsafeIndex ps (i+3)+    next st@(T bs s i) =+      case s of+        S1 a       | U8.validate1 a       -> Yield (unsafeChr8 a)    es+        S2 a b     | U8.validate2 a b     -> Yield (U8.chr2 a b)     es+        S3 a b c   | U8.validate3 a b c   -> Yield (U8.chr3 a b c)   es+        S4 a b c d | U8.validate4 a b c d -> Yield (U8.chr4 a b c d) es+        _ -> consume st+       where es = T bs S0 i+    consume (T bs@(Chunk ps rest) s i)+        | i >= B.length ps = consume (T rest s 0)+        | otherwise =+      case s of+        S0         -> next (T bs (S1 x)       (i+1))+        S1 a       -> next (T bs (S2 a x)     (i+1))+        S2 a b     -> next (T bs (S3 a b x)   (i+1))+        S3 a b c   -> next (T bs (S4 a b c x) (i+1))+        S4 a b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a)+                           (T bs (S4 b c d x) (i+1))+        where x = B.unsafeIndex ps i+    consume (T Empty S0 _) = Done+    consume (T Empty _  i) = decodeError "streamUtf8" "UTF-8" onErr Nothing (T Empty S0 i)+{-# INLINE [0] streamUtf8 #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little+-- endian UTF-16 encoding.+streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize+  where+    next (T bs@(Chunk ps _) S0 i)+      | i + 1 < len && U16.validate1 x1 =+          Yield (unsafeChr16 x1)         (T bs S0 (i+2))+      | i + 3 < len && U16.validate2 x1 x2 =+          Yield (U16.chr2 x1 x2)       (T bs S0 (i+4))+      where len = B.length ps+            x1   = c (idx  i)      (idx (i + 1))+            x2   = c (idx (i + 2)) (idx (i + 3))+            c w1 w2 = w1 + (w2 `shiftL` 8)+            idx = word8ToWord16 . B.unsafeIndex ps :: Int -> Word16+    next st@(T bs s i) =+      case s of+        S2 w1 w2       | U16.validate1 (c w1 w2)           ->+          Yield (unsafeChr16 (c w1 w2))   es+        S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) ->+          Yield (U16.chr2 (c w1 w2) (c w3 w4)) es+        _ -> consume st+       where es = T bs S0 i+             c :: Word8 -> Word8 -> Word16+             c w1 w2 = word8ToWord16 w1 + (word8ToWord16 w2 `shiftL` 8)+    consume (T bs@(Chunk ps rest) s i)+        | i >= B.length ps = consume (T rest s 0)+        | otherwise =+      case s of+        S0             -> next (T bs (S1 x)          (i+1))+        S1 w1          -> next (T bs (S2 w1 x)       (i+1))+        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))+        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))+        S4 w1 w2 w3 w4 -> decodeError "streamUtf16LE" "UTF-16LE" onErr (Just w1)+                           (T bs (S4 w2 w3 w4 x)     (i+1))+        where x = B.unsafeIndex ps i+    consume (T Empty S0 _) = Done+    consume (T Empty _  i) = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (T Empty S0 i)+{-# INLINE [0] streamUtf16LE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big+-- endian UTF-16 encoding.+streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize+  where+    next (T bs@(Chunk ps _) S0 i)+      | i + 1 < len && U16.validate1 x1 =+          Yield (unsafeChr16 x1)         (T bs S0 (i+2))+      | i + 3 < len && U16.validate2 x1 x2 =+          Yield (U16.chr2 x1 x2)       (T bs S0 (i+4))+      where len = B.length ps+            x1   = c (idx  i)      (idx (i + 1))+            x2   = c (idx (i + 2)) (idx (i + 3))+            c w1 w2 = (w1 `shiftL` 8) + w2+            idx = word8ToWord16 . B.unsafeIndex ps :: Int -> Word16+    next st@(T bs s i) =+      case s of+        S2 w1 w2       | U16.validate1 (c w1 w2)           ->+          Yield (unsafeChr16 (c w1 w2))   es+        S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) ->+          Yield (U16.chr2 (c w1 w2) (c w3 w4)) es+        _ -> consume st+       where es = T bs S0 i+             c :: Word8 -> Word8 -> Word16+             c w1 w2 = (word8ToWord16 w1 `shiftL` 8) + word8ToWord16 w2+    consume (T bs@(Chunk ps rest) s i)+        | i >= B.length ps = consume (T rest s 0)+        | otherwise =+      case s of+        S0             -> next (T bs (S1 x)          (i+1))+        S1 w1          -> next (T bs (S2 w1 x)       (i+1))+        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))+        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))+        S4 w1 w2 w3 w4 -> decodeError "streamUtf16BE" "UTF-16BE" onErr (Just w1)+                           (T bs (S4 w2 w3 w4 x)     (i+1))+        where x = B.unsafeIndex ps i+    consume (T Empty S0 _) = Done+    consume (T Empty _  i) = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (T Empty S0 i)+{-# INLINE [0] streamUtf16BE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big+-- endian UTF-32 encoding.+streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32BE onErr bs0 = Stream next (T bs0 S0 0) unknownSize+  where+    next (T bs@(Chunk ps _) S0 i)+      | i + 3 < len && U32.validate x =+          Yield (unsafeChr32 x)       (T bs S0 (i+4))+      where len = B.length ps+            x = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4+            x1    = idx i+            x2    = idx (i+1)+            x3    = idx (i+2)+            x4    = idx (i+3)+            idx = word8ToWord32 . B.unsafeIndex ps :: Int -> Word32+    next st@(T bs s i) =+      case s of+        S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) ->+          Yield (unsafeChr32 (c w1 w2 w3 w4)) es+        _ -> consume st+       where es = T bs S0 i+             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32+             c w1 w2 w3 w4 = shifted+              where+               shifted = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4+               x1 = word8ToWord32 w1+               x2 = word8ToWord32 w2+               x3 = word8ToWord32 w3+               x4 = word8ToWord32 w4+    consume (T bs@(Chunk ps rest) s i)+        | i >= B.length ps = consume (T rest s 0)+        | otherwise =+      case s of+        S0             -> next (T bs (S1 x)          (i+1))+        S1 w1          -> next (T bs (S2 w1 x)       (i+1))+        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))+        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))+        S4 w1 w2 w3 w4 -> decodeError "streamUtf32BE" "UTF-32BE" onErr (Just w1)+                           (T bs (S4 w2 w3 w4 x)     (i+1))+        where x = B.unsafeIndex ps i+    consume (T Empty S0 _) = Done+    consume (T Empty _  i) = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (T Empty S0 i)+{-# INLINE [0] streamUtf32BE #-}++-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little+-- endian UTF-32 encoding.+streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32LE onErr bs0 = Stream next (T bs0 S0 0) unknownSize+  where+    next (T bs@(Chunk ps _) S0 i)+      | i + 3 < len && U32.validate x =+          Yield (unsafeChr32 x)       (T bs S0 (i+4))+      where len = B.length ps+            x = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1+            x1    = idx i+            x2    = idx (i+1)+            x3    = idx (i+2)+            x4    = idx (i+3)+            idx = word8ToWord32 . B.unsafeIndex ps :: Int -> Word32+    next st@(T bs s i) =+      case s of+        S4 w1 w2 w3 w4 | U32.validate (c w1 w2 w3 w4) ->+          Yield (unsafeChr32 (c w1 w2 w3 w4)) es+        _ -> consume st+       where es = T bs S0 i+             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32+             c w1 w2 w3 w4 = shifted+              where+               shifted = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1+               x1 = word8ToWord32 w1+               x2 = word8ToWord32 w2+               x3 = word8ToWord32 w3+               x4 = word8ToWord32 w4+    consume (T bs@(Chunk ps rest) s i)+        | i >= B.length ps = consume (T rest s 0)+        | otherwise =+      case s of+        S0             -> next (T bs (S1 x)          (i+1))+        S1 w1          -> next (T bs (S2 w1 x)       (i+1))+        S2 w1 w2       -> next (T bs (S3 w1 w2 x)    (i+1))+        S3 w1 w2 w3    -> next (T bs (S4 w1 w2 w3 x) (i+1))+        S4 w1 w2 w3 w4 -> decodeError "streamUtf32LE" "UTF-32LE" onErr (Just w1)+                           (T bs (S4 w2 w3 w4 x)     (i+1))+        where x = B.unsafeIndex ps i+    consume (T Empty S0 _) = Done+    consume (T Empty _  i) = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (T Empty S0 i)+{-# INLINE [0] streamUtf32LE #-}++-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.+unstreamChunks :: Int -> Stream Word8 -> ByteString+unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 (upperBound 4 len0)+  where chunk s1 len1 = unsafeDupablePerformIO $ do+          let len = max 4 (min len1 chunkSize)+          mallocByteString len >>= loop len 0 s1+          where+            loop !n !off !s fp = case next s of+                Done | off == 0 -> return Empty+                     | otherwise -> return $! Chunk (trimUp fp off) Empty+                Skip s' -> loop n off s' fp+                Yield x s'+                    | off == chunkSize -> do+                      let !newLen = n - off+                      return $! Chunk (trimUp fp off) (chunk s newLen)+                    | off == n -> realloc fp n off s' x+                    | otherwise -> do+                      unsafeWithForeignPtr fp $ \p -> pokeByteOff p off x+                      loop n (off+1) s' fp+            {-# NOINLINE realloc #-}+            realloc fp n off s x = do+              let n' = min (n+n) chunkSize+              fp' <- copy0 fp n n'+              unsafeWithForeignPtr fp' $ \p -> pokeByteOff p off x+              loop n' (off+1) s fp'+            trimUp fp off = mkBS fp off+            copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)+            copy0 !src !srcLen !destLen =+#if defined(ASSERTS)+              assert (srcLen <= destLen) $+#endif+              do+                dest <- mallocByteString destLen+                unsafeWithForeignPtr src  $ \src'  ->+                    unsafeWithForeignPtr dest $ \dest' ->+                        copyBytes dest' src' srcLen+                return dest++-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.+unstream :: Stream Word8 -> ByteString+unstream = unstreamChunks defaultChunkSize++decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8+            -> s -> Step s Char+decodeError func kind onErr mb i =+    case onErr desc mb of+      Nothing -> Skip i+      Just c  -> Yield c i+    where desc = "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Invalid " +++                 kind ++ " stream"++word8ToWord16 :: Word8 -> Word16+word8ToWord16 = fromIntegral++word8ToWord32 :: Word8 -> Word32+word8ToWord32 = fromIntegral
+ src/Data/Text/Internal/Lazy/Fusion.hs view
@@ -0,0 +1,156 @@+{-# LANGUAGE BangPatterns, CPP #-}+-- |+-- Module      : Data.Text.Lazy.Fusion+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Core stream fusion functionality for text.++module Data.Text.Internal.Lazy.Fusion+    (+      stream+    , streamLn+    , unstream+    , unstreamChunks+    , length+    , unfoldrN+    , index+    , countChar+    ) where++import Prelude hiding (length)+import Data.Bits (shiftL)+import qualified Data.Text.Internal.Fusion.Common as S+import Control.Monad.ST (runST)+import Data.Text.Internal.Fusion.Types+import Data.Text.Internal.Fusion.Size (isEmpty, unknownSize)+import Data.Text.Internal.Lazy+import qualified Data.Text.Internal as I+import qualified Data.Text.Array as A+import Data.Text.Internal.Unsafe.Char (unsafeWrite)+import Data.Text.Unsafe (Iter(..), iter)+import Data.Int (Int64)+import GHC.Stack (HasCallStack)++default(Int64)++-- | /O(n)/ Convert a 'Text' into a 'Stream Char'.+stream ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Stream Char+stream = stream' False+{-# INLINE [0] stream #-}++-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@+--+-- @since 2.1.2+streamLn ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Stream Char+streamLn = stream' True++-- | Shared implementation of 'stream' and 'streamLn'.+stream' ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Bool -> Text -> Stream Char+stream' addNl text = Stream next (text :*: 0) unknownSize+  where+    next (Empty :*: i)+        | addNl && i <= 0 = Yield '\n' (Empty :*: 1)+        | otherwise = Done+    next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i)+        | i >= len  = next (ts :*: 0)+        | otherwise = Yield c (txt :*: i+d)+        where Iter c d = iter t i+{-# INLINE [0] stream' #-}++-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given+-- chunk size.+unstreamChunks ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Int -> Stream Char -> Text+unstreamChunks !chunkSize (Stream next s0 len0)+  | isEmpty len0 = Empty+  | otherwise    = outer s0+  where+    outer so = {-# SCC "unstreamChunks/outer" #-}+              case next so of+                Done       -> Empty+                Skip s'    -> outer s'+                Yield x s' -> runST $ do+                                a <- A.new unknownLength+                                unsafeWrite a 0 x >>= inner a unknownLength s'+                    where unknownLength = 4+      where+        inner marr !len s !i+            | i + 3 >= chunkSize = finish marr i s+            | i + 3 >= len       = {-# SCC "unstreamChunks/resize" #-} do+                let newLen = min (len `shiftL` 1) chunkSize+                marr' <- A.new newLen+                A.copyM marr' 0 marr 0 len+                inner marr' newLen s i+            | otherwise =+                {-# SCC "unstreamChunks/inner" #-}+                case next s of+                  Done        -> finish marr i s+                  Skip s'     -> inner marr len s' i+                  Yield x s'  -> do d <- unsafeWrite marr i x+                                    inner marr len s' (i+d)+        finish marr len s' = do+          A.shrinkM marr len+          arr <- A.unsafeFreeze marr+          return (I.Text arr 0 len `Chunk` outer s')+{-# INLINE [0] unstreamChunks #-}++-- | /O(n)/ Convert a 'Stream Char' into a 'Text', using+-- 'defaultChunkSize'.+unstream ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Stream Char -> Text+unstream = unstreamChunks defaultChunkSize+{-# INLINE [0] unstream #-}++-- | /O(n)/ Returns the number of characters in a text.+length :: Stream Char -> Int64+length = S.lengthI+{-# INLINE[0] length #-}++{-# RULES "LAZY STREAM stream/unstream fusion" forall s.+    stream (unstream s) = s #-}++-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed+-- value. However, the length of the result is limited by the+-- first argument to 'unfoldrN'. This function is more efficient than+-- 'unfoldr' when the length of the result is known.+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Stream Char+unfoldrN n = S.unfoldrNI n+{-# INLINE [0] unfoldrN #-}++-- | /O(n)/ stream index (subscript) operator, starting from 0.+index :: HasCallStack => Stream Char -> Int64 -> Char+index = S.indexI+{-# INLINE [0] index #-}++-- | /O(n)/ The 'count' function returns the number of times the query+-- element appears in the given stream.+countChar :: Char -> Stream Char -> Int64+countChar = S.countCharI+{-# INLINE [0] countChar #-}
+ src/Data/Text/Internal/Lazy/Search.hs view
@@ -0,0 +1,132 @@+{-# LANGUAGE BangPatterns, ScopedTypeVariables #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- |+-- Module      : Data.Text.Lazy.Search+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Fast substring search for lazy 'Text', based on work by Boyer,+-- Moore, Horspool, Sunday, and Lundh.  Adapted from the strict+-- implementation.++module Data.Text.Internal.Lazy.Search+    (+      indices+    ) where++import Data.Bits (unsafeShiftL, (.|.), (.&.))+import qualified Data.Text.Array as A+import Data.Int (Int64)+import Data.Word (Word8, Word64)+import qualified Data.Text.Internal as T+import qualified Data.Text as T (concat, isPrefixOf)+import Data.Text.Internal.ArrayUtils (memchr)+import Data.Text.Internal.Fusion.Types (PairS(..))+import Data.Text.Internal.Lazy (Text(..), foldrChunks)++-- | /O(n+m)/ Find the offsets of all non-overlapping indices of+-- @needle@ within @haystack@.+--+-- This function is strict in @needle@, and lazy (as far as possible)+-- in the chunks of @haystack@.+--+-- In (unlikely) bad cases, this algorithm's complexity degrades+-- towards /O(n*m)/.+indices :: Text              -- ^ Substring to search for (@needle@)+        -> Text              -- ^ Text to search in (@haystack@)+        -> [Int64]+indices needle+    | nlen <= 0  = const []+    | nlen == 1  = indicesOne (A.unsafeIndex narr noff) 0+    | otherwise  = advance 0 0+  where+    T.Text narr noff nlen = T.concat (foldrChunks (:) [] needle)++    advance !_ !_ Empty = []+    advance !(g :: Int64) !(i :: Int) xxs@(Chunk x@(T.Text xarr@(A.ByteArray xarr#) xoff l) xs)+         | i >= l = advance g (i - l) xs+         | lackingHay (i + nlen) x xs  = []+         | c == z && candidateMatch    = g : advance (g + intToInt64 nlen) (i + nlen) xxs+         | otherwise                   = advance (g + intToInt64 delta) (i + delta) xxs+       where+         c = index xxs (i + nlast)+         delta | nextInPattern = nlen + 1+               | c == z        = skip + 1+               | l >= i + nlen = case+                  memchr xarr# (xoff + i + nlen) (l - i - nlen) z of+                    -1 -> max 1 (l - i - nlen)+                    s  -> s + 1+                | otherwise = 1+         nextInPattern         = mask .&. swizzle (index xxs (i + nlen)) == 0++         candidateMatch+          | i + nlen <= l = A.equal narr noff xarr (xoff + i) nlen+          | otherwise     = A.equal narr noff xarr (xoff + i) (l - i) &&+            T.Text narr (noff + l - i) (nlen - l + i) `isPrefixOf` xs++    nlast     = nlen - 1+    z         = A.unsafeIndex narr (noff + nlen - 1)+    (mask :: Word64) :*: skip = buildTable 0 0 0 (nlen-2)++    swizzle :: Word8 -> Word64+    swizzle w = 1 `unsafeShiftL` (word8ToInt w .&. 0x3f)++    buildTable !g !i !msk !skp+            | i >= nlast = (msk .|. swizzle z) :*: skp+            | otherwise = buildTable (g+1) (i+1) msk' skp'+            where c                = A.unsafeIndex narr (noff+i)+                  msk'             = msk .|. swizzle c+                  skp' | c == z    = nlen - g - 2+                       | otherwise = skp++    -- | Check whether an attempt to index into the haystack at the+    -- given offset would fail.+    lackingHay :: Int -> T.Text -> Text -> Bool+    lackingHay q (T.Text _ _ l) ps = l < q && case ps of+      Empty -> True+      Chunk r rs -> lackingHay (q - l) r rs++-- | Fast index into a partly unpacked 'Text'.  We take into account+-- the possibility that the caller might try to access one element+-- past the end.+index :: Text -> Int -> Word8+index Empty !_ = 0+index (Chunk (T.Text arr off len) xs) !i+    | i < len   = A.unsafeIndex arr (off + i)+    | otherwise = index xs (i - len)++-- | A variant of 'indices' that scans linearly for a single 'Word8'.+indicesOne :: Word8 -> Int64 -> Text -> [Int64]+indicesOne c = chunk+  where+    chunk :: Int64 -> Text -> [Int64]+    chunk !_ Empty = []+    chunk !i (Chunk (T.Text oarr ooff olen) os) = go 0+      where+        go h | h >= olen = chunk (i+intToInt64 olen) os+             | on == c = i + intToInt64 h : go (h+1)+             | otherwise = go (h+1)+             where on = A.unsafeIndex oarr (ooff+h)++-- | First argument is a strict Text, and second is a lazy one.+isPrefixOf :: T.Text -> Text -> Bool+isPrefixOf (T.Text _ _ xlen) Empty = xlen == 0+isPrefixOf x@(T.Text xarr xoff xlen) (Chunk y@(T.Text _ _ ylen) ys)+  | xlen <= ylen = x `T.isPrefixOf` y+  | otherwise = y `T.isPrefixOf` x && T.Text xarr (xoff + ylen) (xlen - ylen) `isPrefixOf` ys++intToInt64 :: Int -> Int64+intToInt64 = fromIntegral++word8ToInt :: Word8 -> Int+word8ToInt = fromIntegral
+ src/Data/Text/Internal/Measure.hs view
@@ -0,0 +1,53 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}++#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MultiWayIf #-}+#endif++#if !defined(PURE_HASKELL)+{-# LANGUAGE UnliftedFFITypes #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'measure_off', using efficient C routines by default.+module Data.Text.Internal.Measure+  ( measure_off+  )+where++import GHC.Exts++#if defined(PURE_HASKELL)+import GHC.Word+import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader)+#endif++import Foreign.C.Types (CSize(..))+import System.Posix.Types (CSsize(..))++#if defined(PURE_HASKELL)++measure_off :: ByteArray# -> CSize -> CSize -> CSize -> CSsize+measure_off ba off len cnt = go 0 0+  where+    go !cc !i+      -- return the number of bytes for the first cnt codepoints,+      | cc == cnt = fromIntegral i+      -- return negated number of codepoints if there are fewer than cnt+      | i >= len  = negate (fromIntegral cc)+      | otherwise =+          let !(I# o) = fromIntegral (off+i)+              !b = indexWord8Array# ba o+          in go (cc+1) (i + fromIntegral (utf8LengthByLeader (W8# b)))++#else++-- | The input buffer (arr :: ByteArray#, off :: CSize, len :: CSize)+-- must specify a valid UTF-8 sequence, this condition is not checked.+foreign import ccall unsafe "_hs_text_measure_off" measure_off+    :: ByteArray# -> CSize -> CSize -> CSize -> CSsize++#endif
+ src/Data/Text/Internal/PrimCompat.hs view
@@ -0,0 +1,35 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}++module Data.Text.Internal.PrimCompat+  ( word8ToWord#+  , wordToWord8#++  , word16ToWord#+  , wordToWord16#++  , wordToWord32#+  , word32ToWord#+  ) where++#if MIN_VERSION_base(4,16,0)+import GHC.Exts (wordToWord8#,word8ToWord#,wordToWord16#,word16ToWord#,wordToWord32#,word32ToWord#)+#else+import GHC.Prim (Word#)+#endif++#if !(MIN_VERSION_base(4,16,0))+wordToWord8#,  word8ToWord#  :: Word# -> Word#+wordToWord16#, word16ToWord# :: Word# -> Word#+wordToWord32#, word32ToWord# :: Word# -> Word#+word8ToWord#  w = w+word16ToWord# w = w+word32ToWord# w = w+wordToWord8#  w = w+wordToWord16# w = w+wordToWord32# w = w+{-# INLINE wordToWord16# #-}+{-# INLINE word16ToWord# #-}+{-# INLINE wordToWord32# #-}+{-# INLINE word32ToWord# #-}+#endif
+ src/Data/Text/Internal/Private.hs view
@@ -0,0 +1,61 @@+{-# LANGUAGE BangPatterns, CPP, RankNTypes, UnboxedTuples #-}++-- |+-- Module      : Data.Text.Internal.Private+-- Copyright   : (c) 2011 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC++module Data.Text.Internal.Private+    (+      runText+    , span_+    , spanAscii_+    ) where++import Control.Monad.ST (ST, runST)+import Data.Text.Internal (Text(..), text)+import Data.Text.Unsafe (Iter(..), iter)+import qualified Data.Text.Array as A+import Data.Word (Word8)++#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif++span_ :: (Char -> Bool) -> Text -> (# Text, Text #)+span_ p t@(Text arr off len) = (# hd,tl #)+  where hd = text arr off k+        tl = text arr (off+k) (len-k)+        !k = loop 0+        loop !i | i < len && p c = loop (i+d)+                | otherwise      = i+            where Iter c d       = iter t i+{-# INLINE span_ #-}++-- | For the sake of performance this function does not check+-- that a char is in ASCII range; it is a responsibility of @p@.+--+-- @since 2.0+spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #)+spanAscii_ p (Text arr off len) = (# hd, tl #)+  where hd = text arr off k+        tl = text arr (off + k) (len - k)+        !k = loop 0+        loop !i | i < len && p (A.unsafeIndex arr (off + i)) = loop (i + 1)+                | otherwise = i+{-# INLINE spanAscii_ #-}++runText ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text+runText act = runST (act $ \ !marr !len -> do+                             A.shrinkM marr len+                             arr <- A.unsafeFreeze marr+                             return $! text arr 0 len)+{-# INLINE runText #-}
+ src/Data/Text/Internal/Read.hs view
@@ -0,0 +1,80 @@+-- |+-- Module      : Data.Text.Internal.Read+-- Copyright   : (c) 2014 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- Common internal functions for reading textual data.+module Data.Text.Internal.Read+    (+      IReader+    , IParser(..)+    , T(..)+    , digitToInt+    , hexDigitToInt+    , perhaps+    ) where++import Control.Applicative as App (Applicative(..))+import Control.Arrow (first)+import Control.Monad (ap)+import Data.Char (ord)++type IReader t a = t -> Either String (a,t)++newtype IParser t a = P {+      runP :: IReader t a+    }++instance Functor (IParser t) where+    fmap f m = P $ fmap (first f) . runP m++instance Applicative (IParser t) where+    pure a = P $ \t -> Right (a,t)+    {-# INLINE pure #-}+    (<*>) = ap++instance Monad (IParser t) where+    return = App.pure+    m >>= k  = P $ \t -> case runP m t of+                           Left err     -> Left err+                           Right (a,t') -> runP (k a) t'+    {-# INLINE (>>=) #-}++-- If we ever need a `MonadFail` instance the definition below can be used+--+-- > instance MonadFail (IParser t) where+-- >   fail msg = P $ \_ -> Left msg+--+-- But given the code compiles fine with a post-MFP GHC 8.6+ we don't need+-- one just yet.++data T = T !Integer !Int++perhaps :: a -> IParser t a -> IParser t a+perhaps def m = P $ \t -> case runP m t of+                            Left _      -> Right (def,t)+                            r@(Right _) -> r++hexDigitToInt :: Char -> Int+hexDigitToInt c+    | to0 < 10  = wordToInt to0+    | toa < 6   = wordToInt toa + 10+    | otherwise = wordToInt toA + 10+    where+        ordW = intToWord (ord c)+        to0 = ordW - intToWord (ord '0')+        toa = ordW - intToWord (ord 'a')+        toA = ordW - intToWord (ord 'A')++digitToInt :: Char -> Int+digitToInt c = ord c - ord '0'++intToWord :: Int -> Word+intToWord = fromIntegral++wordToInt :: Word -> Int+wordToInt = fromIntegral
+ src/Data/Text/Internal/Reverse.hs view
@@ -0,0 +1,108 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}+{-# LANGUAGE NoImplicitPrelude #-}+#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'reverse', using efficient C routines by default.+module Data.Text.Internal.Reverse (reverse, reverseNonEmpty) where++#if !defined(PURE_HASKELL)+import GHC.Exts as Exts+import Control.Monad.ST.Unsafe (unsafeIOToST)+import Foreign.C.Types (CSize(..))+#else+import Control.Monad.ST (ST)+import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader)+#endif+#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif+import Prelude hiding (reverse)+import Data.Text.Internal (Text(..), empty)+import Control.Monad.ST (runST)+import qualified Data.Text.Array as A++-- | /O(n)/ Reverse the characters of a string.+--+-- Example:+--+-- $setup+-- >>> T.reverse "desrever"+-- "reversed"+reverse ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Text+reverse (Text _ _ 0) = empty+reverse t            = reverseNonEmpty t+{-# INLINE reverse #-}++-- | /O(n)/ Reverse the characters of a string.+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.+reverseNonEmpty ::+  Text -> Text+#if defined(PURE_HASKELL)+reverseNonEmpty (Text src off len) = runST $ do+    dest <- A.new len+    _ <- reversePoints src off dest len+    result <- A.unsafeFreeze dest+    pure $ Text result 0 len++-- Step 0:+--+-- Input:  R E D R U M+--         ^+--         x+-- Output: _ _ _ _ _ _+--                     ^+--                     y+--+-- Step 1:+--+-- Input:  R E D R U M+--           ^+--           x+--+-- Output: _ _ _ _ _ R+--                   ^+--                   y+reversePoints+    :: A.Array -- ^ Input array+    -> Int -- ^ Input index+    -> A.MArray s -- ^ Output array+    -> Int -- ^ Output index+    -> ST s ()+reversePoints src xx dest yy = go xx yy where+    go !_ y | y <= 0 = pure ()+    go x y =+        let pLen = utf8LengthByLeader (A.unsafeIndex src x)+            -- The next y is also the start of the current point in the output+            yNext = y - pLen+        in do+            A.copyI pLen dest yNext src x+            go (x + pLen) yNext+#else+reverseNonEmpty (Text (A.ByteArray ba) off len) = runST $ do+    marr@(A.MutableByteArray mba) <- A.new len+    unsafeIOToST $ c_reverse mba ba (fromIntegral off) (fromIntegral len)+    brr <- A.unsafeFreeze marr+    return $ Text brr 0 len+#endif+{-# INLINE reverseNonEmpty #-}++#if !defined(PURE_HASKELL)+-- | The input buffer (src :: ByteArray#, off :: CSize, len :: CSize)+-- must specify a valid UTF-8 sequence, this condition is not checked.+foreign import ccall unsafe "_hs_text_reverse" c_reverse+    :: Exts.MutableByteArray# s -> ByteArray# -> CSize -> CSize -> IO ()+#endif++-- $setup+-- >>> :set -XOverloadedStrings+-- >>> import qualified Data.Text.Internal.Reverse as T
+ src/Data/Text/Internal/Search.hs view
@@ -0,0 +1,103 @@+{-# LANGUAGE BangPatterns, ScopedTypeVariables #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- |+-- Module      : Data.Text.Internal.Search+-- Copyright   : (c) Bryan O'Sullivan 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- Fast substring search for 'Text', based on work by Boyer, Moore,+-- Horspool, Sunday, and Lundh.+--+-- References:+--+-- * R. S. Boyer, J. S. Moore: A Fast String Searching Algorithm.+--   Communications of the ACM, 20, 10, 762-772 (1977)+--+-- * R. N. Horspool: Practical Fast Searching in Strings.  Software -+--   Practice and Experience 10, 501-506 (1980)+--+-- * D. M. Sunday: A Very Fast Substring Search Algorithm.+--   Communications of the ACM, 33, 8, 132-142 (1990)+--+-- * F. Lundh:+--   <http://web.archive.org/web/20201107074620/http://effbot.org/zone/stringlib.htm The Fast Search Algorithm>. (2006)++module Data.Text.Internal.Search+    (+      indices+    ) where++import qualified Data.Text.Array as A+import Data.Word (Word64, Word8)+import Data.Text.Internal (Text(..))+import Data.Bits ((.|.), (.&.), unsafeShiftL)+import Data.Text.Internal.ArrayUtils (memchr)++data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int++-- | /O(n+m)/ Find the offsets of all non-overlapping indices of+-- @needle@ within @haystack@.+--+-- In (unlikely) bad cases, this algorithm's complexity degrades+-- towards /O(n*m)/.+indices :: Text                -- ^ Substring to search for (@needle@)+        -> Text                -- ^ Text to search in (@haystack@)+        -> [Int]+indices needle@(Text narr noff nlen)+  | nlen == 1 = scanOne (A.unsafeIndex narr noff)+  | nlen <= 0 = const []+  | otherwise = indices' needle+{-# INLINE indices #-}++-- | nlen must be >= 2, otherwise nindex causes access violation+indices' :: Text -> Text -> [Int]+indices' (Text narr noff nlen) (Text harr@(A.ByteArray harr#) hoff hlen) = loop (hoff + nlen)+  where+    nlast    = nlen - 1+    !z       = nindex nlast+    nindex k = A.unsafeIndex narr (noff+k)+    buildTable !i !msk !skp+        | i >= nlast           = (msk .|. swizzle z) :* skp+        | otherwise            = buildTable (i+1) (msk .|. swizzle c) skp'+        where !c               = nindex i+              skp' | c == z    = nlen - i - 2+                   | otherwise = skp+    !(mask :* skip) = buildTable 0 0 (nlen-2)++    swizzle :: Word8 -> Word64+    swizzle !k = 1 `unsafeShiftL` (word8ToInt k .&. 0x3f)++    loop !i+      | i > hlen + hoff+      = []+      | A.unsafeIndex harr (i - 1) == z+      = if A.equal narr noff harr (i - nlen) nlen+        then i - nlen - hoff : loop (i + nlen)+        else                   loop (i + skip + 1)+      | i == hlen + hoff+      = []+      | mask .&. swizzle (A.unsafeIndex harr i) == 0+      = loop (i + nlen + 1)+      | otherwise+      = case memchr harr# i (hlen + hoff - i) z of+        -1 -> []+        x  -> loop (i + x + 1)+{-# INLINE indices' #-}++scanOne :: Word8 -> Text -> [Int]+scanOne c (Text harr hoff hlen) = loop 0+  where+    loop !i+      | i >= hlen                        = []+      | A.unsafeIndex harr (hoff+i) == c = i : loop (i+1)+      | otherwise                        = loop (i+1)+{-# INLINE scanOne #-}++word8ToInt :: Word8 -> Int+word8ToInt = fromIntegral
+ src/Data/Text/Internal/StrictBuilder.hs view
@@ -0,0 +1,129 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE TypeSynonymInstances #-}++-- |+-- Module      : Data.Text.Internal.Builder+-- License     : BSD-style (see LICENSE)+-- Stability   : experimental+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- @since 2.0.2++module Data.Text.Internal.StrictBuilder+  ( StrictTextBuilder(..)+  , StrictBuilder+  , toText+  , fromChar+  , fromText++    -- * Unsafe+    -- $unsafe+  , unsafeFromByteString+  , unsafeFromWord8+  ) where++import Control.Monad.ST (ST, runST)+import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)+import Data.Functor (void)+import Data.Word (Word8)+import Data.ByteString (ByteString)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup (Semigroup(..))+#endif+import Data.Text.Internal (Text(..), empty, safe)+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Encoding.Utf8 (utf8Length)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import qualified Data.ByteString as B+import qualified Data.Text.Array as A+import qualified Data.Text.Internal.Unsafe.Char as Char++-- | A delayed representation of strict 'Text'.+--+-- @since 2.1.2+data StrictTextBuilder = StrictTextBuilder+  { sbLength :: {-# UNPACK #-} !Int+  , sbWrite :: forall s. A.MArray s -> Int -> ST s ()+  }++-- | A delayed representation of strict 'Text'.+--+-- @since 2.0.2+{-# DEPRECATED StrictBuilder "Use StrictTextBuilder instead" #-}+type StrictBuilder = StrictTextBuilder++-- | Use 'StrictBuilder' to build 'Text'.+--+-- @since 2.0.2+toText :: StrictTextBuilder -> Text+toText (StrictTextBuilder 0 _) = empty+toText (StrictTextBuilder n write) = runST (do+  dst <- A.new n+  write dst 0+  arr <- A.unsafeFreeze dst+  pure (Text arr 0 n))++-- | Concatenation of 'StrictBuilder' is right-biased:+-- the right builder will be run first. This allows a builder to+-- run tail-recursively when it was accumulated left-to-right.+instance Semigroup StrictTextBuilder where+  (<>) = appendRStrictBuilder++instance Monoid StrictTextBuilder where+  mempty = emptyStrictBuilder+  mappend = (<>)++emptyStrictBuilder :: StrictTextBuilder+emptyStrictBuilder = StrictTextBuilder 0 (\_ _ -> pure ())++appendRStrictBuilder :: StrictTextBuilder -> StrictTextBuilder -> StrictTextBuilder+appendRStrictBuilder (StrictTextBuilder 0 _) b2 = b2+appendRStrictBuilder b1 (StrictTextBuilder 0 _) = b1+appendRStrictBuilder (StrictTextBuilder n1 write1) (StrictTextBuilder n2 write2) =+  StrictTextBuilder (n1 + n2) (\dst ofs -> do+    write2 dst (ofs + n1)+    write1 dst ofs)++copyFromByteString :: A.MArray s -> Int -> ByteString -> ST s ()+copyFromByteString dst ofs src = withBS src $ \ srcFPtr len ->+  unsafeIOToST $ unsafeWithForeignPtr srcFPtr $ \ srcPtr -> do+    unsafeSTToIO $ A.copyFromPointer dst ofs srcPtr len++-- | Copy a 'ByteString'.+--+-- Unsafe: This may not be valid UTF-8 text.+--+-- @since 2.0.2+unsafeFromByteString :: ByteString -> StrictTextBuilder+unsafeFromByteString bs =+  StrictTextBuilder (B.length bs) (\dst ofs -> copyFromByteString dst ofs bs)++-- |+-- @since 2.0.2+{-# INLINE fromChar #-}+fromChar :: Char -> StrictTextBuilder+fromChar c =+  StrictTextBuilder (utf8Length c) (\dst ofs -> void (Char.unsafeWrite dst ofs (safe c)))++-- $unsafe+-- For internal purposes, we abuse 'StrictBuilder' as a delayed 'Array' rather+-- than 'Text': it may not actually be valid 'Text'.++-- | Unsafe: This may not be valid UTF-8 text.+--+-- @since 2.0.2+unsafeFromWord8 :: Word8 -> StrictTextBuilder+unsafeFromWord8 !w =+  StrictTextBuilder 1 (\dst ofs -> A.unsafeWrite dst ofs w)++-- | Copy 'Text' in a 'StrictBuilder'+--+-- @since 2.0.2+fromText :: Text -> StrictTextBuilder+fromText (Text src srcOfs n) = StrictTextBuilder n (\dst dstOfs ->+  A.copyI n dst dstOfs src srcOfs)
+ src/Data/Text/Internal/Transformation.hs view
@@ -0,0 +1,340 @@+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE MagicHash #-}
+{-# LANGUAGE PartialTypeSignatures #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE UnboxedTuples #-}
+{-# LANGUAGE UnliftedFFITypes #-}
+
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# OPTIONS_GHC -Wno-partial-type-signatures #-}
+
+-- |
+-- Module      : Data.Text.Internal.Transformation
+-- Copyright   : (c) 2008, 2009 Tom Harper,
+--               (c) 2009, 2010 Bryan O'Sullivan,
+--               (c) 2009 Duncan Coutts
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- This module holds functions shared between the strict and lazy implementations of @Text@ transformations.
+
+module Data.Text.Internal.Transformation
+  ( mapNonEmpty
+  , toCaseFoldNonEmpty
+  , toLowerNonEmpty
+  , toUpperNonEmpty
+  , toTitleNonEmpty
+  , filter_
+  ) where
+
+import Prelude (Char, Bool(..), Int,
+                Ord(..),
+                Monad(..), pure,
+                (+), (-), ($), (&&), (||), (==),
+                not, return, otherwise)
+import Data.Bits ((.&.), shiftR, shiftL)
+import Data.Char (isLetter, isSpace)
+import Control.Monad.ST (ST, runST)
+import qualified Data.Text.Array as A
+import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4)
+import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping)
+import Data.Text.Internal (Text(..), safe)
+import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8)
+import qualified Prelude as P
+import Data.Text.Unsafe (Iter(..), iterArray)
+import Data.Word (Word8)
+import qualified GHC.Exts as Exts
+import GHC.Int (Int64(..))
+
+-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
+-- each element of @t@.
+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.
+mapNonEmpty :: (Char -> Char) -> Text -> Text
+mapNonEmpty f = go
+  where
+    go (Text src o l) = runST $ do
+      marr <- A.new (l + 4)
+      outer marr (l + 4) o 0
+      where
+        outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text
+        outer !dst !dstLen = inner
+          where
+            inner !srcOff !dstOff
+              | srcOff >= l + o = do
+                A.shrinkM dst dstOff
+                arr <- A.unsafeFreeze dst
+                return (Text arr 0 dstOff)
+              | dstOff + 4 > dstLen = do
+                let !dstLen' = dstLen + (l + o) - srcOff + 4
+                dst' <- A.resizeM dst dstLen'
+                outer dst' dstLen' srcOff dstOff
+              | otherwise = do
+                let !(Iter c d) = iterArray src srcOff
+                d' <- unsafeWrite dst dstOff (safe (f c))
+                inner (srcOff + d) (dstOff + d')
+{-# INLINE mapNonEmpty #-}
+
+caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text
+caseConvert ascii remap (Text src o l) = runST $ do
+  -- Case conversion a single code point may produce up to 3 code-points,
+  -- each up to 4 bytes, so 12 in total.
+  dst <- A.new (l + 12)
+  outer dst l o 0
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text
+    outer !dst !dstLen = inner
+      where
+        inner !srcOff !dstOff
+          | srcOff >= o + l = do
+            A.shrinkM dst dstOff
+            arr <- A.unsafeFreeze dst
+            return (Text arr 0 dstOff)
+          | dstOff + 12 > dstLen = do
+            -- Ensure to extend the buffer by at least 12 bytes.
+            let !dstLen' = dstLen + max 12 (l + o - srcOff)
+            dst' <- A.resizeM dst dstLen'
+            outer dst' dstLen' srcOff dstOff
+          -- If a character is to remain unchanged, no need to decode Char back into UTF8,
+          -- just copy bytes from input.
+          | otherwise = do
+            let m0 = A.unsafeIndex src srcOff
+                m1 = A.unsafeIndex src (srcOff + 1)
+                m2 = A.unsafeIndex src (srcOff + 2)
+                m3 = A.unsafeIndex src (srcOff + 3)
+                !d = utf8LengthByLeader m0
+            case d of
+              1 -> do
+                A.unsafeWrite dst dstOff (ascii m0)
+                inner (srcOff + 1) (dstOff + 1)
+              2 -> do
+                let !(Exts.C# c) = chr2 m0 m1
+                dstOff' <- case I64# (remap c) of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    pure $ dstOff + 2
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 2) dstOff'
+              3 -> do
+                let !(Exts.C# c) = chr3 m0 m1 m2
+                dstOff' <- case I64# (remap c) of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    A.unsafeWrite dst (dstOff + 2) m2
+                    pure $ dstOff + 3
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 3) dstOff'
+              _ -> do
+                let !(Exts.C# c) = chr4 m0 m1 m2 m3
+                dstOff' <- case I64# (remap c) of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    A.unsafeWrite dst (dstOff + 2) m2
+                    A.unsafeWrite dst (dstOff + 3) m3
+                    pure $ dstOff + 4
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 4) dstOff'
+
+{-# INLINABLE caseConvert #-}
+
+writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int
+writeMapping !_ 0 !dstOff = pure dstOff
+writeMapping dst i dstOff = do
+  let (ch, j) = chopOffChar i
+  d <- unsafeWrite dst dstOff ch
+  writeMapping dst j (dstOff + d)
+
+chopOffChar :: Int64 -> (Char, Int64)
+chopOffChar ab = (chr a, ab `shiftR` 21)
+  where
+    chr (Exts.I# n) = Exts.C# (Exts.chr# n)
+    mask = (1 `shiftL` 21) - 1
+    a = P.fromIntegral $ ab .&. mask
+
+-- | /O(n)/ Convert a string to folded case.
+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.
+toCaseFoldNonEmpty :: Text -> Text
+toCaseFoldNonEmpty  = \xs -> caseConvert asciiToLower foldMapping xs
+{-# INLINE toCaseFoldNonEmpty #-}
+
+-- | /O(n)/ Convert a string to lower case, using simple case
+-- conversion.
+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.
+toLowerNonEmpty :: Text -> Text
+toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs
+{-# INLINE toLowerNonEmpty #-}
+
+-- | /O(n)/ Convert a string to upper case, using simple case
+-- conversion.
+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.
+toUpperNonEmpty :: Text -> Text
+toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs
+{-# INLINE toUpperNonEmpty #-}
+
+asciiToLower :: Word8 -> Word8
+asciiToLower w = if w - 65 <= 25 then w + 32 else w
+
+asciiToUpper :: Word8 -> Word8
+asciiToUpper w = if w - 97 <= 25 then w - 32 else w
+
+isAsciiLetter :: Word8 -> Bool
+isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25
+
+isAsciiSpace :: Word8 -> Bool
+isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5)
+
+-- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion.
+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.
+toTitleNonEmpty :: Text -> Text
+toTitleNonEmpty (Text src o l) = runST $ do
+  -- Case conversion a single code point may produce up to 3 code-points,
+  -- each up to 4 bytes, so 12 in total.
+  dst <- A.new (l + 12)
+  outer dst l o 0 False
+  where
+    outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text
+    outer !dst !dstLen = inner
+      where
+        inner !srcOff !dstOff !mode
+          | srcOff >= o + l = do
+            A.shrinkM dst dstOff
+            arr <- A.unsafeFreeze dst
+            return (Text arr 0 dstOff)
+          | dstOff + 12 > dstLen = do
+            -- Ensure to extend the buffer by at least 12 bytes.
+            let !dstLen' = dstLen + max 12 (l + o - srcOff)
+            dst' <- A.resizeM dst dstLen'
+            outer dst' dstLen' srcOff dstOff mode
+          -- If a character is to remain unchanged, no need to decode Char back into UTF8,
+          -- just copy bytes from input.
+          | otherwise = do
+            let m0 = A.unsafeIndex src srcOff
+                m1 = A.unsafeIndex src (srcOff + 1)
+                m2 = A.unsafeIndex src (srcOff + 2)
+                m3 = A.unsafeIndex src (srcOff + 3)
+                !d = utf8LengthByLeader m0
+
+            case d of
+              1 -> do
+                let (mode', m0') = asciiAdvance mode m0
+                A.unsafeWrite dst dstOff m0'
+                inner (srcOff + 1) (dstOff + 1) mode'
+              2 -> do
+                let !(Exts.C# c) = chr2 m0 m1
+                    !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c
+                dstOff' <- case I64# c' of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    pure $ dstOff + 2
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 2) dstOff' mode'
+              3 -> do
+                let !(Exts.C# c) = chr3 m0 m1 m2
+                    isSpace3 ch
+                      =  m0 == 0xE1 && m1 == 0x9A && m2 == 0x80
+                      || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F)
+                      || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80
+                    !(# mode', c' #) = advance isSpace3 mode c
+                dstOff' <- case I64# c' of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    A.unsafeWrite dst (dstOff + 2) m2
+                    pure $ dstOff + 3
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 3) dstOff' mode'
+              _ -> do
+                let !(Exts.C# c) = chr4 m0 m1 m2 m3
+                    !(# mode', c' #) = advance (\_ -> False) mode c
+                dstOff' <- case I64# c' of
+                  0 -> do
+                    A.unsafeWrite dst dstOff m0
+                    A.unsafeWrite dst (dstOff + 1) m1
+                    A.unsafeWrite dst (dstOff + 2) m2
+                    A.unsafeWrite dst (dstOff + 3) m3
+                    pure $ dstOff + 4
+                  i -> writeMapping dst i dstOff
+                inner (srcOff + 4) dstOff' mode'
+
+        asciiAdvance :: Bool -> Word8 -> (Bool, Word8)
+        asciiAdvance False w = (isAsciiLetter w, asciiToUpper w)
+        asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w)
+
+        advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #)
+        advance _ False c = (# isLetter (Exts.C# c), titleMapping c #)
+        advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #)
+        {-# INLINE advance #-}
+
+-- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@,
+-- calls the continuation with the @Text@ containing only the characters satisfying the predicate.
+filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a
+filter_ mkText p = go
+  where
+    go (Text src o l) = runST $ do
+      -- It's tempting to allocate l elements at once and avoid resizing.
+      -- However, this can be unacceptable in scenarios where a huge array
+      -- is filtered with a rare predicate, resulting in a much shorter buffer.
+      let !dstLen = min l 64
+      dst <- A.new dstLen
+      outer dst dstLen o 0
+      where
+        outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a
+        outer !dst !dstLen = inner
+          where
+            inner !srcOff !dstOff
+              | srcOff >= o + l = do
+                A.shrinkM dst dstOff
+                arr <- A.unsafeFreeze dst
+                return $ mkText arr 0 dstOff
+              | dstOff + 4 > dstLen = do
+                -- Double size of the buffer, unless it becomes longer than
+                -- source string. Ensure to extend it by least 4 bytes.
+                let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen)
+                dst' <- A.resizeM dst dstLen'
+                outer dst' dstLen' srcOff dstOff
+              -- In case of success, filter writes exactly the same character
+              -- it just read (this is not a case for map, for example).
+              -- We leverage this fact below: no need to decode Char back into UTF8,
+              -- just copy bytes from input.
+              | otherwise = do
+                let m0 = A.unsafeIndex src srcOff
+                    m1 = A.unsafeIndex src (srcOff + 1)
+                    m2 = A.unsafeIndex src (srcOff + 2)
+                    m3 = A.unsafeIndex src (srcOff + 3)
+                    !d = utf8LengthByLeader m0
+                case d of
+                  1 -> do
+                    let !c = unsafeChr8 m0
+                    if not (p c) then inner (srcOff + 1) dstOff else do
+                      A.unsafeWrite dst dstOff m0
+                      inner (srcOff + 1) (dstOff + 1)
+                  2 -> do
+                    let !c = chr2 m0 m1
+                    if not (p c) then inner (srcOff + 2) dstOff else do
+                      A.unsafeWrite dst dstOff m0
+                      A.unsafeWrite dst (dstOff + 1) m1
+                      inner (srcOff + 2) (dstOff + 2)
+                  3 -> do
+                    let !c = chr3 m0 m1 m2
+                    if not (p c) then inner (srcOff + 3) dstOff else do
+                      A.unsafeWrite dst dstOff m0
+                      A.unsafeWrite dst (dstOff + 1) m1
+                      A.unsafeWrite dst (dstOff + 2) m2
+                      inner (srcOff + 3) (dstOff + 3)
+                  _ -> do
+                    let !c = chr4 m0 m1 m2 m3
+                    if not (p c) then inner (srcOff + 4) dstOff else do
+                      A.unsafeWrite dst dstOff m0
+                      A.unsafeWrite dst (dstOff + 1) m1
+                      A.unsafeWrite dst (dstOff + 2) m2
+                      A.unsafeWrite dst (dstOff + 3) m3
+                      inner (srcOff + 4) (dstOff + 4)
+{-# INLINE filter_ #-}
+ src/Data/Text/Internal/Unsafe.hs view
@@ -0,0 +1,65 @@+{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-}+{-# OPTIONS_HADDOCK not-home #-}++-- |+-- Module      : Data.Text.Internal.Unsafe+-- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : portable+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- A module containing /unsafe/ operations, for /very very careful/ use+-- in /heavily tested/ code.+module Data.Text.Internal.Unsafe+    (+      inlineInterleaveST+    , inlinePerformIO+    , unsafeWithForeignPtr+    ) where++import Foreign.Ptr (Ptr)+import Foreign.ForeignPtr (ForeignPtr)+#if MIN_VERSION_base(4,15,0)+import qualified GHC.ForeignPtr (unsafeWithForeignPtr)+#else+import qualified Foreign.ForeignPtr (withForeignPtr)+#endif++import GHC.ST (ST(..))+import GHC.IO (IO(IO))+import GHC.Base (realWorld#)++-- | Just like unsafePerformIO, but we inline it. Big performance gains as+-- it exposes lots of things to further inlining. /Very unsafe/. In+-- particular, you should do no memory allocation inside an+-- 'inlinePerformIO' block.+--+{-# INLINE inlinePerformIO #-}+inlinePerformIO :: IO a -> a+inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r++-- | Allow an 'ST' computation to be deferred lazily. When passed an+-- action of type 'ST' @s@ @a@, the action will only be performed when+-- the value of @a@ is demanded.+--+-- This function is identical to the normal unsafeInterleaveST, but is+-- inlined and hence faster.+--+-- /Note/: This operation is highly unsafe, as it can introduce+-- externally visible non-determinism into an 'ST' action.+inlineInterleaveST :: ST s a -> ST s a+inlineInterleaveST (ST m) = ST $ \ s ->+    let r = case m s of (# _, res #) -> res in (# s, r #)+{-# INLINE inlineInterleaveST #-}++unsafeWithForeignPtr :: ForeignPtr a -> (Ptr a -> IO b) -> IO b+#if MIN_VERSION_base(4,15,0)+unsafeWithForeignPtr = GHC.ForeignPtr.unsafeWithForeignPtr+#else+unsafeWithForeignPtr = Foreign.ForeignPtr.withForeignPtr+#endif
+ src/Data/Text/Internal/Unsafe/Char.hs view
@@ -0,0 +1,88 @@+{-# LANGUAGE CPP, MagicHash #-}++-- |+-- Module      : Data.Text.Internal.Unsafe.Char+-- Copyright   : (c) 2008, 2009 Tom Harper,+--               (c) 2009, 2010 Bryan O'Sullivan,+--               (c) 2009 Duncan Coutts+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Fast character manipulation functions.+module Data.Text.Internal.Unsafe.Char+    (+      ord+    , unsafeChr16+    , unsafeChr8+    , unsafeChr32+    , unsafeWrite+    ) where++import Control.Monad.ST (ST)+import Data.Text.Internal.Encoding.Utf8+import GHC.Exts (Char(..), Int(..), chr#, ord#, word2Int#)+import GHC.Word (Word8(..), Word16(..), Word32(..))+import qualified Data.Text.Array as A+import Data.Text.Internal.PrimCompat ( word8ToWord#, word16ToWord#, word32ToWord# )+#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif++ord :: Char -> Int+ord (C# c#) = I# (ord# c#)+{-# INLINE ord #-}++-- | @since 2.0+unsafeChr16 :: Word16 -> Char+unsafeChr16 (W16# w#) = C# (chr# (word2Int# (word16ToWord# w#)))+{-# INLINE unsafeChr16 #-}++unsafeChr8 :: Word8 -> Char+unsafeChr8 (W8# w#) = C# (chr# (word2Int# (word8ToWord# w#)))+{-# INLINE unsafeChr8 #-}++unsafeChr32 :: Word32 -> Char+unsafeChr32 (W32# w#) = C# (chr# (word2Int# (word32ToWord# w#)))+{-# INLINE unsafeChr32 #-}++-- | Write a character into the array at the given offset.  Returns+-- the number of 'Word8's written.+unsafeWrite ::+#if defined(ASSERTS)+    HasCallStack =>+#endif+    A.MArray s -> Int -> Char -> ST s Int+unsafeWrite marr i c = case utf8Length c of+    1 -> do+        let n0 = intToWord8 (ord c)+        A.unsafeWrite marr i n0+        return 1+    2 -> do+        let (n0, n1) = ord2 c+        A.unsafeWrite marr i     n0+        A.unsafeWrite marr (i+1) n1+        return 2+    3 -> do+        let (n0, n1, n2) = ord3 c+        A.unsafeWrite marr i     n0+        A.unsafeWrite marr (i+1) n1+        A.unsafeWrite marr (i+2) n2+        return 3+    _ -> do+        let (n0, n1, n2, n3) = ord4 c+        A.unsafeWrite marr i     n0+        A.unsafeWrite marr (i+1) n1+        A.unsafeWrite marr (i+2) n2+        A.unsafeWrite marr (i+3) n3+        return 4+{-# INLINE unsafeWrite #-}++intToWord8 :: Int -> Word8+intToWord8 = fromIntegral
+ src/Data/Text/Internal/Validate.hs view
@@ -0,0 +1,107 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- | Test whether or not a sequence of bytes is a valid UTF-8 byte sequence.+-- In the GHC Haskell ecosystem, there are several representations of byte+-- sequences. The only one that the stable @text@ API concerns itself with is+-- 'ByteString'. Part of bytestring-to-text decoding is 'isValidUtf8ByteString',+-- a high-performance UTF-8 validation routine written in C++ with fallbacks+-- for various platforms. The C++ code backing this routine is nontrivial,+-- so in the interest of reuse, this module additionally exports functions+-- for working with the GC-managed @ByteArray@ type. These @ByteArray@+-- functions are not used anywhere else in @text@. They are for the benefit+-- of library and application authors who do not use 'ByteString' but still+-- need to interoperate with @text@.+module Data.Text.Internal.Validate+  (+  -- * ByteString+    isValidUtf8ByteString+  -- * ByteArray+  --+  -- | Is the slice of a byte array a valid UTF-8 byte sequence? These+  -- functions all accept an offset and a length.+  , isValidUtf8ByteArray+  , isValidUtf8ByteArrayUnpinned+  , isValidUtf8ByteArrayPinned+  ) where++import Data.Array.Byte (ByteArray(ByteArray))+import Data.ByteString (ByteString)+import GHC.Exts (isTrue#,isByteArrayPinned#)++#ifdef SIMDUTF+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Internal.Validate.Simd (c_is_valid_utf8_bytearray_safe,c_is_valid_utf8_bytearray_unsafe,c_is_valid_utf8_ptr_unsafe)+#else+import qualified Data.ByteString as B+import qualified Data.Text.Internal.Validate.Native as N+#endif++-- | Is the ByteString a valid UTF-8 byte sequence?+isValidUtf8ByteString :: ByteString -> Bool+#ifdef SIMDUTF+isValidUtf8ByteString bs = withBS bs $ \fp len -> unsafeDupablePerformIO $+  unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> c_is_valid_utf8_ptr_unsafe ptr (fromIntegral len)+#else+-- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0.+-- MIN_VERSION_bytestring does not allow us to differentiate+-- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+.+#if MIN_VERSION_bytestring(0,12,1)+isValidUtf8ByteString = B.isValidUtf8+#else+isValidUtf8ByteString = N.isValidUtf8ByteStringHaskell+#endif+#endif++-- | For pinned byte arrays larger than 128KiB, this switches to the safe FFI+-- so that it does not prevent GC. This threshold (128KiB) was chosen+-- somewhat arbitrarily and may change in the future.+isValidUtf8ByteArray ::+     ByteArray -- ^ Bytes+  -> Int -- ^ Offset+  -> Int -- ^ Length+  -> Bool+isValidUtf8ByteArray b@(ByteArray b#) !off !len+  | len >= 131072 -- 128KiB+  , isTrue# (isByteArrayPinned# b#)+  = isValidUtf8ByteArrayPinned b off len+  | otherwise = isValidUtf8ByteArrayUnpinned b off len++-- | This uses the @unsafe@ FFI. GC waits for all @unsafe@ FFI calls+-- to complete before starting. Consequently, an @unsafe@ FFI call does not+-- run concurrently with GC and is not interrupted by GC. Since relocation+-- cannot happen concurrently with an @unsafe@ FFI call, it is safe+-- to call this function with an unpinned byte array argument.+-- It is also safe to call this with a pinned @ByteArray@ argument.+isValidUtf8ByteArrayUnpinned ::+     ByteArray -- ^ Bytes+  -> Int -- ^ Offset+  -> Int -- ^ Length+  -> Bool+#ifdef SIMDUTF+isValidUtf8ByteArrayUnpinned (ByteArray bs) !off !len =+  unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_unsafe bs (fromIntegral off) (fromIntegral len)+#else+isValidUtf8ByteArrayUnpinned = N.isValidUtf8ByteArrayHaskell+#endif++-- | This uses the @safe@ FFI. GC may run concurrently with @safe@+-- FFI calls. Consequently, unpinned objects may be relocated while a+-- @safe@ FFI call is executing. The byte array argument /must/ be pinned,+-- and the calling context is responsible for enforcing this. If the+-- byte array is not pinned, this function's behavior is undefined.+isValidUtf8ByteArrayPinned ::+     ByteArray -- ^ Bytes+  -> Int -- ^ Offset+  -> Int -- ^ Length+  -> Bool+#ifdef SIMDUTF+isValidUtf8ByteArrayPinned (ByteArray bs) !off !len =+  unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_safe bs (fromIntegral off) (fromIntegral len)+#else+isValidUtf8ByteArrayPinned = N.isValidUtf8ByteArrayHaskell+#endif
+ src/Data/Text/Internal/Validate/Native.hs view
@@ -0,0 +1,60 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MagicHash #-}++-- | Native implementation of 'Data.Text.Internal.Validate'.+module Data.Text.Internal.Validate.Native+  ( isValidUtf8ByteStringHaskell+  , isValidUtf8ByteArrayHaskell+  ) where++import Data.Array.Byte (ByteArray(ByteArray))+import Data.ByteString (ByteString)+import GHC.Exts (ByteArray#,Int(I#),indexWord8Array#)+import GHC.Word (Word8(W8#))+import Data.Text.Internal.Encoding.Utf8 (CodePoint(..),DecoderResult(..),utf8DecodeStart,utf8DecodeContinue)+import qualified Data.ByteString as B+import qualified Data.ByteString.Unsafe as B++-- | Native implementation of 'Data.Text.Internal.Validate.isValidUtf8ByteString'.+isValidUtf8ByteStringHaskell :: ByteString -> Bool+isValidUtf8ByteStringHaskell bs = start 0+  where+    start ix+      | ix >= B.length bs = True+      | otherwise = case utf8DecodeStart (B.unsafeIndex bs ix) of+        Accept{} -> start (ix + 1)+        Reject{} -> False+        Incomplete st _ -> step (ix + 1) st+    step ix st+      | ix >= B.length bs = False+      -- We do not use decoded code point, so passing a dummy value to save an argument.+      | otherwise = case utf8DecodeContinue (B.unsafeIndex bs ix) st (CodePoint 0) of+        Accept{} -> start (ix + 1)+        Reject{} -> False+        Incomplete st' _ -> step (ix + 1) st'++-- | Native implementation of+-- 'Data.Text.Internal.Validate.isValidUtf8ByteArrayUnpinned'+-- and 'Data.Text.Internal.Validate.isValidUtf8ByteArrayPinned'.+isValidUtf8ByteArrayHaskell ::+     ByteArray -- ^ Bytes+  -> Int -- ^ Offset+  -> Int -- ^ Length+  -> Bool+isValidUtf8ByteArrayHaskell (ByteArray b) !off !len = start off+  where+    indexWord8 :: ByteArray# -> Int -> Word8+    indexWord8 !x (I# i) = W8# (indexWord8Array# x i)+    start ix+      | ix >= off + len = True+      | otherwise = case utf8DecodeStart (indexWord8 b ix) of+        Accept{} -> start (ix + 1)+        Reject{} -> False+        Incomplete st _ -> step (ix + 1) st+    step ix st+      | ix >= off + len = False+      -- We do not use decoded code point, so passing a dummy value to save an argument.+      | otherwise = case utf8DecodeContinue (indexWord8 b ix) st (CodePoint 0) of+        Accept{} -> start (ix + 1)+        Reject{} -> False+        Incomplete st' _ -> step (ix + 1) st'
+ src/Data/Text/Internal/Validate/Simd.hs view
@@ -0,0 +1,42 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- | Validate that a byte sequence is UTF-8-encoded text. All of these+-- functions return zero when the byte sequence is not UTF-8-encoded text,+-- and they return an unspecified non-zero value when the byte sequence+-- is UTF-8-encoded text.+--+-- Variants are provided for both @ByteArray#@ and @Ptr@. Additionally,+-- variants are provided that use both the @safe@ and @unsafe@ FFI.+--+-- If compiling with SIMDUTF turned off, this module exports nothing.+module Data.Text.Internal.Validate.Simd+  ( c_is_valid_utf8_ptr_unsafe+  , c_is_valid_utf8_ptr_safe+  , c_is_valid_utf8_bytearray_unsafe+  , c_is_valid_utf8_bytearray_safe+  ) where++import Data.Word (Word8)+import Foreign.C.Types (CSize(..),CInt(..))+import GHC.Exts (Ptr,ByteArray#)++foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_unsafe+    :: Ptr Word8 -- ^ Bytes+    -> CSize -- ^ Length+    -> IO CInt+foreign import ccall safe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_safe+    :: Ptr Word8 -- ^ Bytes+    -> CSize -- ^ Length+    -> IO CInt+foreign import ccall unsafe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_unsafe+    :: ByteArray# -- ^ Bytes+    -> CSize -- ^ Offset into bytes+    -> CSize -- ^ Length+    -> IO CInt+foreign import ccall safe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_safe+    :: ByteArray# -- ^ Bytes+    -> CSize -- ^ Offset into bytes+    -> CSize -- ^ Length+    -> IO CInt
+ src/Data/Text/Lazy.hs view
@@ -0,0 +1,1911 @@+{-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-}
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE TemplateHaskellQuotes #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE PatternSynonyms #-}
+{-# LANGUAGE ViewPatterns #-}
+
+-- |
+-- Module      : Data.Text.Lazy
+-- Copyright   : (c) 2009, 2010, 2012 Bryan O'Sullivan
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : GHC
+--
+-- A time and space-efficient implementation of Unicode text using
+-- lists of packed arrays.
+--
+-- /Note/: Read below the synopsis for important notes on the use of
+-- this module.
+--
+-- The representation used by this module is suitable for high
+-- performance use and for streaming large quantities of data.  It
+-- provides a means to manipulate a large body of text without
+-- requiring that the entire content be resident in memory.
+--
+-- Some operations, such as 'concat', 'append', 'reverse' and 'cons',
+-- have better time complexity than their "Data.Text" equivalents, due
+-- to the underlying representation being a list of chunks. For other
+-- operations, lazy 'Text's are usually within a few percent of strict
+-- ones, but often with better heap usage if used in a streaming
+-- fashion. For data larger than available memory, or if you have
+-- tight memory constraints, this module will be the only option.
+--
+-- This module is intended to be imported @qualified@, to avoid name
+-- clashes with "Prelude" functions.  eg.
+--
+-- > import qualified Data.Text.Lazy as L
+
+module Data.Text.Lazy
+    (
+    -- * Fusion
+    -- $fusion
+
+    -- * Acceptable data
+    -- $replacement
+
+    -- * Types
+      Text
+    , LazyText
+
+    -- * Creation and elimination
+    , pack
+    , unpack
+    , singleton
+    , empty
+    , fromChunks
+    , toChunks
+    , toStrict
+    , fromStrict
+    , foldrChunks
+    , foldlChunks
+
+    -- * Pattern matching
+    , pattern Empty
+    , pattern (:<)
+    , pattern (:>)
+
+    -- * Basic interface
+    , cons
+    , snoc
+    , append
+    , uncons
+    , unsnoc
+    , head
+    , last
+    , tail
+    , init
+    , null
+    , length
+    , compareLength
+
+    -- * Transformations
+    , map
+    , intercalate
+    , intersperse
+    , transpose
+    , reverse
+    , replace
+
+    -- ** Case conversion
+    -- $case
+    , toCaseFold
+    , toLower
+    , toUpper
+    , toTitle
+
+    -- ** Justification
+    , justifyLeft
+    , justifyRight
+    , center
+
+    -- * Folds
+    , foldl
+    , foldl'
+    , foldl1
+    , foldl1'
+    , foldr
+    , foldr1
+    , foldlM'
+
+    -- ** Special folds
+    , concat
+    , concatMap
+    , any
+    , all
+    , maximum
+    , minimum
+    , isAscii
+
+    -- * Construction
+
+    -- ** Scans
+    , scanl
+    , scanl1
+    , scanr
+    , scanr1
+
+    -- ** Accumulating maps
+    , mapAccumL
+    , mapAccumR
+
+    -- ** Generation and unfolding
+    , repeat
+    , replicate
+    , cycle
+    , iterate
+    , unfoldr
+    , unfoldrN
+
+    -- * Substrings
+
+    -- ** Breaking strings
+    , take
+    , takeEnd
+    , drop
+    , dropEnd
+    , takeWhile
+    , takeWhileEnd
+    , dropWhile
+    , dropWhileEnd
+    , dropAround
+    , strip
+    , stripStart
+    , stripEnd
+    , splitAt
+    , span
+    , spanM
+    , spanEndM
+    , breakOn
+    , breakOnEnd
+    , break
+    , group
+    , groupBy
+    , inits
+    , initsNE
+    , tails
+    , tailsNE
+
+    -- ** Breaking into many substrings
+    -- $split
+    , splitOn
+    , split
+    , chunksOf
+    -- , breakSubstring
+
+    -- ** Breaking into lines and words
+    , lines
+    , words
+    , unlines
+    , unwords
+
+    -- * Predicates
+    , isPrefixOf
+    , isSuffixOf
+    , isInfixOf
+
+    -- ** View patterns
+    , stripPrefix
+    , stripSuffix
+    , commonPrefixes
+
+    -- * Searching
+    , filter
+    , find
+    , elem
+    , breakOnAll
+    , partition
+
+    -- , findSubstring
+
+    -- * Indexing
+    , index
+    , count
+
+    -- * Zipping and unzipping
+    , zip
+    , zipWith
+
+    -- * Showing values
+    , show
+
+    -- -* Ordered text
+    -- , sort
+    ) where
+
+import Prelude (Char, Bool(..), Maybe(..), String,
+                Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec),
+                Monad(..), pure, (<$>),
+                (&&), (+), (-), (.), ($), (++),
+                error, flip, fmap, fromIntegral, not, otherwise, quot)
+import qualified Prelude as P
+import Control.Arrow (first)
+import Control.DeepSeq (NFData(..))
+import Data.Bits (finiteBitSize, toIntegralSized)
+import Data.Int (Int64)
+import qualified Data.List as L hiding (head, tail)
+import Data.Char (isSpace)
+import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,
+                  Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))
+import Data.Binary (Binary(get, put))
+import Data.Binary.Put (putBuilder)
+import Data.List.NonEmpty (NonEmpty(..))
+import qualified Data.List.NonEmpty as NE
+import Data.Monoid (Monoid(..))
+import Data.Semigroup (Semigroup(..))
+import Data.String (IsString(..))
+import qualified Data.Text as T
+import qualified Data.Text.Array as A
+import qualified Data.Text.Internal as T
+import qualified Data.Text.Internal.Fusion.Common as S
+import qualified Data.Text.Unsafe as T
+import qualified Data.Text.Internal.Lazy.Fusion as S
+import Data.Text.Internal.Fusion.Types (PairS(..))
+import Data.Text.Internal.Lazy.Fusion (stream, unstream)
+import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,
+                                foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText)
+import Data.Text.Internal (firstf, safe, text)
+import Data.Text.Internal.Reverse (reverseNonEmpty)
+import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_)
+import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder)
+import Data.Text.Internal.Lazy.Search (indices)
+import qualified GHC.CString as GHC
+import qualified GHC.Exts as Exts
+import GHC.Prim (Addr#)
+import GHC.Stack (HasCallStack)
+#if __GLASGOW_HASKELL__ >= 914
+import qualified Language.Haskell.TH.Lift as TH
+#else
+import qualified Language.Haskell.TH.Syntax as TH
+import qualified Language.Haskell.TH.Lib as TH
+#endif
+import Text.Printf (PrintfArg, formatArg, formatString)
+
+-- $fusion
+--
+-- Starting from @text-1.3@ fusion is no longer implicit,
+-- and pipelines of transformations usually allocate intermediate 'Text' values.
+-- Users, who observe significant changes to performances,
+-- are encouraged to use fusion framework explicitly, employing
+-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".
+
+-- $replacement
+--
+-- A 'Text' value is a sequence of Unicode scalar values, as defined
+-- in
+-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.
+-- As such, a 'Text' cannot contain values in the range U+D800 to
+-- U+DFFF inclusive. Haskell implementations admit all Unicode code
+-- points
+-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)
+-- as 'Char' values, including code points from this invalid range.
+-- This means that there are some 'Char' values
+-- (corresponding to 'Data.Char.Surrogate' category) that are not valid
+-- Unicode scalar values, and the functions in this module must handle
+-- those cases.
+--
+-- Within this module, many functions construct a 'Text' from one or
+-- more 'Char' values. Those functions will substitute 'Char' values
+-- that are not valid Unicode scalar values with the replacement
+-- character \"&#xfffd;\" (U+FFFD).  Functions that perform this
+-- inspection and replacement are documented with the phrase
+-- \"Performs replacement on invalid scalar values\". The functions replace
+-- invalid scalar values, instead of dropping them, as a security
+-- measure. For details, see
+-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)
+
+-- $setup
+-- >>> :set -package transformers
+-- >>> import Control.Monad.Trans.State
+-- >>> import Data.Text
+-- >>> import qualified Data.Text as T
+-- >>> :seti -XOverloadedStrings
+
+instance Eq Text where
+    (==) = equal
+    {-# INLINE (==) #-}
+
+instance Ord Text where
+    compare = compareText
+
+compareText :: Text -> Text -> Ordering
+compareText Empty Empty = EQ
+compareText Empty _     = LT
+compareText _     Empty = GT
+compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =
+  A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of
+    LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs)
+    EQ -> compareText as bs
+    GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs
+-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),
+-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering
+-- of underlying bytearrays, no decoding is needed.
+
+instance Show Text where
+    showsPrec p ps r = showsPrec p (unpack ps) r
+
+instance Read Text where
+    readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]
+
+-- | @since 1.2.2.0
+instance Semigroup Text where
+    (<>) = append
+    stimes n _ | n < 0 = P.error "Data.Text.Lazy.stimes: given number is negative!"
+    stimes n a =
+      let nInt64 = fromIntegral n :: Int64
+          len = if n == fromIntegral nInt64 && nInt64 >= 0 then nInt64 else P.maxBound
+          -- We clamp the length to maxBound :: Int64.
+          -- To tell the difference, the caller would have to skip through 2^63 chunks.
+      in replicate len a
+
+instance Monoid Text where
+    mempty  = empty
+    mappend = (<>)
+    mconcat = concat
+
+-- | Performs replacement on invalid scalar values:
+--
+-- >>> :set -XOverloadedStrings
+-- >>> "\55555" :: Data.Text.Lazy.Text
+-- "\65533"
+instance IsString Text where
+    fromString = pack
+
+-- | Performs replacement on invalid scalar values:
+--
+-- >>> :set -XOverloadedLists
+-- >>> ['\55555'] :: Data.Text.Lazy.Text
+-- "\65533"
+--
+-- @since 1.2.0.0
+instance Exts.IsList Text where
+    type Item Text = Char
+    fromList       = pack
+    toList         = unpack
+
+instance NFData Text where
+    rnf Empty        = ()
+    rnf (Chunk _ ts) = rnf ts
+
+-- | @since 1.2.1.0
+instance Binary Text where
+    put t = do
+      -- This needs to be in sync with the Binary instance for ByteString
+      -- in the binary package.
+      put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t)
+      putBuilder (encodeUtf8Builder t)
+    get   = do
+      bs <- get
+      case decodeUtf8' bs of
+        P.Left exn -> P.fail (P.show exn)
+        P.Right a -> P.return a
+
+-- | This instance preserves data abstraction at the cost of inefficiency.
+-- We omit reflection services for the sake of data abstraction.
+--
+-- This instance was created by copying the updated behavior of
+-- @"Data.Text".@'Data.Text.Text'
+instance Data Text where
+  gfoldl f z txt = z pack `f` (unpack txt)
+  toConstr _     = packConstr
+  gunfold k z c  = case constrIndex c of
+    1 -> k (z pack)
+    _ -> error "Data.Text.Lazy.Text.gunfold"
+  dataTypeOf _   = textDataType
+
+-- | @since 1.2.4.0
+instance TH.Lift Text where
+#if __GLASGOW_HASKELL__ >= 900
+  lift x = [| fromStrict $(TH.lift . toStrict $ x) |]
+#else
+  lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict
+#endif
+#if __GLASGOW_HASKELL__ >= 914
+  liftTyped = TH.defaultLiftTyped
+#elif __GLASGOW_HASKELL__ >= 900
+  liftTyped = TH.unsafeCodeCoerce . TH.lift
+#elif __GLASGOW_HASKELL__ >= 810
+  liftTyped = TH.unsafeTExpCoerce . TH.lift
+#endif
+
+-- | @since 1.2.2.0
+instance PrintfArg Text where
+  formatArg txt = formatString $ unpack txt
+
+packConstr :: Constr
+packConstr = mkConstr textDataType "pack" [] Prefix
+
+textDataType :: DataType
+textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr]
+
+-- | /O(n)/ Convert a 'String' into a 'Text'.
+--
+-- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id':
+--
+-- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555")
+-- "\65533"
+pack ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  String -> Text
+pack = unstream . S.streamList . L.map safe
+{-# INLINE [1] pack #-}
+
+-- | /O(n)/ Convert a 'Text' into a 'String'.
+unpack ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Text -> String
+unpack t = S.unstreamList (stream t)
+{-# NOINLINE unpack #-}
+
+foldrFB :: (Char -> b -> b) -> b -> Text -> b
+foldrFB = foldr
+{-# INLINE [0] foldrFB #-}
+
+-- List fusion rules for `unpack`:
+-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build`
+--   fuses if `foldr` is applied to it.
+-- * If it doesn't fuse: In phase 1, `build` inlines to give us `foldrFB (:) []`
+--   and we rewrite that back to `unpack`.
+-- * If it fuses: In phase 0, `foldrFB` inlines and `foldr` inlines. GHC
+--   optimizes the fused code.
+{-# RULES
+"Text.Lazy.unpack"     [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrFB lcons lnil t)
+"Text.Lazy.unpackBack" [1]  foldrFB (:) [] = unpack
+  #-}
+
+-- | /O(n)/ Convert a literal string into a Text.
+unpackCString# :: Addr# -> Text
+unpackCString# addr# = unstream (S.streamCString# addr#)
+{-# NOINLINE unpackCString# #-}
+
+{-# RULES "TEXT literal" forall a.
+    unstream (S.streamList (L.map safe (GHC.unpackCString# a)))
+      = unpackCString# a #-}
+
+{-# RULES "TEXT literal UTF8" forall a.
+    unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a)))
+      = unpackCString# a #-}
+
+{-# RULES "LAZY TEXT empty literal"
+    unstream (S.streamList (L.map safe []))
+      = Empty #-}
+
+{-# RULES "LAZY TEXT empty literal" forall a.
+    unstream (S.streamList (L.map safe [a]))
+      = Chunk (T.singleton a) Empty #-}
+
+-- | /O(1)/ Convert a character into a Text.
+-- Performs replacement on invalid scalar values.
+singleton :: Char -> Text
+singleton c = Chunk (T.singleton c) Empty
+{-# INLINE [1] singleton #-}
+
+-- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'.
+fromChunks :: [T.Text] -> Text
+fromChunks cs = L.foldr chunk Empty cs
+
+-- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's.
+toChunks :: Text -> [T.Text]
+toChunks cs = foldrChunks (:) [] cs
+
+-- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'.
+toStrict :: LazyText -> T.StrictText
+toStrict t = T.concat (toChunks t)
+{-# INLINE [1] toStrict #-}
+
+-- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'.
+fromStrict :: T.StrictText -> LazyText
+fromStrict t = chunk t Empty
+{-# INLINE [1] fromStrict #-}
+
+-- -----------------------------------------------------------------------------
+-- * Basic functions
+
+-- | /O(1)/ Adds a character to the front of a 'Text'.
+cons :: Char -> Text -> Text
+cons c t = Chunk (T.singleton c) t
+{-# INLINE [1] cons #-}
+
+infixr 5 `cons`
+
+-- | /O(n)/ Adds a character to the end of a 'Text'.  This copies the
+-- entire array in the process.
+snoc :: Text -> Char -> Text
+snoc t c = foldrChunks Chunk (singleton c) t
+{-# INLINE [1] snoc #-}
+
+-- | /O(n\/c)/ Appends one 'Text' to another.
+append :: Text -> Text -> Text
+append xs ys = foldrChunks Chunk ys xs
+{-# INLINE [1] append #-}
+
+-- | /O(1)/ Returns the first character and rest of a 'Text', or
+-- 'Nothing' if empty.
+uncons :: Text -> Maybe (Char, Text)
+uncons Empty        = Nothing
+uncons (Chunk t ts) = Just (T.unsafeHead t, ts')
+  where ts' | T.compareLength t 1 == EQ = ts
+            | otherwise                 = Chunk (T.unsafeTail t) ts
+{-# INLINE uncons #-}
+
+-- | /O(1)/ Returns the first character of a 'Text', which must be
+-- non-empty. This is a partial function, consider using 'uncons' instead.
+head :: HasCallStack => Text -> Char
+head t = S.head (stream t)
+{-# INLINE head #-}
+
+-- | /O(1)/ Returns all characters after the head of a 'Text', which
+-- must be non-empty. This is a partial function, consider using 'uncons' instead.
+tail :: HasCallStack => Text -> Text
+tail (Chunk t ts) = chunk (T.tail t) ts
+tail Empty        = emptyError "tail"
+{-# INLINE [1] tail #-}
+
+-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must
+-- be non-empty. This is a partial function, consider using 'unsnoc' instead.
+init :: HasCallStack => Text -> Text
+init (Chunk t0 ts0) = go t0 ts0
+    where go t (Chunk t' ts) = Chunk t (go t' ts)
+          go t Empty         = chunk (T.init t) Empty
+init Empty = emptyError "init"
+{-# INLINE [1] init #-}
+
+-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if
+-- empty.
+--
+-- * It is no faster than using 'init' and 'last'.
+--
+-- @since 1.2.3.0
+unsnoc :: Text -> Maybe (Text, Char)
+unsnoc Empty          = Nothing
+unsnoc ts@(Chunk _ _) = Just (init ts, last ts)
+{-# INLINE unsnoc #-}
+
+-- | /O(1)/ Tests whether a 'Text' is empty or not.
+null :: Text -> Bool
+null Empty = True
+null _     = False
+{-# INLINE [1] null #-}
+
+-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/),
+-- to be used together with 'Empty'.
+--
+-- @since 2.1.2
+pattern (:<) :: Char -> Text -> Text
+pattern x :< xs <- (uncons -> Just (x, xs)) where
+  (:<) = cons
+infixr 5 :<
+{-# COMPLETE Empty, (:<) #-}
+
+-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/)
+-- to be used together with 'Empty'.
+--
+-- @since 2.1.2
+pattern (:>) :: Text -> Char -> Text
+pattern xs :> x <- (unsnoc -> Just (xs, x)) where
+  (:>) = snoc
+infixl 5 :>
+{-# COMPLETE Empty, (:>) #-}
+
+-- | /O(1)/ Tests whether a 'Text' contains exactly one character.
+isSingleton :: Text -> Bool
+isSingleton = S.isSingleton . stream
+{-# INLINE isSingleton #-}
+
+-- | /O(n\/c)/ Returns the last character of a 'Text', which must be
+-- non-empty. This is a partial function, consider using 'unsnoc' instead.
+last :: HasCallStack => Text -> Char
+last Empty        = emptyError "last"
+last (Chunk t ts) = go t ts
+    where go _ (Chunk t' ts') = go t' ts'
+          go t' Empty         = T.last t'
+{-# INLINE [1] last #-}
+
+-- | /O(n)/ Returns the number of characters in a 'Text'.
+length :: Text -> Int64
+length = foldlChunks go 0
+    where
+        go :: Int64 -> T.Text -> Int64
+        go l t = l + intToInt64 (T.length t)
+{-# INLINE [1] length #-}
+
+{-# RULES
+"TEXT length/map -> length" forall f t.
+    length (map f t) = length t
+"TEXT length/zipWith -> length" forall f t1 t2.
+    length (zipWith f t1 t2) = min (length t1) (length t2)
+"TEXT length/replicate -> n" forall n t.
+    length (replicate n t) = max 0 n P.* length t
+"TEXT length/cons -> length+1" forall c t.
+    length (cons c t) = 1 + length t
+"TEXT length/intersperse -> 2*length-1" forall c t.
+    length (intersperse c t) = max 0 (2 P.* length t - 1)
+"TEXT length/intercalate -> n*length" forall s ts.
+    length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)
+  #-}
+
+-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number.
+--
+-- @
+-- 'compareLength' t c = 'P.compare' ('length' t) c
+-- @
+--
+-- This function gives the same answer as comparing against the result
+-- of 'length', but can short circuit if the count of characters is
+-- greater than the number, and hence be more efficient.
+compareLength :: Text -> Int64 -> Ordering
+compareLength t c = S.compareLengthI (stream t) c
+{-# INLINE [1] compareLength #-}
+
+-- We don't apply those otherwise appealing length-to-compareLength
+-- rewrite rules here, because they can change the strictness
+-- properties of code.
+
+-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to
+-- each element of @t@. Performs replacement on
+-- invalid scalar values.
+map :: (Char -> Char) -> Text -> Text
+map f = foldrChunks (Chunk . mapNonEmpty f) Empty
+{-# INLINE [1] map #-}
+
+{-# RULES
+"TEXT map/map -> map" forall f g t.
+    map f (map g t) = map (f . safe . g) t
+#-}
+
+-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of
+-- 'Text's and concatenates the list after interspersing the first
+-- argument between each element of the list.
+intercalate :: Text -> [Text] -> Text
+intercalate t = concat . L.intersperse t
+{-# INLINE [1] intercalate #-}
+
+-- | /O(n)/ The 'intersperse' function takes a character and places it
+-- between the characters of a 'Text'. Performs
+-- replacement on invalid scalar values.
+intersperse :: Char -> Text -> Text
+intersperse c t = unstream (S.intersperse (safe c) (stream t))
+{-# INLINE [1] intersperse #-}
+
+-- | /O(n)/ Left-justify a string to the given length, using the
+-- specified fill character on the right. Performs
+-- replacement on invalid scalar values.
+--
+-- Examples:
+--
+-- > justifyLeft 7 'x' "foo"    == "fooxxxx"
+-- > justifyLeft 3 'x' "foobar" == "foobar"
+justifyLeft :: Int64 -> Char -> Text -> Text
+justifyLeft k c t
+    | len >= k  = t
+    | otherwise = t `append` replicateChunk (k-len) (T.singleton c)
+  where len = length t
+{-# INLINE [1] justifyLeft #-}
+
+-- | /O(n)/ Right-justify a string to the given length, using the
+-- specified fill character on the left.  Performs replacement on
+-- invalid scalar values.
+--
+-- Examples:
+--
+-- > justifyRight 7 'x' "bar"    == "xxxxbar"
+-- > justifyRight 3 'x' "foobar" == "foobar"
+justifyRight :: Int64 -> Char -> Text -> Text
+justifyRight k c t
+    | len >= k  = t
+    | otherwise = replicateChunk (k-len) (T.singleton c) `append` t
+  where len = length t
+{-# INLINE justifyRight #-}
+
+-- | /O(n)/ Center a string to the given length, using the specified
+-- fill character on either side.  Performs replacement on invalid
+-- scalar values.
+--
+-- Examples:
+--
+-- > center 8 'x' "HS" = "xxxHSxxx"
+center :: Int64 -> Char -> Text -> Text
+center k c t
+    | len >= k  = t
+    | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c)
+  where len = length t
+        d   = k - len
+        r   = d `quot` 2
+        l   = d - r
+{-# INLINE center #-}
+
+-- | /O(n)/ The 'transpose' function transposes the rows and columns
+-- of its 'Text' argument.  Note that this function uses 'pack',
+-- 'unpack', and the list version of transpose, and is thus not very
+-- efficient.
+transpose :: [Text] -> [Text]
+transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty)
+                     (L.transpose (L.map unpack ts))
+-- TODO: make this fast
+
+-- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order.
+reverse ::
+#if defined(ASSERTS)
+  HasCallStack =>
+#endif
+  Text -> Text
+reverse = rev Empty
+  where rev a Empty        = a
+        rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts
+
+-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in
+-- @haystack@ with @replacement@.
+--
+-- This function behaves as though it was defined as follows:
+--
+-- @
+-- replace needle replacement haystack =
+--   'intercalate' replacement ('splitOn' needle haystack)
+-- @
+--
+-- As this suggests, each occurrence is replaced exactly once.  So if
+-- @needle@ occurs in @replacement@, that occurrence will /not/ itself
+-- be replaced recursively:
+--
+-- > replace "oo" "foo" "oo" == "foo"
+--
+-- In cases where several instances of @needle@ overlap, only the
+-- first one will be replaced:
+--
+-- > replace "ofo" "bar" "ofofo" == "barfo"
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+replace :: HasCallStack
+        => Text
+        -- ^ @needle@ to search for.  If this string is empty, an
+        -- error will occur.
+        -> Text
+        -- ^ @replacement@ to replace @needle@ with.
+        -> Text
+        -- ^ @haystack@ in which to search.
+        -> Text
+replace s d = intercalate d . splitOn s
+{-# INLINE replace #-}
+
+-- ----------------------------------------------------------------------------
+-- ** Case conversions (folds)
+
+-- $case
+--
+-- With Unicode text, it is incorrect to use combinators like @map
+-- toUpper@ to case convert each character of a string individually.
+-- Instead, use the whole-string case conversion functions from this
+-- module.  For correctness in different writing systems, these
+-- functions may map one input character to two or three output
+-- characters.
+
+-- | /O(n)/ Convert a string to folded case.
+--
+-- This function is mainly useful for performing caseless (or case
+-- insensitive) string comparisons.
+--
+-- A string @x@ is a caseless match for a string @y@ if and only if:
+--
+-- @toCaseFold x == toCaseFold y@
+--
+-- The result string may be longer than the input string, and may
+-- differ from applying 'toLower' to the input string.  For instance,
+-- the Armenian small ligature men now (U+FB13) is case folded to the
+-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is
+-- case folded to the Greek small letter letter mu (U+03BC) instead of
+-- itself.
+toCaseFold :: Text -> Text
+toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty
+{-# INLINE toCaseFold #-}
+
+-- | /O(n)/ Convert a string to lower case, using simple case
+-- conversion.
+--
+-- The result string may be longer than the input string.  For
+-- instance, the Latin capital letter I with dot above (U+0130) maps
+-- to the sequence Latin small letter i (U+0069) followed by combining
+-- dot above (U+0307).
+toLower :: Text -> Text
+toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty
+{-# INLINE toLower #-}
+
+-- | /O(n)/ Convert a string to upper case, using simple case
+-- conversion.
+--
+-- The result string may be longer than the input string.  For
+-- instance, the German eszett (U+00DF) maps to the two-letter
+-- sequence SS.
+toUpper :: Text -> Text
+toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty
+{-# INLINE toUpper #-}
+
+
+-- | /O(n)/ Convert a string to title case, using simple case
+-- conversion.
+--
+-- The first letter (as determined by 'Data.Char.isLetter')
+-- of the input is converted to title case, as is
+-- every subsequent letter that immediately follows a non-letter.
+-- Every letter that immediately follows another letter is converted
+-- to lower case.
+--
+-- The result string may be longer than the input string. For example,
+-- the Latin small ligature &#xfb02; (U+FB02) is converted to the
+-- sequence Latin capital letter F (U+0046) followed by Latin small
+-- letter l (U+006C).
+--
+-- This function is not idempotent.
+-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).
+-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input
+-- is converted to title case, becoming two letters.
+-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter
+-- and as such is recognised as a letter by 'Data.Char.isLetter',
+-- so 'T.toTitle' @"ʼN"@ = @"'n"@.
+--
+-- /Note/: this function does not take language or culture specific
+-- rules into account. For instance, in English, different style
+-- guides disagree on whether the book name \"The Hill of the Red
+-- Fox\" is correctly title cased&#x2014;but this function will
+-- capitalize /every/ word.
+--
+-- @since 1.0.0.0
+toTitle :: Text -> Text
+toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty
+{-# INLINE toTitle #-}
+
+-- | /O(n)/ 'foldl', applied to a binary operator, a starting value
+-- (typically the left-identity of the operator), and a 'Text',
+-- reduces the 'Text' using the binary operator, from left to right.
+foldl :: (a -> Char -> a) -> a -> Text -> a
+foldl f z t = S.foldl f z (stream t)
+{-# INLINE foldl #-}
+
+-- | /O(n)/ A strict version of 'foldl'.
+--
+foldl' :: (a -> Char -> a) -> a -> Text -> a
+foldl' f z t = S.foldl' f z (stream t)
+{-# INLINE foldl' #-}
+
+-- | /O(n)/ A variant of 'foldl' that has no starting value argument,
+-- and thus must be applied to a non-empty 'Text'.
+foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldl1 f t = S.foldl1 f (stream t)
+{-# INLINE foldl1 #-}
+
+-- | /O(n)/ A strict version of 'foldl1'.
+foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldl1' f t = S.foldl1' f (stream t)
+{-# INLINE foldl1' #-}
+
+-- | /O(n)/ A monadic version of 'foldl''.
+--
+-- @since 2.1.2
+foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a
+foldlM' f z t = S.foldlM' f z (stream t)
+{-# INLINE foldlM' #-}
+
+-- | /O(n)/ 'foldr', applied to a binary operator, a starting value
+-- (typically the right-identity of the operator), and a 'Text',
+-- reduces the 'Text' using the binary operator, from right to left.
+--
+-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually
+-- traverses the 'Text' from left to right, only as far as it needs to.
+--
+-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr':
+--
+-- @
+-- head :: Text -> Char
+-- head = foldr const (error "head empty")
+-- @
+foldr :: (Char -> a -> a) -> a -> Text -> a
+foldr f z t = S.foldr f z (stream t)
+{-# INLINE foldr #-}
+
+-- | /O(n)/ A variant of 'foldr' that has no starting value argument,
+-- and thus must be applied to a non-empty 'Text'.
+foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char
+foldr1 f t = S.foldr1 f (stream t)
+{-# INLINE foldr1 #-}
+
+-- | /O(n)/ Concatenate a list of 'Text's.
+concat :: [Text] -> Text
+concat []                    = Empty
+concat (Empty : css)         = concat css
+concat (Chunk c Empty : css) = Chunk c (concat css)
+concat (Chunk c cs : css)    = Chunk c (concat (cs : css))
+{-# INLINE concat #-}
+
+-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and
+-- concatenate the results.
+concatMap :: (Char -> Text) -> Text -> Text
+concatMap f = concat . foldr ((:) . f) []
+{-# INLINE concatMap #-}
+
+-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
+-- 'Text' @t@ satisfies the predicate @p@.
+any :: (Char -> Bool) -> Text -> Bool
+any p t = S.any p (stream t)
+{-# INLINE any #-}
+
+-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
+-- 'Text' @t@ satisfy the predicate @p@.
+all :: (Char -> Bool) -> Text -> Bool
+all p t = S.all p (stream t)
+{-# INLINE all #-}
+
+-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which
+-- must be non-empty.
+maximum :: HasCallStack => Text -> Char
+maximum t = S.maximum (stream t)
+{-# INLINE maximum #-}
+
+-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which
+-- must be non-empty.
+minimum :: HasCallStack => Text -> Char
+minimum t = S.minimum (stream t)
+{-# INLINE minimum #-}
+
+-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only
+--   U+0000 through U+007F).
+--
+-- This is a more efficient version of @'all' 'Data.Char.isAscii'@.
+--
+-- >>> isAscii ""
+-- True
+--
+-- >>> isAscii "abc\NUL"
+-- True
+--
+-- >>> isAscii "abcd€"
+-- False
+--
+-- prop> isAscii t == all (< '\x80') t
+--
+-- @since 2.0.2
+isAscii :: Text -> Bool
+isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True
+
+-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of
+-- successive reduced values from the left.
+-- Performs replacement on invalid scalar values.
+--
+-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
+--
+-- Note that
+--
+-- > last (scanl f z xs) == foldl f z xs.
+scanl :: (Char -> Char -> Char) -> Char -> Text -> Text
+scanl f z t = cons z $ P.snd $
+  mapAccumL (\acc c -> let c' = f acc c in (c', c')) (safe z) t
+-- This is a bit suboptimal: we could have used
+-- Data.Text.scanl for the first chunk and mapAccumL
+-- for subsequent ones, but but I doubt anyone cares
+-- about the performance of 'scanl' much.
+{-# INLINE scanl #-}
+
+-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting
+-- value argument.  Performs replacement on invalid scalar values.
+--
+-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]
+scanl1 :: (Char -> Char -> Char) -> Text -> Text
+scanl1 f t0 = case uncons t0 of
+                Nothing -> empty
+                Just (t,ts) -> scanl f t ts
+{-# INLINE scanl1 #-}
+
+-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'.  Performs
+-- replacement on invalid scalar values.
+--
+-- > scanr f v == reverse . scanl (flip f) v . reverse
+scanr :: (Char -> Char -> Char) -> Char -> Text -> Text
+scanr f z t = (`snoc` z) $ P.snd $
+  mapAccumR (\acc c -> let c' = f c acc in (c', c')) (safe z) t
+-- See the comment for 'scanl' above.
+{-# INLINE scanr #-}
+
+-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting
+-- value argument.  Performs replacement on invalid scalar values.
+scanr1 :: (Char -> Char -> Char) -> Text -> Text
+scanr1 f t | null t    = empty
+           | otherwise = scanr f (last t) (init t)
+
+-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a
+-- function to each element of a 'Text', passing an accumulating
+-- parameter from left to right, and returns a final 'Text'.  Performs
+-- replacement on invalid scalar values.
+mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
+mapAccumL f = go
+  where
+    go z (Chunk c cs)    = (z'', Chunk c' cs')
+        where (z',  c')  = T.mapAccumL f z c
+              (z'', cs') = go z' cs
+    go z Empty           = (z, Empty)
+{-# INLINE mapAccumL #-}
+
+-- | The 'mapAccumR' function behaves like a combination of 'map' and
+-- a strict 'foldr'; it applies a function to each element of a
+-- 'Text', passing an accumulating parameter from right to left, and
+-- returning a final value of this accumulator together with the new
+-- 'Text'.  Performs replacement on invalid scalar values.
+mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)
+mapAccumR f = go
+  where
+    go z (Chunk c cs)   = (z'', Chunk c' cs')
+        where (z'', c') = T.mapAccumR f z' c
+              (z', cs') = go z cs
+    go z Empty          = (z, Empty)
+{-# INLINE mapAccumR #-}
+
+-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every
+-- element.
+--
+-- @since 1.2.0.5
+repeat :: Char -> Text
+repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t
+            in t
+
+-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input
+-- @t@ repeated @n@ times.
+replicate :: Int64 -> Text -> Text
+replicate n
+  | n <= 0 = P.const Empty
+  | otherwise = \case
+    Empty -> Empty
+    Chunk t Empty -> replicateChunk n t
+    t -> concat (rep n)
+      where
+        rep 0 = []
+        rep i = t : rep (i - 1)
+{-# INLINE [1] replicate #-}
+
+replicateChunk :: Int64 -> T.Text -> Text
+replicateChunk !n !t@(T.Text _ _ len)
+  | n <= 0 = Empty
+  | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk)
+  where
+    perChunk = defaultChunkSize `quot` len
+    normalChunk = T.replicate perChunk t
+    (q, r) = n `P.quotRem` intToInt64 perChunk
+    headChunk = T.replicate (int64ToInt r) t
+{-# INLINE replicateChunk #-}
+
+-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or
+-- equivalently, the infinite repetition of the original 'Text'.
+--
+-- @since 1.2.0.5
+cycle :: HasCallStack => Text -> Text
+cycle Empty = emptyError "cycle"
+cycle t     = let t' = foldrChunks Chunk t' t
+               in t'
+
+-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications
+-- of @f@ to @x@:
+--
+-- > iterate f x == [x, f x, f (f x), ...]
+--
+-- @since 1.2.0.5
+iterate :: (Char -> Char) -> Char -> Text
+iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))
+               in t c
+
+-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'
+-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a
+-- 'Text' from a seed value. The function takes the element and
+-- returns 'Nothing' if it is done producing the 'Text', otherwise
+-- 'Just' @(a,b)@.  In this case, @a@ is the next 'Char' in the
+-- string, and @b@ is the seed value for further production.
+-- Performs replacement on invalid scalar values.
+unfoldr :: (a -> Maybe (Char,a)) -> a -> Text
+unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)
+{-# INLINE unfoldr #-}
+
+-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed
+-- value. However, the length of the result should be limited by the
+-- first argument to 'unfoldrN'. This function is more efficient than
+-- 'unfoldr' when the maximum length of the result is known and
+-- correct, otherwise its performance is similar to 'unfoldr'.
+-- Performs replacement on invalid scalar values.
+unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text
+unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)
+{-# INLINE unfoldrN #-}
+
+-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the
+-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than
+-- the length of the Text.
+take :: Int64 -> Text -> Text
+take i _ | i <= 0 = Empty
+take i t0         = take' i t0
+  where
+    take' :: Int64 -> Text -> Text
+    take' 0 _            = Empty
+    take' _ Empty        = Empty
+    take' n (Chunk t@(T.Text arr off _) ts)
+        | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =
+          if m >= 0
+          then fromStrict (T.Text arr off m)
+          else Chunk t (take' (n + intToInt64 m) ts)
+
+        | n < l     = Chunk (T.take (int64ToInt n) t) Empty
+        | otherwise = Chunk t (take' (n - l) ts)
+        where l = intToInt64 (T.length t)
+{-# INLINE [1] take #-}
+
+-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after
+-- taking @n@ characters from the end of @t@.
+--
+-- Examples:
+--
+-- > takeEnd 3 "foobar" == "bar"
+--
+-- @since 1.1.1.0
+takeEnd :: Int64 -> Text -> Text
+takeEnd n t0
+    | n <= 0    = empty
+    | otherwise = takeChunk n empty . L.reverse . toChunks $ t0
+  where
+    takeChunk :: Int64 -> Text -> [T.Text] -> Text
+    takeChunk _ acc [] = acc
+    takeChunk i acc (t:ts)
+      | i <= l    = chunk (T.takeEnd (int64ToInt i) t) acc
+      | otherwise = takeChunk (i-l) (Chunk t acc) ts
+      where l = intToInt64 (T.length t)
+
+-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the
+-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@
+-- is greater than the length of the 'Text'.
+drop :: Int64 -> Text -> Text
+drop i t0
+    | i <= 0    = t0
+    | otherwise = drop' i t0
+  where
+    drop' :: Int64 -> Text -> Text
+    drop' 0 ts           = ts
+    drop' _ Empty        = Empty
+    drop' n (Chunk t@(T.Text arr off len) ts)
+        | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =
+          if m >= 0
+          then chunk (T.Text arr (off + m) (len - m)) ts
+          else drop' (n + intToInt64 m) ts
+
+        | n < l     = Chunk (T.drop (int64ToInt n) t) ts
+        | otherwise = drop' (n - l) ts
+        where l   = intToInt64 (T.length t)
+{-# INLINE [1] drop #-}
+
+-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after
+-- dropping @n@ characters from the end of @t@.
+--
+-- Examples:
+--
+-- > dropEnd 3 "foobar" == "foo"
+--
+-- @since 1.1.1.0
+dropEnd :: Int64 -> Text -> Text
+dropEnd n t0
+    | n <= 0    = t0
+    | otherwise = dropChunk n . L.reverse . toChunks $ t0
+  where
+    dropChunk :: Int64 -> [T.Text] -> Text
+    dropChunk _ [] = empty
+    dropChunk m (t:ts)
+      | m >= l    = dropChunk (m-l) ts
+      | otherwise = fromChunks . L.reverse $
+                    T.dropEnd (int64ToInt m) t : ts
+      where l = intToInt64 (T.length t)
+
+-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8'
+-- values dropped, or the empty 'Text' if @n@ is greater than the
+-- number of 'Word8' values present.
+dropWords :: Int64 -> Text -> Text
+dropWords i t0
+    | i <= 0    = t0
+    | otherwise = drop' i t0
+  where
+    drop' :: Int64 -> Text -> Text
+    drop' 0 ts           = ts
+    drop' _ Empty        = Empty
+    drop' n (Chunk (T.Text arr off len) ts)
+        | n < len'  = chunk (text arr (off+n') (len-n')) ts
+        | otherwise = drop' (n - len') ts
+        where len'  = intToInt64 len
+              n'    = int64ToInt n
+
+-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',
+-- returns the longest prefix (possibly empty) of elements that
+-- satisfy @p@.
+takeWhile :: (Char -> Bool) -> Text -> Text
+takeWhile p t0 = takeWhile' t0
+  where takeWhile' Empty        = Empty
+        takeWhile' (Chunk t ts) =
+          case T.findIndex (not . p) t of
+            Just n | n > 0     -> Chunk (T.take n t) Empty
+                   | otherwise -> Empty
+            Nothing            -> Chunk t (takeWhile' ts)
+{-# INLINE [1] takeWhile #-}
+
+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',
+-- returns the longest suffix (possibly empty) of elements that
+-- satisfy @p@.
+-- Examples:
+--
+-- > takeWhileEnd (=='o') "foo" == "oo"
+--
+-- @since 1.2.2.0
+takeWhileEnd :: (Char -> Bool) -> Text -> Text
+takeWhileEnd p = takeChunk empty . L.reverse . toChunks
+  where takeChunk acc []     = acc
+        takeChunk acc (t:ts)
+          | T.lengthWord8 t' < T.lengthWord8 t
+                             = chunk t' acc
+          | otherwise        = takeChunk (Chunk t' acc) ts
+          where t' = T.takeWhileEnd p t
+{-# INLINE takeWhileEnd #-}
+
+-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
+-- 'takeWhile' @p@ @t@.
+dropWhile :: (Char -> Bool) -> Text -> Text
+dropWhile p t0 = dropWhile' t0
+  where dropWhile' Empty        = Empty
+        dropWhile' (Chunk t ts) =
+          case T.findIndex (not . p) t of
+            Just n  -> Chunk (T.drop n t) ts
+            Nothing -> dropWhile' ts
+{-# INLINE [1] dropWhile #-}
+
+-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after
+-- dropping characters that satisfy the predicate @p@ from the end of
+-- @t@.
+--
+-- Examples:
+--
+-- > dropWhileEnd (=='.') "foo..." == "foo"
+dropWhileEnd :: (Char -> Bool) -> Text -> Text
+dropWhileEnd p = go
+  where go Empty = Empty
+        go (Chunk t Empty) = if T.null t'
+                             then Empty
+                             else Chunk t' Empty
+            where t' = T.dropWhileEnd p t
+        go (Chunk t ts) = case go ts of
+                            Empty -> go (Chunk t Empty)
+                            ts' -> Chunk t ts'
+{-# INLINE dropWhileEnd #-}
+
+-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after
+-- dropping characters that satisfy the predicate @p@ from both the
+-- beginning and end of @t@.
+dropAround :: (Char -> Bool) -> Text -> Text
+dropAround p = dropWhile p . dropWhileEnd p
+{-# INLINE [1] dropAround #-}
+
+-- | /O(n)/ Remove leading white space from a string.  Equivalent to:
+--
+-- > dropWhile isSpace
+stripStart :: Text -> Text
+stripStart = dropWhile isSpace
+{-# INLINE stripStart #-}
+
+-- | /O(n)/ Remove trailing white space from a string.  Equivalent to:
+--
+-- > dropWhileEnd isSpace
+stripEnd :: Text -> Text
+stripEnd = dropWhileEnd isSpace
+{-# INLINE [1] stripEnd #-}
+
+-- | /O(n)/ Remove leading and trailing white space from a string.
+-- Equivalent to:
+--
+-- > dropAround isSpace
+strip :: Text -> Text
+strip = dropAround isSpace
+{-# INLINE [1] strip #-}
+
+-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a
+-- prefix of @t@ of length @n@, and whose second is the remainder of
+-- the string. It is equivalent to @('take' n t, 'drop' n t)@.
+splitAt :: Int64 -> Text -> (Text, Text)
+splitAt = loop
+  where
+    loop :: Int64 -> Text -> (Text, Text)
+    loop !_ Empty     = (empty, empty)
+    loop n t | n <= 0 = (empty, t)
+    loop n (Chunk t@(T.Text arr off len) ts)
+         | n > mx = let (ts', ts'') = loop (n - intToInt64 (T.length t)) ts
+                    in (Chunk t ts', ts'')
+         | m > 0, m >= len = (Chunk t Empty, ts)
+         | m > 0 = let t' = T.Text arr off m
+                       t'' = T.Text arr (off+m) (len-m)
+                   in (Chunk t' Empty, Chunk t'' ts)
+         | otherwise = let (ts', ts'') = loop (n + intToInt64 m) ts
+                       in (Chunk t ts', ts'')
+         where
+         mx = intToInt64 P.maxBound
+         m = T.measureOff (int64ToInt n) t
+
+
+-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first
+-- element is a prefix of @t@ whose chunks contain @n@ 'Word8'
+-- values, and whose second is the remainder of the string.
+splitAtWord :: Int64 -> Text -> PairS Text Text
+splitAtWord !_ Empty = empty :*: empty
+splitAtWord x (Chunk c@(T.Text arr off len) cs)
+    | y >= len  = let h :*: t = splitAtWord (x-intToInt64 len) cs
+                  in  Chunk c h :*: t
+    | otherwise = chunk (text arr off y) empty :*:
+                  chunk (text arr (off+y) (len-y)) cs
+    where y = int64ToInt x
+
+-- | /O(n+m)/ Find the first instance of @needle@ (which must be
+-- non-'null') in @haystack@.  The first element of the returned tuple
+-- is the prefix of @haystack@ before @needle@ is matched.  The second
+-- is the remainder of @haystack@, starting with the match.
+--
+-- Examples:
+--
+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")
+-- > breakOn "/" "foobar"   ==> ("foobar", "")
+--
+-- Laws:
+--
+-- > append prefix match == haystack
+-- >   where (prefix, match) = breakOn needle haystack
+--
+-- If you need to break a string by a substring repeatedly (e.g. you
+-- want to break on every instance of a substring), use 'breakOnAll'
+-- instead, as it has lower startup overhead.
+--
+-- This function is strict in its first argument, and lazy in its
+-- second.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+breakOn :: HasCallStack => Text -> Text -> (Text, Text)
+breakOn pat src
+    | null pat  = emptyError "breakOn"
+    | otherwise = case indices pat src of
+                    []    -> (src, empty)
+                    (x:_) -> let h :*: t = splitAtWord x src
+                             in  (h, t)
+
+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.
+--
+-- The first element of the returned tuple is the prefix of @haystack@
+-- up to and including the last match of @needle@.  The second is the
+-- remainder of @haystack@, following the match.
+--
+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")
+breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)
+breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)
+                   in  (reverse b, reverse a)
+{-# INLINE breakOnEnd #-}
+
+-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in
+-- @haystack@.  Each element of the returned list consists of a pair:
+--
+-- * The entire string prior to the /k/th match (i.e. the prefix)
+--
+-- * The /k/th match, followed by the remainder of the string
+--
+-- Examples:
+--
+-- > breakOnAll "::" ""
+-- > ==> []
+-- > breakOnAll "/" "a/b/c/"
+-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]
+--
+-- This function is strict in its first argument, and lazy in its
+-- second.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+--
+-- The @needle@ parameter may not be empty.
+breakOnAll :: HasCallStack
+           => Text              -- ^ @needle@ to search for
+           -> Text              -- ^ @haystack@ in which to search
+           -> [(Text, Text)]
+breakOnAll pat src
+    | null pat  = emptyError "breakOnAll"
+    | otherwise = go 0 empty src (indices pat src)
+  where
+    go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s
+                           h'      = append p h
+                       in (h',t) : go x h' t xs
+    go _  _ _ _      = []
+
+-- | /O(n)/ 'break' is like 'span', but the prefix returned is over
+-- elements that fail the predicate @p@.
+--
+-- >>> T.break (=='c') "180cm"
+-- ("180","cm")
+break :: (Char -> Bool) -> Text -> (Text, Text)
+break p t0 = break' t0
+  where break' Empty          = (empty, empty)
+        break' c@(Chunk t ts) =
+          case T.findIndex p t of
+            Nothing      -> let (ts', ts'') = break' ts
+                            in (Chunk t ts', ts'')
+            Just n | n == 0    -> (Empty, c)
+                   | otherwise -> let (a,b) = T.splitAt n t
+                                  in (Chunk a Empty, Chunk b ts)
+
+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns
+-- a pair whose first element is the longest prefix (possibly empty)
+-- of @t@ of elements that satisfy @p@, and whose second is the
+-- remainder of the text.
+--
+-- >>> T.span (=='0') "000AB"
+-- ("000","AB")
+span :: (Char -> Bool) -> Text -> (Text, Text)
+span p = break (not . p)
+{-# INLINE span #-}
+
+-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@,
+-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of
+-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text.
+--
+-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97
+-- (("abc","efg"),101)
+--
+-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity':
+--
+-- @
+-- -- for all p :: Char -> Bool
+-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p)
+-- @
+--
+-- @since 2.0.1
+spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)
+spanM p t0 = go t0
+  where
+    go Empty = pure (empty, empty)
+    go (Chunk t ts) = do
+        (t1, t2) <- T.spanM p t
+        if T.null t2 then first (chunk t) <$> go ts
+        else pure (chunk t1 empty, Chunk t2 ts)
+{-# INLINE spanM #-}
+
+-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@,
+-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of
+-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text.
+--
+-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122
+-- (("tuv","xyz"),118)
+--
+-- @
+-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p
+-- @
+--
+-- @since 2.0.1
+spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)
+spanEndM p t0 = go t0
+  where
+    go Empty = pure (empty, empty)
+    go (Chunk t ts) = do
+        (t3, t4) <- go ts
+        if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t
+        else pure (Chunk t t3, t4)
+{-# INLINE spanEndM #-}
+
+-- | The 'group' function takes a 'Text' and returns a list of 'Text's
+-- such that the concatenation of the result is equal to the argument.
+-- Moreover, each sublist in the result contains only equal elements.
+-- For example,
+--
+-- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"]
+--
+-- It is a special case of 'groupBy', which allows the programmer to
+-- supply their own equality test.
+group :: Text -> [Text]
+group =  groupBy (==)
+{-# INLINE group #-}
+
+-- | The 'groupBy' function is the non-overloaded version of 'group'.
+groupBy :: (Char -> Char -> Bool) -> Text -> [Text]
+groupBy _  Empty        = []
+groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs
+                          where (ys,zs) = span (eq x) xs
+                                x  = T.unsafeHead t
+                                xs = chunk (T.unsafeTail t) ts
+
+-- | /O(n²)/ Return all initial segments of the given 'Text',
+-- shortest first.
+inits :: Text -> [Text]
+inits = (NE.toList P.$!) . initsNE
+
+-- | /O(n²)/ Return all initial segments of the given 'Text',
+-- shortest first.
+--
+-- @since 2.1.2
+initsNE :: Text -> NonEmpty Text
+initsNE ts0 = Empty NE.:| inits' 0 ts0
+  where
+    inits' :: Int64  -- Number of previous chunks i
+           -> Text   -- The remainder after dropping i chunks from ts0
+           -> [Text] -- Prefixes longer than the first i chunks of ts0.
+    inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t))
+                          ++ inits' (i + 1) ts
+    inits' _ Empty         = []
+
+takeChunks :: Int64 -> Text -> T.Text -> Text
+takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk)
+takeChunks _ _ lastChunk = Chunk lastChunk Empty
+
+-- | /O(n)/ Return all final segments of the given 'Text', longest
+-- first.
+tails :: Text -> [Text]
+tails = (NE.toList P.$!) . tailsNE
+
+-- | /O(n)/ Return all final segments of the given 'Text', longest
+-- first.
+--
+-- @since 2.1.2
+tailsNE :: Text -> NonEmpty Text
+tailsNE Empty = Empty :| []
+tailsNE ts@(Chunk t ts')
+  | T.length t == 1 = ts :| tails ts'
+  | otherwise       = ts :| tails (Chunk (T.unsafeTail t) ts')
+
+-- $split
+--
+-- Splitting functions in this library do not perform character-wise
+-- copies to create substrings; they just construct new 'Text's that
+-- are slices of the original.
+
+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'
+-- argument (which cannot be an empty string), consuming the
+-- delimiter. An empty delimiter is invalid, and will cause an error
+-- to be raised.
+--
+-- Examples:
+--
+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]
+-- > splitOn "aaa"  "aaaXaaaXaaaXaaa"  == ["","X","X","X",""]
+-- > splitOn "x"    "x"                == ["",""]
+--
+-- and
+--
+-- > intercalate s . splitOn s         == id
+-- > splitOn (singleton c)             == split (==c)
+--
+-- (Note: the string @s@ to split on above cannot be empty.)
+--
+-- This function is strict in its first argument, and lazy in its
+-- second.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+splitOn :: HasCallStack
+        => Text
+        -- ^ String to split on. If this string is empty, an error
+        -- will occur.
+        -> Text
+        -- ^ Input text.
+        -> [Text]
+splitOn pat src
+    | null pat        = emptyError "splitOn"
+    | isSingleton pat = split (== head pat) src
+    | otherwise       = go 0 (indices pat src) src
+  where
+    go  _ []     cs = [cs]
+    go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs
+                      in  h : go (x+l) xs (dropWords l t)
+    l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat
+{-# INLINE [1] splitOn #-}
+
+{-# RULES
+"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.
+    splitOn (singleton c) t = split (==c) t
+  #-}
+
+-- | /O(n)/ Splits a 'Text' into components delimited by separators,
+-- where the predicate returns True for a separator element.  The
+-- resulting components do not contain the separators.  Two adjacent
+-- separators result in an empty component in the output.  eg.
+--
+-- > split (=='a') "aabbaca" == ["","","bb","c",""]
+-- > split (=='a') []        == [""]
+split :: (Char -> Bool) -> Text -> [Text]
+split _ Empty = [Empty]
+split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0
+  where comb acc (s:[]) Empty        = revChunks (s:acc) : []
+        comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts
+        comb acc (s:ss) ts           = revChunks (s:acc) : comb [] ss ts
+        comb _   []     _            = impossibleError "split"
+{-# INLINE split #-}
+
+-- | /O(n)/ Splits a 'Text' into components of length @k@.  The last
+-- element may be shorter than the other chunks, depending on the
+-- length of the input. Examples:
+--
+-- > chunksOf 3 "foobarbaz"   == ["foo","bar","baz"]
+-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]
+chunksOf :: Int64 -> Text -> [Text]
+chunksOf k = go
+  where
+    go t = case splitAt k t of
+             (a,b) | null a    -> []
+                   | otherwise -> a : go b
+{-# INLINE chunksOf #-}
+
+-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters
+-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.
+--
+-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.
+lines :: Text -> [Text]
+lines Empty = []
+lines t = NE.toList $ go t
+  where
+    go :: Text -> NonEmpty Text
+    go Empty = Empty :| []
+    go (Chunk x xs)
+      -- x is non-empty, so T.lines x is non-empty as well
+      | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs
+      | otherwise = case unsnocList (T.lines x) of
+      Nothing -> impossibleError "lines"
+      Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls
+
+prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text
+prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial!
+
+unsnocList :: [a] -> Maybe ([a], a)
+unsnocList [] = Nothing
+unsnocList (x : xs) = Just $ go x xs
+  where
+    go y [] = ([], y)
+    go y (z : zs) = first (y :) (go z zs)
+
+hasNlEnd :: T.Text -> Bool
+hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A
+
+-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's
+-- representing white space.
+words :: Text -> [Text]
+words = L.filter (not . null) . split isSpace
+{-# INLINE words #-}
+
+-- | /O(n)/ Joins lines, after appending a terminating newline to
+-- each.
+unlines :: [Text] -> Text
+unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []
+{-# INLINE unlines #-}
+
+-- | /O(n)/ Joins words using single space characters.
+unwords :: [Text] -> Text
+unwords = intercalate (singleton ' ')
+{-# INLINE unwords #-}
+
+-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is a prefix of the second.
+isPrefixOf :: Text -> Text -> Bool
+isPrefixOf Empty _  = True
+isPrefixOf _ Empty  = False
+isPrefixOf (Chunk x xs) (Chunk y ys)
+    | lx == ly  = x == y  && isPrefixOf xs ys
+    | lx <  ly  = x == yh && isPrefixOf xs (Chunk yt ys)
+    | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys
+  where (xh,xt) = T.splitAt ly x
+        (yh,yt) = T.splitAt lx y
+        lx = T.length x
+        ly = T.length y
+
+-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is a suffix of the second.
+isSuffixOf :: Text -> Text -> Bool
+isSuffixOf x y = reverse x `isPrefixOf` reverse y
+{-# INLINE isSuffixOf #-}
+-- TODO: a better implementation
+
+-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns
+-- 'True' if and only if the first is contained, wholly and intact, anywhere
+-- within the second.
+--
+-- This function is strict in its first argument, and lazy in its
+-- second.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+isInfixOf :: Text -> Text -> Bool
+isInfixOf needle haystack
+    | null needle        = True
+    | isSingleton needle = S.elem (head needle) . S.stream $ haystack
+    | otherwise          = not . L.null . indices needle $ haystack
+{-# INLINE [1] isInfixOf #-}
+
+-------------------------------------------------------------------------------
+-- * View patterns
+
+-- | /O(n)/ Return the suffix of the second string if its prefix
+-- matches the entire first string.
+--
+-- Examples:
+--
+-- > stripPrefix "foo" "foobar" == Just "bar"
+-- > stripPrefix ""    "baz"    == Just "baz"
+-- > stripPrefix "foo" "quux"   == Nothing
+--
+-- This is particularly useful with the @ViewPatterns@ extension to
+-- GHC, as follows:
+--
+-- > {-# LANGUAGE ViewPatterns #-}
+-- > import Data.Text.Lazy as T
+-- >
+-- > fnordLength :: Text -> Int
+-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf
+-- > fnordLength _                                 = -1
+stripPrefix :: Text -> Text -> Maybe Text
+stripPrefix p t
+    | null p    = Just t
+    | otherwise = case commonPrefixes p t of
+                    Just (_,c,r) | null c -> Just r
+                    _                     -> Nothing
+
+-- | /O(n)/ Find the longest non-empty common prefix of two strings
+-- and return it, along with the suffixes of each string at which they
+-- no longer match.
+--
+-- If the strings do not have a common prefix or either one is empty,
+-- this function returns 'Nothing'.
+--
+-- Examples:
+--
+-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")
+-- > commonPrefixes "veeble" "fetzer"  == Nothing
+-- > commonPrefixes "" "baz"           == Nothing
+commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)
+commonPrefixes Empty _ = Nothing
+commonPrefixes _ Empty = Nothing
+commonPrefixes a0 b0   = Just (go a0 b0 [])
+  where
+    go t0@(Chunk x xs) t1@(Chunk y ys) ps
+        = case T.commonPrefixes x y of
+            Just (p,a,b)
+              | T.null a  -> go xs (chunk b ys) (p:ps)
+              | T.null b  -> go (chunk a xs) ys (p:ps)
+              | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys)
+            Nothing       -> (fromChunks (L.reverse ps),t0,t1)
+    go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1)
+
+-- | /O(n)/ Return the prefix of the second string if its suffix
+-- matches the entire first string.
+--
+-- Examples:
+--
+-- > stripSuffix "bar" "foobar" == Just "foo"
+-- > stripSuffix ""    "baz"    == Just "baz"
+-- > stripSuffix "foo" "quux"   == Nothing
+--
+-- This is particularly useful with the @ViewPatterns@ extension to
+-- GHC, as follows:
+--
+-- > {-# LANGUAGE ViewPatterns #-}
+-- > import Data.Text.Lazy as T
+-- >
+-- > quuxLength :: Text -> Int
+-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre
+-- > quuxLength _                                = -1
+stripSuffix :: Text -> Text -> Maybe Text
+stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t)
+
+-- | /O(n)/ 'filter', applied to a predicate and a 'Text',
+-- returns a 'Text' containing those characters that satisfy the
+-- predicate.
+filter :: (Char -> Bool) -> Text -> Text
+filter p = foldrChunks (chunk . filter_ T.Text p) Empty
+{-# INLINE [1] filter #-}
+
+{-# RULES
+"TEXT filter/filter -> filter" forall p q t.
+    filter p (filter q t) = filter (\c -> q c && p c) t
+#-}
+
+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and
+-- returns the first element in matching the predicate, or 'Nothing'
+-- if there is no such element.
+find :: (Char -> Bool) -> Text -> Maybe Char
+find p t = S.findBy p (stream t)
+{-# INLINE find #-}
+
+-- | /O(n)/ The 'elem' function takes a character and a 'Text', and
+-- returns 'True' if the element is found in the given 'Text', or
+-- 'False' otherwise.
+elem :: Char -> Text -> Bool
+elem c t = S.any (== c) (stream t)
+{-# INLINE elem #-}
+
+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',
+-- and returns the pair of 'Text's with elements which do and do not
+-- satisfy the predicate, respectively; i.e.
+--
+-- > partition p t == (filter p t, filter (not . p) t)
+partition :: (Char -> Bool) -> Text -> (Text, Text)
+partition p t = (filter p t, filter (not . p) t)
+{-# INLINE partition #-}
+
+-- | /O(n)/ 'Text' index (subscript) operator, starting from 0.
+index :: HasCallStack => Text -> Int64 -> Char
+index lazyText ix
+  | ix < 0 = P.error $ "Data.Text.Lazy.index: negative index " ++ P.show ix
+  | otherwise = go lazyText ix
+  where
+    go :: Text -> Int64 -> Char
+    go Empty _ = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large"
+    go (Chunk t@(T.Text _ _ lenInBytes) ts) n = case toIntegralSized n of
+      Nothing ->
+        go ts (n - fromIntegral (T.length t))
+      Just n'
+        | off < 0 -> go ts (n + fromIntegral off)
+        | off == lenInBytes -> go ts 0
+        | otherwise -> ch
+        where
+          off = T.measureOff n' t
+          T.Iter ch _ = T.iter t off
+{-# INLINE index #-}
+
+-- | /O(n+m)/ The 'count' function returns the number of times the
+-- query string appears in the given 'Text'. An empty query string is
+-- invalid, and will cause an error to be raised.
+--
+-- In (unlikely) bad cases, this function's time complexity degrades
+-- towards /O(n*m)/.
+count :: HasCallStack => Text -> Text -> Int64
+count pat
+    | null pat        = emptyError "count"
+    | otherwise       = go 0  . indices pat
+  where go !n []     = n
+        go !n (_:xs) = go (n+1) xs
+{-# INLINE [1] count #-}
+
+{-# RULES
+"LAZY TEXT count/singleton -> countChar" [~1] forall c t.
+    count (singleton c) t = countChar c t
+  #-}
+
+-- | /O(n)/ The 'countChar' function returns the number of times the
+-- query element appears in the given 'Text'.
+countChar :: Char -> Text -> Int64
+countChar c t = S.countChar c (stream t)
+
+-- | /O(n)/ 'zip' takes two 'Text's and returns a list of
+-- corresponding pairs of bytes. If one input 'Text' is short,
+-- excess elements of the longer 'Text' are discarded. This is
+-- equivalent to a pair of 'unpack' operations.
+zip :: Text -> Text -> [(Char,Char)]
+zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
+{-# INLINE [0] zip #-}
+
+-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
+-- given as the first argument, instead of a tupling function.
+-- Performs replacement on invalid scalar values.
+zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
+zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))
+    where g a b = safe (f a b)
+{-# INLINE [0] zipWith #-}
+
+-- | Convert a value to lazy 'Text'.
+--
+-- @since 2.1.2
+show :: Show a => a -> Text
+show = pack . P.show
+
+revChunks :: [T.Text] -> Text
+revChunks = L.foldl' (flip chunk) Empty
+
+emptyError :: HasCallStack => String -> a
+emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input")
+
+impossibleError :: HasCallStack => String -> a
+impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case")
+
+intToInt64 :: Exts.Int -> Int64
+intToInt64 = fromIntegral
+
+int64ToInt :: Int64 -> Exts.Int
+int64ToInt = fromIntegral
+ src/Data/Text/Lazy/Builder.hs view
@@ -0,0 +1,56 @@+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-}+{-# LANGUAGE Trustworthy #-}++-----------------------------------------------------------------------------+-- |+-- Module      : Data.Text.Lazy.Builder+-- Copyright   : (c) 2013 Bryan O'Sullivan+--               (c) 2010 Johan Tibell+-- License     : BSD-style (see LICENSE)+--+-- Maintainer  : Johan Tibell <johan.tibell@gmail.com>+-- Portability : portable to Hugs and GHC+--+-- Efficient construction of lazy @Text@ values.  The principal+-- operations on a @Builder@ are @singleton@, @fromText@, and+-- @fromLazyText@, which construct new builders, and 'mappend', which+-- concatenates two builders.+--+-- To get maximum performance when building lazy @Text@ values using a+-- builder, associate @mappend@ calls to the right.  For example,+-- prefer+--+-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')+--+-- to+--+-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'+--+-- as the latter associates @mappend@ to the left. Or, equivalently,+-- prefer+--+--  > singleton 'a' <> singleton 'b' <> singleton 'c'+--+-- since the '<>' from recent versions of 'Data.Monoid' associates+-- to the right.++-----------------------------------------------------------------------------++module Data.Text.Lazy.Builder+   ( -- * The Builder type+     Builder+   , LazyTextBuilder+   , toLazyText+   , toLazyTextWith++     -- * Constructing Builders+   , singleton+   , fromText+   , fromLazyText+   , fromString++     -- * Flushing the buffer state+   , flush+   ) where++import Data.Text.Internal.Builder
+ src/Data/Text/Lazy/Builder/Int.hs view
@@ -0,0 +1,244 @@+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, ScopedTypeVariables,+    UnboxedTuples #-}+{-# LANGUAGE Trustworthy #-}++-- Module:      Data.Text.Lazy.Builder.Int+-- Copyright:   (c) 2013 Bryan O'Sullivan+--              (c) 2011 MailRank, Inc.+-- License:     BSD-style+-- Maintainer:  Bryan O'Sullivan <bos@serpentine.com>+-- Portability: portable+--+-- Efficiently write an integral value to a 'Builder'.++module Data.Text.Lazy.Builder.Int+    (+      decimal+    , hexadecimal+    ) where++import Data.Int (Int8, Int16, Int32, Int64)+import Data.Monoid (mempty)+import qualified Data.ByteString.Unsafe as B+import Data.Text.Internal.Builder.Functions ((<>), i2d)+import Data.Text.Internal.Builder+import Data.Text.Internal.Builder.Int.Digits (digits)+import Data.Text.Array+import Data.Word (Word, Word8, Word16, Word32, Word64)+import GHC.Base (quotInt, remInt)+import Control.Monad.ST+#if MIN_VERSION_base(4,11,0)+import Prelude hiding ((<>))+#endif++decimal :: Integral a => a -> Builder+{-# RULES "decimal/Int8" decimal = boundedDecimal :: Int8 -> Builder #-}+{-# RULES "decimal/Int" decimal = boundedDecimal :: Int -> Builder #-}+{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-}+{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-}+{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-}+{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-}+{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-}+{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-}+{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-}+{-# RULES "decimal/Word64" decimal = positive :: Word64 -> Builder #-}+{-# RULES "decimal/Integer" decimal = integer 10 :: Integer -> Builder #-}+decimal i = decimal' (<= -128) i+{-# NOINLINE decimal #-}++boundedDecimal :: (Integral a, Bounded a) => a -> Builder+{-# SPECIALIZE boundedDecimal :: Int -> Builder #-}+{-# SPECIALIZE boundedDecimal :: Int8 -> Builder #-}+{-# SPECIALIZE boundedDecimal :: Int16 -> Builder #-}+{-# SPECIALIZE boundedDecimal :: Int32 -> Builder #-}+{-# SPECIALIZE boundedDecimal :: Int64 -> Builder #-}+boundedDecimal i = decimal' (== minBound) i++decimal' :: (Integral a) => (a -> Bool) -> a -> Builder+{-# INLINE decimal' #-}+decimal' p i+    | i < 0 = if p i+              then let (q, r) = i `quotRem` 10+                       qq = -q+                       !n = countDigits qq+                   in writeN (n + 2) $ \marr off -> do+                       unsafeWrite marr off minus+                       posDecimal marr (off+1) n qq+                       unsafeWrite marr (off+n+1) (i2w (-r))+              else let j = -i+                       !n = countDigits j+                   in writeN (n + 1) $ \marr off ->+                       unsafeWrite marr off minus >> posDecimal marr (off+1) n j+    | otherwise = positive i++positive :: (Integral a) => a -> Builder+{-# SPECIALIZE positive :: Int -> Builder #-}+{-# SPECIALIZE positive :: Int8 -> Builder #-}+{-# SPECIALIZE positive :: Int16 -> Builder #-}+{-# SPECIALIZE positive :: Int32 -> Builder #-}+{-# SPECIALIZE positive :: Int64 -> Builder #-}+{-# SPECIALIZE positive :: Word -> Builder #-}+{-# SPECIALIZE positive :: Word8 -> Builder #-}+{-# SPECIALIZE positive :: Word16 -> Builder #-}+{-# SPECIALIZE positive :: Word32 -> Builder #-}+{-# SPECIALIZE positive :: Word64 -> Builder #-}+positive i+    | i < 10    = writeN 1 $ \marr off -> unsafeWrite marr off (i2w i)+    | otherwise = let !n = countDigits i+                  in writeN n $ \marr off -> posDecimal marr off n i++posDecimal :: (Integral a) =>+              forall s. MArray s -> Int -> Int -> a -> ST s ()+{-# INLINE posDecimal #-}+posDecimal marr off0 ds v0 = go (off0 + ds - 1) v0+  where go off v+           | v >= 100 = do+               let (q, r) = v `quotRem` 100+               write2 off r+               go (off - 2) q+           | v < 10    = unsafeWrite marr off (i2w v)+           | otherwise = write2 off v+        write2 off i0 = do+          let i = fromIntegral i0; j = i + i+          unsafeWrite marr off $ get (j + 1)+          unsafeWrite marr (off - 1) $ get j+        get = B.unsafeIndex digits++minus, zero :: Word8+{-# INLINE minus #-}+{-# INLINE zero #-}+minus = 45+zero = 48++i2w :: (Integral a) => a -> Word8+{-# INLINE i2w #-}+i2w v = zero + fromIntegral v++countDigits :: (Integral a) => a -> Int+{-# INLINE countDigits #-}+countDigits v0+  | fromIntegral v64 == v0 = go 1 v64+  | otherwise              = goBig 1 (toInteger v0)+  where v64 = fromIntegral v0+        goBig !k (v :: Integer)+           | v > big   = goBig (k + 19) (v `quot` big)+           | otherwise = go k (fromInteger v)+        big = 10000000000000000000+        go !k (v :: Word64)+           | v < 10    = k+           | v < 100   = k + 1+           | v < 1000  = k + 2+           | v < 1000000000000 =+               k + if v < 100000000+                   then if v < 1000000+                        then if v < 10000+                             then 3+                             else 4 + fin v 100000+                        else 6 + fin v 10000000+                   else if v < 10000000000+                        then 8 + fin v 1000000000+                        else 10 + fin v 100000000000+           | otherwise = go (k + 12) (v `quot` 1000000000000)+        fin v n = if v >= n then 1 else 0++hexadecimal :: Integral a => a -> Builder+{-# SPECIALIZE hexadecimal :: Int -> Builder #-}+{-# SPECIALIZE hexadecimal :: Int8 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Int16 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Int32 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Int64 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Word -> Builder #-}+{-# SPECIALIZE hexadecimal :: Word8 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Word16 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Word32 -> Builder #-}+{-# SPECIALIZE hexadecimal :: Word64 -> Builder #-}+{-# RULES "hexadecimal/Integer"+    hexadecimal = hexInteger :: Integer -> Builder #-}+hexadecimal i+    | i < 0     = error hexErrMsg+    | otherwise = go i+  where+    go n | n < 16    = hexDigit n+         | otherwise = go (n `quot` 16) <> hexDigit (n `rem` 16)+{-# NOINLINE[0] hexadecimal #-}++hexInteger :: Integer -> Builder+hexInteger i+    | i < 0     = error hexErrMsg+    | otherwise = integer 16 i++hexErrMsg :: String+hexErrMsg = "Data.Text.Lazy.Builder.Int.hexadecimal: applied to negative number"++hexDigit :: Integral a => a -> Builder+hexDigit n+    | n <= 9    = singleton $! i2d (fromIntegral n)+    | otherwise = singleton $! toEnum (fromIntegral n + 87)+{-# INLINE hexDigit #-}++data T = T !Integer !Int++integer :: Int -> Integer -> Builder+integer 10 i+    | i' <- fromInteger i, toInteger i' == i = decimal (i' :: Int)+integer 16 i+    | i' <- fromInteger i, toInteger i' == i = hexadecimal (i' :: Int)+integer base i+    | i < 0     = singleton '-' <> go (-i)+    | otherwise = go i+  where+    go n | n < maxInt = int (fromInteger n)+         | otherwise  = putH (splitf (maxInt * maxInt) n)++    splitf p n+      | p > n       = [n]+      | otherwise   = splith p (splitf (p*p) n)++    splith p (n:ns) = case n `quotRem` p of+                        (q, r) | q > 0     -> q : r : splitb p ns+                               | otherwise -> r : splitb p ns+    splith _ _      = error "splith: the impossible happened."++    splitb p (n:ns) = case n `quotRem` p of+                        (q, r) -> q : r : splitb p ns+    splitb _ _      = []++    T maxInt10 maxDigits10 =+        until ((>mi) . (*10) . fstT) (\(T n d) -> T (n*10) (d+1)) (T 10 1)+      where mi = toInteger (maxBound :: Int)+    T maxInt16 maxDigits16 =+        until ((>mi) . (*16) . fstT) (\(T n d) -> T (n*16) (d+1)) (T 16 1)+      where mi = toInteger (maxBound :: Int)++    fstT (T a _) = a++    maxInt | base == 10 = maxInt10+           | otherwise  = maxInt16+    maxDigits | base == 10 = maxDigits10+              | otherwise  = maxDigits16++    putH (n:ns) = case n `quotRem` maxInt of+                    (x, y)+                        | q > 0     -> int q <> pblock r <> putB ns+                        | otherwise -> int r <> putB ns+                        where q = fromInteger x+                              r = fromInteger y+    putH _ = error "putH: the impossible happened"++    putB (n:ns) = case n `quotRem` maxInt of+                    (x, y) -> pblock q <> pblock r <> putB ns+                        where q = fromInteger x+                              r = fromInteger y+    putB _ = Data.Monoid.mempty++    int :: Int -> Builder+    int x | base == 10 = decimal x+          | otherwise  = hexadecimal x++    pblock = loop maxDigits+      where+        loop !d !n+            | d == 1    = hexDigit n+            | otherwise = loop (d-1) q <> hexDigit r+            where q = n `quotInt` base+                  r = n `remInt` base
+ src/Data/Text/Lazy/Builder/RealFloat.hs view
@@ -0,0 +1,257 @@+{-# LANGUAGE CPP, OverloadedStrings #-}+{-# LANGUAGE Trustworthy #-}++{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}++-- |+-- Module:    Data.Text.Lazy.Builder.RealFloat+-- Copyright: (c) The University of Glasgow 1994-2002+-- License:   see libraries/base/LICENSE+--+-- Write a floating point value to a 'Builder'.++module Data.Text.Lazy.Builder.RealFloat+    (+      FPFormat(..)+    , realFloat+    , formatRealFloat+    ) where++import Data.Array.Base (unsafeAt)+import Data.Array.IArray+import Data.Text.Internal.Builder.Functions ((<>), i2d)+import Data.Text.Lazy.Builder.Int (decimal)+import Data.Text.Internal.Builder.RealFloat.Functions (roundTo)+import Data.Text.Lazy.Builder+import qualified Data.Text as T+#if MIN_VERSION_base(4,11,0)+import Prelude hiding ((<>))+#endif++-- | Control the rendering of floating point numbers.+data FPFormat = Exponent+              -- ^ Scientific notation (e.g. @2.3e123@).+              | Fixed+              -- ^ Standard decimal notation.+              | Generic+              -- ^ Use decimal notation for values between @0.1@ and+              -- @9,999,999@, and scientific notation otherwise.+                deriving (Enum, Read, Show, Bounded)++-- | Show a signed 'RealFloat' value to full precision,+-- using standard decimal notation for arguments whose absolute value lies+-- between @0.1@ and @9,999,999@, and scientific notation otherwise.+realFloat :: (RealFloat a) => a -> Builder+{-# SPECIALIZE realFloat :: Float -> Builder #-}+{-# SPECIALIZE realFloat :: Double -> Builder #-}+realFloat x = formatRealFloat Generic Nothing x++-- | Encode a signed 'RealFloat' according to 'FPFormat' and optionally requested precision.+--+-- This corresponds to the @show{E,F,G}Float@ operations provided by @base@'s "Numeric" module.+--+-- __NOTE__: The functions in @base-4.12@ changed the serialisation in+-- case of a @Just 0@ precision; this version of @text@ still provides+-- the serialisation as implemented in @base-4.11@. The next major+-- version of @text@ will switch to the more correct @base-4.12@ serialisation.+formatRealFloat :: (RealFloat a) =>+                   FPFormat+                -> Maybe Int  -- ^ Number of decimal places to render.+                -> a+                -> Builder+{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Float -> Builder #-}+{-# SPECIALIZE formatRealFloat :: FPFormat -> Maybe Int -> Double -> Builder #-}+formatRealFloat fmt decs x+   | isNaN x                   = "NaN"+   | isInfinite x              = if x < 0 then "-Infinity" else "Infinity"+   | x < 0 || isNegativeZero x = singleton '-' <> doFmt fmt (floatToDigits (-x))+   | otherwise                 = doFmt fmt (floatToDigits x)+ where+  doFmt format (is, e) =+    let ds = map i2d is in+    case format of+     Generic ->+      doFmt (if e < 0 || e > 7 then Exponent else Fixed)+            (is,e)+     Exponent ->+      case decs of+       Nothing ->+        let show_e' = decimal (e-1) in+        case ds of+          "0"     -> "0.0e0"+          [d]     -> singleton d <> ".0e" <> show_e'+          (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> show_e'+          []      -> error "formatRealFloat/doFmt/Exponent/Nothing: []"+       Just dec ->+        let dec' = max dec 1 in+        case is of+         [0] -> "0." <> fromText (T.replicate dec' "0") <> "e0"+         _ ->+          let (ei,is') = roundTo (dec'+1) is+              is'' = map i2d (if ei > 0 then init is' else is')+          in case is'' of+               [] -> error "formatRealFloat/doFmt/Exponent/Just: []"+               (d:ds') -> singleton d <> singleton '.' <> fromString ds' <> singleton 'e' <> decimal (e-1+ei)+     Fixed ->+      let+       mk0 ls = case ls of { "" -> "0" ; _ -> fromString ls}+      in+      case decs of+       Nothing+          | e <= 0    -> "0." <> fromText (T.replicate (-e) "0") <> fromString ds+          | otherwise ->+             let+                f 0 s    rs  = mk0 (reverse s) <> singleton '.' <> mk0 rs+                f n s    ""  = f (n-1) ('0':s) ""+                f n s (r:rs) = f (n-1) (r:s) rs+             in+                f e "" ds+       Just dec ->+        let dec' = max dec 0 in+        if e >= 0 then+         let+          (ei,is') = roundTo (dec' + e) is+          (ls,rs)  = splitAt (e+ei) (map i2d is')+         in+         mk0 ls <> (if null rs then "" else singleton '.' <> fromString rs)+        else+         let (ei,is') = roundTo dec' (replicate (-e) 0 ++ is)+             is'' = map i2d (if ei > 0 then is' else 0:is')+         in case is'' of+              [] -> error "formatRealFloat/doFmt/Fixed: []"+              (d:ds') -> singleton d <> (if null ds' then "" else singleton '.' <> fromString ds')+++-- Based on "Printing Floating-Point Numbers Quickly and Accurately"+-- by R.G. Burger and R.K. Dybvig in PLDI 96.+-- This version uses a much slower logarithm estimator. It should be improved.++-- | 'floatToDigits' takes a base and a non-negative 'RealFloat' number,+-- and returns a list of digits and an exponent.+-- In particular, if @x>=0@, and+--+-- > floatToDigits base x = ([d1,d2,...,dn], e)+--+-- then+--+--      (1) @n >= 1@+--+--      (2) @x = 0.d1d2...dn * (base**e)@+--+--      (3) @0 <= di <= base-1@++floatToDigits :: (RealFloat a) => a -> ([Int], Int)+{-# SPECIALIZE floatToDigits :: Float -> ([Int], Int) #-}+{-# SPECIALIZE floatToDigits :: Double -> ([Int], Int) #-}+floatToDigits 0 = ([0], 0)+floatToDigits x =+ let+  (f0, e0) = decodeFloat x+  (minExp0, _) = floatRange x+  p = floatDigits x+  b = floatRadix x+  minExp = minExp0 - p -- the real minimum exponent+  -- Haskell requires that f be adjusted so denormalized numbers+  -- will have an impossibly low exponent.  Adjust for this.+  (f, e) =+   let n = minExp - e0 in+   if n > 0 then (f0 `quot` (expt b n), e0+n) else (f0, e0)+  (r, s, mUp, mDn) =+   if e >= 0 then+    let be = expt b e in+    if f == expt b (p-1) then+      (f*be*b*2, 2*b, be*b, be)     -- according to Burger and Dybvig+    else+      (f*be*2, 2, be, be)+   else+    if e > minExp && f == expt b (p-1) then+      (f*b*2, expt b (-e+1)*2, b, 1)+    else+      (f*2, expt b (-e)*2, 1, 1)+  k :: Int+  k =+   let+    k0 :: Int+    k0 =+     if b == 2 then+        -- logBase 10 2 is very slightly larger than 8651/28738+        -- (about 5.3558e-10), so if log x >= 0, the approximation+        -- k1 is too small, hence we add one and need one fixup step less.+        -- If log x < 0, the approximation errs rather on the high side.+        -- That is usually more than compensated for by ignoring the+        -- fractional part of logBase 2 x, but when x is a power of 1/2+        -- or slightly larger and the exponent is a multiple of the+        -- denominator of the rational approximation to logBase 10 2,+        -- k1 is larger than logBase 10 x. If k1 > 1 + logBase 10 x,+        -- we get a leading zero-digit we don't want.+        -- With the approximation 3/10, this happened for+        -- 0.5^1030, 0.5^1040, ..., 0.5^1070 and values close above.+        -- The approximation 8651/28738 guarantees k1 < 1 + logBase 10 x+        -- for IEEE-ish floating point types with exponent fields+        -- <= 17 bits and mantissae of several thousand bits, earlier+        -- convergents to logBase 10 2 would fail for long double.+        -- Using quot instead of div is a little faster and requires+        -- fewer fixup steps for negative lx.+        let lx = p - 1 + e0+            k1 = (lx * 8651) `quot` 28738+        in if lx >= 0 then k1 + 1 else k1+     else+        -- f :: Integer, log :: Float -> Float,+        --               ceiling :: Float -> Int+        ceiling ((log (fromInteger (f+1) :: Float) ++                 intToFloat e * log (fromInteger b)) /+                   log 10)+--WAS:            fromInt e * log (fromInteger b))++    fixup n =+      if n >= 0 then+        if r + mUp <= expt 10 n * s then n else fixup (n+1)+      else+        if expt 10 (-n) * (r + mUp) <= s then n else fixup (n+1)+   in+   fixup k0++  gen ds rn sN mUpN mDnN =+   let+    (dn, rn') = (rn * 10) `quotRem` sN+    mUpN' = mUpN * 10+    mDnN' = mDnN * 10+   in+   case (rn' < mDnN', rn' + mUpN' > sN) of+    (True,  False) -> dn : ds+    (False, True)  -> dn+1 : ds+    (True,  True)  -> if rn' * 2 < sN then dn : ds else dn+1 : ds+    (False, False) -> gen (dn:ds) rn' sN mUpN' mDnN'++  rds =+   if k >= 0 then+      gen [] r (s * expt 10 k) mUp mDn+   else+     let bk = expt 10 (-k) in+     gen [] (r * bk) s (mUp * bk) (mDn * bk)+ in+ (map fromInteger (reverse rds), k)++-- Exponentiation with a cache for the most common numbers.+minExpt, maxExpt :: Int+minExpt = 0+maxExpt = 1100++expt :: Integer -> Int -> Integer+expt base n+    | base == 2 && n >= minExpt && n <= maxExpt = expts `unsafeAt` n+    | base == 10 && n <= maxExpt10              = expts10 `unsafeAt` n+    | otherwise                                 = base^n++expts :: Array Int Integer+expts = array (minExpt,maxExpt) [(n,2^n) | n <- [minExpt .. maxExpt]]++maxExpt10 :: Int+maxExpt10 = 324++expts10 :: Array Int Integer+expts10 = array (minExpt,maxExpt10) [(n,10^n) | n <- [minExpt .. maxExpt10]]++intToFloat :: Int -> Float+intToFloat = fromIntegral
+ src/Data/Text/Lazy/Encoding.hs view
@@ -0,0 +1,254 @@+{-# LANGUAGE BangPatterns,CPP #-}
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE ViewPatterns #-}
+
+-- |
+-- Module      : Data.Text.Lazy.Encoding
+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Portability : portable
+--
+-- Functions for converting lazy 'Text' values to and from lazy
+-- 'ByteString', using several standard encodings.
+--
+-- To gain access to a much larger family of encodings, use the
+-- <http://hackage.haskell.org/package/text-icu text-icu package>.
+
+module Data.Text.Lazy.Encoding
+    (
+    -- * Decoding ByteStrings to Text
+    -- $strict
+
+    -- ** Total Functions #total#
+    -- $total
+      decodeLatin1
+    , decodeUtf8Lenient
+
+    -- *** Catchable failure
+    , decodeUtf8'
+
+    -- *** Controllable error handling
+    , decodeUtf8With
+    , decodeUtf16LEWith
+    , decodeUtf16BEWith
+    , decodeUtf32LEWith
+    , decodeUtf32BEWith
+
+    -- ** Partial Functions
+    -- $partial
+    , decodeASCII
+    , decodeUtf8
+    , decodeUtf16LE
+    , decodeUtf16BE
+    , decodeUtf32LE
+    , decodeUtf32BE
+
+    -- * Encoding Text to ByteStrings
+    , encodeUtf8
+    , encodeUtf16LE
+    , encodeUtf16BE
+    , encodeUtf32LE
+    , encodeUtf32BE
+
+    -- * Encoding Text using ByteString Builders
+    , encodeUtf8Builder
+    , encodeUtf8BuilderEscaped
+    ) where
+
+import Control.Exception (evaluate, try)
+import Data.Monoid (Monoid(..))
+import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode)
+import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)
+import Data.Word (Word8)
+import qualified Data.ByteString.Builder as B
+import qualified Data.ByteString.Builder.Prim as BP
+import qualified Data.ByteString.Lazy as B
+import qualified Data.ByteString.Lazy.Internal as B
+import qualified Data.Text.Encoding as TE
+import qualified Data.Text.Internal.Encoding as TE
+import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E
+import qualified Data.Text.Internal.Lazy.Fusion as F
+import qualified Data.Text.Internal.StrictBuilder as SB
+import Data.Text.Unsafe (unsafeDupablePerformIO)
+
+-- $strict
+--
+-- All of the single-parameter functions for decoding bytestrings
+-- encoded in one of the Unicode Transformation Formats (UTF) operate
+-- in a /strict/ mode: each will throw an exception if given invalid
+-- input.
+--
+-- Each function has a variant, whose name is suffixed with -'With',
+-- that gives greater control over the handling of decoding errors.
+-- For instance, 'decodeUtf8' will throw an exception, but
+-- 'decodeUtf8With' allows the programmer to determine what to do on a
+-- decoding error.
+
+-- $total
+--
+-- These functions facilitate total decoding and should be preferred
+-- over their partial counterparts.
+
+-- $partial
+--
+-- These functions are partial and should only be used with great caution
+-- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better
+-- solutions.
+
+-- | Decode a 'ByteString' containing 7-bit ASCII
+-- encoded text.
+decodeASCII :: B.ByteString -> Text
+decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks
+
+-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.
+decodeLatin1 :: B.ByteString -> Text
+decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text.
+decodeUtf8With :: OnDecodeError -> B.ByteString -> Text
+decodeUtf8With onErr = loop TE.startUtf8State
+  where
+    chunkb builder t | SB.sbLength builder == 0 = t
+                    | otherwise = Chunk (TE.strictBuilderToText builder) t
+    loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of
+      (builder, _, s') -> chunkb builder (loop s' bs)
+    loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty
+    msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream"
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text that is known
+-- to be valid.
+--
+-- If the input contains any invalid UTF-8 data, an exception will be
+-- thrown that cannot be caught in pure code.  For more control over
+-- the handling of invalid data, use 'decodeUtf8'' or
+-- 'decodeUtf8With'.
+decodeUtf8 :: B.ByteString -> Text
+decodeUtf8 = decodeUtf8With strictDecode
+{-# INLINE[0] decodeUtf8 #-}
+
+-- | Decode a 'ByteString' containing UTF-8 encoded text..
+--
+-- If the input contains any invalid UTF-8 data, the relevant
+-- exception will be returned, otherwise the decoded text.
+--
+-- /Note/: this function is /not/ lazy, as it must decode its entire
+-- input before it can return a result.  If you need lazy (streaming)
+-- decoding, use 'decodeUtf8With' in lenient mode.
+decodeUtf8' :: B.ByteString -> Either UnicodeException Text
+decodeUtf8' bs = unsafeDupablePerformIO $ do
+                   let t = decodeUtf8 bs
+                   try (evaluate (rnf t `seq` t))
+  where
+    rnf Empty        = ()
+    rnf (Chunk _ ts) = rnf ts
+{-# INLINE decodeUtf8' #-}
+
+-- | Decode a lazy 'ByteString' containing UTF-8 encoded text.
+--
+-- Any invalid input bytes will be replaced with the Unicode replacement
+-- character U+FFFD.
+--
+-- @since 2.1.4
+decodeUtf8Lenient :: B.ByteString -> Text
+decodeUtf8Lenient = decodeUtf8With lenientDecode
+
+-- | Encode text using UTF-8 encoding.
+encodeUtf8 :: Text -> B.ByteString
+encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty
+
+-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
+--
+-- @since 1.1.0.0
+encodeUtf8Builder :: Text -> B.Builder
+encodeUtf8Builder =
+    foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty
+
+-- | Encode text using UTF-8 encoding and escape the ASCII characters using
+-- a 'BP.BoundedPrim'.
+--
+-- Use this function is to implement efficient encoders for text-based formats
+-- like JSON or HTML.
+--
+-- @since 1.1.0.0
+{-# INLINE encodeUtf8BuilderEscaped #-}
+encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
+encodeUtf8BuilderEscaped prim =
+    foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty
+
+-- | Decode text from little endian UTF-16 encoding.
+decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text
+decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
+{-# INLINE decodeUtf16LEWith #-}
+
+-- | Decode text from little endian UTF-16 encoding.
+--
+-- If the input contains any invalid little endian UTF-16 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf16LEWith'.
+decodeUtf16LE :: B.ByteString -> Text
+decodeUtf16LE = decodeUtf16LEWith strictDecode
+{-# INLINE decodeUtf16LE #-}
+
+-- | Decode text from big endian UTF-16 encoding.
+decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text
+decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)
+{-# INLINE decodeUtf16BEWith #-}
+
+-- | Decode text from big endian UTF-16 encoding.
+--
+-- If the input contains any invalid big endian UTF-16 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf16BEWith'.
+decodeUtf16BE :: B.ByteString -> Text
+decodeUtf16BE = decodeUtf16BEWith strictDecode
+{-# INLINE decodeUtf16BE #-}
+
+-- | Encode text using little endian UTF-16 encoding.
+encodeUtf16LE :: Text -> B.ByteString
+encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt)
+{-# INLINE encodeUtf16LE #-}
+
+-- | Encode text using big endian UTF-16 encoding.
+encodeUtf16BE :: Text -> B.ByteString
+encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt)
+{-# INLINE encodeUtf16BE #-}
+
+-- | Decode text from little endian UTF-32 encoding.
+decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text
+decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)
+{-# INLINE decodeUtf32LEWith #-}
+
+-- | Decode text from little endian UTF-32 encoding.
+--
+-- If the input contains any invalid little endian UTF-32 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf32LEWith'.
+decodeUtf32LE :: B.ByteString -> Text
+decodeUtf32LE = decodeUtf32LEWith strictDecode
+{-# INLINE decodeUtf32LE #-}
+
+-- | Decode text from big endian UTF-32 encoding.
+decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text
+decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)
+{-# INLINE decodeUtf32BEWith #-}
+
+-- | Decode text from big endian UTF-32 encoding.
+--
+-- If the input contains any invalid big endian UTF-32 data, an
+-- exception will be thrown.  For more control over the handling of
+-- invalid data, use 'decodeUtf32BEWith'.
+decodeUtf32BE :: B.ByteString -> Text
+decodeUtf32BE = decodeUtf32BEWith strictDecode
+{-# INLINE decodeUtf32BE #-}
+
+-- | Encode text using little endian UTF-32 encoding.
+encodeUtf32LE :: Text -> B.ByteString
+encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt)
+{-# INLINE encodeUtf32LE #-}
+
+-- | Encode text using big endian UTF-32 encoding.
+encodeUtf32BE :: Text -> B.ByteString
+encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt)
+{-# INLINE encodeUtf32BE #-}
+ src/Data/Text/Lazy/IO.hs view
@@ -0,0 +1,159 @@+{-# LANGUAGE BangPatterns, CPP, RecordWildCards #-}+{-# LANGUAGE Trustworthy #-}+-- |+-- Module      : Data.Text.Lazy.IO+-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan,+--               (c) 2009 Simon Marlow+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Portability : GHC+--+-- Efficient locale-sensitive support for lazy text I\/O.+--+-- The functions in this module obey the runtime system's locale,+-- character set encoding, and line ending conversion settings.+--+-- If you know in advance that you will be working with data that has+-- a specific encoding (e.g. UTF-8), and your application is highly+-- performance sensitive, you may find that it is faster to perform+-- I\/O with bytestrings and to encode and decode yourself than to use+-- the functions in this module.++module Data.Text.Lazy.IO+    (+    -- * File-at-a-time operations+      readFile+    , writeFile+    , appendFile+    -- * Operations on handles+    , hGetContents+    , hGetLine+    , hPutStr+    , hPutStrLn+    -- * Special cases for standard input and output+    , interact+    , getContents+    , getLine+    , putStr+    , putStrLn+    ) where++import Data.Text.Lazy (Text)+import Prelude hiding (appendFile, getContents, getLine, interact,+                       putStr, putStrLn, readFile, writeFile)+import System.IO (Handle, IOMode(..), openFile, stdin, stdout,+                  withFile)+import qualified Data.Text.Lazy as L+import qualified Control.Exception as E+import Control.Monad (when)+import Data.IORef (readIORef)+import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStream)+import Data.Text.Internal.Lazy (chunk, empty)+import Data.Text.Internal.Lazy.Fusion (stream, streamLn)+import GHC.IO.Buffer (isEmptyBuffer)+import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException)+import GHC.IO.Handle.Internals (augmentIOError, hClose_help,+                                wantReadableHandle, withHandle)+import GHC.IO.Handle.Types (Handle__(..), HandleType(..))+import System.IO (BufferMode(..), hGetBuffering, hSetBuffering)+import System.IO.Error (isEOFError)+import System.IO.Unsafe (unsafeInterleaveIO)++-- | Read a file and return its contents as a string.  The file is+-- read lazily, as with 'getContents'.+--+-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent.+-- Unexpected system locale may cause your application to read corrupted data or+-- throw runtime exceptions about "invalid argument (invalid byte sequence)"+-- or "invalid argument (invalid character)". This is also slow, because GHC+-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8.+--+-- If your data is UTF-8,+-- using 'Data.Text.Lazy.Encoding.decodeUtf8' '.' 'Data.ByteString.Lazy.readFile'+-- is a much faster and safer alternative.+readFile :: FilePath -> IO Text+readFile name = openFile name ReadMode >>= hGetContents++-- | Write a string to a file.  The file is truncated to zero length+-- before writing begins.+writeFile :: FilePath -> Text -> IO ()+writeFile p = withFile p WriteMode . flip hPutStr++-- | Write a string to the end of a file.+appendFile :: FilePath -> Text -> IO ()+appendFile p = withFile p AppendMode . flip hPutStr++-- | Lazily read the remaining contents of a 'Handle'.  The 'Handle'+-- will be closed after the read completes, or on error.+hGetContents :: Handle -> IO Text+hGetContents h = do+  chooseGoodBuffering h+  wantReadableHandle "hGetContents" h $ \hh -> do+    ts <- lazyRead h+    return (hh{haType=SemiClosedHandle}, ts)++-- | Use a more efficient buffer size if we're reading in+-- block-buffered mode with the default buffer size.+chooseGoodBuffering :: Handle -> IO ()+chooseGoodBuffering h = do+  bufMode <- hGetBuffering h+  when (bufMode == BlockBuffering Nothing) $+    hSetBuffering h (BlockBuffering (Just 16384))++lazyRead :: Handle -> IO Text+lazyRead h = unsafeInterleaveIO $+  withHandle "hGetContents" h $ \hh -> do+    case haType hh of+      ClosedHandle     -> return (hh, L.empty)+      SemiClosedHandle -> lazyReadBuffered h hh+      _                -> ioException+                          (IOError (Just h) IllegalOperation "hGetContents"+                           "illegal handle type" Nothing Nothing)++lazyReadBuffered :: Handle -> Handle__ -> IO (Handle__, Text)+lazyReadBuffered h hh@Handle__{..} = do+   buf <- readIORef haCharBuffer+   (do t <- readChunk hh buf+       ts <- lazyRead h+       return (hh, chunk t ts)) `E.catch` \e -> do+         (hh', _) <- hClose_help hh+         if isEOFError e+           then return $ if isEmptyBuffer buf+                         then (hh', empty)+                         else (hh', L.singleton '\r')+           else E.throwIO (augmentIOError e "hGetContents" h)++-- | Read a single line from a handle.+hGetLine :: Handle -> IO Text+hGetLine = hGetLineWith L.fromChunks++-- | Write a string to a handle.+hPutStr :: Handle -> Text -> IO ()+hPutStr h = hPutStream h . stream++-- | Write a string to a handle, followed by a newline.+hPutStrLn :: Handle -> Text -> IO ()+hPutStrLn h = hPutStream h . streamLn++-- | The 'interact' function takes a function of type @Text -> Text@+-- as its argument. The entire input from the standard input device is+-- passed (lazily) to this function as its argument, and the resulting+-- string is output on the standard output device.+interact :: (Text -> Text) -> IO ()+interact f = putStr . f =<< getContents++-- | Lazily read all user input on 'stdin' as a single string.+getContents :: IO Text+getContents = hGetContents stdin++-- | Read a single line of user input from 'stdin'.+getLine :: IO Text+getLine = hGetLine stdin++-- | Write a string to 'stdout'.+putStr :: Text -> IO ()+putStr = hPutStr stdout++-- | Write a string to 'stdout', followed by a newline.+putStrLn :: Text -> IO ()+putStrLn = hPutStrLn stdout
+ src/Data/Text/Lazy/Internal.hs view
@@ -0,0 +1,19 @@+-- |+-- Module      : Data.Text.Lazy.Internal+-- Copyright   : (c) 2013 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC+--+-- This module has been renamed to 'Data.Text.Internal.Lazy'. This+-- name for the module will be removed in the next major release.++module Data.Text.Lazy.Internal+    {-# DEPRECATED "Use Data.Text.Internal.Lazy instead" #-}+    (+      module Data.Text.Internal.Lazy+    ) where++import Data.Text.Internal.Lazy
+ src/Data/Text/Lazy/Read.hs view
@@ -0,0 +1,221 @@+{-# LANGUAGE OverloadedStrings, CPP #-}+{-# LANGUAGE Trustworthy #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE UnboxedTuples #-}++-- |+-- Module      : Data.Text.Lazy.Read+-- Copyright   : (c) 2010, 2011 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Portability : GHC+--+-- Functions used frequently when reading textual data.+module Data.Text.Lazy.Read+    (+      Reader+    , decimal+    , hexadecimal+    , signed+    , rational+    , double+    ) where++import Control.Monad (liftM)+import Data.Char (ord)+import Data.Int (Int8, Int16, Int32, Int64)+import Data.Ratio ((%))+import Data.Text.Internal.Read+import Data.Text.Array as A+import Data.Text.Lazy as T+import Data.Text.Internal.Lazy as T (Text(..))+import qualified Data.Text.Internal as T (Text(..))+import qualified Data.Text.Internal.Private as T (spanAscii_)+import Data.Word (Word, Word8, Word16, Word32, Word64)++-- | Read some text.  If the read succeeds, return its value and the+-- remaining text, otherwise an error message.+type Reader a = Text -> Either String (a, Text)+type Parser = IParser Text++-- | Read a decimal integer.  The input must begin with at least one+-- decimal digit, and is consumed until a non-digit or end of string+-- is reached.+--+-- This function does not handle leading sign characters.  If you need+-- to handle signed input, use @'signed' 'decimal'@.+--+-- /Note/: For fixed-width integer types, this function does not+-- attempt to detect overflow, so a sufficiently long input may give+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type.+decimal :: Integral a => Reader a+{-# SPECIALIZE decimal :: Reader Int #-}+{-# SPECIALIZE decimal :: Reader Int8 #-}+{-# SPECIALIZE decimal :: Reader Int16 #-}+{-# SPECIALIZE decimal :: Reader Int32 #-}+{-# SPECIALIZE decimal :: Reader Int64 #-}+{-# SPECIALIZE decimal :: Reader Integer #-}+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}+{-# SPECIALIZE decimal :: Reader Word8 #-}+{-# SPECIALIZE decimal :: Reader Word16 #-}+{-# SPECIALIZE decimal :: Reader Word32 #-}+{-# SPECIALIZE decimal :: Reader Word64 #-}+decimal txt+    | T.null h  = Left "input does not start with a digit"+    | otherwise = Right (T.foldl' go 0 h, t)+  where (# h, t #)  = spanAscii_ (\w -> w - ord8 '0' < 10) txt+        go n d = (n * 10 + fromIntegral (digitToInt d))++-- | Read a hexadecimal integer, consisting of an optional leading+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is+-- consumed until a non-hex-digit or end of string is reached.+-- This function is case insensitive.+--+-- This function does not handle leading sign characters.  If you need+-- to handle signed input, use @'signed' 'hexadecimal'@.+--+-- /Note/: For fixed-width integer types, this function does not+-- attempt to detect overflow, so a sufficiently long input may give+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type.+hexadecimal :: Integral a => Reader a+{-# SPECIALIZE hexadecimal :: Reader Int #-}+{-# SPECIALIZE hexadecimal :: Reader Integer #-}+hexadecimal txt+    | h == "0x" || h == "0X" = hex t+    | otherwise              = hex txt+ where (h,t) = T.splitAt 2 txt++hex :: Integral a => Reader a+{-# SPECIALIZE hexadecimal :: Reader Int #-}+{-# SPECIALIZE hexadecimal :: Reader Int8 #-}+{-# SPECIALIZE hexadecimal :: Reader Int16 #-}+{-# SPECIALIZE hexadecimal :: Reader Int32 #-}+{-# SPECIALIZE hexadecimal :: Reader Int64 #-}+{-# SPECIALIZE hexadecimal :: Reader Integer #-}+{-# SPECIALIZE hexadecimal :: Reader Word #-}+{-# SPECIALIZE hexadecimal :: Reader Word8 #-}+{-# SPECIALIZE hexadecimal :: Reader Word16 #-}+{-# SPECIALIZE hexadecimal :: Reader Word32 #-}+{-# SPECIALIZE hexadecimal :: Reader Word64 #-}+hex txt+    | T.null h  = Left "input does not start with a hexadecimal digit"+    | otherwise = Right (T.foldl' go 0 h, t)+  where (# h, t #)  = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt+        go n d = (n * 16 + fromIntegral (hexDigitToInt d))++-- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and+-- apply it to the result of applying the given reader.+signed :: Num a => Reader a -> Reader a+{-# INLINE signed #-}+signed f = runP (signa (P f))++-- | Read a rational number.+--+-- This function accepts an optional leading sign character, followed+-- by at least one decimal digit.  The syntax similar to that accepted+-- by the 'read' function, with the exception that a trailing @\'.\'@+-- or @\'e\'@ /not/ followed by a number is not consumed.+--+-- Examples:+--+-- >rational "3"     == Right (3.0, "")+-- >rational "3.1"   == Right (3.1, "")+-- >rational "3e4"   == Right (30000.0, "")+-- >rational "3.1e4" == Right (31000.0, "")+-- >rational ".3"    == Left "input does not start with a digit"+-- >rational "e3"    == Left "input does not start with a digit"+--+-- Examples of differences from 'read':+--+-- >rational "3.foo" == Right (3.0, ".foo")+-- >rational "3e"    == Right (3.0, "e")+rational :: Fractional a => Reader a+{-# SPECIALIZE rational :: Reader Double #-}+rational = floaty $ \real frac fracDenom power ->+  -- We must be careful to prevent DDoS attacks: if the return type is 'Double',+  -- a client rightfully expects 'rational' to operate within bounded memory.+  -- Thus if power is small, we can compute fraction with full precision and divide.+  -- Otherwise divide first, apply fromRational and scale last:+  -- the small loss of precision for Double does not matter much because the result is+  -- likely infinity or zero anyway.+  if abs power < 1000+  then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power))+  else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power)++-- | Read a rational number.+--+-- The syntax accepted by this function is the same as for 'rational'.+--+-- /Note/: This function is almost ten times faster than 'rational',+-- but is slightly less accurate.+--+-- The 'Double' type supports about 16 decimal places of accuracy.+-- For 94.2% of numbers, this function and 'rational' give identical+-- results, but for the remaining 5.8%, this function loses precision+-- around the 15th decimal place.  For 0.001% of numbers, this+-- function will lose precision at the 13th or 14th decimal place.+double :: Reader Double+double = floaty $ \real frac fracDenom power ->+                   (fromInteger real ++                   fromInteger frac / fromInteger fracDenom) * (10 ^^ power)++signa :: Num a => Parser a -> Parser a+{-# SPECIALIZE signa :: Parser Int -> Parser Int #-}+{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-}+{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-}+{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-}+{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-}+{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-}+signa p = do+  sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')+  if sign == ord8 '+' then p else negate `liftM` p++charAscii :: (Word8 -> Bool) -> Parser Word8+charAscii p = P $ \case+  Empty -> Left "character does not match"+  -- len is > 0, unless the internal invariant of Text is violated+  Chunk (T.Text arr off len) ts -> let c = A.unsafeIndex arr off in+    if p c+    then Right (c, if len <= 1 then ts else Chunk (T.Text arr (off + 1) (len - 1)) ts)+    else Left "character does not match"++floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a+{-# INLINE floaty #-}+floaty f = runP $ do+  sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')+  real <- P decimal+  T fraction fracDigits <- perhaps (T 0 0) $ do+    _ <- charAscii (== ord8 '.')+    digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in int64ToInt (T.length hd), t)+    n <- P decimal+    return $ T n digits+  let e c = c == ord8 'e' || c == ord8 'E'+  power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int)+  let n = if fracDigits == 0+          then if power == 0+               then fromInteger real+               else fromInteger real * (10 ^^ power)+          else f real fraction (10 ^ fracDigits) power+  return $! if sign == ord8 '+'+            then n+            else -n++int64ToInt :: Int64 -> Int+int64ToInt = fromIntegral++ord8 :: Char -> Word8+ord8 = fromIntegral . ord++-- | For the sake of performance this function does not check+-- that a char is in ASCII range; it is a responsibility of @p@.+spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #)+spanAscii_ p = loop+  where+    loop Empty = (# Empty, Empty #)+    loop (Chunk t ts) = let (# t', t''@(T.Text _ _ len) #) = T.spanAscii_ p t in+      if len == 0+      then let (# ts', ts'' #) = loop ts in (# Chunk t ts', ts'' #)+      else (# Chunk t' Empty, Chunk t'' ts #)
+ src/Data/Text/Read.hs view
@@ -0,0 +1,210 @@+{-# LANGUAGE OverloadedStrings, UnboxedTuples, CPP #-}+{-# LANGUAGE Trustworthy #-}++-- |+-- Module      : Data.Text.Read+-- Copyright   : (c) 2010, 2011 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Portability : GHC+--+-- Functions used frequently when reading textual data.+module Data.Text.Read+    (+      Reader+    , decimal+    , hexadecimal+    , signed+    , rational+    , double+    ) where++import Control.Monad (liftM)+import Data.Char (ord)+import Data.Int (Int8, Int16, Int32, Int64)+import Data.Ratio ((%))+import Data.Text as T+import Data.Text.Internal as T (Text(..))+import Data.Text.Array as A+import Data.Text.Internal.Private (spanAscii_)+import Data.Text.Internal.Read+import Data.Word (Word, Word8, Word16, Word32, Word64)++-- | Read some text.  If the read succeeds, return its value and the+-- remaining text, otherwise an error message.+type Reader a = Text -> Either String (a, Text)+type Parser a = IParser Text a++-- | Read a decimal integer.  The input must begin with at least one+-- decimal digit, and is consumed until a non-digit or end of string+-- is reached.+--+-- This function does not handle leading sign characters.  If you need+-- to handle signed input, use @'signed' 'decimal'@.+--+-- /Note/: For fixed-width integer types, this function does not+-- attempt to detect overflow, so a sufficiently long input may give+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type.+decimal :: Integral a => Reader a+{-# SPECIALIZE decimal :: Reader Int #-}+{-# SPECIALIZE decimal :: Reader Int8 #-}+{-# SPECIALIZE decimal :: Reader Int16 #-}+{-# SPECIALIZE decimal :: Reader Int32 #-}+{-# SPECIALIZE decimal :: Reader Int64 #-}+{-# SPECIALIZE decimal :: Reader Integer #-}+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}+{-# SPECIALIZE decimal :: Reader Word8 #-}+{-# SPECIALIZE decimal :: Reader Word16 #-}+{-# SPECIALIZE decimal :: Reader Word32 #-}+{-# SPECIALIZE decimal :: Reader Word64 #-}+decimal txt+    | T.null h  = Left "input does not start with a digit"+    | otherwise = Right (T.foldl' go 0 h, t)+  where (# h,t #)  = spanAscii_ (\w -> w - ord8 '0' < 10) txt+        go n d = (n * 10 + fromIntegral (digitToInt d))++-- | Read a hexadecimal integer, consisting of an optional leading+-- @\"0x\"@ followed by at least one hexadecimal digit. Input is+-- consumed until a non-hex-digit or end of string is reached.+-- This function is case insensitive.+--+-- This function does not handle leading sign characters.  If you need+-- to handle signed input, use @'signed' 'hexadecimal'@.+--+-- /Note/: For fixed-width integer types, this function does not+-- attempt to detect overflow, so a sufficiently long input may give+-- incorrect results.  If you are worried about overflow, use+-- 'Integer' for your result type.+hexadecimal :: Integral a => Reader a+{-# SPECIALIZE hexadecimal :: Reader Int #-}+{-# SPECIALIZE hexadecimal :: Reader Int8 #-}+{-# SPECIALIZE hexadecimal :: Reader Int16 #-}+{-# SPECIALIZE hexadecimal :: Reader Int32 #-}+{-# SPECIALIZE hexadecimal :: Reader Int64 #-}+{-# SPECIALIZE hexadecimal :: Reader Integer #-}+{-# SPECIALIZE hexadecimal :: Reader Word #-}+{-# SPECIALIZE hexadecimal :: Reader Word8 #-}+{-# SPECIALIZE hexadecimal :: Reader Word16 #-}+{-# SPECIALIZE hexadecimal :: Reader Word32 #-}+{-# SPECIALIZE hexadecimal :: Reader Word64 #-}+hexadecimal txt+    | h == "0x" || h == "0X" = hex t+    | otherwise              = hex txt+ where (h,t) = T.splitAt 2 txt++hex :: Integral a => Reader a+{-# SPECIALIZE hex :: Reader Int #-}+{-# SPECIALIZE hex :: Reader Int8 #-}+{-# SPECIALIZE hex :: Reader Int16 #-}+{-# SPECIALIZE hex :: Reader Int32 #-}+{-# SPECIALIZE hex :: Reader Int64 #-}+{-# SPECIALIZE hex :: Reader Integer #-}+{-# SPECIALIZE hex :: Reader Word #-}+{-# SPECIALIZE hex :: Reader Word8 #-}+{-# SPECIALIZE hex :: Reader Word16 #-}+{-# SPECIALIZE hex :: Reader Word32 #-}+{-# SPECIALIZE hex :: Reader Word64 #-}+hex txt+    | T.null h  = Left "input does not start with a hexadecimal digit"+    | otherwise = Right (T.foldl' go 0 h, t)+  where (# h,t #)  = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt+        go n d = (n * 16 + fromIntegral (hexDigitToInt d))++-- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and+-- apply it to the result of applying the given reader.+signed :: Num a => Reader a -> Reader a+{-# INLINE signed #-}+signed f = runP (signa (P f))++-- | Read a rational number.+--+-- This function accepts an optional leading sign character, followed+-- by at least one decimal digit.  The syntax similar to that accepted+-- by the 'read' function, with the exception that a trailing @\'.\'@+-- or @\'e\'@ /not/ followed by a number is not consumed.+--+-- Examples (with behaviour identical to 'read'):+--+-- >rational "3"     == Right (3.0, "")+-- >rational "3.1"   == Right (3.1, "")+-- >rational "3e4"   == Right (30000.0, "")+-- >rational "3.1e4" == Right (31000.0, "")+-- >rational ".3"    == Left "input does not start with a digit"+-- >rational "e3"    == Left "input does not start with a digit"+--+-- Examples of differences from 'read':+--+-- >rational "3.foo" == Right (3.0, ".foo")+-- >rational "3e"    == Right (3.0, "e")+rational :: Fractional a => Reader a+{-# SPECIALIZE rational :: Reader Double #-}+rational = floaty $ \real frac fracDenom power ->+  -- We must be careful to prevent DDoS attacks: if the return type is 'Double',+  -- a client rightfully expects 'rational' to operate within bounded memory.+  -- Thus if power is small, we can compute fraction with full precision and divide.+  -- Otherwise divide first, apply fromRational and scale last:+  -- the small loss of precision for Double does not matter much because the result is+  -- likely infinity or zero anyway.+  if abs power < 1000+  then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power))+  else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power)++-- | Read a rational number.+--+-- The syntax accepted by this function is the same as for 'rational'.+--+-- /Note/: This function is almost ten times faster than 'rational',+-- but is slightly less accurate.+--+-- The 'Double' type supports about 16 decimal places of accuracy.+-- For 94.2% of numbers, this function and 'rational' give identical+-- results, but for the remaining 5.8%, this function loses precision+-- around the 15th decimal place.  For 0.001% of numbers, this+-- function will lose precision at the 13th or 14th decimal place.+double :: Reader Double+double = floaty $ \real frac fracDenom power ->+                   (fromInteger real ++                   fromInteger frac / fromInteger fracDenom) * (10 ^^ power)++signa :: Num a => Parser a -> Parser a+{-# SPECIALIZE signa :: Parser Int -> Parser Int #-}+{-# SPECIALIZE signa :: Parser Int8 -> Parser Int8 #-}+{-# SPECIALIZE signa :: Parser Int16 -> Parser Int16 #-}+{-# SPECIALIZE signa :: Parser Int32 -> Parser Int32 #-}+{-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-}+{-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-}+signa p = do+  sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')+  if sign == ord8 '+' then p else negate `liftM` p++charAscii :: (Word8 -> Bool) -> Parser Word8+charAscii p = P $ \(Text arr off len) -> let c = A.unsafeIndex arr off in+  if len > 0 && p c+  then Right (c, Text arr (off + 1) (len - 1))+  else Left "character does not match"++floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a+{-# INLINE floaty #-}+floaty f = runP $ do+  sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')+  real <- P decimal+  T fraction fracDigits <- perhaps (T 0 0) $ do+    _ <- charAscii (== ord8 '.')+    digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in T.length hd, t)+    n <- P decimal+    return $ T n digits+  let e c = c == ord8 'e' || c == ord8 'E'+  power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int)+  let n = if fracDigits == 0+          then if power == 0+               then fromInteger real+               else fromInteger real * (10 ^^ power)+          else f real fraction (10 ^ fracDigits) power+  return $! if sign == ord8 '+'+            then n+            else -n++ord8 :: Char -> Word8+ord8 = fromIntegral . ord
+ src/Data/Text/Show.hs view
@@ -0,0 +1,171 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE Trustworthy #-}+{-# LANGUAGE ViewPatterns #-}++{-# OPTIONS_GHC -fno-warn-orphans #-}++-- |+-- Module      : Data.Text.Show+-- Copyright   : (c) 2009-2015 Bryan O'Sullivan+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Stability   : experimental+-- Portability : GHC++module Data.Text.Show+    (+      addrLen+    , singleton+    , unpack+    , unpackCString#+    , unpackCStringAscii#+    ) where++import Control.Monad.ST (ST, runST)+import Data.Text.Internal (Text(..), empty, safe, pack)+import Data.Text.Internal.Encoding.Utf8 (utf8Length)+import Data.Text.Internal.Unsafe.Char (unsafeWrite)+import Data.Text.Unsafe (Iter(..), iterArray)+import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#)+import qualified GHC.Exts as Exts+import GHC.Word (Word8(..))+import qualified Data.Text.Array as A+#if !MIN_VERSION_ghc_prim(0,7,0)+import Foreign.C.String (CString)+import Foreign.C.Types (CSize(..))+#endif++import qualified GHC.CString as GHC++#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif++instance Show Text where+    showsPrec p ps r = showsPrec p (unpack ps) r++-- | /O(n)/ Convert a 'Text' into a 'String'.+unpack ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> String+unpack t = foldrText (:) [] t+{-# NOINLINE unpack #-}++foldrText :: (Char -> b -> b) -> b -> Text -> b+foldrText f z (Text arr off len) = go off+  where+    go !i+      | i >= off + len = z+      | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l))+{-# INLINE foldrText #-}++foldrTextFB :: (Char -> b -> b) -> b -> Text -> b+foldrTextFB = foldrText+{-# INLINE [0] foldrTextFB #-}++-- List fusion rules for `unpack`:+-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build`+--   fuses if `foldr` is applied to it.+-- * If it doesn't fuse: In phase 1, `build` inlines to give us+--   `foldrTextFB (:) []` and we rewrite that back to `unpack`.+-- * If it fuses: In phase 0, `foldrTextFB` inlines and `foldrText` inlines. GHC+--   optimizes the fused code.+{-# RULES+"Text.unpack"     [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrTextFB lcons lnil t)+"Text.unpackBack" [1]  foldrTextFB (:) [] = unpack+  #-}++-- | /O(n)/ Convert a null-terminated+-- <https://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 modified UTF-8>+-- (but with a standard UTF-8 representation of characters from supplementary planes)+-- string to a 'Text'. Counterpart to 'GHC.unpackCStringUtf8#'.+-- No validation is performed, malformed input can lead to memory access violation.+--+-- @since 1.2.1.1+unpackCString# :: Addr# -> Text+unpackCString# addr# = runST $ do+  let l = addrLen addr#+      at (I# i#) = W8# (indexWord8OffAddr# addr# i#)+  marr <- A.new l+  let go srcOff@(at -> w8) dstOff+        | srcOff >= l+        = return dstOff+        -- Surrogate halves take 3 bytes and are replaced by \xfffd (also 3 bytes long).+        -- Cf. Data.Text.Internal.safe+        | w8 == 0xed, at (srcOff + 1) >= 0xa0 = do+          A.unsafeWrite marr  dstOff      0xef+          A.unsafeWrite marr (dstOff + 1) 0xbf+          A.unsafeWrite marr (dstOff + 2) 0xbd+          go (srcOff + 3) (dstOff + 3)+        -- Byte sequence "\xc0\x80" is used to represent NUL+        | w8 == 0xc0, at (srcOff + 1) == 0x80+        = A.unsafeWrite marr dstOff 0  >> go (srcOff + 2) (dstOff + 1)+        | otherwise+        = A.unsafeWrite marr dstOff w8 >> go (srcOff + 1) (dstOff + 1)+  actualLen <- go 0 0+  A.shrinkM marr actualLen+  arr <- A.unsafeFreeze marr+  return $ Text arr 0 actualLen++-- When a module contains many literal strings, 'unpackCString#' can easily+-- bloat generated code to insane size. There is also very little to gain+-- from inlining. Thus explicit NOINLINE is desired.+{-# NOINLINE unpackCString# #-}++-- | /O(n)/ Convert a null-terminated ASCII string to a 'Text'.+-- Counterpart to 'GHC.unpackCString#'.+-- No validation is performed, malformed input can lead to memory access violation.+--+-- @since 2.0+unpackCStringAscii# :: Addr# -> Text+unpackCStringAscii# addr# = Text ba 0 l+  where+    l = addrLen addr#+    ba = runST $ do+      marr <- A.new l+      A.copyFromPointer marr 0 (Ptr addr#) l+      A.unsafeFreeze marr+{-# NOINLINE unpackCStringAscii# #-}++addrLen :: Addr# -> Int+#if MIN_VERSION_ghc_prim(0,7,0)+addrLen addr# = I# (GHC.cstringLength# addr#)+#else+addrLen addr# = fromIntegral (c_strlen (Ptr addr#))++foreign import capi unsafe "string.h strlen" c_strlen :: CString -> CSize+#endif++{-# RULES "TEXT literal" forall a.+    pack (GHC.unpackCString# a) = unpackCStringAscii# a #-}++{-# RULES "TEXT literal UTF8" forall a.+    pack (GHC.unpackCStringUtf8# a) = unpackCString# a #-}++{-# RULES "TEXT empty literal"+    pack [] = empty #-}++{-# RULES "TEXT singleton literal" forall a.+    pack [a] = singleton a #-}++-- | /O(1)/ Convert a character into a Text.+-- Performs replacement on invalid scalar values.+singleton ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Char -> Text+singleton c = Text (A.run x) 0 len+  where x :: ST s (A.MArray s)+        x = do arr <- A.new len+               _ <- unsafeWrite arr 0 d+               return arr+        len = utf8Length d+        d = safe c+{-# NOINLINE singleton #-}
+ src/Data/Text/Unsafe.hs view
@@ -0,0 +1,166 @@+{-# LANGUAGE CPP, MagicHash, UnboxedTuples #-}+-- |+-- Module      : Data.Text.Unsafe+-- Copyright   : (c) 2009, 2010, 2011 Bryan O'Sullivan+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com+-- Portability : portable+--+-- A module containing unsafe 'Text' operations, for very very careful+-- use in heavily tested code.+module Data.Text.Unsafe+    (+      inlineInterleaveST+    , inlinePerformIO+    , unsafeDupablePerformIO+    , Iter(..)+    , iter+    , iterArray+    , iter_+    , reverseIter+    , reverseIterArray+    , reverseIter_+    , unsafeHead+    , unsafeTail+    , lengthWord8+    , takeWord8+    , dropWord8+    ) where++#if defined(ASSERTS)+import Control.Exception (assert)+import GHC.Stack (HasCallStack)+#endif+import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader)+import Data.Text.Internal (Text(..))+import Data.Text.Internal.Unsafe (inlineInterleaveST, inlinePerformIO)+import Data.Text.Internal.Unsafe.Char (unsafeChr8)+import qualified Data.Text.Array as A+import GHC.IO (unsafeDupablePerformIO)++-- | /O(1)/ A variant of 'head' for non-empty 'Text'. 'unsafeHead'+-- omits the check for the empty case, so there is an obligation on+-- the programmer to provide a proof that the 'Text' is non-empty.+unsafeHead :: Text -> Char+unsafeHead (Text arr off _len) = case utf8LengthByLeader m0 of+    1 -> unsafeChr8 m0+    2 -> chr2 m0 m1+    3 -> chr3 m0 m1 m2+    _ -> chr4 m0 m1 m2 m3+    where m0 = A.unsafeIndex arr off+          m1 = A.unsafeIndex arr (off+1)+          m2 = A.unsafeIndex arr (off+2)+          m3 = A.unsafeIndex arr (off+3)+{-# INLINE unsafeHead #-}++-- | /O(1)/ A variant of 'tail' for non-empty 'Text'. 'unsafeTail'+-- omits the check for the empty case, so there is an obligation on+-- the programmer to provide a proof that the 'Text' is non-empty.+unsafeTail :: Text -> Text+unsafeTail t@(Text arr off len) =+#if defined(ASSERTS)+    assert (d <= len) $+#endif+    Text arr (off+d) (len-d)+  where d = iter_ t 0+{-# INLINE unsafeTail #-}++data Iter = Iter {-# UNPACK #-} !Char {-# UNPACK #-} !Int+  deriving (Show)++-- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-8+-- array, returning the current character and the delta to add to give+-- the next offset to iterate at.+iter ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  Text -> Int -> Iter+iter (Text arr off _len) i = iterArray arr (off + i)+{-# INLINE iter #-}++-- | @since 2.0+iterArray ::+#if defined(ASSERTS)+  HasCallStack =>+#endif+  A.Array -> Int -> Iter+iterArray arr j = Iter chr l+  where m0 = A.unsafeIndex arr j+        m1 = A.unsafeIndex arr (j+1)+        m2 = A.unsafeIndex arr (j+2)+        m3 = A.unsafeIndex arr (j+3)+        l = utf8LengthByLeader m0+        chr = case l of+            1 -> unsafeChr8 m0+            2 -> chr2 m0 m1+            3 -> chr3 m0 m1 m2+            _ -> chr4 m0 m1 m2 m3+{-# INLINE iterArray #-}++-- | /O(1)/ Iterate one step through a UTF-8 array, returning the+-- delta to add to give the next offset to iterate at.+iter_ :: Text -> Int -> Int+iter_ (Text arr off _len) i = utf8LengthByLeader m+  where m = A.unsafeIndex arr (off+i)+{-# INLINE iter_ #-}++-- | /O(1)/ Iterate one step backwards through a UTF-8 array,+-- returning the current character and the delta to add (i.e. a+-- negative number) to give the next offset to iterate at.+reverseIter :: Text -> Int -> Iter+reverseIter (Text arr off _len) i = reverseIterArray arr (off + i)+{-# INLINE reverseIter #-}++-- | @since 2.0+reverseIterArray :: A.Array -> Int -> Iter+reverseIterArray arr j+    | m0 <  0x80 = Iter (unsafeChr8 m0) (-1)+    | m1 >= 0xC0 = Iter (chr2 m1 m0) (-2)+    | m2 >= 0xC0 = Iter (chr3 m2 m1 m0) (-3)+    | otherwise  = Iter (chr4 m3 m2 m1 m0) (-4)+  where m0 = A.unsafeIndex arr j+        m1 = A.unsafeIndex arr (j-1)+        m2 = A.unsafeIndex arr (j-2)+        m3 = A.unsafeIndex arr (j-3)+{-# INLINE reverseIterArray #-}++-- | /O(1)/ Iterate one step backwards through a UTF-8 array,+-- returning the delta to add (i.e. a negative number) to give the+-- next offset to iterate at.+--+-- @since 1.1.1.0+reverseIter_ :: Text -> Int -> Int+reverseIter_ (Text arr off _len) i+    | m0 <  0x80 = -1+    | m1 >= 0xC0 = -2+    | m2 >= 0xC0 = -3+    | otherwise  = -4+  where m0 = A.unsafeIndex arr j+        m1 = A.unsafeIndex arr (j-1)+        m2 = A.unsafeIndex arr (j-2)+        j = off + i+{-# INLINE reverseIter_ #-}++-- | /O(1)/ Return the length of a 'Text' in units of 'Word8'.  This+-- is useful for sizing a target array appropriately before using+-- 'unsafeCopyToPtr'.+--+-- @since 2.0+lengthWord8 :: Text -> Int+lengthWord8 (Text _arr _off len) = len+{-# INLINE lengthWord8 #-}++-- | /O(1)/ Unchecked take of 'k' 'Word8's from the front of a 'Text'.+--+-- @since 2.0+takeWord8 :: Int -> Text -> Text+takeWord8 k (Text arr off _len) = Text arr off k+{-# INLINE takeWord8 #-}++-- | /O(1)/ Unchecked drop of 'k' 'Word8's from the front of a 'Text'.+--+-- @since 2.0+dropWord8 :: Int -> Text -> Text+dropWord8 k (Text arr off len) = Text arr (off+k) (len-k)+{-# INLINE dropWord8 #-}
− tests-and-benchmarks.markdown 
@@ -1,68 +0,0 @@-Tests and benchmarks-====================--Prerequisites----------------To run the tests and benchmarks, you will need the test data, which-you can clone from one of the following locations:--* Mercurial master repository:-  [bitbucket.org/bos/text-test-data](https://bitbucket.org/bos/text-test-data)--* Git mirror repository:-  [github.com/bos/text-test-data](https://github.com/bos/text-test-data)--You can clone either repository into the `tests` subdirectory using--    cd tests/-    make text-test-data         # to clone from mercurial, OR-    make VCS=git text-test-data # to clone from git--Many tests and benchmarks will fail if the test files are missing.--Functional tests-------------------The functional tests are located in the `tests` subdirectory. An overview of-what's in that directory:--    Makefile          Has targets for common tasks-    Tests             Source files of the testing code-    scripts           Various utility scripts-    text-tests.cabal  Cabal file that compiles all benchmarks--The `text-tests.cabal` builds:--- A copy of the text library, sharing the source code, but exposing all internal-  modules, for testing purposes-- The different test suites--To compile, run all tests, and generate a coverage report, simply use `make`.--Benchmarks-------------The benchmarks are located in the `benchmarks` subdirectory. An overview of-what's in that directory:--    Makefile               Has targets for common tasks-    haskell                Source files of the haskell benchmarks-    python                 Python implementations of some benchmarks-    ruby                   Ruby implementations of some benchmarks-    text-benchmarks.cabal  Cabal file which compiles all benchmarks--To compile the benchmarks, navigate to the `benchmarks` subdirectory and run-`cabal configure && cabal build`. Then, you can run the benchmarks using:--    ./dist/build/text-benchmarks/text-benchmarks--Or if you have a recent enough `cabal`, you can build and run the-benchmarks via--    cabal new-run exe:text-benchmarks -- --help--However, since there's quite a lot of benchmarks, you usually don't want to-run them all. Instead, use the `-l` flag to get a list of benchmarks-and run the ones you want to inspect. If you want to configure the benchmarks-further, the exact parameters can be changed in `Benchmarks.hs`.
− tests/.ghci 
@@ -1,1 +0,0 @@-:set -isrc -i../..
− tests/Makefile 
@@ -1,45 +0,0 @@-VCS = hg-count = 1000--all: coverage literal-rule-test--literal-rule-test:-	./literal-rule-test.sh--coverage: build coverage/hpc_index.html--build: text-test-data-	cabal configure -fhpc-	cabal build--text-test-data:-ifeq ($(VCS),git)-	git clone https://github.com/bos/text-test-data.git-else-	hg clone https://bitbucket.org/bos/text-test-data-endif-	$(MAKE) -C text-test-data--coverage/text-tests.tix:-	-mkdir -p coverage-	./dist/build/text-tests/text-tests -a $(count)-	mv text-tests.tix $@--coverage/text-tests-stdio.tix:-	-mkdir -p coverage-	./scripts/cover-stdio.sh ./dist/build/text-tests-stdio/text-tests-stdio-	mv text-tests-stdio.tix $@--coverage/coverage.tix: coverage/text-tests.tix coverage/text-tests-stdio.tix-	hpc combine --output=$@ \-        --exclude=Main \-        coverage/text-tests.tix \-        coverage/text-tests-stdio.tix--coverage/hpc_index.html: coverage/coverage.tix-	hpc markup --destdir=coverage coverage/coverage.tix--clean:-	rm -rf dist coverage .hpc--.PHONY: all build clean coverage literal-rule-test
tests/Tests.hs view
@@ -1,13 +1,22 @@--- | Provides a simple main function which runs all the tests---+{-# LANGUAGE CPP #-}+ module Main     ( main     ) where -import Test.Framework (defaultMain)+import Test.Tasty (defaultMain, testGroup) +import qualified Tests.Lift as Lift import qualified Tests.Properties as Properties import qualified Tests.Regressions as Regressions+import qualified Tests.ShareEmpty as ShareEmpty+import qualified Tests.RebindableSyntaxTest as RST  main :: IO ()-main = defaultMain [Properties.tests, Regressions.tests]+main = defaultMain $ testGroup "All"+  [ Lift.tests+  , Properties.tests+  , Regressions.tests+  , ShareEmpty.tests+  , RST.tests+  ]
− tests/Tests/IO.hs 
@@ -1,34 +0,0 @@--- | Program which exposes some haskell functions as an exutable. The results--- and coverage of this module is meant to be checked using a shell script.----module Main-    (-      main-    ) where--import System.Environment (getArgs)-import System.Exit (exitFailure)-import System.IO (hPutStrLn, stderr)-import qualified Data.Text as T-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.IO as TL--main :: IO ()-main = do-  args <- getArgs-  case args of-    ["T.readFile", name] -> T.putStr =<< T.readFile name-    ["T.writeFile", name, t] -> T.writeFile name (T.pack t)-    ["T.appendFile", name, t] -> T.appendFile name (T.pack t)-    ["T.interact"] -> T.interact id-    ["T.getContents"] -> T.putStr =<< T.getContents-    ["T.getLine"] -> T.putStrLn =<< T.getLine--    ["TL.readFile", name] -> TL.putStr =<< TL.readFile name-    ["TL.writeFile", name, t] -> TL.writeFile name (TL.pack t)-    ["TL.appendFile", name, t] -> TL.appendFile name (TL.pack t)-    ["TL.interact"] -> TL.interact id-    ["TL.getContents"] -> TL.putStr =<< TL.getContents-    ["TL.getLine"] -> TL.putStrLn =<< TL.getLine-    _ -> hPutStrLn stderr "invalid directive!" >> exitFailure
+ tests/Tests/Lift.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE CPP #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+module Tests.Lift
+  ( tests
+  )
+  where
+
+import qualified Data.Text as S
+import qualified Data.Text.Lazy as L
+#if __GLASGOW_HASKELL__ >= 914
+import Language.Haskell.TH.Lift (lift)
+#else
+import Language.Haskell.TH.Syntax (lift)
+#endif
+import Test.Tasty.HUnit (testCase, assertEqual)
+import Test.Tasty (TestTree, testGroup)
+
+tests :: TestTree
+tests = testGroup "TH lifting Text"
+  [ testCase "strict" $ assertEqual "strict"
+      $(lift ("foo" :: S.Text))
+      ("foo" :: S.Text)
+  , testCase "strict0" $ assertEqual "strict0"
+      $(lift ("f\0o\1o\2" :: S.Text))
+      ("f\0o\1o\2" :: S.Text)
+  , testCase "strict-nihao" $ assertEqual "strict-nihao"
+      $(lift ("\20320\22909" :: S.Text))
+      ("\20320\22909" :: S.Text)
+  , testCase "lazy" $ assertEqual "lazy"
+      $(lift ("foo" :: L.Text))
+      ("foo" :: L.Text)
+  , testCase "lazy0" $ assertEqual "lazy0"
+      $(lift ("f\0o\1o\2" :: L.Text))
+      ("f\0o\1o\2" :: L.Text)
+  , testCase "lazy-nihao" $ assertEqual "lazy-nihao"
+      $(lift ("\20320\22909" :: L.Text))
+      ("\20320\22909" :: L.Text)
+  ]
tests/Tests/Properties.hs view
@@ -1,1460 +1,36 @@ -- | QuickCheck properties for the text library. -{-# LANGUAGE CPP #-}-{-# LANGUAGE BangPatterns, FlexibleInstances, OverloadedStrings,-             ScopedTypeVariables, TypeSynonymInstances #-}-{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-}-module Tests.Properties-    (-      tests-    ) where--import Control.Applicative ((<$>), (<*>))-import Control.Arrow ((***), first, second)-import Data.Bits ((.&.))-import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isLetter, isUpper, ord)-import Data.Int (Int8, Int16, Int32, Int64)-import Data.Monoid (Monoid(..))-import Data.String (IsString(fromString))-import Data.Text.Encoding.Error-import Data.Text.Foreign-import Data.Text.Internal.Encoding.Utf8-import Data.Text.Internal.Fusion.Size-import Data.Text.Internal.Search (indices)-import Data.Text.Lazy.Read as TL-import Data.Text.Read as T-import Data.Word (Word, Word8, Word16, Word32, Word64)-import Data.Maybe (mapMaybe)-import Numeric (showEFloat, showFFloat, showGFloat, showHex)-import Prelude hiding (replicate)-import Test.Framework (Test, testGroup)-import Test.Framework.Providers.QuickCheck2 (testProperty)-import Test.QuickCheck hiding ((.&.))-import Test.QuickCheck.Monadic-import Test.QuickCheck.Property (Property(..))-import Test.QuickCheck.Unicode (char)-import Tests.QuickCheckUtils-import Tests.Utils-import Text.Show.Functions ()-import qualified Control.Exception as Exception-import qualified Data.Bits as Bits (shiftL, shiftR)-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL-import qualified Data.Char as C-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Encoding as E-import qualified Data.Text.IO as T-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Internal.Lazy.Fusion as SL-import qualified Data.Text.Internal.Lazy.Search as S (indices)-import qualified Data.Text.Internal.Unsafe.Shift as U-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.Builder as TB-import qualified Data.Text.Lazy.Builder.Int as TB-import qualified Data.Text.Lazy.Builder.RealFloat as TB-import qualified Data.Text.Lazy.Encoding as EL-import qualified Data.Text.Lazy.IO as TL-import qualified System.IO as IO-import qualified Tests.Properties.Mul as Mul-import qualified Tests.SlowFunctions as Slow--t_pack_unpack       = (T.unpack . T.pack) `eq` id-tl_pack_unpack      = (TL.unpack . TL.pack) `eq` id-t_stream_unstream   = (S.unstream . S.stream) `eq` id-tl_stream_unstream  = (SL.unstream . SL.stream) `eq` id-t_reverse_stream t  = (S.reverse . S.reverseStream) t === t-t_singleton c       = [c] === (T.unpack . T.singleton) c-tl_singleton c      = [c] === (TL.unpack . TL.singleton) c-tl_unstreamChunks x = f 11 x === f 1000 x-    where f n = SL.unstreamChunks n . S.streamList-tl_chunk_unchunk    = (TL.fromChunks . TL.toChunks) `eq` id-tl_from_to_strict   = (TL.fromStrict . TL.toStrict) `eq` id---- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack'-encodeL1 :: T.Text -> B.ByteString-encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack-encodeLazyL1 :: TL.Text -> BL.ByteString-encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks--t_ascii t    = E.decodeASCII (E.encodeUtf8 a) === a-    where a  = T.map (\c -> chr (ord c `mod` 128)) t-tl_ascii t   = EL.decodeASCII (EL.encodeUtf8 a) === a-    where a  = TL.map (\c -> chr (ord c `mod` 128)) t-t_latin1 t   = E.decodeLatin1 (encodeL1 a) === a-    where a  = T.map (\c -> chr (ord c `mod` 256)) t-tl_latin1 t  = EL.decodeLatin1 (encodeLazyL1 a) === a-    where a  = TL.map (\c -> chr (ord c `mod` 256)) t-t_utf8       = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id-t_utf8'      = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)-tl_utf8      = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id-tl_utf8'     = forAll genUnicode $ (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right)-t_utf16LE    = forAll genUnicode $ (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id-tl_utf16LE   = forAll genUnicode $ (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id-t_utf16BE    = forAll genUnicode $ (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id-tl_utf16BE   = forAll genUnicode $ (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id-t_utf32LE    = forAll genUnicode $ (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id-tl_utf32LE   = forAll genUnicode $ (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id-t_utf32BE    = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id-tl_utf32BE   = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id--t_utf8_incr = forAll genUnicode $ \s (Positive n) -> (recode n `eq` id) s-    where recode n = T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 .-                     E.encodeUtf8--feedChunksOf :: Int -> (B.ByteString -> E.Decoding) -> B.ByteString-             -> [(T.Text, B.ByteString)]-feedChunksOf n f bs-  | B.null bs  = []-  | otherwise  = let (x,y) = B.splitAt n bs-                     E.Some t b f' = f x-                 in (t,b) : feedChunksOf n f' y--t_utf8_undecoded = forAll genUnicode $ \t ->-  let b = E.encodeUtf8 t-      ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t-      leftover = (++ [B.empty]) . init . tail . B.inits-  in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls--data Badness = Solo | Leading | Trailing-             deriving (Eq, Show)--instance Arbitrary Badness where-    arbitrary = elements [Solo, Leading, Trailing]--t_utf8_err :: Badness -> Maybe DecodeErr -> Property-t_utf8_err bad mde = do-  let gen = case bad of-        Solo     -> genInvalidUTF8-        Leading  -> B.append <$> genInvalidUTF8 <*> genUTF8-        Trailing -> B.append <$> genUTF8 <*> genInvalidUTF8-      genUTF8 = E.encodeUtf8 <$> genUnicode-  forAll gen $ \bs -> MkProperty $-    case mde of-      -- generate an invalid character-      Nothing -> do-        c <- choose ('\x10000', maxBound)-        let onErr _ _ = Just c-        unProperty . monadicIO $ do-        l <- run $ let len = T.length (E.decodeUtf8With onErr bs)-                   in (len `seq` return (Right len)) `Exception.catch`-                      (\(e::Exception.SomeException) -> return (Left e))-        assert $ case l of-          Left err ->-            "non-BMP replacement characters not supported" `T.isInfixOf` T.pack (show err)-          Right _  -> False--      -- generate a valid onErr-      Just de -> do-        onErr <- genDecodeErr de-        unProperty . monadicIO $ do-        l <- run $ let len = T.length (E.decodeUtf8With onErr bs)-                   in (len `seq` return (Right len)) `Exception.catch`-                      (\(e::UnicodeException) -> return (Left e))-        assert $ case l of-          Left err -> length (show err) >= 0-          Right _  -> de /= Strict--t_utf8_err' :: B.ByteString -> Property-t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of-                                        Left err -> length (show err) >= 0-                                        Right t  -> T.length t >= 0--genInvalidUTF8 :: Gen B.ByteString-genInvalidUTF8 = B.pack <$> oneof [-    -- invalid leading byte of a 2-byte sequence-    (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte-    -- invalid leading byte of a 4-byte sequence-  , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte-    -- 4-byte sequence greater than U+10FFFF-  , do k <- choose (0x11, 0x13)-       let w0 = 0xF0 + (k `Bits.shiftR` 2)-           w1 = 0x80 + ((k .&. 3) `Bits.shiftL` 4)-       ([w0,w1]++) <$> vectorOf 2 contByte-    -- continuation bytes without a start byte-  , listOf1 contByte-    -- short 2-byte sequence-  , (:[]) <$> choose (0xC2, 0xDF)-    -- short 3-byte sequence-  , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte-    -- short 4-byte sequence-  , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte-    -- overlong encoding-  , do k <- choose (0,0xFFFF)-       let c = chr k-       case k of-         _ | k < 0x80   -> oneof [ let (w,x)     = ord2 c in return [w,x]-                                 , let (w,x,y)   = ord3 c in return [w,x,y]-                                 , let (w,x,y,z) = ord4 c in return [w,x,y,z] ]-           | k < 0x7FF  -> oneof [ let (w,x,y)   = ord3 c in return [w,x,y]-                                 , let (w,x,y,z) = ord4 c in return [w,x,y,z] ]-           | otherwise  ->         let (w,x,y,z) = ord4 c in return [w,x,y,z]-  ]-  where-    contByte = (0x80 +) <$> choose (0, 0x3f)-    upTo n gen = do-      k <- choose (0,n)-      vectorOf k gen---- See http://unicode.org/faq/utf_bom.html#gen8--- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...--- When faced with this illegal byte sequence ... a UTF-8 conformant process--- must treat the first byte 110xxxxx2 as an illegal termination error--- (e.g. filter it out or replace by 0xFFFD) ...--- ... and continue processing at the second byte 0xxxxxxx2-t_decode_with_error2 =-  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"-t_decode_with_error3 =-  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"-t_decode_with_error4 =-  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"--t_decode_with_error2' =-  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of-    E.Some x _ _ -> x === "xa"-t_decode_with_error3' =-  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of-    E.Some x _ _ -> x === "xaa"-t_decode_with_error4' =-  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of-    E.Some x _ _ -> x === "xaaa"--t_infix_concat bs1 text bs2 =-  forAll (genDecodeErr Replace) $ \onErr ->-  text `T.isInfixOf`-    E.decodeUtf8With onErr (B.concat [bs1, E.encodeUtf8 text, bs2])--s_Eq s            = (s==)    `eq` ((S.streamList s==) . S.streamList)-    where _types = s :: String-sf_Eq p s =-    ((L.filter p s==) . L.filter p) `eq`-    (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList)-t_Eq s            = (s==)    `eq` ((T.pack s==) . T.pack)-tl_Eq s           = (s==)    `eq` ((TL.pack s==) . TL.pack)-s_Ord s           = (compare s) `eq` (compare (S.streamList s) . S.streamList)-    where _types = s :: String-sf_Ord p s =-    ((compare $ L.filter p s) . L.filter p) `eq`-    (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList)-t_Ord s           = (compare s) `eq` (compare (T.pack s) . T.pack)-tl_Ord s          = (compare s) `eq` (compare (TL.pack s) . TL.pack)-t_Read            = id       `eq` (T.unpack . read . show)-tl_Read           = id       `eq` (TL.unpack . read . show)-t_Show            = show     `eq` (show . T.pack)-tl_Show           = show     `eq` (show . TL.pack)-t_mappend s       = mappend s`eqP` (unpackS . mappend (T.pack s))-tl_mappend s      = mappend s`eqP` (unpackS . mappend (TL.pack s))-t_mconcat         = unsquare $-                    mconcat `eq` (unpackS . mconcat . L.map T.pack)-tl_mconcat        = unsquare $-                    mconcat `eq` (unpackS . mconcat . L.map TL.pack)-t_mempty          = mempty === (unpackS (mempty :: T.Text))-tl_mempty         = mempty === (unpackS (mempty :: TL.Text))-t_IsString        = fromString  `eqP` (T.unpack . fromString)-tl_IsString       = fromString  `eqP` (TL.unpack . fromString)--s_cons x          = (x:)     `eqP` (unpackS . S.cons x)-s_cons_s x        = (x:)     `eqP` (unpackS . S.unstream . S.cons x)-sf_cons p x       = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p)-t_cons x          = (x:)     `eqP` (unpackS . T.cons x)-tl_cons x         = (x:)     `eqP` (unpackS . TL.cons x)-s_snoc x          = (++ [x]) `eqP` (unpackS . (flip S.snoc) x)-t_snoc x          = (++ [x]) `eqP` (unpackS . (flip T.snoc) x)-tl_snoc x         = (++ [x]) `eqP` (unpackS . (flip TL.snoc) x)-s_append s        = (s++)    `eqP` (unpackS . S.append (S.streamList s))-s_append_s s      = (s++)    `eqP`-                    (unpackS . S.unstream . S.append (S.streamList s))-sf_append p s     = (L.filter p s++) `eqP`-                    (unpackS . S.append (S.filter p $ S.streamList s))-t_append s        = (s++)    `eqP` (unpackS . T.append (packS s))--uncons (x:xs) = Just (x,xs)-uncons _      = Nothing--s_uncons          = uncons   `eqP` (fmap (second unpackS) . S.uncons)-sf_uncons p       = (uncons . L.filter p) `eqP`-                    (fmap (second unpackS) . S.uncons . S.filter p)-t_uncons          = uncons   `eqP` (fmap (second unpackS) . T.uncons)-tl_uncons         = uncons   `eqP` (fmap (second unpackS) . TL.uncons)--unsnoc xs@(_:_) = Just (init xs, last xs)-unsnoc []       = Nothing--t_unsnoc          = unsnoc   `eqP` (fmap (first unpackS) . T.unsnoc)-tl_unsnoc         = unsnoc   `eqP` (fmap (first unpackS) . TL.unsnoc)--s_head            = head   `eqP` S.head-sf_head p         = (head . L.filter p) `eqP` (S.head . S.filter p)-t_head            = head   `eqP` T.head-tl_head           = head   `eqP` TL.head-s_last            = last   `eqP` S.last-sf_last p         = (last . L.filter p) `eqP` (S.last . S.filter p)-t_last            = last   `eqP` T.last-tl_last           = last   `eqP` TL.last-s_tail            = tail   `eqP` (unpackS . S.tail)-s_tail_s          = tail   `eqP` (unpackS . S.unstream . S.tail)-sf_tail p         = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p)-t_tail            = tail   `eqP` (unpackS . T.tail)-tl_tail           = tail   `eqP` (unpackS . TL.tail)-s_init            = init   `eqP` (unpackS . S.init)-s_init_s          = init   `eqP` (unpackS . S.unstream . S.init)-sf_init p         = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p)-t_init            = init   `eqP` (unpackS . T.init)-tl_init           = init   `eqP` (unpackS . TL.init)-s_null            = null   `eqP` S.null-sf_null p         = (null . L.filter p) `eqP` (S.null . S.filter p)-t_null            = null   `eqP` T.null-tl_null           = null   `eqP` TL.null-s_length          = length `eqP` S.length-sf_length p       = (length . L.filter p) `eqP` (S.length . S.filter p)-sl_length         = (fromIntegral . length) `eqP` SL.length-t_length          = length `eqP` T.length-tl_length         = L.genericLength `eqP` TL.length-t_compareLength t = (compare (T.length t)) `eq` T.compareLength t-tl_compareLength t= (compare (TL.length t)) `eq` TL.compareLength t--s_map f           = map f  `eqP` (unpackS . S.map f)-s_map_s f         = map f  `eqP` (unpackS . S.unstream . S.map f)-sf_map p f        = (map f . L.filter p)  `eqP` (unpackS . S.map f . S.filter p)-t_map f           = map f  `eqP` (unpackS . T.map f)-tl_map f          = map f  `eqP` (unpackS . TL.map f)-s_intercalate c   = unsquare $-                    L.intercalate c `eq`-                    (unpackS . S.intercalate (packS c) . map packS)-t_intercalate c   = unsquare $-                    L.intercalate c `eq`-                    (unpackS . T.intercalate (packS c) . map packS)-tl_intercalate c  = unsquare $-                    L.intercalate c `eq`-                    (unpackS . TL.intercalate (TL.pack c) . map TL.pack)-s_intersperse c   = L.intersperse c `eqP`-                    (unpackS . S.intersperse c)-s_intersperse_s c = L.intersperse c `eqP`-                    (unpackS . S.unstream . S.intersperse c)-sf_intersperse p c= (L.intersperse c . L.filter p) `eqP`-                   (unpackS . S.intersperse c . S.filter p)-t_intersperse c   = unsquare $-                    L.intersperse c `eqP` (unpackS . T.intersperse c)-tl_intersperse c  = unsquare $-                    L.intersperse c `eqP` (unpackS . TL.intersperse c)-t_transpose       = unsquare $-                    L.transpose `eq` (map unpackS . T.transpose . map packS)-tl_transpose      = unsquare $-                    L.transpose `eq` (map unpackS . TL.transpose . map TL.pack)-t_reverse         = L.reverse `eqP` (unpackS . T.reverse)-tl_reverse        = L.reverse `eqP` (unpackS . TL.reverse)-t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)--t_replace s d     = (L.intercalate d . splitOn s) `eqP`-                    (unpackS . T.replace (T.pack s) (T.pack d))-tl_replace s d     = (L.intercalate d . splitOn s) `eqP`-                     (unpackS . TL.replace (TL.pack s) (TL.pack d))--splitOn :: (Eq a) => [a] -> [a] -> [[a]]-splitOn pat src0-    | l == 0    = error "splitOn: empty"-    | otherwise = go src0-  where-    l           = length pat-    go src      = search 0 src-      where-        search _ [] = [src]-        search !n s@(_:s')-            | pat `L.isPrefixOf` s = take n src : go (drop l s)-            | otherwise            = search (n+1) s'--s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs-    where s = S.streamList xs-sf_toCaseFold_length p xs =-    (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs)-    where s = S.streamList xs-t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t-tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t-t_toLower_length t = T.length (T.toLower t) >= T.length t-t_toLower_lower t = p (T.toLower t) >= p t-    where p = T.length . T.filter isLower-tl_toLower_lower t = p (TL.toLower t) >= p t-    where p = TL.length . TL.filter isLower-t_toUpper_length t = T.length (T.toUpper t) >= T.length t-t_toUpper_upper t = p (T.toUpper t) >= p t-    where p = T.length . T.filter isUpper-tl_toUpper_upper t = p (TL.toUpper t) >= p t-    where p = TL.length . TL.filter isUpper-t_toTitle_title t = all (<= 1) (caps w)-    where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle-          -- TIL: there exist uppercase-only letters-          w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t-t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable-    where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words-          -- Surprise! The Spanish/Portuguese ordinal indicators changed-          -- from category Ll (letter, lowercase) to Lo (letter, other)-          -- in Unicode 7.0-          -- Oh, and there exist lowercase-only letters (see previous test)-          stable c = if isLower c-                     then C.toUpper c /= c-                     else c /= '\170' && c /= '\186'--justifyLeft k c xs  = xs ++ L.replicate (k - length xs) c-justifyRight m n xs = L.replicate (m - length xs) n ++ xs-center k c xs-    | len >= k  = xs-    | otherwise = L.replicate l c ++ xs ++ L.replicate r c-   where len = length xs-         d   = k - len-         r   = d `div` 2-         l   = d - r--s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c)-    where j = fromIntegral (k :: Word8)-s_justifyLeft_s k c = justifyLeft j c `eqP`-                      (unpackS . S.unstream . S.justifyLeftI j c)-    where j = fromIntegral (k :: Word8)-sf_justifyLeft p k c = (justifyLeft j c . L.filter p) `eqP`-                       (unpackS . S.justifyLeftI j c . S.filter p)-    where j = fromIntegral (k :: Word8)-t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c)-    where j = fromIntegral (k :: Word8)-tl_justifyLeft k c = justifyLeft j c `eqP`-                     (unpackS . TL.justifyLeft (fromIntegral j) c)-    where j = fromIntegral (k :: Word8)-t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c)-    where j = fromIntegral (k :: Word8)-tl_justifyRight k c = justifyRight j c `eqP`-                      (unpackS . TL.justifyRight (fromIntegral j) c)-    where j = fromIntegral (k :: Word8)-t_center k c = center j c `eqP` (unpackS . T.center j c)-    where j = fromIntegral (k :: Word8)-tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c)-    where j = fromIntegral (k :: Word8)--sf_foldl p f z    = (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p)-    where _types  = f :: Char -> Char -> Char-t_foldl f z       = L.foldl f z  `eqP` (T.foldl f z)-    where _types  = f :: Char -> Char -> Char-tl_foldl f z      = L.foldl f z  `eqP` (TL.foldl f z)-    where _types  = f :: Char -> Char -> Char-sf_foldl' p f z   = (L.foldl' f z . L.filter p) `eqP`-                    (S.foldl' f z . S.filter p)-    where _types  = f :: Char -> Char -> Char-t_foldl' f z      = L.foldl' f z `eqP` T.foldl' f z-    where _types  = f :: Char -> Char -> Char-tl_foldl' f z     = L.foldl' f z `eqP` TL.foldl' f z-    where _types  = f :: Char -> Char -> Char-sf_foldl1 p f     = (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p)-t_foldl1 f        = L.foldl1 f   `eqP` T.foldl1 f-tl_foldl1 f       = L.foldl1 f   `eqP` TL.foldl1 f-sf_foldl1' p f    = (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p)-t_foldl1' f       = L.foldl1' f  `eqP` T.foldl1' f-tl_foldl1' f      = L.foldl1' f  `eqP` TL.foldl1' f-sf_foldr p f z    = (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p)-    where _types  = f :: Char -> Char -> Char-t_foldr f z       = L.foldr f z  `eqP` T.foldr f z-    where _types  = f :: Char -> Char -> Char-tl_foldr f z      = unsquare $-                    L.foldr f z  `eqP` TL.foldr f z-    where _types  = f :: Char -> Char -> Char-sf_foldr1 p f     = unsquare $-                    (L.foldr1 f . L.filter p) `eqP` (S.foldr1 f . S.filter p)-t_foldr1 f        = L.foldr1 f   `eqP` T.foldr1 f-tl_foldr1 f       = unsquare $-                    L.foldr1 f   `eqP` TL.foldr1 f--s_concat_s        = unsquare $-                    L.concat `eq` (unpackS . S.unstream . S.concat . map packS)-sf_concat p       = unsquare $-                    (L.concat . map (L.filter p)) `eq`-                    (unpackS . S.concat . map (S.filter p . packS))-t_concat          = unsquare $-                    L.concat `eq` (unpackS . T.concat . map packS)-tl_concat         = unsquare $-                    L.concat `eq` (unpackS . TL.concat . map TL.pack)-sf_concatMap p f  = unsquare $ (L.concatMap f . L.filter p) `eqP`-                               (unpackS . S.concatMap (packS . f) . S.filter p)-t_concatMap f     = unsquare $-                    L.concatMap f `eqP` (unpackS . T.concatMap (packS . f))-tl_concatMap f    = unsquare $-                    L.concatMap f `eqP` (unpackS . TL.concatMap (TL.pack . f))-sf_any q p        = (L.any p . L.filter q) `eqP` (S.any p . S.filter q)-t_any p           = L.any p       `eqP` T.any p-tl_any p          = L.any p       `eqP` TL.any p-sf_all q p        = (L.all p . L.filter q) `eqP` (S.all p . S.filter q)-t_all p           = L.all p       `eqP` T.all p-tl_all p          = L.all p       `eqP` TL.all p-sf_maximum p      = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p)-t_maximum         = L.maximum     `eqP` T.maximum-tl_maximum        = L.maximum     `eqP` TL.maximum-sf_minimum p      = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p)-t_minimum         = L.minimum     `eqP` T.minimum-tl_minimum        = L.minimum     `eqP` TL.minimum--sf_scanl p f z    = (L.scanl f z . L.filter p) `eqP`-                    (unpackS . S.scanl f z . S.filter p)-t_scanl f z       = L.scanl f z   `eqP` (unpackS . T.scanl f z)-tl_scanl f z      = L.scanl f z   `eqP` (unpackS . TL.scanl f z)-t_scanl1 f        = L.scanl1 f    `eqP` (unpackS . T.scanl1 f)-tl_scanl1 f       = L.scanl1 f    `eqP` (unpackS . TL.scanl1 f)-t_scanr f z       = L.scanr f z   `eqP` (unpackS . T.scanr f z)-tl_scanr f z      = L.scanr f z   `eqP` (unpackS . TL.scanr f z)-t_scanr1 f        = L.scanr1 f    `eqP` (unpackS . T.scanr1 f)-tl_scanr1 f       = L.scanr1 f    `eqP` (unpackS . TL.scanr1 f)--t_mapAccumL f z   = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z)-    where _types  = f :: Int -> Char -> (Int,Char)-tl_mapAccumL f z  = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z)-    where _types  = f :: Int -> Char -> (Int,Char)-t_mapAccumR f z   = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)-    where _types  = f :: Int -> Char -> (Int,Char)-tl_mapAccumR f z  = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)-    where _types  = f :: Int -> Char -> (Int,Char)--tl_repeat n       = (L.take m . L.repeat) `eq`-                    (unpackS . TL.take (fromIntegral m) . TL.repeat)-    where m = fromIntegral (n :: Word8)--replicate n l = concat (L.replicate n l)--s_replicate n     = replicate m `eq`-                    (unpackS . S.replicateI (fromIntegral m) . packS)-    where m = fromIntegral (n :: Word8)-t_replicate n     = replicate m `eq` (unpackS . T.replicate m . packS)-    where m = fromIntegral (n :: Word8)-tl_replicate n    = replicate m `eq`-                    (unpackS . TL.replicate (fromIntegral m) . packS)-    where m = fromIntegral (n :: Word8)--tl_cycle n        = (L.take m . L.cycle) `eq`-                    (unpackS . TL.take (fromIntegral m) . TL.cycle . packS)-    where m = fromIntegral (n :: Word8)--tl_iterate f n    = (L.take m . L.iterate f) `eq`-                    (unpackS . TL.take (fromIntegral m) . TL.iterate f)-    where m = fromIntegral (n :: Word8)--unf :: Int -> Char -> Maybe (Char, Char)-unf n c | fromEnum c * 100 > n = Nothing-        | otherwise            = Just (c, succ c)--t_unfoldr n       = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m))-    where m = fromIntegral (n :: Word16)-tl_unfoldr n      = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m))-    where m = fromIntegral (n :: Word16)-t_unfoldrN n m    = (L.take i . L.unfoldr (unf j)) `eq`-                         (unpackS . T.unfoldrN i (unf j))-    where i = fromIntegral (n :: Word16)-          j = fromIntegral (m :: Word16)-tl_unfoldrN n m   = (L.take i . L.unfoldr (unf j)) `eq`-                         (unpackS . TL.unfoldrN (fromIntegral i) (unf j))-    where i = fromIntegral (n :: Word16)-          j = fromIntegral (m :: Word16)--unpack2 :: (Stringy s) => (s,s) -> (String,String)-unpack2 = unpackS *** unpackS--s_take n          = L.take n      `eqP` (unpackS . S.take n)-s_take_s m        = L.take n      `eqP` (unpackS . S.unstream . S.take n)-  where n = small m-sf_take p n       = (L.take n . L.filter p) `eqP`-                    (unpackS . S.take n . S.filter p)-t_take n          = L.take n      `eqP` (unpackS . T.take n)-t_takeEnd n       = (L.reverse . L.take n . L.reverse) `eqP`-                    (unpackS . T.takeEnd n)-tl_take n         = L.take n      `eqP` (unpackS . TL.take (fromIntegral n))-tl_takeEnd n      = (L.reverse . L.take (fromIntegral n) . L.reverse) `eqP`-                    (unpackS . TL.takeEnd n)-s_drop n          = L.drop n      `eqP` (unpackS . S.drop n)-s_drop_s m        = L.drop n      `eqP` (unpackS . S.unstream . S.drop n)-  where n = small m-sf_drop p n       = (L.drop n . L.filter p) `eqP`-                    (unpackS . S.drop n . S.filter p)-t_drop n          = L.drop n      `eqP` (unpackS . T.drop n)-t_dropEnd n       = (L.reverse . L.drop n . L.reverse) `eqP`-                    (unpackS . T.dropEnd n)-tl_drop n         = L.drop n      `eqP` (unpackS . TL.drop (fromIntegral n))-tl_dropEnd n      = (L.reverse . L.drop n . L.reverse) `eqP`-                    (unpackS . TL.dropEnd (fromIntegral n))-s_take_drop m     = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n)-  where n = small m-s_take_drop_s m   = (L.take n . L.drop n) `eqP`-                    (unpackS . S.unstream . S.take n . S.drop n)-  where n = small m-s_takeWhile p     = L.takeWhile p `eqP` (unpackS . S.takeWhile p)-s_takeWhile_s p   = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)-sf_takeWhile q p  = (L.takeWhile p . L.filter q) `eqP`-                    (unpackS . S.takeWhile p . S.filter q)-noMatch = do-  c <- char-  d <- suchThat char (/= c)-  return (c,d)-t_takeWhile p     = L.takeWhile p `eqP` (unpackS . T.takeWhile p)-tl_takeWhile p    = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)-t_takeWhileEnd p  = (L.reverse . L.takeWhile p . L.reverse) `eqP`-                    (unpackS . T.takeWhileEnd p)-t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $-                    T.takeWhileEnd (==d) (T.snoc t c)-tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`-                    (unpackS . TL.takeWhileEnd p)-tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $-                    TL.takeWhileEnd (==d) (TL.snoc t c)-s_dropWhile p     = L.dropWhile p `eqP` (unpackS . S.dropWhile p)-s_dropWhile_s p   = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)-sf_dropWhile q p  = (L.dropWhile p . L.filter q) `eqP`-                    (unpackS . S.dropWhile p . S.filter q)-t_dropWhile p     = L.dropWhile p `eqP` (unpackS . T.dropWhile p)-tl_dropWhile p    = L.dropWhile p `eqP` (unpackS . S.dropWhile p)-t_dropWhileEnd p  = (L.reverse . L.dropWhile p . L.reverse) `eqP`-                    (unpackS . T.dropWhileEnd p)-tl_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP`-                    (unpackS . TL.dropWhileEnd p)-t_dropAround p    = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse)-                    `eqP` (unpackS . T.dropAround p)-tl_dropAround p   = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse)-                    `eqP` (unpackS . TL.dropAround p)-t_stripStart      = T.dropWhile isSpace `eq` T.stripStart-tl_stripStart     = TL.dropWhile isSpace `eq` TL.stripStart-t_stripEnd        = T.dropWhileEnd isSpace `eq` T.stripEnd-tl_stripEnd       = TL.dropWhileEnd isSpace `eq` TL.stripEnd-t_strip           = T.dropAround isSpace `eq` T.strip-tl_strip          = TL.dropAround isSpace `eq` TL.strip-t_splitAt n       = L.splitAt n   `eqP` (unpack2 . T.splitAt n)-tl_splitAt n      = L.splitAt n   `eqP` (unpack2 . TL.splitAt (fromIntegral n))-t_span p        = L.span p      `eqP` (unpack2 . T.span p)-tl_span p       = L.span p      `eqP` (unpack2 . TL.span p)--t_breakOn_id s      = squid `eq` (uncurry T.append . T.breakOn s)-  where squid t | T.null s  = error "empty"-                | otherwise = t-tl_breakOn_id s     = squid `eq` (uncurry TL.append . TL.breakOn s)-  where squid t | TL.null s  = error "empty"-                | otherwise = t-t_breakOn_start (NotEmpty s) t =-    let (k,m) = T.breakOn s t-    in k `T.isPrefixOf` t && (T.null m || s `T.isPrefixOf` m)-tl_breakOn_start (NotEmpty s) t =-    let (k,m) = TL.breakOn s t-    in k `TL.isPrefixOf` t && TL.null m || s `TL.isPrefixOf` m-t_breakOnEnd_end (NotEmpty s) t =-    let (m,k) = T.breakOnEnd s t-    in k `T.isSuffixOf` t && (T.null m || s `T.isSuffixOf` m)-tl_breakOnEnd_end (NotEmpty s) t =-    let (m,k) = TL.breakOnEnd s t-    in k `TL.isSuffixOf` t && (TL.null m || s `TL.isSuffixOf` m)-t_break p       = L.break p     `eqP` (unpack2 . T.break p)-tl_break p      = L.break p     `eqP` (unpack2 . TL.break p)-t_group           = L.group       `eqP` (map unpackS . T.group)-tl_group          = L.group       `eqP` (map unpackS . TL.group)-t_groupBy p       = L.groupBy p   `eqP` (map unpackS . T.groupBy p)-tl_groupBy p      = L.groupBy p   `eqP` (map unpackS . TL.groupBy p)-t_inits           = L.inits       `eqP` (map unpackS . T.inits)-tl_inits          = L.inits       `eqP` (map unpackS . TL.inits)-t_tails           = L.tails       `eqP` (map unpackS . T.tails)-tl_tails          = unsquare $-                    L.tails       `eqP` (map unpackS . TL.tails)-t_findAppendId = unsquare $ \(NotEmpty s) ts ->-    let t = T.intercalate s ts-    in all (==t) $ map (uncurry T.append) (T.breakOnAll s t)-tl_findAppendId = unsquare $ \(NotEmpty s) ts ->-    let t = TL.intercalate s ts-    in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t)-t_findContains = unsquare $ \(NotEmpty s) ->-    all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s-tl_findContains = unsquare $ \(NotEmpty s) -> all (TL.isPrefixOf s . snd) .-                               TL.breakOnAll s . TL.intercalate s-sl_filterCount c  = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c-t_findCount s     = (L.length . T.breakOnAll s) `eq` T.count s-tl_findCount s    = (L.genericLength . TL.breakOnAll s) `eq` TL.count s--t_splitOn_split s  = unsquare $-                     (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s-tl_splitOn_split s = unsquare $-                     ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`-                      (map TL.fromStrict . T.splitOn s)) . T.intercalate s-t_splitOn_i (NotEmpty t)  = id `eq` (T.intercalate t . T.splitOn t)-tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t)--t_split p       = split p `eqP` (map unpackS . T.split p)-t_split_count c = (L.length . T.split (==c)) `eq`-                  ((1+) . T.count (T.singleton c))-t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)-tl_split p      = split p `eqP` (map unpackS . TL.split p)--split :: (a -> Bool) -> [a] -> [[a]]-split _ [] =  [[]]-split p xs = loop xs-    where loop s | null s'   = [l]-                 | otherwise = l : loop (tail s')-              where (l, s') = break p s--t_chunksOf_same_lengths k = all ((==k) . T.length) . ini . T.chunksOf k-  where ini [] = []-        ini xs = init xs--t_chunksOf_length k t = len == T.length t || (k <= 0 && len == 0)-  where len = L.sum . L.map T.length $ T.chunksOf k t--tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) .-                                   TL.chunksOf (fromIntegral k) . TL.fromStrict)--t_lines           = L.lines       `eqP` (map unpackS . T.lines)-tl_lines          = L.lines       `eqP` (map unpackS . TL.lines)-{--t_lines'          = lines'        `eqP` (map unpackS . T.lines')-    where lines' "" =  []-          lines' s =  let (l, s') = break eol s-                      in  l : case s' of-                                []      -> []-                                ('\r':'\n':s'') -> lines' s''-                                (_:s'') -> lines' s''-          eol c = c == '\r' || c == '\n'--}-t_words           = L.words       `eqP` (map unpackS . T.words)--tl_words          = L.words       `eqP` (map unpackS . TL.words)-t_unlines         = unsquare $-                    L.unlines `eq` (unpackS . T.unlines . map packS)-tl_unlines        = unsquare $-                    L.unlines `eq` (unpackS . TL.unlines . map packS)-t_unwords         = unsquare $-                    L.unwords `eq` (unpackS . T.unwords . map packS)-tl_unwords        = unsquare $-                    L.unwords `eq` (unpackS . TL.unwords . map packS)--s_isPrefixOf s    = L.isPrefixOf s `eqP`-                    (S.isPrefixOf (S.stream $ packS s) . S.stream)-sf_isPrefixOf p s = (L.isPrefixOf s . L.filter p) `eqP`-                    (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream)-t_isPrefixOf s    = L.isPrefixOf s`eqP` T.isPrefixOf (packS s)-tl_isPrefixOf s   = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s)-t_isSuffixOf s    = L.isSuffixOf s`eqP` T.isSuffixOf (packS s)-tl_isSuffixOf s   = L.isSuffixOf s`eqP` TL.isSuffixOf (packS s)-t_isInfixOf s     = L.isInfixOf s `eqP` T.isInfixOf (packS s)-tl_isInfixOf s    = L.isInfixOf s `eqP` TL.isInfixOf (packS s)--t_stripPrefix s      = (fmap packS . L.stripPrefix s) `eqP` T.stripPrefix (packS s)-tl_stripPrefix s     = (fmap packS . L.stripPrefix s) `eqP` TL.stripPrefix (packS s)--stripSuffix p t = reverse `fmap` L.stripPrefix (reverse p) (reverse t)--t_stripSuffix s      = (fmap packS . stripSuffix s) `eqP` T.stripSuffix (packS s)-tl_stripSuffix s     = (fmap packS . stripSuffix s) `eqP` TL.stripSuffix (packS s)--commonPrefixes a0@(_:_) b0@(_:_) = Just (go a0 b0 [])-    where go (a:as) (b:bs) ps-              | a == b = go as bs (a:ps)-          go as bs ps  = (reverse ps,as,bs)-commonPrefixes _ _ = Nothing--t_commonPrefixes a b (NonEmpty p)-    = commonPrefixes pa pb ==-      repack `fmap` T.commonPrefixes (packS pa) (packS pb)-  where repack (x,y,z) = (unpackS x,unpackS y,unpackS z)-        pa = p ++ a-        pb = p ++ b--tl_commonPrefixes a b (NonEmpty p)-    = commonPrefixes pa pb ==-      repack `fmap` TL.commonPrefixes (packS pa) (packS pb)-  where repack (x,y,z) = (unpackS x,unpackS y,unpackS z)-        pa = p ++ a-        pb = p ++ b--sf_elem p c       = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p)-sf_filter q p     = (L.filter p . L.filter q) `eqP`-                    (unpackS . S.filter p . S.filter q)-t_filter p        = L.filter p    `eqP` (unpackS . T.filter p)-tl_filter p       = L.filter p    `eqP` (unpackS . TL.filter p)-sf_findBy q p     = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)-t_find p          = L.find p      `eqP` T.find p-tl_find p         = L.find p      `eqP` TL.find p-t_partition p     = L.partition p `eqP` (unpack2 . T.partition p)-tl_partition p    = L.partition p `eqP` (unpack2 . TL.partition p)--sf_index p s      = forAll (choose (-l,l*2))-                    ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s))-    where l = L.length s-t_index s         = forAll (choose (-l,l*2)) ((s L.!!) `eq` T.index (packS s))-    where l = L.length s--tl_index s        = forAll (choose (-l,l*2))-                    ((s L.!!) `eq` (TL.index (packS s) . fromIntegral))-    where l = L.length s--t_findIndex p     = L.findIndex p `eqP` T.findIndex p-t_count (NotEmpty t)  = (subtract 1 . L.length . T.splitOn t) `eq` T.count t-tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`-                        TL.count t-t_zip s           = L.zip s `eqP` T.zip (packS s)-tl_zip s          = L.zip s `eqP` TL.zip (packS s)-sf_zipWith p c s  = (L.zipWith c (L.filter p s) . L.filter p) `eqP`-                    (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p)-t_zipWith c s     = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))-tl_zipWith c s    = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))--t_indices  (NotEmpty s) = Slow.indices s `eq` indices s-tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s-    where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t)-          conc = T.concat . TL.toChunks-t_indices_occurs = unsquare $ \(NotEmpty t) ts ->-    let s = T.intercalate t ts-    in Slow.indices t s === indices t s---- Bit shifts.-shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k-    where width = round (log (fromIntegral m) / log 2 :: Double)-          (m,_) = (maxBound, m == w)-shiftR w = forAll (choose (0,width-1)) $ \k -> Bits.shiftR w k == U.shiftR w k-    where width = round (log (fromIntegral m) / log 2 :: Double)-          (m,_) = (maxBound, m == w)--shiftL_Int    = shiftL :: Int -> Property-shiftL_Word16 = shiftL :: Word16 -> Property-shiftL_Word32 = shiftL :: Word32 -> Property-shiftR_Int    = shiftR :: Int -> Property-shiftR_Word16 = shiftR :: Word16 -> Property-shiftR_Word32 = shiftR :: Word32 -> Property---- Builder.--tb_singleton = id `eqP`-               (unpackS . TB.toLazyText . mconcat . map TB.singleton)-tb_fromText = L.concat `eq` (unpackS . TB.toLazyText . mconcat .-                                   map (TB.fromText . packS))-tb_associative s1 s2 s3 =-    TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) ==-    TB.toLazyText ((b1 `mappend` b2) `mappend` b3)-  where b1 = TB.fromText (packS s1)-        b2 = TB.fromText (packS s2)-        b3 = TB.fromText (packS s3)---- Numeric builder stuff.--tb_decimal :: (Integral a, Show a) => a -> Bool-tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show)--tb_decimal_integer (a::Integer) = tb_decimal a-tb_decimal_integer_big (Big a) = tb_decimal a-tb_decimal_int (a::Int) = tb_decimal a-tb_decimal_int8 (a::Int8) = tb_decimal a-tb_decimal_int16 (a::Int16) = tb_decimal a-tb_decimal_int32 (a::Int32) = tb_decimal a-tb_decimal_int64 (a::Int64) = tb_decimal a-tb_decimal_word (a::Word) = tb_decimal a-tb_decimal_word8 (a::Word8) = tb_decimal a-tb_decimal_word16 (a::Word16) = tb_decimal a-tb_decimal_word32 (a::Word32) = tb_decimal a-tb_decimal_word64 (a::Word64) = tb_decimal a--tb_decimal_big_int (BigBounded (a::Int)) = tb_decimal a-tb_decimal_big_int64 (BigBounded (a::Int64)) = tb_decimal a-tb_decimal_big_word (BigBounded (a::Word)) = tb_decimal a-tb_decimal_big_word64 (BigBounded (a::Word64)) = tb_decimal a--tb_hex :: (Integral a, Show a) => a -> Bool-tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "")--tb_hexadecimal_integer (a::Integer) = tb_hex a-tb_hexadecimal_int (a::Int) = tb_hex a-tb_hexadecimal_int8 (a::Int8) = tb_hex a-tb_hexadecimal_int16 (a::Int16) = tb_hex a-tb_hexadecimal_int32 (a::Int32) = tb_hex a-tb_hexadecimal_int64 (a::Int64) = tb_hex a-tb_hexadecimal_word (a::Word) = tb_hex a-tb_hexadecimal_word8 (a::Word8) = tb_hex a-tb_hexadecimal_word16 (a::Word16) = tb_hex a-tb_hexadecimal_word32 (a::Word32) = tb_hex a-tb_hexadecimal_word64 (a::Word64) = tb_hex a--tb_realfloat :: (RealFloat a, Show a) => a -> Bool-tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show)--tb_realfloat_float (a::Float) = tb_realfloat a-tb_realfloat_double (a::Double) = tb_realfloat a--showFloat :: (RealFloat a) => TB.FPFormat -> Maybe Int -> a -> ShowS-showFloat TB.Exponent (Just 0) = showEFloat (Just 1) -- see gh-231-showFloat TB.Exponent p = showEFloat p-showFloat TB.Fixed    p = showFFloat p-showFloat TB.Generic  p = showGFloat p--tb_formatRealFloat :: (RealFloat a, Show a) =>-                      a -> TB.FPFormat -> Precision a -> Property-tb_formatRealFloat a fmt prec = cond ==>-    TB.formatRealFloat fmt p a ===-    TB.fromString (showFloat fmt p a "")-  where p = precision a prec-        cond = case (p,fmt) of-#if MIN_VERSION_base(4,12,0)-                  (Just 0, TB.Generic) -> False -- skipping due to gh-231-#endif-                  _                    -> True--tb_formatRealFloat_float (a::Float) = tb_formatRealFloat a-tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a---- Reading.--t_decimal (n::Int) s =-    T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)-    where t = T.dropWhile isDigit s-tl_decimal (n::Int) s =-    TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)-    where t = TL.dropWhile isDigit s-t_hexadecimal m s ox =-    T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)-    where t = T.dropWhile isHexDigit s-          p = if ox then "0x" else ""-          n = getPositive m :: Int-tl_hexadecimal m s ox =-    TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)-    where t = TL.dropWhile isHexDigit s-          p = if ox then "0x" else ""-          n = getPositive m :: Int--isFloaty c = c `elem` ("+-.0123456789eE" :: String)--t_read_rational p tol (n::Double) s =-    case p (T.pack (show n) `T.append` t) of-      Left _err     -> False-      Right (n',t') -> t == t' && abs (n-n') <= tol-    where t = T.dropWhile isFloaty s--tl_read_rational p tol (n::Double) s =-    case p (TL.pack (show n) `TL.append` t) of-      Left _err     -> False-      Right (n',t') -> t == t' && abs (n-n') <= tol-    where t = TL.dropWhile isFloaty s--t_double = t_read_rational T.double 1e-13-tl_double = tl_read_rational TL.double 1e-13-t_rational = t_read_rational T.rational 1e-16-tl_rational = tl_read_rational TL.rational 1e-16---- Input and output.--t_put_get = write_read T.unlines T.filter put get-  where put h = withRedirect h IO.stdout . T.putStr-        get h = withRedirect h IO.stdin T.getContents-tl_put_get = write_read TL.unlines TL.filter put get-  where put h = withRedirect h IO.stdout . TL.putStr-        get h = withRedirect h IO.stdin TL.getContents-t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents-tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents--t_write_read_line e m b t = write_read head T.filter T.hPutStrLn-                            T.hGetLine e m b [t]-tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn-                             TL.hGetLine e m b [t]---- Low-level.--t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t-t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t-t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t-  where n = small m-t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr)--t_copy t = T.copy t === t---- Regression tests.-s_filter_eq s = S.filter p t == S.streamList (filter p s)-    where p = (/= S.last t)-          t = S.streamList s---- Make a stream appear shorter than it really is, to ensure that--- functions that consume inaccurately sized streams behave--- themselves.-shorten :: Int -> S.Stream a -> S.Stream a-shorten n t@(S.Stream arr off len)-    | n > 0     = S.Stream arr off (smaller (exactSize n) len)-    | otherwise = t--tests :: Test-tests =-  testGroup "Properties" [-    testGroup "creation/elimination" [-      testProperty "t_pack_unpack" t_pack_unpack,-      testProperty "tl_pack_unpack" tl_pack_unpack,-      testProperty "t_stream_unstream" t_stream_unstream,-      testProperty "tl_stream_unstream" tl_stream_unstream,-      testProperty "t_reverse_stream" t_reverse_stream,-      testProperty "t_singleton" t_singleton,-      testProperty "tl_singleton" tl_singleton,-      testProperty "tl_unstreamChunks" tl_unstreamChunks,-      testProperty "tl_chunk_unchunk" tl_chunk_unchunk,-      testProperty "tl_from_to_strict" tl_from_to_strict-    ],--    testGroup "transcoding" [-      testProperty "t_ascii" t_ascii,-      testProperty "tl_ascii" tl_ascii,-      testProperty "t_latin1" t_latin1,-      testProperty "tl_latin1" tl_latin1,-      testProperty "t_utf8" t_utf8,-      testProperty "t_utf8'" t_utf8',-      testProperty "t_utf8_incr" t_utf8_incr,-      testProperty "t_utf8_undecoded" t_utf8_undecoded,-      testProperty "tl_utf8" tl_utf8,-      testProperty "tl_utf8'" tl_utf8',-      testProperty "t_utf16LE" t_utf16LE,-      testProperty "tl_utf16LE" tl_utf16LE,-      testProperty "t_utf16BE" t_utf16BE,-      testProperty "tl_utf16BE" tl_utf16BE,-      testProperty "t_utf32LE" t_utf32LE,-      testProperty "tl_utf32LE" tl_utf32LE,-      testProperty "t_utf32BE" t_utf32BE,-      testProperty "tl_utf32BE" tl_utf32BE,-      testGroup "errors" [-        testProperty "t_utf8_err" t_utf8_err,-        testProperty "t_utf8_err'" t_utf8_err'-      ],-      testGroup "error recovery" [-        testProperty "t_decode_with_error2" t_decode_with_error2,-        testProperty "t_decode_with_error3" t_decode_with_error3,-        testProperty "t_decode_with_error4" t_decode_with_error4,-        testProperty "t_decode_with_error2'" t_decode_with_error2',-        testProperty "t_decode_with_error3'" t_decode_with_error3',-        testProperty "t_decode_with_error4'" t_decode_with_error4',-        testProperty "t_infix_concat" t_infix_concat-      ]-    ],--    testGroup "instances" [-      testProperty "s_Eq" s_Eq,-      testProperty "sf_Eq" sf_Eq,-      testProperty "t_Eq" t_Eq,-      testProperty "tl_Eq" tl_Eq,-      testProperty "s_Ord" s_Ord,-      testProperty "sf_Ord" sf_Ord,-      testProperty "t_Ord" t_Ord,-      testProperty "tl_Ord" tl_Ord,-      testProperty "t_Read" t_Read,-      testProperty "tl_Read" tl_Read,-      testProperty "t_Show" t_Show,-      testProperty "tl_Show" tl_Show,-      testProperty "t_mappend" t_mappend,-      testProperty "tl_mappend" tl_mappend,-      testProperty "t_mconcat" t_mconcat,-      testProperty "tl_mconcat" tl_mconcat,-      testProperty "t_mempty" t_mempty,-      testProperty "tl_mempty" tl_mempty,-      testProperty "t_IsString" t_IsString,-      testProperty "tl_IsString" tl_IsString-    ],--    testGroup "basics" [-      testProperty "s_cons" s_cons,-      testProperty "s_cons_s" s_cons_s,-      testProperty "sf_cons" sf_cons,-      testProperty "t_cons" t_cons,-      testProperty "tl_cons" tl_cons,-      testProperty "s_snoc" s_snoc,-      testProperty "t_snoc" t_snoc,-      testProperty "tl_snoc" tl_snoc,-      testProperty "s_append" s_append,-      testProperty "s_append_s" s_append_s,-      testProperty "sf_append" sf_append,-      testProperty "t_append" t_append,-      testProperty "s_uncons" s_uncons,-      testProperty "sf_uncons" sf_uncons,-      testProperty "t_uncons" t_uncons,-      testProperty "tl_uncons" tl_uncons,-      testProperty "t_unsnoc" t_unsnoc,-      testProperty "tl_unsnoc" tl_unsnoc,-      testProperty "s_head" s_head,-      testProperty "sf_head" sf_head,-      testProperty "t_head" t_head,-      testProperty "tl_head" tl_head,-      testProperty "s_last" s_last,-      testProperty "sf_last" sf_last,-      testProperty "t_last" t_last,-      testProperty "tl_last" tl_last,-      testProperty "s_tail" s_tail,-      testProperty "s_tail_s" s_tail_s,-      testProperty "sf_tail" sf_tail,-      testProperty "t_tail" t_tail,-      testProperty "tl_tail" tl_tail,-      testProperty "s_init" s_init,-      testProperty "s_init_s" s_init_s,-      testProperty "sf_init" sf_init,-      testProperty "t_init" t_init,-      testProperty "tl_init" tl_init,-      testProperty "s_null" s_null,-      testProperty "sf_null" sf_null,-      testProperty "t_null" t_null,-      testProperty "tl_null" tl_null,-      testProperty "s_length" s_length,-      testProperty "sf_length" sf_length,-      testProperty "sl_length" sl_length,-      testProperty "t_length" t_length,-      testProperty "tl_length" tl_length,-      testProperty "t_compareLength" t_compareLength,-      testProperty "tl_compareLength" tl_compareLength-    ],--    testGroup "transformations" [-      testProperty "s_map" s_map,-      testProperty "s_map_s" s_map_s,-      testProperty "sf_map" sf_map,-      testProperty "t_map" t_map,-      testProperty "tl_map" tl_map,-      testProperty "s_intercalate" s_intercalate,-      testProperty "t_intercalate" t_intercalate,-      testProperty "tl_intercalate" tl_intercalate,-      testProperty "s_intersperse" s_intersperse,-      testProperty "s_intersperse_s" s_intersperse_s,-      testProperty "sf_intersperse" sf_intersperse,-      testProperty "t_intersperse" t_intersperse,-      testProperty "tl_intersperse" tl_intersperse,-      testProperty "t_transpose" t_transpose,-      testProperty "tl_transpose" tl_transpose,-      testProperty "t_reverse" t_reverse,-      testProperty "tl_reverse" tl_reverse,-      testProperty "t_reverse_short" t_reverse_short,-      testProperty "t_replace" t_replace,-      testProperty "tl_replace" tl_replace,--      testGroup "case conversion" [-        testProperty "s_toCaseFold_length" s_toCaseFold_length,-        testProperty "sf_toCaseFold_length" sf_toCaseFold_length,-        testProperty "t_toCaseFold_length" t_toCaseFold_length,-        testProperty "tl_toCaseFold_length" tl_toCaseFold_length,-        testProperty "t_toLower_length" t_toLower_length,-        testProperty "t_toLower_lower" t_toLower_lower,-        testProperty "tl_toLower_lower" tl_toLower_lower,-        testProperty "t_toUpper_length" t_toUpper_length,-        testProperty "t_toUpper_upper" t_toUpper_upper,-        testProperty "tl_toUpper_upper" tl_toUpper_upper,-        testProperty "t_toTitle_title" t_toTitle_title,-        testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower-      ],--      testGroup "justification" [-        testProperty "s_justifyLeft" s_justifyLeft,-        testProperty "s_justifyLeft_s" s_justifyLeft_s,-        testProperty "sf_justifyLeft" sf_justifyLeft,-        testProperty "t_justifyLeft" t_justifyLeft,-        testProperty "tl_justifyLeft" tl_justifyLeft,-        testProperty "t_justifyRight" t_justifyRight,-        testProperty "tl_justifyRight" tl_justifyRight,-        testProperty "t_center" t_center,-        testProperty "tl_center" tl_center-      ]-    ],--    testGroup "folds" [-      testProperty "sf_foldl" sf_foldl,-      testProperty "t_foldl" t_foldl,-      testProperty "tl_foldl" tl_foldl,-      testProperty "sf_foldl'" sf_foldl',-      testProperty "t_foldl'" t_foldl',-      testProperty "tl_foldl'" tl_foldl',-      testProperty "sf_foldl1" sf_foldl1,-      testProperty "t_foldl1" t_foldl1,-      testProperty "tl_foldl1" tl_foldl1,-      testProperty "t_foldl1'" t_foldl1',-      testProperty "sf_foldl1'" sf_foldl1',-      testProperty "tl_foldl1'" tl_foldl1',-      testProperty "sf_foldr" sf_foldr,-      testProperty "t_foldr" t_foldr,-      testProperty "tl_foldr" tl_foldr,-      testProperty "sf_foldr1" sf_foldr1,-      testProperty "t_foldr1" t_foldr1,-      testProperty "tl_foldr1" tl_foldr1,--      testGroup "special" [-        testProperty "s_concat_s" s_concat_s,-        testProperty "sf_concat" sf_concat,-        testProperty "t_concat" t_concat,-        testProperty "tl_concat" tl_concat,-        testProperty "sf_concatMap" sf_concatMap,-        testProperty "t_concatMap" t_concatMap,-        testProperty "tl_concatMap" tl_concatMap,-        testProperty "sf_any" sf_any,-        testProperty "t_any" t_any,-        testProperty "tl_any" tl_any,-        testProperty "sf_all" sf_all,-        testProperty "t_all" t_all,-        testProperty "tl_all" tl_all,-        testProperty "sf_maximum" sf_maximum,-        testProperty "t_maximum" t_maximum,-        testProperty "tl_maximum" tl_maximum,-        testProperty "sf_minimum" sf_minimum,-        testProperty "t_minimum" t_minimum,-        testProperty "tl_minimum" tl_minimum-      ]-    ],--    testGroup "construction" [-      testGroup "scans" [-        testProperty "sf_scanl" sf_scanl,-        testProperty "t_scanl" t_scanl,-        testProperty "tl_scanl" tl_scanl,-        testProperty "t_scanl1" t_scanl1,-        testProperty "tl_scanl1" tl_scanl1,-        testProperty "t_scanr" t_scanr,-        testProperty "tl_scanr" tl_scanr,-        testProperty "t_scanr1" t_scanr1,-        testProperty "tl_scanr1" tl_scanr1-      ],--      testGroup "mapAccum" [-        testProperty "t_mapAccumL" t_mapAccumL,-        testProperty "tl_mapAccumL" tl_mapAccumL,-        testProperty "t_mapAccumR" t_mapAccumR,-        testProperty "tl_mapAccumR" tl_mapAccumR-      ],--      testGroup "unfolds" [-        testProperty "tl_repeat" tl_repeat,-        testProperty "s_replicate" s_replicate,-        testProperty "t_replicate" t_replicate,-        testProperty "tl_replicate" tl_replicate,-        testProperty "tl_cycle" tl_cycle,-        testProperty "tl_iterate" tl_iterate,-        testProperty "t_unfoldr" t_unfoldr,-        testProperty "tl_unfoldr" tl_unfoldr,-        testProperty "t_unfoldrN" t_unfoldrN,-        testProperty "tl_unfoldrN" tl_unfoldrN-      ]-    ],--    testGroup "substrings" [-      testGroup "breaking" [-        testProperty "s_take" s_take,-        testProperty "s_take_s" s_take_s,-        testProperty "sf_take" sf_take,-        testProperty "t_take" t_take,-        testProperty "t_takeEnd" t_takeEnd,-        testProperty "tl_take" tl_take,-        testProperty "tl_takeEnd" tl_takeEnd,-        testProperty "s_drop" s_drop,-        testProperty "s_drop_s" s_drop_s,-        testProperty "sf_drop" sf_drop,-        testProperty "t_drop" t_drop,-        testProperty "t_dropEnd" t_dropEnd,-        testProperty "tl_drop" tl_drop,-        testProperty "tl_dropEnd" tl_dropEnd,-        testProperty "s_take_drop" s_take_drop,-        testProperty "s_take_drop_s" s_take_drop_s,-        testProperty "s_takeWhile" s_takeWhile,-        testProperty "s_takeWhile_s" s_takeWhile_s,-        testProperty "sf_takeWhile" sf_takeWhile,-        testProperty "t_takeWhile" t_takeWhile,-        testProperty "tl_takeWhile" tl_takeWhile,-        testProperty "t_takeWhileEnd" t_takeWhileEnd,-        testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,-        testProperty "tl_takeWhileEnd" tl_takeWhileEnd,-        testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,-        testProperty "sf_dropWhile" sf_dropWhile,-        testProperty "s_dropWhile" s_dropWhile,-        testProperty "s_dropWhile_s" s_dropWhile_s,-        testProperty "t_dropWhile" t_dropWhile,-        testProperty "tl_dropWhile" tl_dropWhile,-        testProperty "t_dropWhileEnd" t_dropWhileEnd,-        testProperty "tl_dropWhileEnd" tl_dropWhileEnd,-        testProperty "t_dropAround" t_dropAround,-        testProperty "tl_dropAround" tl_dropAround,-        testProperty "t_stripStart" t_stripStart,-        testProperty "tl_stripStart" tl_stripStart,-        testProperty "t_stripEnd" t_stripEnd,-        testProperty "tl_stripEnd" tl_stripEnd,-        testProperty "t_strip" t_strip,-        testProperty "tl_strip" tl_strip,-        testProperty "t_splitAt" t_splitAt,-        testProperty "tl_splitAt" tl_splitAt,-        testProperty "t_span" t_span,-        testProperty "tl_span" tl_span,-        testProperty "t_breakOn_id" t_breakOn_id,-        testProperty "tl_breakOn_id" tl_breakOn_id,-        testProperty "t_breakOn_start" t_breakOn_start,-        testProperty "tl_breakOn_start" tl_breakOn_start,-        testProperty "t_breakOnEnd_end" t_breakOnEnd_end,-        testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,-        testProperty "t_break" t_break,-        testProperty "tl_break" tl_break,-        testProperty "t_group" t_group,-        testProperty "tl_group" tl_group,-        testProperty "t_groupBy" t_groupBy,-        testProperty "tl_groupBy" tl_groupBy,-        testProperty "t_inits" t_inits,-        testProperty "tl_inits" tl_inits,-        testProperty "t_tails" t_tails,-        testProperty "tl_tails" tl_tails-      ],--      testGroup "breaking many" [-        testProperty "t_findAppendId" t_findAppendId,-        testProperty "tl_findAppendId" tl_findAppendId,-        testProperty "t_findContains" t_findContains,-        testProperty "tl_findContains" tl_findContains,-        testProperty "sl_filterCount" sl_filterCount,-        testProperty "t_findCount" t_findCount,-        testProperty "tl_findCount" tl_findCount,-        testProperty "t_splitOn_split" t_splitOn_split,-        testProperty "tl_splitOn_split" tl_splitOn_split,-        testProperty "t_splitOn_i" t_splitOn_i,-        testProperty "tl_splitOn_i" tl_splitOn_i,-        testProperty "t_split" t_split,-        testProperty "t_split_count" t_split_count,-        testProperty "t_split_splitOn" t_split_splitOn,-        testProperty "tl_split" tl_split,-        testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths,-        testProperty "t_chunksOf_length" t_chunksOf_length,-        testProperty "tl_chunksOf" tl_chunksOf-      ],--      testGroup "lines and words" [-        testProperty "t_lines" t_lines,-        testProperty "tl_lines" tl_lines,-      --testProperty "t_lines'" t_lines',-        testProperty "t_words" t_words,-        testProperty "tl_words" tl_words,-        testProperty "t_unlines" t_unlines,-        testProperty "tl_unlines" tl_unlines,-        testProperty "t_unwords" t_unwords,-        testProperty "tl_unwords" tl_unwords-      ]-    ],--    testGroup "predicates" [-      testProperty "s_isPrefixOf" s_isPrefixOf,-      testProperty "sf_isPrefixOf" sf_isPrefixOf,-      testProperty "t_isPrefixOf" t_isPrefixOf,-      testProperty "tl_isPrefixOf" tl_isPrefixOf,-      testProperty "t_isSuffixOf" t_isSuffixOf,-      testProperty "tl_isSuffixOf" tl_isSuffixOf,-      testProperty "t_isInfixOf" t_isInfixOf,-      testProperty "tl_isInfixOf" tl_isInfixOf,--      testGroup "view" [-        testProperty "t_stripPrefix" t_stripPrefix,-        testProperty "tl_stripPrefix" tl_stripPrefix,-        testProperty "t_stripSuffix" t_stripSuffix,-        testProperty "tl_stripSuffix" tl_stripSuffix,-        testProperty "t_commonPrefixes" t_commonPrefixes,-        testProperty "tl_commonPrefixes" tl_commonPrefixes-      ]-    ],--    testGroup "searching" [-      testProperty "sf_elem" sf_elem,-      testProperty "sf_filter" sf_filter,-      testProperty "t_filter" t_filter,-      testProperty "tl_filter" tl_filter,-      testProperty "sf_findBy" sf_findBy,-      testProperty "t_find" t_find,-      testProperty "tl_find" tl_find,-      testProperty "t_partition" t_partition,-      testProperty "tl_partition" tl_partition-    ],--    testGroup "indexing" [-      testProperty "sf_index" sf_index,-      testProperty "t_index" t_index,-      testProperty "tl_index" tl_index,-      testProperty "t_findIndex" t_findIndex,-      testProperty "t_count" t_count,-      testProperty "tl_count" tl_count,-      testProperty "t_indices" t_indices,-      testProperty "tl_indices" tl_indices,-      testProperty "t_indices_occurs" t_indices_occurs-    ],--    testGroup "zips" [-      testProperty "t_zip" t_zip,-      testProperty "tl_zip" tl_zip,-      testProperty "sf_zipWith" sf_zipWith,-      testProperty "t_zipWith" t_zipWith,-      testProperty "tl_zipWith" tl_zipWith-    ],--    testGroup "regressions" [-      testProperty "s_filter_eq" s_filter_eq-    ],--    testGroup "shifts" [-      testProperty "shiftL_Int" shiftL_Int,-      testProperty "shiftL_Word16" shiftL_Word16,-      testProperty "shiftL_Word32" shiftL_Word32,-      testProperty "shiftR_Int" shiftR_Int,-      testProperty "shiftR_Word16" shiftR_Word16,-      testProperty "shiftR_Word32" shiftR_Word32-    ],--    testGroup "builder" [-      testProperty "tb_associative" tb_associative,-      testGroup "decimal" [-        testProperty "tb_decimal_int" tb_decimal_int,-        testProperty "tb_decimal_int8" tb_decimal_int8,-        testProperty "tb_decimal_int16" tb_decimal_int16,-        testProperty "tb_decimal_int32" tb_decimal_int32,-        testProperty "tb_decimal_int64" tb_decimal_int64,-        testProperty "tb_decimal_integer" tb_decimal_integer,-        testProperty "tb_decimal_integer_big" tb_decimal_integer_big,-        testProperty "tb_decimal_word" tb_decimal_word,-        testProperty "tb_decimal_word8" tb_decimal_word8,-        testProperty "tb_decimal_word16" tb_decimal_word16,-        testProperty "tb_decimal_word32" tb_decimal_word32,-        testProperty "tb_decimal_word64" tb_decimal_word64,-        testProperty "tb_decimal_big_int" tb_decimal_big_int,-        testProperty "tb_decimal_big_word" tb_decimal_big_word,-        testProperty "tb_decimal_big_int64" tb_decimal_big_int64,-        testProperty "tb_decimal_big_word64" tb_decimal_big_word64-      ],-      testGroup "hexadecimal" [-        testProperty "tb_hexadecimal_int" tb_hexadecimal_int,-        testProperty "tb_hexadecimal_int8" tb_hexadecimal_int8,-        testProperty "tb_hexadecimal_int16" tb_hexadecimal_int16,-        testProperty "tb_hexadecimal_int32" tb_hexadecimal_int32,-        testProperty "tb_hexadecimal_int64" tb_hexadecimal_int64,-        testProperty "tb_hexadecimal_integer" tb_hexadecimal_integer,-        testProperty "tb_hexadecimal_word" tb_hexadecimal_word,-        testProperty "tb_hexadecimal_word8" tb_hexadecimal_word8,-        testProperty "tb_hexadecimal_word16" tb_hexadecimal_word16,-        testProperty "tb_hexadecimal_word32" tb_hexadecimal_word32,-        testProperty "tb_hexadecimal_word64" tb_hexadecimal_word64-      ],-      testGroup "realfloat" [-        testProperty "tb_realfloat_double" tb_realfloat_double,-        testProperty "tb_realfloat_float" tb_realfloat_float,-        testProperty "tb_formatRealFloat_float" tb_formatRealFloat_float,-        testProperty "tb_formatRealFloat_double" tb_formatRealFloat_double-      ],-      testProperty "tb_fromText" tb_fromText,-      testProperty "tb_singleton" tb_singleton-    ],--    testGroup "read" [-      testProperty "t_decimal" t_decimal,-      testProperty "tl_decimal" tl_decimal,-      testProperty "t_hexadecimal" t_hexadecimal,-      testProperty "tl_hexadecimal" tl_hexadecimal,-      testProperty "t_double" t_double,-      testProperty "tl_double" tl_double,-      testProperty "t_rational" t_rational,-      testProperty "tl_rational" tl_rational-    ],--    {--    testGroup "input-output" [-      testProperty "t_write_read" t_write_read,-      testProperty "tl_write_read" tl_write_read,-      testProperty "t_write_read_line" t_write_read_line,-      testProperty "tl_write_read_line" tl_write_read_line-      -- These tests are subject to I/O race conditions when run under-      -- test-framework-quickcheck2.-      -- testProperty "t_put_get" t_put_get-      -- testProperty "tl_put_get" tl_put_get-    ],-    -}--    testGroup "lowlevel" [-      testProperty "t_dropWord16" t_dropWord16,-      testProperty "t_takeWord16" t_takeWord16,-      testProperty "t_take_drop_16" t_take_drop_16,-      testProperty "t_use_from" t_use_from,-      testProperty "t_copy" t_copy-    ],--    testGroup "mul" Mul.tests+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+module Tests.Properties+    (+      tests+    ) where++import Test.Tasty (TestTree, testGroup)+import Tests.Properties.Basics (testBasics)+import Tests.Properties.Builder (testBuilder)+import Tests.Properties.Folds (testFolds)+import Tests.Properties.LowLevel (testLowLevel)+import Tests.Properties.Instances (testInstances)+import Tests.Properties.Substrings (testSubstrings)+import Tests.Properties.Read (testRead)+import Tests.Properties.Text (testText)+import Tests.Properties.Transcoding (testTranscoding)+import Tests.Properties.Validate (testValidate)+import Tests.Properties.CornerCases (testCornerCases)++tests :: TestTree+tests =+  testGroup "Properties" [+    testTranscoding,+    testInstances,+    testBasics,+    testFolds,+    testText,+    testSubstrings,+    testBuilder,+    testLowLevel,+    testRead,+    testCornerCases,+    testValidate   ]
+ tests/Tests/Properties/Basics.hs view
@@ -0,0 +1,157 @@+-- | Test basic text functions++{-# LANGUAGE ViewPatterns #-}++{-# OPTIONS_GHC -Wno-missing-signatures    #-}+{-# OPTIONS_GHC -Wno-warnings-deprecations #-}+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}++module Tests.Properties.Basics+    ( testBasics+    ) where++import Control.Arrow (first, second)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty, applyFun)+import Tests.QuickCheckUtils+import qualified Data.List as L+import qualified Data.Text as T+import qualified Data.Text.Internal.Fusion as S+import qualified Data.Text.Internal.Fusion.Common as S+import qualified Data.Text.Internal.Lazy.Fusion as SL+import qualified Data.Text.Lazy as TL++s_cons x          = (x:)     `eqP` (unpackS . S.cons x)+s_cons_s x        = (x:)     `eqP` (unpackS . S.unstream . S.cons x)+sf_cons (applyFun -> p) x+                  = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p)+t_cons x          = (x:)     `eqP` (unpackS . T.cons x)+tl_cons x         = (x:)     `eqP` (unpackS . TL.cons x)+t_length_cons x   = (L.length . (x:)) `eqP` (T.length . T.cons x)+tl_length_cons x  = (L.genericLength . (x:)) `eqP` (TL.length . TL.cons x)++s_snoc x          = (++ [x]) `eqP` (unpackS . flip S.snoc x)+t_snoc x          = (++ [x]) `eqP` (unpackS . flip T.snoc x)+tl_snoc x         = (++ [x]) `eqP` (unpackS . flip TL.snoc x)+t_length_snoc x   = (L.length . (++ [x])) `eqP` (T.length . flip T.snoc x)+tl_length_snoc x  = (L.genericLength . (++ [x])) `eqP` (TL.length . flip TL.snoc x)++s_append s        = (s++)    `eqP` (unpackS . S.append (S.streamList s))+s_append_s s      = (s++)    `eqP`+                    (unpackS . S.unstream . S.append (S.streamList s))+sf_append (applyFun -> p) s+                  = (L.filter p s++) `eqP`+                    (unpackS . S.append (S.filter p $ S.streamList s))+t_append s        = (s++)    `eqP` (unpackS . T.append (packS s))++uncons (x:xs) = Just (x,xs)+uncons _      = Nothing++s_uncons          = uncons   `eqP` (fmap (second unpackS) . S.uncons)+sf_uncons (applyFun -> p)+                  = (uncons . L.filter p) `eqP`+                    (fmap (second unpackS) . S.uncons . S.filter p)+t_uncons          = uncons   `eqP` (fmap (second unpackS) . T.uncons)+tl_uncons         = uncons   `eqP` (fmap (second unpackS) . TL.uncons)++unsnoc xs@(_:_) = Just (init xs, last xs)+unsnoc []       = Nothing++t_unsnoc          = unsnoc   `eqP` (fmap (first unpackS) . T.unsnoc)+tl_unsnoc         = unsnoc   `eqP` (fmap (first unpackS) . TL.unsnoc)++s_head            = head   `eqP` S.head+sf_head (applyFun -> p) = (head . L.filter p) `eqP` (S.head . S.filter p)+t_head            = head   `eqP` T.head+tl_head           = head   `eqP` TL.head+s_last            = last   `eqP` S.last+sf_last (applyFun -> p) = (last . L.filter p) `eqP` (S.last . S.filter p)+t_last            = last   `eqP` T.last+tl_last           = last   `eqP` TL.last+s_tail            = tail   `eqP` (unpackS . S.tail)+s_tail_s          = tail   `eqP` (unpackS . S.unstream . S.tail)+sf_tail (applyFun -> p) = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p)+t_tail            = tail   `eqP` (unpackS . T.tail)+tl_tail           = tail   `eqP` (unpackS . TL.tail)+s_init            = init   `eqP` (unpackS . S.init)+s_init_s          = init   `eqP` (unpackS . S.unstream . S.init)+sf_init (applyFun -> p) = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p)+t_init            = init   `eqP` (unpackS . T.init)+tl_init           = init   `eqP` (unpackS . TL.init)+s_null            = null   `eqP` S.null+sf_null (applyFun -> p) = (null . L.filter p) `eqP` (S.null . S.filter p)+t_null            = null   `eqP` T.null+tl_null           = null   `eqP` TL.null+s_length          = length `eqP` S.length+sf_length (applyFun -> p) = (length . L.filter p) `eqP` (S.length . S.filter p)+sl_length         = (fromIntegral . length) `eqP` SL.length+t_length          = length `eqP` T.length+tl_length         = L.genericLength `eqP` TL.length+t_compareLength t = (compare (T.length t)) `eq` T.compareLength t+tl_compareLength t= (compare (TL.length t)) `eq` TL.compareLength t++-- Regression tests.+s_filter_eq s = S.filter p t == S.streamList (filter p s)+    where p = (/= S.last t)+          t = S.streamList s++testBasics :: TestTree+testBasics =+  testGroup "basics" [+    testProperty "s_cons" s_cons,+    testProperty "s_cons_s" s_cons_s,+    testProperty "sf_cons" sf_cons,+    testProperty "t_cons" t_cons,+    testProperty "tl_cons" tl_cons,+    testProperty "t_length_cons" t_length_cons,+    testProperty "tl_length_cons" tl_length_cons,+    testProperty "s_snoc" s_snoc,+    testProperty "t_snoc" t_snoc,+    testProperty "tl_snoc" tl_snoc,+    testProperty "t_length_snoc" t_length_snoc,+    testProperty "tl_length_snoc" tl_length_snoc,+    testProperty "s_append" s_append,+    testProperty "s_append_s" s_append_s,+    testProperty "sf_append" sf_append,+    testProperty "t_append" t_append,+    testProperty "s_uncons" s_uncons,+    testProperty "sf_uncons" sf_uncons,+    testProperty "t_uncons" t_uncons,+    testProperty "tl_uncons" tl_uncons,+    testProperty "t_unsnoc" t_unsnoc,+    testProperty "tl_unsnoc" tl_unsnoc,+    testProperty "s_head" s_head,+    testProperty "sf_head" sf_head,+    testProperty "t_head" t_head,+    testProperty "tl_head" tl_head,+    testProperty "s_last" s_last,+    testProperty "sf_last" sf_last,+    testProperty "t_last" t_last,+    testProperty "tl_last" tl_last,+    testProperty "s_tail" s_tail,+    testProperty "s_tail_s" s_tail_s,+    testProperty "sf_tail" sf_tail,+    testProperty "t_tail" t_tail,+    testProperty "tl_tail" tl_tail,+    testProperty "s_init" s_init,+    testProperty "s_init_s" s_init_s,+    testProperty "sf_init" sf_init,+    testProperty "t_init" t_init,+    testProperty "tl_init" tl_init,+    testProperty "s_null" s_null,+    testProperty "sf_null" sf_null,+    testProperty "t_null" t_null,+    testProperty "tl_null" tl_null,+    testProperty "s_length" s_length,+    testProperty "sf_length" sf_length,+    testProperty "sl_length" sl_length,+    testProperty "t_length" t_length,+    testProperty "tl_length" tl_length,+    testProperty "t_compareLength" t_compareLength,+    testProperty "tl_compareLength" tl_compareLength,++    testGroup "regressions" [+      testProperty "s_filter_eq" s_filter_eq+    ]+  ]
+ tests/Tests/Properties/Builder.hs view
@@ -0,0 +1,149 @@+-- | Test @Builder@++{-# LANGUAGE CPP #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+module Tests.Properties.Builder+    ( testBuilder+    ) where++import Data.Int (Int8, Int16, Int32, Int64)+import Data.Word+import Numeric (showEFloat, showFFloat, showGFloat, showHex)+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils+import qualified Data.List as L+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Builder as TB+import qualified Data.Text.Lazy.Builder.Int as TB+import qualified Data.Text.Lazy.Builder.RealFloat as TB++-- Builder.++tb_singleton   = id `eqP` (unpackS . TB.toLazyText . mconcat . map TB.singleton)+tb_fromString  = id `eq` (TL.unpack . TB.toLazyText . TB.fromString)+tb_fromText    = id `eqP` (unpackS . TB.toLazyText . TB.fromText)+tb_fromStrings = L.concat `eq` (TL.unpack . TB.toLazyText . mconcat . map TB.fromString)+tb_fromTexts   = L.concat `eq` (unpackS . TB.toLazyText . mconcat . map (TB.fromText . packS))++tb_associative s1 s2 s3 =+    TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) ===+    TB.toLazyText ((b1 `mappend` b2) `mappend` b3)+  where b1 = TB.fromText (packS s1)+        b2 = TB.fromText (packS s2)+        b3 = TB.fromText (packS s3)++-- Numeric builder stuff.++tb_decimal :: (Integral a, Show a) => a -> Property+tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show)++tb_decimal_integer (a::Integer) = tb_decimal a+tb_decimal_integer_big (Big a) = tb_decimal a+tb_decimal_int (a::Int) = tb_decimal a+tb_decimal_int8 (a::Int8) = tb_decimal a+tb_decimal_int16 (a::Int16) = tb_decimal a+tb_decimal_int32 (a::Int32) = tb_decimal a+tb_decimal_int64 (a::Int64) = tb_decimal a+tb_decimal_word (a::Word) = tb_decimal a+tb_decimal_word8 (a::Word8) = tb_decimal a+tb_decimal_word16 (a::Word16) = tb_decimal a+tb_decimal_word32 (a::Word32) = tb_decimal a+tb_decimal_word64 (a::Word64) = tb_decimal a++tb_decimal_big_int (Large (a::Int)) = tb_decimal a+tb_decimal_big_int64 (Large (a::Int64)) = tb_decimal a+tb_decimal_big_word (Large (a::Word)) = tb_decimal a+tb_decimal_big_word64 (Large (a::Word64)) = tb_decimal a++tb_hex :: (Integral a, Show a) => a -> Property+tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "")++tb_hexadecimal_integer (a::Integer) = tb_hex a+tb_hexadecimal_int (a::Int) = tb_hex a+tb_hexadecimal_int8 (a::Int8) = tb_hex a+tb_hexadecimal_int16 (a::Int16) = tb_hex a+tb_hexadecimal_int32 (a::Int32) = tb_hex a+tb_hexadecimal_int64 (a::Int64) = tb_hex a+tb_hexadecimal_word (a::Word) = tb_hex a+tb_hexadecimal_word8 (a::Word8) = tb_hex a+tb_hexadecimal_word16 (a::Word16) = tb_hex a+tb_hexadecimal_word32 (a::Word32) = tb_hex a+tb_hexadecimal_word64 (a::Word64) = tb_hex a++tb_realfloat :: (RealFloat a, Show a) => a -> Property+tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show)++tb_realfloat_float (a::Float) = tb_realfloat a+tb_realfloat_double (a::Double) = tb_realfloat a++showFloat :: (RealFloat a) => TB.FPFormat -> Maybe Int -> a -> ShowS+showFloat TB.Exponent (Just 0) = showEFloat (Just 1) -- see gh-231+showFloat TB.Exponent p = showEFloat p+showFloat TB.Fixed    p = showFFloat p+showFloat TB.Generic  p = showGFloat p++tb_formatRealFloat :: (RealFloat a, Show a) =>+                      a -> TB.FPFormat -> Precision a -> Property+tb_formatRealFloat a fmt prec = cond ==>+    TB.formatRealFloat fmt p a ===+    TB.fromString (showFloat fmt p a "")+  where p = precision a prec+        cond = case (p,fmt) of+#if MIN_VERSION_base(4,12,0)+                  (Just 0, TB.Generic) -> False -- skipping due to gh-231+#endif+                  _                    -> True++tb_formatRealFloat_float (a::Float) = tb_formatRealFloat a+tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a++testBuilder :: TestTree+testBuilder =+  testGroup "builder" [+    testProperty "tb_singleton" tb_singleton,+    testProperty "tb_fromString" tb_fromString,+    testProperty "tb_fromText" tb_fromText,+    testProperty "tb_fromStrings" tb_fromStrings,+    testProperty "tb_fromTexts" tb_fromTexts,+    testProperty "tb_associative" tb_associative,+    testGroup "decimal" [+      testProperty "tb_decimal_int" tb_decimal_int,+      testProperty "tb_decimal_int8" tb_decimal_int8,+      testProperty "tb_decimal_int16" tb_decimal_int16,+      testProperty "tb_decimal_int32" tb_decimal_int32,+      testProperty "tb_decimal_int64" tb_decimal_int64,+      testProperty "tb_decimal_integer" tb_decimal_integer,+      testProperty "tb_decimal_integer_big" tb_decimal_integer_big,+      testProperty "tb_decimal_word" tb_decimal_word,+      testProperty "tb_decimal_word8" tb_decimal_word8,+      testProperty "tb_decimal_word16" tb_decimal_word16,+      testProperty "tb_decimal_word32" tb_decimal_word32,+      testProperty "tb_decimal_word64" tb_decimal_word64,+      testProperty "tb_decimal_big_int" tb_decimal_big_int,+      testProperty "tb_decimal_big_word" tb_decimal_big_word,+      testProperty "tb_decimal_big_int64" tb_decimal_big_int64,+      testProperty "tb_decimal_big_word64" tb_decimal_big_word64+    ],+    testGroup "hexadecimal" [+      testProperty "tb_hexadecimal_int" tb_hexadecimal_int,+      testProperty "tb_hexadecimal_int8" tb_hexadecimal_int8,+      testProperty "tb_hexadecimal_int16" tb_hexadecimal_int16,+      testProperty "tb_hexadecimal_int32" tb_hexadecimal_int32,+      testProperty "tb_hexadecimal_int64" tb_hexadecimal_int64,+      testProperty "tb_hexadecimal_integer" tb_hexadecimal_integer,+      testProperty "tb_hexadecimal_word" tb_hexadecimal_word,+      testProperty "tb_hexadecimal_word8" tb_hexadecimal_word8,+      testProperty "tb_hexadecimal_word16" tb_hexadecimal_word16,+      testProperty "tb_hexadecimal_word32" tb_hexadecimal_word32,+      testProperty "tb_hexadecimal_word64" tb_hexadecimal_word64+    ],+    testGroup "realfloat" [+      testProperty "tb_realfloat_double" tb_realfloat_double,+      testProperty "tb_realfloat_float" tb_realfloat_float,+      testProperty "tb_formatRealFloat_float" tb_formatRealFloat_float,+      testProperty "tb_formatRealFloat_double" tb_formatRealFloat_double+    ]+  ]
+ tests/Tests/Properties/CornerCases.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++-- | Check that the definitions that are partial crash in the expected ways or+-- return sensible defaults.+module Tests.Properties.CornerCases (testCornerCases) where++import Control.Exception+import Data.Either+import Data.Semigroup+import Data.Text+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils ()++testCornerCases :: TestTree+testCornerCases =+  testGroup+    "corner cases"+    [ testGroup+        "stimes"+        $ let specimen = stimes :: Integer -> Text -> Text+          in  [ testProperty+                  "given a negative number, evaluate to error call"+                  $ \(Negative number) text ->+                    (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+                      specimen+                        (fromIntegral (number :: Int))+                        text+              , testProperty+                  "given a number that does not fit into Int, evaluate to error call"+                  $ \(NonNegative number) text ->+                    (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+                      specimen+                        (fromIntegral (number :: Int) + fromIntegral (maxBound :: Int) + 1)+                        text+              ]+    ]
+ tests/Tests/Properties/Folds.hs view
@@ -0,0 +1,357 @@+-- | Test folds, scans, and unfolds
+
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE ViewPatterns #-}
+
+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+{-# LANGUAGE TemplateHaskell #-}
+{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-}
+#endif
+
+module Tests.Properties.Folds
+    ( testFolds
+    ) where
+
+import Control.Arrow (second)
+import Control.Exception (ErrorCall, evaluate, try)
+import Data.Functor.Identity (Identity(..))
+import Control.Monad.Trans.State (runState, state)
+import Data.Word (Word8, Word16)
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.HUnit (testCase, assertFailure, assertBool)
+import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2)
+import Tests.QuickCheckUtils
+import qualified Data.List as L
+import qualified Data.Text as T
+import qualified Data.Text.Internal.Fusion as S
+import qualified Data.Text.Internal.Fusion.Common as S
+import qualified Data.Text.Lazy as TL
+import qualified Data.Char as Char
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+import Test.Tasty.Inspection (inspectTest, (==~))
+import GHC.Exts (inline)
+#endif
+
+-- Folds
+
+sf_foldl (applyFun -> p) (applyFun2 -> f) z =
+    (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p)
+    where _types  = f :: Char -> Char -> Char
+t_foldl (applyFun2 -> f) z       = L.foldl f z  `eqP` (T.foldl f z)
+    where _types  = f :: Char -> Char -> Char
+tl_foldl (applyFun2 -> f) z      = L.foldl f z  `eqP` (TL.foldl f z)
+    where _types  = f :: Char -> Char -> Char
+sf_foldl' (applyFun -> p) (applyFun2 -> f) z =
+    (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p)
+    where _types  = f :: Char -> Char -> Char
+t_foldl' (applyFun2 -> f) z      = L.foldl' f z `eqP` T.foldl' f z
+    where _types  = f :: Char -> Char -> Char
+tl_foldl' (applyFun2 -> f) z     = L.foldl' f z `eqP` TL.foldl' f z
+    where _types  = f :: Char -> Char -> Char
+sf_foldl1 (applyFun -> p) (applyFun2 -> f) =
+    (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p)
+t_foldl1 (applyFun2 -> f)        = L.foldl1 f   `eqP` T.foldl1 f
+tl_foldl1 (applyFun2 -> f)       = L.foldl1 f   `eqP` TL.foldl1 f
+sf_foldl1' (applyFun -> p) (applyFun2 -> f) =
+    (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p)
+t_foldl1' (applyFun2 -> f)       = L.foldl1' f  `eqP` T.foldl1' f
+tl_foldl1' (applyFun2 -> f)      = L.foldl1' f  `eqP` TL.foldl1' f
+sf_foldr (applyFun -> p) (applyFun2 -> f) z =
+    (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p)
+    where _types  = f :: Char -> Char -> Char
+t_foldr (applyFun2 -> f) z       = L.foldr f z  `eqP` T.foldr f z
+    where _types  = f :: Char -> Char -> Char
+t_foldr' (applyFun2 -> f) z       = L.foldr f z  `eqP` T.foldr' f z
+    where _types  = f :: Char -> Char -> Char
+tl_foldr (applyFun2 -> f) z      = L.foldr f z  `eqPSqrt` TL.foldr f z
+    where _types  = f :: Char -> Char -> Char
+sf_foldr1 (applyFun -> p) (applyFun2 -> f) =
+    (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p)
+t_foldr1 (applyFun2 -> f)        = L.foldr1 f   `eqP` T.foldr1 f
+tl_foldr1 (applyFun2 -> f)       = L.foldr1 f   `eqPSqrt` TL.foldr1 f
+
+-- Distinguish foldl/foldr from foldl'/foldr'
+
+fold_apart :: IO ()
+fold_apart = do
+    ok (T.foldr  f () (T.pack "az"))
+    ko (T.foldr' f () (T.pack "az"))
+    ok (T.foldl  (flip f) () (T.pack "za"))
+    ko (T.foldl' (flip f) () (T.pack "za"))
+  where
+    f c _ = if c == 'z' then error "catchme" else ()
+    ok = evaluate
+    ko t = do
+        x <- try (evaluate t)
+        case x :: Either ErrorCall () of
+            Left _ -> pure ()
+            Right _ -> assertFailure "test should have failed but didn't"
+
+-- Special folds
+
+s_concat_s        = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt)
+sf_concat (applyFun -> p)
+                  = (L.concat . map (L.filter p) . unSqrt) `eq`
+                    (unpackS . S.concat . map (S.filter p . packS) . unSqrt)
+t_concat          = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt)
+tl_concat         = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt)
+sf_concatMap (applyFun -> p) (applyFun -> f) =
+    (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p)
+t_concatMap (applyFun -> f)
+                  = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f))
+tl_concatMap (applyFun -> f)
+                  = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f))
+sf_any (applyFun -> q) (applyFun -> p)
+                  = (L.any p . L.filter q) `eqP` (S.any p . S.filter q)
+t_any (applyFun -> p)
+                  = L.any p       `eqP` T.any p
+tl_any (applyFun -> p)
+                  = L.any p       `eqP` TL.any p
+sf_all (applyFun -> q) (applyFun -> p)
+                  = (L.all p . L.filter q) `eqP` (S.all p . S.filter q)
+t_all (applyFun -> p)
+                  = L.all p       `eqP` T.all p
+tl_all (applyFun -> p)
+                  = L.all p       `eqP` TL.all p
+sf_maximum (applyFun -> p)
+                  = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p)
+t_maximum         = L.maximum     `eqP` T.maximum
+tl_maximum        = L.maximum     `eqP` TL.maximum
+sf_minimum (applyFun -> p)
+                  = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p)
+t_minimum         = L.minimum     `eqP` T.minimum
+tl_minimum        = L.minimum     `eqP` TL.minimum
+t_isAscii         = L.all Char.isAscii `eqP` T.isAscii
+tl_isAscii        = L.all Char.isAscii `eqP` TL.isAscii
+
+-- Scans
+
+sf_scanl (applyFun -> p) (applyFun2 -> f) z =
+    (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p)
+t_scanl (applyFun2 -> f) z       = L.scanl f z   `eqP` (unpackS . T.scanl f z)
+tl_scanl (applyFun2 -> f) z      = L.scanl f z   `eqP` (unpackS . TL.scanl f z)
+t_scanl1 (applyFun2 -> f)        = L.scanl1 f    `eqP` (unpackS . T.scanl1 f)
+tl_scanl1 (applyFun2 -> f)       = L.scanl1 f    `eqP` (unpackS . TL.scanl1 f)
+t_scanr (applyFun2 -> f) z       = L.scanr f z   `eqP` (unpackS . T.scanr f z)
+tl_scanr (applyFun2 -> f) z      = L.scanr f z   `eqP` (unpackS . TL.scanr f z)
+t_scanr1 (applyFun2 -> f)        = L.scanr1 f    `eqP` (unpackS . T.scanr1 f)
+tl_scanr1 (applyFun2 -> f)       = L.scanr1 f    `eqP` (unpackS . TL.scanr1 f)
+
+t_scanl_is_safe = let c = '\55296' in
+    T.scanl undefined c mempty === T.singleton c
+tl_scanl_is_safe = let c = '\55296' in
+    TL.scanl undefined c mempty === TL.singleton c
+t_scanr_is_safe = let c = '\55296' in
+    T.scanr undefined c mempty === T.singleton c
+tl_scanr_is_safe = let c = '\55296' in
+    TL.scanr undefined c mempty === TL.singleton c
+
+t_mapAccumL_char c t =
+    snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)
+t_mapAccumL (applyFun2 -> f) z   = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z)
+    where _types  = f :: Int -> Char -> (Int,Char)
+tl_mapAccumL_char c t =
+    snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)
+tl_mapAccumL (applyFun2 -> f) z  = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z)
+    where _types  = f :: Int -> Char -> (Int,Char)
+t_mapAccumR_char c t =
+    snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)
+t_mapAccumR (applyFun2 -> f) z   = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)
+    where _types  = f :: Int -> Char -> (Int,Char)
+tl_mapAccumR_char c t =
+    snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)
+tl_mapAccumR (applyFun2 -> f) z  = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)
+    where _types  = f :: Int -> Char -> (Int,Char)
+
+-- Unfolds
+
+tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat)
+
+s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS)
+
+t_replicate_char (Small n) c =
+    L.replicate n c === T.unpack (T.replicate n (T.singleton c))
+tl_replicate_char (Small n) c =
+    L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c))
+t_length_replicate_char (Small n) c =
+    L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c))
+tl_length_replicate_char (Small n) c =
+    L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c))
+
+t_replicate (Small n) =
+    (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n)
+tl_replicate (Small n) =
+    (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n))
+t_length_replicate (Small n) =
+    (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n)
+tl_length_replicate (Small n) =
+    (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n))
+
+tl_cycle n        = (L.take m . L.cycle) `eq`
+                    (unpackS . TL.take (fromIntegral m) . TL.cycle . packS)
+    where m = fromIntegral (n :: Word8)
+
+tl_iterate (applyFun -> f) n
+                  = (L.take m . L.iterate f) `eq`
+                    (unpackS . TL.take (fromIntegral m) . TL.iterate f)
+    where m = fromIntegral (n :: Word8)
+
+unf :: Int -> Char -> Maybe (Char, Char)
+unf n c | fromEnum c * 100 > n = Nothing
+        | otherwise            = Just (c, succ c)
+
+t_unfoldr n       = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m))
+    where m = fromIntegral (n :: Word16)
+tl_unfoldr n      = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m))
+    where m = fromIntegral (n :: Word16)
+t_unfoldrN n m    = (L.take i . L.unfoldr (unf j)) `eq`
+                         (unpackS . T.unfoldrN i (unf j))
+    where i = fromIntegral (n :: Word16)
+          j = fromIntegral (m :: Word16)
+tl_unfoldrN n m   = (L.take i . L.unfoldr (unf j)) `eq`
+                         (unpackS . TL.unfoldrN (fromIntegral i) (unf j))
+    where i = fromIntegral (n :: Word16)
+          j = fromIntegral (m :: Word16)
+
+-- Monadic folds
+
+-- Parametric polymorphism allows us to only test foldlM' specialized to
+-- one function in the state monad (called @logger@ in the following tests)
+-- that just logs the arguments it was applied to and produces a fresh
+-- accumulator. That alone determines the general behavior of foldlM' with an
+-- arbitrary function in any monad.
+-- Reference: "Testing Polymorphic Properties" by Bernardy et al.
+-- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf
+
+t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0)
+  where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order
+tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0)
+  where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+-- As a sanity check for performance, the simplified Core
+-- foldlM' specialized to Identity is the same as foldl'.
+
+_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a
+_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x
+
+_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a
+_S_foldl' = inline S.foldl'
+#endif
+
+isAscii_border :: IO ()
+isAscii_border = do
+    let text  = T.drop 2 $ T.pack "XX1234五"
+    assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text
+
+testFolds :: TestTree
+testFolds =
+  testGroup "folds-unfolds" [
+    testGroup "folds" [
+      testProperty "sf_foldl" sf_foldl,
+      testProperty "t_foldl" t_foldl,
+      testProperty "tl_foldl" tl_foldl,
+      testProperty "sf_foldl'" sf_foldl',
+      testProperty "t_foldl'" t_foldl',
+      testProperty "tl_foldl'" tl_foldl',
+      testProperty "sf_foldl1" sf_foldl1,
+      testProperty "t_foldl1" t_foldl1,
+      testProperty "tl_foldl1" tl_foldl1,
+      testProperty "t_foldl1'" t_foldl1',
+      testProperty "sf_foldl1'" sf_foldl1',
+      testProperty "tl_foldl1'" tl_foldl1',
+      testProperty "sf_foldr" sf_foldr,
+      testProperty "t_foldr" t_foldr,
+      testProperty "t_foldr'" t_foldr',
+      testProperty "tl_foldr" tl_foldr,
+      testProperty "sf_foldr1" sf_foldr1,
+      testProperty "t_foldr1" t_foldr1,
+      testProperty "tl_foldr1" tl_foldr1,
+      testProperty "t_foldlM'" t_foldlM',
+      testProperty "tl_foldlM'" tl_foldlM',
+#ifdef MIN_VERSION_tasty_inspection_testing
+      let _unused = ['_S_foldl'_from_foldlM', '_S_foldl'] in
+      $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')),
+#endif
+      testCase "fold_apart" fold_apart,
+
+      testGroup "special" [
+        testProperty "s_concat_s" s_concat_s,
+        testProperty "sf_concat" sf_concat,
+        testProperty "t_concat" t_concat,
+        testProperty "tl_concat" tl_concat,
+        testProperty "sf_concatMap" sf_concatMap,
+        testProperty "t_concatMap" t_concatMap,
+        testProperty "tl_concatMap" tl_concatMap,
+        testProperty "sf_any" sf_any,
+        testProperty "t_any" t_any,
+        testProperty "tl_any" tl_any,
+        testProperty "sf_all" sf_all,
+        testProperty "t_all" t_all,
+        testProperty "tl_all" tl_all,
+        testProperty "sf_maximum" sf_maximum,
+        testProperty "t_maximum" t_maximum,
+        testProperty "tl_maximum" tl_maximum,
+        testProperty "sf_minimum" sf_minimum,
+        testProperty "t_minimum" t_minimum,
+        testProperty "tl_minimum" tl_minimum,
+        testProperty "t_isAscii " t_isAscii,
+        testProperty "tl_isAscii " tl_isAscii,
+        testCase "isAscii_border" isAscii_border
+      ]
+    ],
+
+    testGroup "scans" [
+      testProperty "sf_scanl" sf_scanl,
+      testProperty "t_scanl" t_scanl,
+      testProperty "tl_scanl" tl_scanl,
+      testProperty "t_scanl1" t_scanl1,
+      testProperty "tl_scanl1" tl_scanl1,
+      testProperty "t_scanr" t_scanr,
+      testProperty "tl_scanr" tl_scanr,
+      testProperty "t_scanr1" t_scanr1,
+      testProperty "tl_scanr1" tl_scanr1,
+
+      testProperty "t_scanl_is_safe" t_scanl_is_safe,
+      testProperty "tl_scanl_is_safe" tl_scanl_is_safe,
+      testProperty "t_scanr_is_safe" t_scanr_is_safe,
+      testProperty "tl_scanr_is_safe" tl_scanr_is_safe
+    ],
+
+    testGroup "mapAccum" [
+      testProperty "t_mapAccumL_char" t_mapAccumL_char,
+      testProperty "t_mapAccumL" t_mapAccumL,
+      testProperty "tl_mapAccumL_char" tl_mapAccumL_char,
+      testProperty "tl_mapAccumL" tl_mapAccumL,
+      testProperty "t_mapAccumR_char" t_mapAccumR_char,
+      testProperty "t_mapAccumR" t_mapAccumR,
+      testProperty "tl_mapAccumR_char" tl_mapAccumR_char,
+      testProperty "tl_mapAccumR" tl_mapAccumR
+    ],
+
+    testGroup "unfolds" [
+      testProperty "tl_cycle" tl_cycle,
+      testProperty "tl_iterate" tl_iterate,
+      testProperty "t_unfoldr" t_unfoldr,
+      testProperty "tl_unfoldr" tl_unfoldr,
+      testProperty "t_unfoldrN" t_unfoldrN,
+      testProperty "tl_unfoldrN" tl_unfoldrN
+    ],
+
+    testGroup "replicate" [
+      testProperty "tl_repeat" tl_repeat,
+      testProperty "s_replicate" s_replicate,
+      testProperty "t_replicate_char" t_replicate_char,
+      testProperty "tl_replicate_char" tl_replicate_char,
+      testProperty "t_length_replicate_char" t_length_replicate_char,
+      testProperty "tl_length_replicate_char" tl_length_replicate_char,
+      testProperty "t_replicate" t_replicate,
+      testProperty "tl_replicate" tl_replicate,
+      testProperty "t_length_replicate" t_length_replicate,
+      testProperty "tl_length_replicate" tl_length_replicate
+    ]
+
+  ]
+ tests/Tests/Properties/Instances.hs view
@@ -0,0 +1,91 @@+-- | Test instances++{-# LANGUAGE ViewPatterns #-}+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+module Tests.Properties.Instances+    ( testInstances+    ) where++import Data.Binary (encode, decodeOrFail)+import Data.Semigroup+import Data.String (IsString(fromString))+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils+import qualified Data.List as L+import qualified Data.Text as T+import qualified Data.Text.Internal.Fusion.Common as S+import qualified Data.Text.Lazy as TL++s_Eq s            = (s==)    `eq` ((S.streamList s==) . S.streamList)+    where _types = s :: String+sf_Eq (applyFun -> p) s =+    ((L.filter p s==) . L.filter p) `eq`+    (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList)+t_Eq s            = (s==)    `eq` ((T.pack s==) . T.pack)+tl_Eq s           = (s==)    `eq` ((TL.pack s==) . TL.pack)+s_Ord s           = (compare s) `eq` (compare (S.streamList s) . S.streamList)+    where _types = s :: String+sf_Ord (applyFun -> p) s =+    ((compare $ L.filter p s) . L.filter p) `eq`+    (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList)+t_Ord s           = (compare s) `eq` (compare (T.pack s) . T.pack)+tl_Ord s          = (compare s) `eq` (compare (TL.pack s) . TL.pack)+t_Read            = id       `eq` (T.unpack . read . show)+tl_Read           = id       `eq` (TL.unpack . read . show)+t_Show            = show     `eq` (show . T.pack)+tl_Show           = show     `eq` (show . TL.pack)+t_mappend s       = mappend s`eqP` (unpackS . mappend (T.pack s))+tl_mappend s      = mappend s`eqP` (unpackS . mappend (TL.pack s))+t_stimes          = \ number -> eq+  ((stimes :: Int -> String -> String) number . unSqrt)+  (unpackS . (stimes :: Int -> T.Text -> T.Text) number . T.pack . unSqrt)+tl_stimes         = \ number -> eq+  ((stimes :: Int -> String -> String) number . unSqrt)+  (unpackS . (stimes :: Int -> TL.Text -> TL.Text) number . TL.pack . unSqrt)+t_mconcat         = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt)+tl_mconcat        = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt)+t_mempty          = mempty === (unpackS (mempty :: T.Text))+tl_mempty         = mempty === (unpackS (mempty :: TL.Text))+t_IsString        = fromString  `eqP` (T.unpack . fromString)+tl_IsString       = fromString  `eqP` (TL.unpack . fromString)++t_Binary s        =+  case decodeOrFail . encode $ (s :: T.Text) of+    Left _   -> counterexample (show (T.unpack s)) (property False)+    Right (_, _, s') -> s === s'++tl_Binary s       =+  case decodeOrFail . encode $ (s :: TL.Text) of+    Left _   -> counterexample (show (TL.unpack s)) (property False)+    Right (_, _, s') -> s === s'++testInstances :: TestTree+testInstances =+  testGroup "instances" [+    testProperty "s_Eq" s_Eq,+    testProperty "sf_Eq" sf_Eq,+    testProperty "t_Eq" t_Eq,+    testProperty "tl_Eq" tl_Eq,+    testProperty "s_Ord" s_Ord,+    testProperty "sf_Ord" sf_Ord,+    testProperty "t_Ord" t_Ord,+    testProperty "tl_Ord" tl_Ord,+    testProperty "t_Read" t_Read,+    testProperty "tl_Read" tl_Read,+    testProperty "t_Show" t_Show,+    testProperty "tl_Show" tl_Show,+    testProperty "t_mappend" t_mappend,+    testProperty "tl_mappend" tl_mappend,+    testProperty "t_stimes" t_stimes,+    testProperty "tl_stimes" tl_stimes,+    testProperty "t_mconcat" t_mconcat,+    testProperty "tl_mconcat" tl_mconcat,+    testProperty "t_mempty" t_mempty,+    testProperty "tl_mempty" tl_mempty,+    testProperty "t_IsString" t_IsString,+    testProperty "tl_IsString" tl_IsString,+    testProperty "t_Binary" t_Binary,+    testProperty "tl_Binary" tl_Binary+  ]
+ tests/Tests/Properties/LowLevel.hs view
@@ -0,0 +1,176 @@+-- | Test low-level operations
+
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE MagicHash #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-}
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+{-# LANGUAGE TemplateHaskell #-}
+{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-}
+#endif
+
+module Tests.Properties.LowLevel (testLowLevel) where
+
+import Prelude hiding (head, tail)
+import Control.Applicative ((<$>), pure)
+import Control.Exception as E (SomeException, catch, evaluate)
+import Data.Functor.Identity (Identity(..))
+import Data.Int (Int32, Int64)
+import Data.Text.Foreign
+import Data.Text.Internal (Text(..), mul, mul32, mul64, safe)
+import Data.Word (Word8, Word16, Word32)
+import System.IO.Unsafe (unsafePerformIO)
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.HUnit (testCase, assertEqual)
+import Test.Tasty.QuickCheck (testProperty)
+import Test.QuickCheck hiding ((.&.))
+import Tests.QuickCheckUtils
+import Tests.Utils
+import qualified Data.Text as T
+import qualified Data.Text.Foreign as T
+import qualified Data.Text.IO as T
+import qualified Data.Text.Lazy as TL
+import qualified Data.Text.Lazy.IO as TL
+import qualified Data.Text.IO.Utf8 as TU
+import qualified System.IO as IO
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf)
+import qualified Data.Text.Internal.Fusion as S
+import qualified Data.Text.Internal.Fusion.Common as S
+import qualified GHC.CString as GHC
+#endif
+
+mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a
+mulRef a b
+  | ab < bot || ab > top = Nothing
+  | otherwise            = Just (fromIntegral ab)
+  where ab  = fromIntegral a * fromIntegral b
+        top = fromIntegral (maxBound `asTypeOf` a) :: Integer
+        bot = fromIntegral (minBound `asTypeOf` a) :: Integer
+
+eval :: (a -> b -> c) -> a -> b -> Maybe c
+eval f a b = unsafePerformIO $
+  (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing)
+
+t_mul32 :: Int32 -> Int32 -> Property
+t_mul32 a b = mulRef a b === eval mul32 a b
+
+t_mul64 :: Int64 -> Int64 -> Property
+t_mul64 a b = mulRef a b === eval mul64 a b
+
+t_mul :: Int -> Int -> Property
+t_mul a b = mulRef a b === eval mul a b
+
+-- Misc.
+
+t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t
+t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t
+t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t
+t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr
+t_use_from0 t = ioProperty $ do
+  let t' = t `T.snoc` '\0'
+  (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0)
+
+t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do
+  roundTrip <- T.withCString t T.peekCString
+  assertEqual "cstring" t roundTrip
+
+t_peek_cstring_len t = ioProperty $ do
+  roundTrip <- T.withCStringLen t T.peekCStringLen
+  assertEqual "cstring_len" t roundTrip
+
+t_copy t = T.copy t === t
+
+t_literal_length1 = assertEqual xs (length xs) byteLen
+  where
+    xs = "\0\1\0\1\0"
+    Text _ _ byteLen = T.pack xs
+t_literal_length2 = assertEqual xs (length xs) byteLen
+  where
+    xs = "\1\2\3\4\5"
+    Text _ _ byteLen = T.pack xs
+t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys)
+  where
+    ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000"
+    xs = map safe ys
+
+#ifdef MIN_VERSION_tasty_inspection_testing
+t_literal_foo :: Text
+t_literal_foo = T.pack "foo"
+#endif
+
+-- Input and output.
+
+-- t_put_get = write_read T.unlines T.filter put get
+--   where put h = withRedirect h IO.stdout . T.putStr
+--         get h = withRedirect h IO.stdin T.getContents
+-- tl_put_get = write_read TL.unlines TL.filter put get
+--   where put h = withRedirect h IO.stdout . TL.putStr
+--         get h = withRedirect h IO.stdin TL.getContents
+
+inputOutput :: TestTree
+inputOutput = testGroup "input-output" [
+    testProperty "t_write_read" $ write_read arbitrary shrink (T.replace "\n" "\r\n") T.hPutStr T.hGetContents,
+    testProperty "tl_write_read" $ write_read arbitrary shrink (TL.replace "\n" "\r\n") TL.hPutStr TL.hGetContents,
+    testProperty "t_write_read_line" $ write_read genTLine shrinkTLine (`T.append` "\r") T.hPutStrLn T.hGetLine,
+    testProperty "tl_write_read_line" $ write_read genTLLine shrinkTLLine (`TL.append` "\r") TL.hPutStrLn TL.hGetLine,
+    -- Note: Data.Text.IO.Utf8 does NO newline translation
+    testProperty "utf8_write_read" $ write_read arbitrary shrink id TU.hPutStr TU.hGetContents,
+    testProperty "utf8_write_read_line" $ write_read genTLine shrinkTLine id TU.hPutStrLn TU.hGetLine
+    -- These tests are subject to I/O race conditions
+    -- testProperty "t_put_get" t_put_get,
+    -- testProperty "tl_put_get" tl_put_get
+  ]
+
+genTLine :: Gen T.Text
+genTLine = T.filter (`notElem` ("\r\n" :: String)) <$> arbitrary
+
+genTLLine :: Gen TL.Text
+genTLLine = TL.filter (`notElem` ("\r\n" :: String)) <$> arbitrary
+
+shrinkTLine :: T.Text -> [T.Text]
+shrinkTLine = filter (T.all (/= '\n')) . shrink
+
+shrinkTLLine :: TL.Text -> [TL.Text]
+shrinkTLLine = filter (TL.all (/= '\n')) . shrink
+
+testLowLevel :: TestTree
+testLowLevel =
+  testGroup "lowlevel" [
+    testGroup "mul" [
+      testProperty "t_mul" t_mul,
+      testProperty "t_mul32" t_mul32,
+      testProperty "t_mul64" t_mul64
+    ],
+
+    testGroup "misc" [
+      testProperty "t_dropWord8" t_dropWord8,
+      testProperty "t_takeWord8" t_takeWord8,
+      testProperty "t_take_drop_8" t_take_drop_8,
+      testProperty "t_use_from" t_use_from,
+      testProperty "t_use_from0" t_use_from0,
+      testProperty "t_copy" t_copy,
+      testProperty "t_peek_cstring" t_peek_cstring,
+      testProperty "t_peek_cstring_len" t_peek_cstring_len,
+      testCase "t_literal_length1" t_literal_length1,
+      testCase "t_literal_length2" t_literal_length2,
+      testCase "t_literal_surrogates" t_literal_surrogates
+#ifdef MIN_VERSION_tasty_inspection_testing
+        -- Hack to force GHC to keep the binding around until this is fixed: https://gitlab.haskell.org/ghc/ghc/-/issues/26436
+      , let _unused = 't_literal_foo in
+        $(inspectObligations
+        [ (`hasNoTypes` [''Char, ''[]])
+        , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList])
+        , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#])
+        , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#])
+        ]
+        't_literal_foo)
+#endif
+    ],
+
+    inputOutput
+  ]
− tests/Tests/Properties/Mul.hs 
@@ -1,40 +0,0 @@-{-# LANGUAGE ScopedTypeVariables #-}--module Tests.Properties.Mul (tests) where--import Control.Applicative ((<$>), pure)-import Control.Exception as E (SomeException, catch, evaluate)-import Data.Int (Int32, Int64)-import Data.Text.Internal (mul, mul32, mul64)-import System.IO.Unsafe (unsafePerformIO)-import Test.Framework (Test)-import Test.Framework.Providers.QuickCheck2 (testProperty)-import Test.QuickCheck hiding ((.&.))--mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a-mulRef a b-  | ab < bot || ab > top = Nothing-  | otherwise            = Just (fromIntegral ab)-  where ab  = fromIntegral a * fromIntegral b-        top = fromIntegral (maxBound `asTypeOf` a) :: Integer-        bot = fromIntegral (minBound `asTypeOf` a) :: Integer--eval :: (a -> b -> c) -> a -> b -> Maybe c-eval f a b = unsafePerformIO $-  (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing)--t_mul32 :: Int32 -> Int32 -> Property-t_mul32 a b = mulRef a b === eval mul32 a b--t_mul64 :: Int64 -> Int64 -> Property-t_mul64 a b = mulRef a b === eval mul64 a b--t_mul :: Int -> Int -> Property-t_mul a b = mulRef a b === eval mul a b--tests :: [Test]-tests = [-   testProperty "t_mul" t_mul- , testProperty "t_mul32" t_mul32- , testProperty "t_mul64" t_mul64- ]
+ tests/Tests/Properties/Read.hs view
@@ -0,0 +1,111 @@+-- | Tests for readers++{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-}+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+module Tests.Properties.Read+    ( testRead+    ) where++import Data.Char (isDigit, isHexDigit)+import Numeric (showHex)+import Test.Tasty (TestTree, testGroup, localOption, mkTimeout)+import Test.Tasty.QuickCheck (testProperty)+import Test.QuickCheck+import Tests.QuickCheckUtils ()+import qualified Data.Text as T+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Read as TL+import qualified Data.Text.Read as T++-- Reading.++t_decimal (n::Int) s =+    T.signed T.decimal (T.pack (show n) `T.append` t) === Right (n,t)+    where t = T.dropWhile isDigit s+tl_decimal (n::Int) s =+    TL.signed TL.decimal (TL.pack (show n) `TL.append` t) === Right (n,t)+    where t = TL.dropWhile isDigit s+t_hexadecimal m s ox =+    T.hexadecimal (T.concat [p, T.pack (showHex n ""), t]) === Right (n,t)+    where t = T.dropWhile isHexDigit s+          p = if ox then "0x" else ""+          n = getPositive m :: Int+tl_hexadecimal m s ox =+    TL.hexadecimal (TL.concat [p, TL.pack (showHex n ""), t]) === Right (n,t)+    where t = TL.dropWhile isHexDigit s+          p = if ox then "0x" else ""+          n = getPositive m :: Int++isFloaty c = c `elem` ("+-.0123456789eE" :: String)++t_read_rational :: Double -> T.Text -> Property+t_read_rational n s =+  case T.rational (T.pack (show n) `T.append` t) of+    Left err       -> counterexample err $ property False+    Right (n', t') -> t === t' .&&. n' === n''+  where+    t = T.dropWhile isFloaty s+    n'' = read (show n) :: Double++t_read_double :: Double -> Double -> T.Text -> Property+t_read_double tol n s =+  case T.double (T.pack (show n) `T.append` t) of+    Left err       -> counterexample err $ property False+    Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol)+  where+    t = T.dropWhile isFloaty s++tl_read_rational :: Double -> TL.Text -> Property+tl_read_rational n s =+  case TL.rational (TL.pack (show n) `TL.append` t) of+    Left err       -> counterexample err $ property False+    Right (n', t') -> t === t' .&&. n' === n''+  where+    t = TL.dropWhile isFloaty s+    n'' = read (show n) :: Double++tl_read_double :: Double -> Double -> TL.Text -> Property+tl_read_double tol n s =+  case TL.rational (TL.pack (show n) `TL.append` t) of+    Left err       -> counterexample err $ property False+    Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol)+  where+    t = TL.dropWhile isFloaty s++testRead :: TestTree+testRead =+  testGroup "read" [+    testProperty "t_decimal" t_decimal,+    testProperty "tl_decimal" tl_decimal,+    testProperty "t_hexadecimal" t_hexadecimal,+    testProperty "tl_hexadecimal" tl_hexadecimal,++    testProperty "t_double" $ t_read_double 1e-13,+    testProperty "tl_double" $ tl_read_double 1e-13,++    testProperty "t_rational" t_read_rational,+    testProperty "t_rational 1.3e-2" (t_read_rational 1.3e-2),+    testProperty "tl_rational" tl_read_rational,+    testProperty "tl_rational 9e-3" (tl_read_rational 9e-3),++    localOption (mkTimeout 100000) $ testGroup "DDoS attacks" [+      testProperty "t_double large positive exponent" $+        T.double (T.pack "1.1e1000000000") === Right (1 / 0, mempty),+      testProperty "t_double large negative exponent" $+        T.double (T.pack "1.1e-1000000000") === Right (0.0, mempty),+      testProperty "tl_double large positive exponent" $+        TL.double (TL.pack "1.1e1000000000") === Right (1 / 0, mempty),+      testProperty "tl_double large negative exponent" $+        TL.double (TL.pack "1.1e-1000000000") === Right (0.0, mempty),++      testProperty "t_rational large positive exponent" $+        T.rational (T.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty),+      testProperty "t_rational large negative exponent" $+        T.rational (T.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty),+      testProperty "tl_rational large positive exponent" $+        TL.rational (TL.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty),+      testProperty "tl_rational large negative exponent" $+        TL.rational (TL.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty)+      ]++    ]
+ tests/Tests/Properties/Substrings.hs view
@@ -0,0 +1,438 @@+-- | Tests for substring functions (@take@, @split@, @isInfixOf@, etc.)++{-# LANGUAGE ViewPatterns #-}++{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+module Tests.Properties.Substrings+    ( testSubstrings+    ) where++import Prelude hiding (head, tail)+import Control.Monad.Trans.State (State, state, runState)+import Data.Char (isSpace)+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils+import qualified Data.List as L+import qualified Data.List.NonEmpty as NonEmptyList+import qualified Data.Text as T+import qualified Data.Text.Internal.Fusion as S+import qualified Data.Text.Internal.Fusion.Common as S+import qualified Data.Text.Internal.Lazy as TL (Text(..))+import qualified Data.Text.Internal.Lazy.Fusion as SL+import qualified Data.Text.Lazy as TL+import qualified Tests.SlowFunctions as Slow++s_take n          = L.take n      `eqP` (unpackS . S.take n)+s_take_s (Small n) = L.take n      `eqP` (unpackS . S.unstream . S.take n)+sf_take (applyFun -> p) n+                  = (L.take n . L.filter p) `eqP`+                    (unpackS . S.take n . S.filter p)+t_take n          = L.take n      `eqP` (unpackS . T.take n)+t_takeEnd n       = (L.reverse . L.take n . L.reverse) `eqP`+                    (unpackS . T.takeEnd n)+tl_take n         = L.genericTake n      `eqP` (unpackS . TL.take n)+tl_take_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in+                    L.genericTake n      `eqP` (unpackS . TL.take n)+tl_takeEnd n      = (L.reverse . L.genericTake n . L.reverse) `eqP`+                    (unpackS . TL.takeEnd n)+tl_takeEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in+                    (L.reverse . L.genericTake n . L.reverse) `eqP`+                    (unpackS . TL.takeEnd n)++s_drop n          = L.drop n      `eqP` (unpackS . S.drop n)+s_drop_s (Small n) = L.drop n      `eqP` (unpackS . S.unstream . S.drop n)+sf_drop (applyFun -> p) n+                  = (L.drop n . L.filter p) `eqP` (unpackS . S.drop n . S.filter p)+t_drop n          = L.drop n      `eqP` (unpackS . T.drop n)+t_dropEnd n       = (L.reverse . L.drop n . L.reverse) `eqP`+                    (unpackS . T.dropEnd n)+tl_drop n         = L.genericDrop n `eqP` (unpackS . TL.drop n)+tl_drop_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in                  L.genericDrop n `eqP` (unpackS . TL.drop n)+tl_dropEnd n      = (L.reverse . L.genericDrop n . L.reverse) `eqP`+                    (unpackS . TL.dropEnd n)+tl_dropEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in                  (L.reverse . L.genericDrop n . L.reverse) `eqP`+                    (unpackS . TL.dropEnd n)++s_take_drop (Small n) = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n)+s_take_drop_s (Small n) = (L.take n . L.drop n) `eqP`+                    (unpackS . S.unstream . S.take n . S.drop n)+s_takeWhile (applyFun -> p)+                  = L.takeWhile p `eqP` (unpackS . S.takeWhile p)+s_takeWhile_s (applyFun -> p)+                  = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p)+sf_takeWhile (applyFun -> q) (applyFun -> p)+                  = (L.takeWhile p . L.filter q) `eqP` (unpackS . S.takeWhile p . S.filter q)++data NoMatch = NoMatch Char Char+  deriving (Eq, Show)++instance Arbitrary NoMatch where+  arbitrary = do+    c <- arbitraryUnicodeChar+    d <- suchThat arbitraryUnicodeChar (/= c)+    return $ NoMatch c d+  shrink (NoMatch c d) = fmap (NoMatch c)   (filter (/= c) (shrink d))+                      ++ fmap (`NoMatch` d) (filter (/= d) (shrink c))++t_takeWhile (applyFun -> p)+                  = L.takeWhile p `eqP` (unpackS . T.takeWhile p)+tl_takeWhile (applyFun -> p)+                  = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)+t_takeWhileEnd (applyFun -> p)+                  = (L.reverse . L.takeWhile p . L.reverse) `eqP`+                    (unpackS . T.takeWhileEnd p)+t_takeWhileEnd_null t (NoMatch c d)+                  = T.null $ T.takeWhileEnd (==d) (T.snoc t c)+tl_takeWhileEnd (applyFun -> p)+                  = (L.reverse . L.takeWhile p . L.reverse) `eqP`+                    (unpackS . TL.takeWhileEnd p)+tl_takeWhileEnd_null t (NoMatch c d)+                  = TL.null $ TL.takeWhileEnd (==d) (TL.snoc t c)+s_dropWhile (applyFun -> p)+                  = L.dropWhile p `eqP` (unpackS . S.dropWhile p)+s_dropWhile_s (applyFun -> p)+                  = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)+sf_dropWhile (applyFun -> q) (applyFun -> p)+                  = (L.dropWhile p . L.filter q) `eqP`+                    (unpackS . S.dropWhile p . S.filter q)+t_dropWhile (applyFun -> p)+                  = L.dropWhile p `eqP` (unpackS . T.dropWhile p)+tl_dropWhile (applyFun -> p)+                  = L.dropWhile p `eqP` (unpackS . S.dropWhile p)+t_dropWhileEnd (applyFun -> p)+                  = (L.reverse . L.dropWhile p . L.reverse) `eqP`+                    (unpackS . T.dropWhileEnd p)+tl_dropWhileEnd (applyFun -> p)+                  = (L.reverse . L.dropWhile p . L.reverse) `eqP`+                    (unpackS . TL.dropWhileEnd p)+t_dropAround (applyFun -> p)+                  = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse)+                    `eqP` (unpackS . T.dropAround p)+tl_dropAround (applyFun -> p)+                  = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse)+                    `eqP` (unpackS . TL.dropAround p)+t_stripStart      = T.dropWhile isSpace `eq` T.stripStart+tl_stripStart     = TL.dropWhile isSpace `eq` TL.stripStart+t_stripEnd        = T.dropWhileEnd isSpace `eq` T.stripEnd+tl_stripEnd       = TL.dropWhileEnd isSpace `eq` TL.stripEnd+t_strip           = T.dropAround isSpace `eq` T.strip+tl_strip          = TL.dropAround isSpace `eq` TL.strip++t_splitAt n       = L.splitAt n   `eqP` (unpack2 . T.splitAt n)+tl_splitAt n      = L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n)+tl_splitAt_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in+                    L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n)++t_span (applyFun -> p)  = L.span p `eqP` (unpack2 . T.span p)+tl_span (applyFun -> p) = L.span p `eqP` (unpack2 . TL.span p)++t_breakOn_id s      = squid `eq` (uncurry T.append . T.breakOn s)+  where squid t | T.null s  = error "empty"+                | otherwise = t+tl_breakOn_id s     = squid `eq` (uncurry TL.append . TL.breakOn s)+  where squid t | TL.null s  = error "empty"+                | otherwise = t+t_breakOn_start (NotEmpty s) t =+    let (k,m) = T.breakOn s t+    in k `T.isPrefixOf` t && (T.null m || s `T.isPrefixOf` m)+tl_breakOn_start (NotEmpty s) t =+    let (k,m) = TL.breakOn s t+    in k `TL.isPrefixOf` t && TL.null m || s `TL.isPrefixOf` m+t_breakOnEnd_end (NotEmpty s) t =+    let (m,k) = T.breakOnEnd s t+    in k `T.isSuffixOf` t && (T.null m || s `T.isSuffixOf` m)+tl_breakOnEnd_end (NotEmpty s) t =+    let (m,k) = TL.breakOnEnd s t+    in k `TL.isSuffixOf` t && (TL.null m || s `TL.isSuffixOf` m)+t_break (applyFun -> p)+                  = L.break p     `eqP` (unpack2 . T.break p)+tl_break (applyFun -> p)+                  = L.break p     `eqP` (unpack2 . TL.break p)+t_group           = L.group       `eqP` (map unpackS . T.group)+tl_group          = L.group       `eqP` (map unpackS . TL.group)+t_groupBy (applyFun2 -> p)+                  = L.groupBy p   `eqP` (map unpackS . T.groupBy p)+tl_groupBy (applyFun2 -> p)+                  = L.groupBy p   `eqP` (map unpackS . TL.groupBy p)+t_inits           = L.inits       `eqP` (map unpackS . T.inits)+t_initsNE         = NonEmptyList.inits `eqP` (fmap unpackS . T.initsNE)+tl_inits          = L.inits       `eqP` (map unpackS . TL.inits)+tl_initsNE        = NonEmptyList.inits `eqP` (fmap unpackS . TL.initsNE)+t_tails           = L.tails       `eqP` (map unpackS . T.tails)+t_tailsNE         = NonEmptyList.tails `eqP` (fmap unpackS . T.tailsNE)+tl_tails          = L.tails       `eqPSqrt` (map unpackS . TL.tails)+tl_tailsNE        = NonEmptyList.tails `eqP` (fmap unpackS . TL.tailsNE)++spanML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b])+spanML p s = go [] s+  where+    go acc [] = pure (reverse acc, [])+    go acc ccs@(c : cs) = do+      continue <- p c+      if continue then go (c : acc) cs else pure (reverse acc, ccs)++spanEndML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b])+spanEndML p s = (\(s1, s2) -> (reverse s2, reverse s1)) <$> spanML p (reverse s)++eqSP :: Stringy s => (String -> State Int (String, String)) -> (s -> State Int (s, s)) -> Property+eqSP a b = property $ ((`runState` 0) . a) `eqP` ((`runState` 0) . (fmap unpack2 . b))++t_spanM (applyFun2 -> f')+  = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+    spanML f `eqSP` T.spanM f++t_spanEndM (applyFun2 -> f')+  = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+        spanEndML f `eqSP` T.spanEndM f++tl_spanM (applyFun2 -> f')+  = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+    spanML f `eqSP` TL.spanM f++tl_spanEndM (applyFun2 -> f')+  = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+    spanEndML f `eqSP` TL.spanEndM f++t_findAppendId = \(Sqrt (NotEmpty s)) ts ->+    let t = T.intercalate s ts+    in conjoin $ map (=== t) $ map (uncurry T.append) (T.breakOnAll s t)+tl_findAppendId = \(Sqrt (NotEmpty s)) ts ->+    let t = TL.intercalate s ts+    in conjoin $ map (=== t) $ map (uncurry TL.append) (TL.breakOnAll s t)+t_findContains = \(Sqrt (NotEmpty s)) ->+    all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s+tl_findContains = \(Sqrt (NotEmpty s)) -> all (TL.isPrefixOf s . snd) .+    TL.breakOnAll s . TL.intercalate s+sl_filterCount c  = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c+t_findCount s     = (L.length . T.breakOnAll s) `eq` T.count s+tl_findCount s    = (L.genericLength . TL.breakOnAll s) `eq` TL.count s++t_splitOn_split s  = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s . unSqrt+tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`+                      (map TL.fromStrict . T.splitOn s)) . T.intercalate s . unSqrt+t_splitOn_i (NotEmpty t)  = id `eq` (T.intercalate t . T.splitOn t)+tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t)++t_split (applyFun -> p) = split p `eqP` (map unpackS . T.split p)+t_split_count c = (L.length . T.split (==c)) `eq`+                  ((1+) . T.count (T.singleton c))+t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)+tl_split (applyFun -> p) = split p `eqP` (map unpackS . TL.split p)++split :: (a -> Bool) -> [a] -> [[a]]+split _ [] =  [[]]+split p xs = loop xs+  where+    loop s = case break p s of+      (l, []) -> [l]+      (l, _ : s') -> l : loop s'++t_chunksOf_same_lengths k =+  conjoin . map ((===k) . T.length) . drop 1 . reverse . T.chunksOf k++t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0)+  where len = L.sum . L.map T.length $ T.chunksOf k t++tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) .+                                   TL.chunksOf (fromIntegral k) . TL.fromStrict)++t_lines           = L.lines       `eqP` (map unpackS . T.lines)+tl_lines          = L.lines       `eqP` (map unpackS . TL.lines)+t_lines_spacy     = (L.lines      `eqP` (map unpackS . T.lines))  . getSpacyString+tl_lines_spacy    = (L.lines      `eqP` (map unpackS . TL.lines)) . getSpacyString++tl_lines_laziness = case TL.lines (TL.replicate 1000000000000000 (TL.singleton 'a')) of+  [] -> property False+  hd : _ -> TL.head hd === 'a'++tl_lines_specialCase = TL.lines (TL.Chunk (T.pack "foo") $ TL.Chunk (T.pack "bar\nbaz\n") $ TL.Empty) === [TL.pack "foobar", TL.pack "baz"]++t_words           = L.words       `eqP` (map unpackS . T.words)+tl_words          = L.words       `eqP` (map unpackS . TL.words)+t_words_spacy     = (L.words      `eqP` (map unpackS . T.words))  . getSpacyString+tl_words_spacy    = (L.words      `eqP` (map unpackS . TL.words)) . getSpacyString++t_unlines         = (L.unlines . unSqrt) `eq` (unpackS . T.unlines . map packS . unSqrt)+tl_unlines        = (L.unlines . unSqrt) `eq` (unpackS . TL.unlines . map packS . unSqrt)+t_unwords         = (L.unwords . unSqrt) `eq` (unpackS . T.unwords . map packS . unSqrt)+tl_unwords        = (L.unwords . unSqrt) `eq` (unpackS . TL.unwords . map packS . unSqrt)++s_isPrefixOf s    = L.isPrefixOf s `eqP`+                    (S.isPrefixOf (S.stream $ packS s) . S.stream)+sf_isPrefixOf (applyFun -> p) s+                  = (L.isPrefixOf s . L.filter p) `eqP`+                    (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream)+t_isPrefixOf s    = L.isPrefixOf s`eqP` T.isPrefixOf (packS s)+tl_isPrefixOf s   = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s)+t_isSuffixOf s    = L.isSuffixOf s`eqP` T.isSuffixOf (packS s)+tl_isSuffixOf s   = L.isSuffixOf s`eqP` TL.isSuffixOf (packS s)+t_isInfixOf s     = L.isInfixOf s `eqP` T.isInfixOf (packS s)+tl_isInfixOf s    = L.isInfixOf s `eqP` TL.isInfixOf (packS s)++t_stripPrefix s      = (fmap packS . L.stripPrefix s) `eqP` T.stripPrefix (packS s)+tl_stripPrefix s     = (fmap packS . L.stripPrefix s) `eqP` TL.stripPrefix (packS s)++stripSuffix p t = reverse `fmap` L.stripPrefix (reverse p) (reverse t)++t_stripSuffix s      = (fmap packS . stripSuffix s) `eqP` T.stripSuffix (packS s)+tl_stripSuffix s     = (fmap packS . stripSuffix s) `eqP` TL.stripSuffix (packS s)++commonPrefixes a0@(_:_) b0@(_:_) = Just (go a0 b0 [])+    where go (a:as) (b:bs) ps+              | a == b = go as bs (a:ps)+          go as bs ps  = (reverse ps,as,bs)+commonPrefixes _ _ = Nothing++t_commonPrefixes a b (NonEmpty p)+    = commonPrefixes pa pb ===+      repack `fmap` T.commonPrefixes (packS pa) (packS pb)+  where repack (x,y,z) = (unpackS x,unpackS y,unpackS z)+        pa = p ++ a+        pb = p ++ b++tl_commonPrefixes a b (NonEmpty p)+    = commonPrefixes pa pb ===+      repack `fmap` TL.commonPrefixes (packS pa) (packS pb)+  where repack (x,y,z) = (unpackS x,unpackS y,unpackS z)+        pa = p ++ a+        pb = p ++ b++testSubstrings :: TestTree+testSubstrings =+  testGroup "substrings" [+    testGroup "breaking" [+      testProperty "s_take" s_take,+      testProperty "s_take_s" s_take_s,+      testProperty "sf_take" sf_take,+      testProperty "t_take" t_take,+      testProperty "t_takeEnd" t_takeEnd,+      testProperty "tl_take" tl_take,+      testProperty "tl_take_maxBound" tl_take_maxBound,+      testProperty "tl_takeEnd" tl_takeEnd,+      testProperty "tl_takeEnd_maxBound" tl_takeEnd_maxBound,+      testProperty "s_drop" s_drop,+      testProperty "s_drop_s" s_drop_s,+      testProperty "sf_drop" sf_drop,+      testProperty "t_drop" t_drop,+      testProperty "t_dropEnd" t_dropEnd,+      testProperty "tl_drop" tl_drop,+      testProperty "tl_drop_maxBound" tl_drop_maxBound,+      testProperty "tl_dropEnd" tl_dropEnd,+      testProperty "tl_dropEnd_maxBound" tl_dropEnd_maxBound,+      testProperty "s_take_drop" s_take_drop,+      testProperty "s_take_drop_s" s_take_drop_s,+      testProperty "s_takeWhile" s_takeWhile,+      testProperty "s_takeWhile_s" s_takeWhile_s,+      testProperty "sf_takeWhile" sf_takeWhile,+      testProperty "t_takeWhile" t_takeWhile,+      testProperty "tl_takeWhile" tl_takeWhile,+      testProperty "t_takeWhileEnd" t_takeWhileEnd,+      testProperty "t_takeWhileEnd_null" t_takeWhileEnd_null,+      testProperty "tl_takeWhileEnd" tl_takeWhileEnd,+      testProperty "tl_takeWhileEnd_null" tl_takeWhileEnd_null,+      testProperty "sf_dropWhile" sf_dropWhile,+      testProperty "s_dropWhile" s_dropWhile,+      testProperty "s_dropWhile_s" s_dropWhile_s,+      testProperty "t_dropWhile" t_dropWhile,+      testProperty "tl_dropWhile" tl_dropWhile,+      testProperty "t_dropWhileEnd" t_dropWhileEnd,+      testProperty "tl_dropWhileEnd" tl_dropWhileEnd,+      testProperty "t_dropAround" t_dropAround,+      testProperty "tl_dropAround" tl_dropAround,+      testProperty "t_stripStart" t_stripStart,+      testProperty "tl_stripStart" tl_stripStart,+      testProperty "t_stripEnd" t_stripEnd,+      testProperty "tl_stripEnd" tl_stripEnd,+      testProperty "t_strip" t_strip,+      testProperty "tl_strip" tl_strip,+      testProperty "t_splitAt" t_splitAt,+      testProperty "tl_splitAt" tl_splitAt,+      testProperty "tl_splitAt_maxBound" tl_splitAt_maxBound,+      testProperty "t_span" t_span,+      testProperty "tl_span" tl_span,+      testProperty "t_breakOn_id" t_breakOn_id,+      testProperty "tl_breakOn_id" tl_breakOn_id,+      testProperty "t_breakOn_start" t_breakOn_start,+      testProperty "tl_breakOn_start" tl_breakOn_start,+      testProperty "t_breakOnEnd_end" t_breakOnEnd_end,+      testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,+      testProperty "t_break" t_break,+      testProperty "tl_break" tl_break,+      testProperty "t_group" t_group,+      testProperty "tl_group" tl_group,+      testProperty "t_groupBy" t_groupBy,+      testProperty "tl_groupBy" tl_groupBy,+      testProperty "t_inits" t_inits,+      testProperty "t_initsNE" t_initsNE,+      testProperty "tl_inits" tl_inits,+      testProperty "tl_initsNE" tl_initsNE,+      testProperty "t_tails" t_tails,+      testProperty "t_tailsNE" t_tailsNE,+      testProperty "tl_tails" tl_tails,+      testProperty "tl_tailsNE" tl_tailsNE,+      testProperty "t_spanM" t_spanM,+      testProperty "t_spanEndM" t_spanEndM,+      testProperty "tl_spanM" tl_spanM,+      testProperty "tl_spanEndM" tl_spanEndM+    ],++    testGroup "breaking many" [+      testProperty "t_findAppendId" t_findAppendId,+      testProperty "tl_findAppendId" tl_findAppendId,+      testProperty "t_findContains" t_findContains,+      testProperty "tl_findContains" tl_findContains,+      testProperty "sl_filterCount" sl_filterCount,+      testProperty "t_findCount" t_findCount,+      testProperty "tl_findCount" tl_findCount,+      testProperty "t_splitOn_split" t_splitOn_split,+      testProperty "tl_splitOn_split" tl_splitOn_split,+      testProperty "t_splitOn_i" t_splitOn_i,+      testProperty "tl_splitOn_i" tl_splitOn_i,+      testProperty "t_split" t_split,+      testProperty "t_split_count" t_split_count,+      testProperty "t_split_splitOn" t_split_splitOn,+      testProperty "tl_split" tl_split,+      testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths,+      testProperty "t_chunksOf_length" t_chunksOf_length,+      testProperty "tl_chunksOf" tl_chunksOf+    ],++    testGroup "lines and words" [+      testProperty "t_lines" t_lines,+      testProperty "tl_lines" tl_lines,+      testProperty "t_lines_spacy" t_lines_spacy,+      testProperty "tl_lines_spacy" tl_lines_spacy,+      testProperty "tl_lines_laziness" tl_lines_laziness,+      testProperty "tl_lines_specialCase" tl_lines_specialCase,+      testProperty "t_words" t_words,+      testProperty "tl_words" tl_words,+      testProperty "t_words_spacy" t_words_spacy,+      testProperty "tl_words_spacy" tl_words_spacy,+      testProperty "t_unlines" t_unlines,+      testProperty "tl_unlines" tl_unlines,+      testProperty "t_unwords" t_unwords,+      testProperty "tl_unwords" tl_unwords+    ],++    testGroup "predicates" [+      testProperty "s_isPrefixOf" s_isPrefixOf,+      testProperty "sf_isPrefixOf" sf_isPrefixOf,+      testProperty "t_isPrefixOf" t_isPrefixOf,+      testProperty "tl_isPrefixOf" tl_isPrefixOf,+      testProperty "t_isSuffixOf" t_isSuffixOf,+      testProperty "tl_isSuffixOf" tl_isSuffixOf,+      testProperty "t_isInfixOf" t_isInfixOf,+      testProperty "tl_isInfixOf" tl_isInfixOf,++      testGroup "view" [+        testProperty "t_stripPrefix" t_stripPrefix,+        testProperty "tl_stripPrefix" tl_stripPrefix,+        testProperty "t_stripSuffix" t_stripSuffix,+        testProperty "tl_stripSuffix" tl_stripSuffix,+        testProperty "t_commonPrefixes" t_commonPrefixes,+        testProperty "tl_commonPrefixes" tl_commonPrefixes+      ]+    ]+  ]
+ tests/Tests/Properties/Text.hs view
@@ -0,0 +1,491 @@+-- | Tests for operations that don't fit in the other @Test.Properties.*@ modules.
+
+{-# LANGUAGE BangPatterns        #-}
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE ViewPatterns        #-}
+
+{-# OPTIONS_GHC  -fno-warn-missing-signatures #-}
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+{-# HLINT ignore "Avoid restricted function" #-}
+
+module Tests.Properties.Text
+    ( testText
+    ) where
+
+import Control.Exception (SomeException, evaluate, try)
+import Data.Char (isLower, isLetter, isUpper)
+import Data.Maybe (mapMaybe)
+import Data.Text.Internal.Fusion.Size
+import Data.Word (Word8)
+import Test.QuickCheck
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.QuickCheck (testProperty)
+import Tests.QuickCheckUtils
+import qualified Data.Char as C
+import qualified Data.List as L
+import qualified Data.Text as T
+import qualified Data.Text.Internal.Fusion as S
+import qualified Data.Text.Internal.Fusion.Common as S
+import qualified Data.Text.Internal.Lazy.Fusion as SL
+import qualified Data.Text.Internal.Lazy.Search as S (indices)
+import qualified Data.Text.Internal.Search as T (indices)
+import qualified Data.Text.Lazy as TL
+import qualified Tests.SlowFunctions as Slow
+#if MIN_VERSION_base(4, 15, 0)
+import qualified GHC.Unicode as G (unicodeVersion)
+import qualified Data.Text.Internal.Fusion.CaseMapping as T (unicodeVersion)
+#endif
+
+t_pack_unpack       = (T.unpack . T.pack) `eq` id
+tl_pack_unpack      = (TL.unpack . TL.pack) `eq` id
+t_stream_unstream   = (S.unstream . S.stream) `eq` id
+tl_stream_unstream  = (SL.unstream . SL.stream) `eq` id
+t_reverse_stream t  = (S.reverse . S.reverseStream) t === t
+t_singleton c       = [c] === (T.unpack . T.singleton) c
+tl_singleton c      = [c] === (TL.unpack . TL.singleton) c
+tl_unstreamChunks x = f 11 x === f 1000 x
+    where f n = SL.unstreamChunks n . S.streamList
+tl_chunk_unchunk    = (TL.fromChunks . TL.toChunks) `eq` id
+tl_from_to_strict   = (TL.fromStrict . TL.toStrict) `eq` id
+
+s_map (applyFun -> f)   = map f  `eqP` (unpackS . S.map f)
+s_map_s (applyFun -> f) = map f  `eqP` (unpackS . S.unstream . S.map f)
+sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p)  `eqP` (unpackS . S.map f . S.filter p)
+
+t_map (applyFun -> f)                      = map f  `eqP` (unpackS . T.map f)
+tl_map (applyFun -> f)                     = map f  `eqP` (unpackS . TL.map f)
+t_map_map (applyFun -> f) (applyFun -> g)  = (map f . map g) `eqP` (unpackS . T.map f . T.map g)
+tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g)  `eqP` (unpackS . TL.map f . TL.map g)
+t_length_map (applyFun -> f)               = (L.length . map f)  `eqP` (T.length . T.map f)
+tl_length_map (applyFun -> f)              = (L.genericLength . map f)  `eqP` (TL.length . TL.map f)
+
+s_intercalate c   = (L.intercalate c . unSqrt) `eq`
+                    (unpackS . S.intercalate (packS c) . map packS . unSqrt)
+t_intercalate c   = (L.intercalate c . unSqrt) `eq`
+                    (unpackS . T.intercalate (packS c) . map packS . unSqrt)
+tl_intercalate c  = (L.intercalate c . unSqrt) `eq`
+                    (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)
+t_length_intercalate c  = (L.length . L.intercalate c . unSqrt) `eq`
+                    (T.length . T.intercalate (packS c) . map packS . unSqrt)
+tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq`
+                    (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)
+s_intersperse c   = L.intersperse c `eqP`
+                    (unpackS . S.intersperse c)
+s_intersperse_s c = L.intersperse c `eqP`
+                    (unpackS . S.unstream . S.intersperse c)
+sf_intersperse (applyFun -> p) c
+                  = (L.intersperse c . L.filter p) `eqP`
+                   (unpackS . S.intersperse c . S.filter p)
+t_intersperse c   = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c)
+tl_intersperse c  = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c)
+t_length_intersperse c  = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c)
+tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c)
+t_transpose       = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt)
+tl_transpose      = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt)
+t_reverse         = L.reverse `eqP` (unpackS . T.reverse)
+tl_reverse        = L.reverse `eqP` (unpackS . TL.reverse)
+t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)
+
+t_replace s d     = (L.intercalate d . splitOn s) `eqP`
+                    (unpackS . T.replace (T.pack s) (T.pack d))
+tl_replace s d     = (L.intercalate d . splitOn s) `eqP`
+                     (unpackS . TL.replace (TL.pack s) (TL.pack d))
+
+splitOn :: (Eq a) => [a] -> [a] -> [[a]]
+splitOn pat src0
+    | l == 0    = error "splitOn: empty"
+    | otherwise = go src0
+  where
+    l           = length pat
+    go src      = search 0 src
+      where
+        search _ [] = [src]
+        search !n s@(_:s')
+            | pat `L.isPrefixOf` s = take n src : go (drop l s)
+            | otherwise            = search (n+1) s'
+
+s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs
+    where s = S.streamList xs
+sf_toCaseFold_length (applyFun -> p) xs =
+    (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs)
+    where s = S.streamList xs
+t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t
+
+tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t
+t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==>
+    T.toCaseFold (T.singleton c) === T.singleton (C.toLower c)
+
+-- | Baseline generated with GHC 9.2 + text-1.2.5.0,
+t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389"
+t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower))
+t_toCaseFold_cherokeeUpper = conjoin $
+    map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper
+
+-- | Generated with GHC 9.2 + text-1.2.5.0,
+-- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $
+--   filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c))
+--     [minBound .. maxBound]
+toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"
+cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase
+cherokeeLower = ['\xAB70'..'\xABBF']
+
+t_toLower_length t = T.length (T.toLower t) >= T.length t
+t_toLower_lower t = p (T.toLower t) >= p t
+    where p = T.length . T.filter isLower
+tl_toLower_lower t = p (TL.toLower t) >= p t
+    where p = TL.length . TL.filter isLower
+t_toLower_char c = c /= '\304' ==>
+    T.toLower (T.singleton c) === T.singleton (C.toLower c)
+t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775"
+
+t_toUpper_length t = T.length (T.toUpper t) >= T.length t
+t_toUpper_upper t = p (T.toUpper t) >= p t
+    where p = T.length . T.filter isUpper
+tl_toUpper_upper t = p (TL.toUpper t) >= p t
+    where p = TL.length . TL.filter isUpper
+t_toUpper_char c = c `notElem` toUpperExceptions ==>
+    T.toUpper (T.singleton c) === T.singleton (C.toUpper c)
+
+-- | Baseline generated with GHC 9.2 + text-1.2.5.0,
+t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341"
+
+-- | Generated with GHC 9.2 + text-1.2.5.0,
+-- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c))
+--   [minBound .. maxBound]
+toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"
+
+t_toTitle_title t = all (<= 1) (caps w)
+    where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle
+          -- TIL: there exist uppercase-only letters
+          w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t
+t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian)
+    where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words
+          -- Surprise! The Spanish/Portuguese ordinal indicators changed
+          -- from category Ll (letter, lowercase) to Lo (letter, other)
+          -- in Unicode 7.0
+          -- Oh, and there exist lowercase-only letters (see previous test)
+          stable c = if isLower c
+                     then C.toUpper c /= c
+                     else c /= '\170' && c /= '\186'
+          -- Georgian text does not have a concept of title case
+          -- https://en.wikipedia.org/wiki/Georgian_Extended
+          isGeorgian c = c >= '\4256' && c < '\4352'
+t_toTitle_char c = c `notElem` toTitleExceptions ==>
+    T.toTitle (T.singleton c) === T.singleton (C.toUpper c)
+
+-- | Baseline generated with GHC 9.2 + text-1.2.5.0,
+t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389"
+
+-- | Generated with GHC 9.2 + text-1.2.5.0,
+-- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c))
+--   [minBound .. maxBound]
+toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"
+
+t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t
+t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t
+t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t
+
+ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs))
+ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs))
+ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs))
+
+ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs))
+  where
+    referenceToTitle _ [] = []
+    referenceToTitle False (y : ys)
+      | C.isLetter y = C.toUpper y : referenceToTitle True ys
+      | otherwise = y : referenceToTitle False ys
+    referenceToTitle True (y : ys)
+      | C.isLetter y = C.toLower y : referenceToTitle True ys
+      | otherwise = y : referenceToTitle (not (C.isSpace y)) ys
+
+justifyLeft k c xs  = xs ++ L.replicate (k - length xs) c
+justifyRight m n xs = L.replicate (m - length xs) n ++ xs
+center k c xs
+    | len >= k  = xs
+    | otherwise = L.replicate l c ++ xs ++ L.replicate r c
+   where len = length xs
+         d   = k - len
+         r   = d `div` 2
+         l   = d - r
+
+s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c)
+    where j = fromIntegral (k :: Word8)
+s_justifyLeft_s k c = justifyLeft j c `eqP`
+                      (unpackS . S.unstream . S.justifyLeftI j c)
+    where j = fromIntegral (k :: Word8)
+sf_justifyLeft (applyFun -> p) k c
+                    = (justifyLeft j c . L.filter p) `eqP`
+                       (unpackS . S.justifyLeftI j c . S.filter p)
+    where j = fromIntegral (k :: Word8)
+t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c)
+    where j = fromIntegral (k :: Word8)
+tl_justifyLeft k c = justifyLeft j c `eqP`
+                     (unpackS . TL.justifyLeft (fromIntegral j) c)
+    where j = fromIntegral (k :: Word8)
+t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c)
+    where j = fromIntegral (k :: Word8)
+tl_justifyRight k c = justifyRight j c `eqP`
+                      (unpackS . TL.justifyRight (fromIntegral j) c)
+    where j = fromIntegral (k :: Word8)
+t_center k c = center j c `eqP` (unpackS . T.center j c)
+    where j = fromIntegral (k :: Word8)
+tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c)
+    where j = fromIntegral (k :: Word8)
+
+t_elem c          = L.elem c `eqP` T.elem c
+tl_elem c         = L.elem c `eqP` TL.elem c
+sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p)
+sf_filter (applyFun -> q) (applyFun -> p)
+                  = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q)
+
+t_filter (applyFun -> p)
+                  = L.filter p    `eqP` (unpackS . T.filter p)
+tl_filter (applyFun -> p)
+                  = L.filter p    `eqP` (unpackS . TL.filter p)
+t_filter_filter (applyFun -> p) (applyFun -> q)
+                  = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q)
+tl_filter_filter (applyFun -> p) (applyFun -> q)
+                  = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q)
+t_length_filter (applyFun -> p)
+                  = (L.length . L.filter p) `eqP` (T.length . T.filter p)
+tl_length_filter (applyFun -> p)
+                  = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p)
+
+sf_findBy (applyFun -> q) (applyFun -> p)
+                             = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)
+t_find (applyFun -> p)       = L.find p      `eqP` T.find p
+tl_find (applyFun -> p)      = L.find p      `eqP` TL.find p
+t_partition (applyFun -> p)  = L.partition p `eqP` (unpack2 . T.partition p)
+tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p)
+
+sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j
+    where l = L.length s
+          j = if l == 0 then 0 else i `mod` (3 * l) - l
+
+t_index :: T.Text -> Int -> Property
+t_index xs i = ioProperty $ do
+    ch <- try (evaluate (T.index xs i))
+    pure $ case ch of
+        Left (_ :: SomeException) -> i < 0 .||. i >= T.length xs
+        Right c -> i >= 0 .&&. i < T.length xs .&&. c === T.unpack xs L.!! i
+
+tl_index :: TL.Text -> Int -> Property
+tl_index xs i = ioProperty $ do
+    let i' = fromIntegral i
+    ch <- try (evaluate (TL.index xs i'))
+    pure $ case ch of
+        Left (_ :: SomeException) -> i' < 0 .||. i' >= TL.length xs
+        Right c -> i >= 0 .&&. i' < TL.length xs .&&. c === TL.unpack xs L.!! i
+
+t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p
+t_count (NotEmpty t)  = (subtract 1 . L.length . T.splitOn t) `eq` T.count t
+tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`
+                        TL.count t
+t_zip s           = L.zip s `eqP` T.zip (packS s)
+tl_zip s          = L.zip s `eqP` TL.zip (packS s)
+sf_zipWith (applyFun -> p) (applyFun2 -> c) s
+                  = (L.zipWith c (L.filter p s) . L.filter p) `eqP`
+                    (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p)
+t_zipWith (applyFun2 -> c) s         = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))
+tl_zipWith (applyFun2 -> c) s        = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))
+t_length_zipWith (applyFun2 -> c) s  = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s))
+tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s))
+
+t_indices  (NotEmpty s) = Slow.indices s `eq` T.indices s
+tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s
+    where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t)
+          conc = T.concat . TL.toChunks
+t_indices_occurs = \(Sqrt (NotEmpty t)) ts ->
+    let s = T.intercalate t ts
+    in Slow.indices t s === T.indices t s
+
+t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8]
+tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8]
+
+t_indices_drop n s pref suff = T.indices s t === Slow.indices s t
+  where
+    t = T.drop n $ pref `T.append` s `T.append` suff
+tl_indices_drop n s pref suff =
+  map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)
+  where
+    t = TL.drop n $ pref `TL.append` s `TL.append` suff
+
+tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0]
+tl_indices_drop_chunked n s pref suff =
+  map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)
+  where
+    -- constructing a pathologically chunked haystack
+    t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff
+
+t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t
+  where
+    s = T.singleton c
+    t = T.drop n $ pref `T.append` s `T.append` suff
+tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)
+  where
+    s = TL.singleton c
+    t = TL.drop n $ pref `TL.append` s `TL.append` suff
+
+-- Make a stream appear shorter than it really is, to ensure that
+-- functions that consume inaccurately sized streams behave
+-- themselves.
+shorten :: Int -> S.Stream a -> S.Stream a
+shorten n t@(S.Stream arr off len)
+    | n > 0     = S.Stream arr off (smaller (exactSize n) len)
+    | otherwise = t
+
+testText :: TestTree
+testText =
+  testGroup "Text" [
+    testGroup "creation/elimination" [
+      testProperty "t_pack_unpack" t_pack_unpack,
+      testProperty "tl_pack_unpack" tl_pack_unpack,
+      testProperty "t_stream_unstream" t_stream_unstream,
+      testProperty "tl_stream_unstream" tl_stream_unstream,
+      testProperty "t_reverse_stream" t_reverse_stream,
+      testProperty "t_singleton" t_singleton,
+      testProperty "tl_singleton" tl_singleton,
+      testProperty "tl_unstreamChunks" tl_unstreamChunks,
+      testProperty "tl_chunk_unchunk" tl_chunk_unchunk,
+      testProperty "tl_from_to_strict" tl_from_to_strict
+    ],
+
+    testGroup "transformations" [
+      testProperty "s_map" s_map,
+      testProperty "s_map_s" s_map_s,
+      testProperty "sf_map" sf_map,
+
+      testProperty "t_map" t_map,
+      testProperty "tl_map" tl_map,
+      testProperty "t_map_map" t_map_map,
+      testProperty "tl_map_map" tl_map_map,
+      testProperty "t_length_map" t_length_map,
+      testProperty "tl_length_map" tl_length_map,
+
+      testProperty "s_intercalate" s_intercalate,
+      testProperty "t_intercalate" t_intercalate,
+      testProperty "tl_intercalate" tl_intercalate,
+      testProperty "t_length_intercalate" t_length_intercalate,
+      testProperty "tl_length_intercalate" tl_length_intercalate,
+      testProperty "s_intersperse" s_intersperse,
+      testProperty "s_intersperse_s" s_intersperse_s,
+      testProperty "sf_intersperse" sf_intersperse,
+      testProperty "t_intersperse" t_intersperse,
+      testProperty "tl_intersperse" tl_intersperse,
+      testProperty "t_length_intersperse" t_length_intersperse,
+      testProperty "tl_length_intersperse" tl_length_intersperse,
+      testProperty "t_transpose" t_transpose,
+      testProperty "tl_transpose" tl_transpose,
+      testProperty "t_reverse" t_reverse,
+      testProperty "tl_reverse" tl_reverse,
+      testProperty "t_reverse_short" t_reverse_short,
+      testProperty "t_replace" t_replace,
+      testProperty "tl_replace" tl_replace,
+
+      testGroup "case conversion" [
+        testProperty "s_toCaseFold_length" s_toCaseFold_length,
+        testProperty "sf_toCaseFold_length" sf_toCaseFold_length,
+        testProperty "t_toCaseFold_length" t_toCaseFold_length,
+        testProperty "tl_toCaseFold_length" tl_toCaseFold_length,
+        testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions,
+        testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower,
+        testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper,
+
+        testProperty "t_toLower_length" t_toLower_length,
+        testProperty "t_toLower_lower" t_toLower_lower,
+        testProperty "tl_toLower_lower" tl_toLower_lower,
+        testProperty "t_toLower_dotted_i" t_toLower_dotted_i,
+
+        testProperty "t_toUpper_length" t_toUpper_length,
+        testProperty "t_toUpper_upper" t_toUpper_upper,
+        testProperty "tl_toUpper_upper" tl_toUpper_upper,
+        testProperty "t_toUpper_exceptions" t_toUpper_exceptions,
+
+        testProperty "t_toTitle_title" t_toTitle_title,
+        testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower,
+        testProperty "t_toTitle_exceptions" t_toTitle_exceptions,
+
+        testProperty "t_toUpper_idempotent" t_toUpper_idempotent,
+        testProperty "t_toLower_idempotent" t_toLower_idempotent,
+        testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent,
+
+        testProperty "ascii_toLower" ascii_toLower,
+        testProperty "ascii_toUpper" ascii_toUpper,
+        testProperty "ascii_toTitle" ascii_toTitle,
+        testProperty "ascii_toCaseFold" ascii_toCaseFold
+      ],
+
+#if MIN_VERSION_base(4, 15, 0)
+      -- Requires matching version of Unicode in base and text
+      testGroup "char case conversion" $ if T.unicodeVersion == G.unicodeVersion then [
+        testProperty "t_toCaseFold_char" t_toCaseFold_char,
+        testProperty "t_toLower_char" t_toLower_char,
+        testProperty "t_toUpper_char" t_toUpper_char,
+        testProperty "t_toTitle_char" t_toTitle_char
+      ] else [],
+#endif
+
+      testGroup "justification" [
+        testProperty "s_justifyLeft" s_justifyLeft,
+        testProperty "s_justifyLeft_s" s_justifyLeft_s,
+        testProperty "sf_justifyLeft" sf_justifyLeft,
+        testProperty "t_justifyLeft" t_justifyLeft,
+        testProperty "tl_justifyLeft" tl_justifyLeft,
+        testProperty "t_justifyRight" t_justifyRight,
+        testProperty "tl_justifyRight" tl_justifyRight,
+        testProperty "t_center" t_center,
+        testProperty "tl_center" tl_center
+      ]
+    ],
+
+    testGroup "searching" [
+      testProperty "t_elem" t_elem,
+      testProperty "tl_elem" tl_elem,
+      testProperty "sf_elem" sf_elem,
+      testProperty "sf_filter" sf_filter,
+      testProperty "t_filter" t_filter,
+      testProperty "tl_filter" tl_filter,
+      testProperty "t_filter_filter" t_filter_filter,
+      testProperty "tl_filter_filter" tl_filter_filter,
+      testProperty "t_length_filter" t_length_filter,
+      testProperty "tl_length_filter" tl_length_filter,
+      testProperty "sf_findBy" sf_findBy,
+      testProperty "t_find" t_find,
+      testProperty "tl_find" tl_find,
+      testProperty "t_partition" t_partition,
+      testProperty "tl_partition" tl_partition
+    ],
+
+    testGroup "indexing" [
+      testProperty "sf_index" sf_index,
+      testProperty "t_index" t_index,
+      testProperty "tl_index" tl_index,
+      testProperty "t_findIndex" t_findIndex,
+      testProperty "t_count" t_count,
+      testProperty "tl_count" tl_count,
+      testProperty "t_indices" t_indices,
+      testProperty "tl_indices" tl_indices,
+      testProperty "t_indices_occurs" t_indices_occurs,
+
+      testProperty "t_indices_drop5" t_indices_drop5,
+      testProperty "tl_indices_drop5" tl_indices_drop5,
+      testProperty "t_indices_drop" t_indices_drop,
+      testProperty "tl_indices_drop" tl_indices_drop,
+      testProperty "tl_indices_chunked" tl_indices_chunked,
+      testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked,
+      testProperty "t_indices_char_drop" t_indices_char_drop,
+      testProperty "tl_indices_char_drop" tl_indices_char_drop
+    ],
+
+    testGroup "zips" [
+      testProperty "t_zip" t_zip,
+      testProperty "tl_zip" tl_zip,
+      testProperty "sf_zipWith" sf_zipWith,
+      testProperty "t_zipWith" t_zipWith,
+      testProperty "tl_zipWith" tl_zipWith,
+      testProperty "t_length_zipWith" t_length_zipWith,
+      testProperty "tl_length_zipWith" tl_length_zipWith
+    ]
+  ]
+ tests/Tests/Properties/Transcoding.hs view
@@ -0,0 +1,560 @@+-- | Tests for encoding and decoding++{-# LANGUAGE CPP, OverloadedStrings, ScopedTypeVariables #-}+{-# OPTIONS_GHC -Wno-missing-signatures #-}+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}++module Tests.Properties.Transcoding+    ( testTranscoding+    ) where++import Prelude hiding (head, tail)+import Data.Bits ((.&.), shiftR)+import Data.Char (chr, ord)+import Data.Functor (void)+import Data.Maybe (isNothing)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup ((<>))+#endif+import Data.Word (Word8)+import Test.QuickCheck hiding ((.&.))+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Test.Tasty.HUnit ((@?=), assertBool, assertFailure, testCase)+import Tests.QuickCheckUtils+import qualified Control.Exception as Exception+import qualified Data.Bits as Bits (shiftL, shiftR)+import qualified Data.ByteString as B+import qualified Data.ByteString.Builder as B+import qualified Data.ByteString.Builder.Extra as B+import qualified Data.ByteString.Builder.Prim as BP+import qualified Data.ByteString.Char8 as BC+import qualified Data.ByteString.Lazy as BL+import qualified Data.ByteString.Lazy.Char8 as BLC+import qualified Data.Text as T+import qualified Data.Text.Encoding as E+import qualified Data.Text.Encoding.Error as E+import qualified Data.Text.Internal.Encoding as E+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Encoding as EL++t_ascii t    = E.decodeASCII (E.encodeUtf8 a) === a+    where a  = T.map (\c -> chr (ord c `mod` 128)) t+tl_ascii t   = EL.decodeASCII (EL.encodeUtf8 a) === a+    where a  = TL.map (\c -> chr (ord c `mod` 128)) t++t_latin1     = E.decodeLatin1 `eq` (T.pack . BC.unpack)+tl_latin1    = EL.decodeLatin1 `eq` (TL.pack . BLC.unpack)++t_p_utf8_1   = testValidateUtf8_ [0x63] 1+t_p_utf8_2   = testValidateUtf8_ [0x63, 0x63, 0x63] 3+t_p_utf8_3   = testValidateUtf8_ [0x63, 0x63, 0xc2, 0x80, 0x63] 5+t_p_utf8_4   = testValidateUtf8_ [0x63, 0xe1, 0x80, 0x80, 0x63] 5+t_p_utf8_5   = testValidateUtf8_ [0xF0, 0x90, 0x80, 0x80, 0x63] 5+t_p_utf8_6   = testValidateUtf8_ [0x63, 0x63, 0xF0, 0x90, 0x80] 2+t_p_utf8_7   = testValidateUtf8_ [0x63, 0x63, 0x63, 0xF0, 0x90] 3+t_p_utf8_8   = testValidateUtf8Fail [0xF0, 0x90, 0x80, 0x63, 0x63] 0+t_p_utf8_9   = testValidateUtf8Fail [0x63, 0x63, 0x80, 0x63, 0x63] 2+t_p_utf8_0   = testValidateUtf8Fail [0x63, 0x63, 0xe1, 0x63, 0x63] 2++testValidateUtf8With ::+  (B.ByteString -> (Int, Maybe E.Utf8State)) ->+  (Maybe E.Utf8State -> IO r) ->+  [Word8] -> Int -> IO r+testValidateUtf8With validate k xs expectedLen = case validate (B.pack xs) of+  (len, s) -> do+    len @?= expectedLen+    k s++expectJust :: Maybe a -> IO a+expectJust Nothing = assertFailure "Unexpected Nothing"+expectJust (Just s) = pure s++expectNothing :: Maybe a -> IO ()+expectNothing Nothing = pure ()+expectNothing (Just _) = assertFailure "Unexpected Just"++testValidateUtf8 :: [Word8] -> Int -> IO E.Utf8State+testValidateUtf8 = testValidateUtf8With E.validateUtf8Chunk expectJust++testValidateUtf8_ :: [Word8] -> Int -> IO ()+testValidateUtf8_ = testValidateUtf8With E.validateUtf8Chunk (void . expectJust)++testValidateUtf8Fail :: [Word8] -> Int -> IO ()+testValidateUtf8Fail = testValidateUtf8With E.validateUtf8Chunk expectNothing++testValidateUtf8More :: E.Utf8State -> [Word8] -> Int -> IO E.Utf8State+testValidateUtf8More s =  testValidateUtf8With (E.validateUtf8More s) expectJust++testValidateUtf8MoreFail :: E.Utf8State -> [Word8] -> Int -> IO ()+testValidateUtf8MoreFail s = testValidateUtf8With (E.validateUtf8More s) expectNothing++t_pn_utf8_1 = do+  s <- testValidateUtf8 [0xF0, 0x90, 0x80] 0+  _ <- testValidateUtf8More s [0x80] 1+  testValidateUtf8MoreFail s [0x7f] (-3)+t_pn_utf8_2 = do+  s0 <- testValidateUtf8 [0xF0] 0+  testValidateUtf8MoreFail s0 [0x7f] (-1)+  s1 <- testValidateUtf8More s0 [0x90] (-1)+  testValidateUtf8MoreFail s1 [0x7f] (-2)+  s2 <- testValidateUtf8More s1 [0x80] (-2)+  testValidateUtf8MoreFail s2 [0x7f] (-3)+  _ <- testValidateUtf8More s2 [0x80] 1+  pure ()+t_pn_utf8_3 = do+  s1 <- testValidateUtf8 [0xc2] 0+  assertBool "PartialUtf8 must be partial" $ B.length (E.getPartialUtf8 s1) < E.getCompleteLen s1+  testValidateUtf8MoreFail s1 [0x80, 0x80] 1++-- Precondition: (i, ms1) = E.validateUtf8More s chunk+--+-- The index points to the end of the longest valid prefix+-- of prechunk `B.append` chunk+pre_validateUtf8More_validPrefix s chunk i =+  let prechunk = E.getPartialUtf8 s in+  -- Note: i <= 0 implies take i = id+  let (j, ms2) = E.validateUtf8Chunk (B.take (B.length prechunk + i) (prechunk `B.append` chunk)) in+  counterexample (show prechunk) $+    (B.length prechunk + i, ms2) === (j, Just E.startUtf8State)++-- Precondition: (i, Nothing) = E.validateUtf8More s chunk+--+-- Appending to an invalid chunk yields another invalid chunk.+pre_validateUtf8More_maximalPrefix s chunk i more =+  E.validateUtf8More s (chunk `B.append` more) === (i, Nothing)++-- Precondition: (i, Just s1) = E.validateUtf8More s chunk+pre_validateUtf8More_suffix s chunk i s1 =+  if 0 <= i+  then B.drop i chunk === p2b s1         -- The state s1 contains a suffix of the chunk.+  else p2b s `B.append` chunk === p2b s1 -- Or the chunk extends the incomplete code point in s1.++-- Precondition: (i, Just s1) = E.validateUtf8More s chunk1+--+-- Validating two chunks sequentially is equivalent to validating them at once.+pre_validateUtf8More_append s chunk1 s1 chunk2 =+  let (j, ms2) = E.validateUtf8More s1 chunk2 in+  (B.length chunk1 + j, ms2) === E.validateUtf8More s (chunk1 `B.append` chunk2)++-- These wrappers use custom generators to satisfy the above properties.++t_validateUtf8More_validPrefix = property $ do+  cex@(s, chunk, i, _ms1) <- randomMoreChunk+  pure $ counterexample (show cex) $+    pre_validateUtf8More_validPrefix s chunk i++t_validateUtf8More_maximalPrefix = property $ do+  -- We want chunks that fail validation: force their size to be big,..+  cex@(s, chunk, i, ms1) <- scale (* 3) arbitraryMoreChunk+  pure $ counterexample (show cex) $+    -- ... and just use rejection sampling+    isNothing ms1 ==>+    pre_validateUtf8More_maximalPrefix s chunk i++t_validateUtf8More_valid = property $ do+  cex@(s, chunk1, i, s1, chunk2) <- validMoreChunks+  pure $ counterexample (show cex) $+    pre_validateUtf8More_suffix s chunk1 i s1 .&&.+    pre_validateUtf8More_append s chunk1 s1 chunk2++randomMoreChunk, arbitraryMoreChunk, validMoreChunk :: Gen (E.Utf8State, B.ByteString, Int, Maybe E.Utf8State)+randomMoreChunk = oneof [arbitraryMoreChunk, validMoreChunk]++arbitraryMoreChunk = do+  s <- randomUtf8State+  chunk <- arbitrary+  let (i, ms1) = E.validateUtf8More s chunk+  pure (s, chunk, i, ms1)++-- | Generate a random state by parsing a prefix of a Char+randomUtf8State :: Gen E.Utf8State+randomUtf8State = do+  c <- arbitrary+  chunk <- elements (B.inits (E.encodeUtf8 (T.singleton c)))+  case E.validateUtf8Chunk chunk of+    (_, Just s) -> pure s+    (_, Nothing) -> error "should not happen"++-- | Make a valid chunk, i.e., (s, chunk) such that+--+-- validateUtf8More s chunk = (i, Just s1)+--+-- Also returning i and s1 to not repeat work.+validMoreChunk = do+  (s, chunk, i, s1, _chunk2) <- validMoreChunks+  pure (s, chunk, i, Just s1)++-- | Make a valid chunk by slicing a valid UTF8 bs,+-- and also provide a second chunk which is a valid extension+-- with 0.5 probability.+validMoreChunks :: Gen (E.Utf8State, B.ByteString, Int, E.Utf8State, B.ByteString)+validMoreChunks = do+  bs <- E.encodeUtf8 <$> scale (* 3) arbitrary+  -- Take an intermediate state.+  -- No need to go too far since code points are at most 4 bytes long+  i <- choose (0, 3)+  let (bs0, bs1) = B.splitAt i bs+  case E.validateUtf8Chunk bs0 of+    (_, Just s) -> do+      j <- choose (0, B.length bs1)+      let (chunk1, chunk2') = B.splitAt j bs1+      case E.validateUtf8More s chunk1 of+        (n1, Just s1) -> do+          chunk2 <- oneof [pure chunk2', arbitrary]+          pure (s, chunk1, n1, s1, chunk2)+        (_, Nothing) -> error "should not happen"+    (_, Nothing) -> error "should not happen"++t_utf8_c     = (E.strictBuilderToText . fst3 . E.decodeUtf8Chunk . E.encodeUtf8) `eq` id+t_utf8       = (E.decodeUtf8 . E.encodeUtf8) `eq` id+t_utf8'      = (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right)+tl_utf8      = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id+tl_utf8'     = (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right)+t_utf16LE    = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id+tl_utf16LE   = (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id+t_utf16BE    = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id+tl_utf16BE   = (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id+t_utf32LE    = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id+tl_utf32LE   = (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id+t_utf32BE    = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id+tl_utf32BE   = (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id++fst3 :: (a, b, c) -> a+fst3 (a, _, _) = a++runBuilder :: B.Builder -> B.ByteString+runBuilder =+  -- Use smallish buffers to exercise bufferFull case as well+  BL.toStrict . B.toLazyByteStringWith (B.safeStrategy 5 5) ""++t_encodeUtf8Builder_ toBuilder = (runBuilder . toBuilder) `eq` E.encodeUtf8++t_encodeUtf8Builder_nonZeroOffset_ toBuilder (Positive n) =+  (runBuilder . toBuilder . T.drop n) `eq` (E.encodeUtf8 . T.drop n)++t_encodeUtf8Builder = t_encodeUtf8Builder_ E.encodeUtf8Builder+t_encodeUtf8Builder_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ E.encodeUtf8Builder++t_encodeUtf8BuilderEscaped = t_encodeUtf8Builder_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8))+t_encodeUtf8BuilderEscaped_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8))++t_encodeUtf8Builder_sanity t =+  (runBuilder . E.encodeUtf8Builder) t ===+    (runBuilder . E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) t++t_utf8_incr (Positive n) =+  (T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 . E.encodeUtf8) `eq` id++feedChunksOf :: Int -> (B.ByteString -> E.Decoding) -> B.ByteString+             -> [(T.Text, B.ByteString)]+feedChunksOf n f bs+  | B.null bs  = []+  | otherwise  = let (x,y) = B.splitAt n bs+                     E.Some t b f' = f x+                 in (t,b) : feedChunksOf n f' y++t_utf8_undecoded t =+  let b = E.encodeUtf8 t+      ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t+      leftover = (++ [B.empty]) . init . drop 1 . B.inits+  in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls++data InvalidUtf8 = InvalidUtf8+  { iu8Prefix  :: T.Text+  , iu8Invalid :: B.ByteString+  , iu8Suffix  :: T.Text+  } deriving (Eq)++instance Show InvalidUtf8 where+  show i = "InvalidUtf8 {prefix = "  ++ show (iu8Prefix i)+                   ++ ", invalid = " ++ show (iu8Invalid i)+                   ++ ", suffix = "  ++ show (iu8Suffix i)+                   ++ ", asBS = "    ++ show (toByteString i)+                   ++ ", length = "  ++ show (B.length (toByteString i))+                   ++ "}"++toByteString :: InvalidUtf8 -> B.ByteString+toByteString (InvalidUtf8 a b c) =+  E.encodeUtf8 a `B.append` b `B.append` E.encodeUtf8 c++instance Arbitrary InvalidUtf8 where+  arbitrary = oneof+    [ InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> pure mempty+    , InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> arbitrary+    , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> pure mempty+    , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> arbitrary+    ]+  shrink (InvalidUtf8 a b c)+    =  map (\c' -> InvalidUtf8 a b c') (shrink c)+    ++ map (\a' -> InvalidUtf8 a' b c) (shrink a)++t_utf8_err :: InvalidUtf8 -> DecodeErr -> Property+t_utf8_err bad de = forAll (Blind <$> genDecodeErr de) $ \(Blind onErr) -> ioProperty $ do+  let decoded = E.decodeUtf8With onErr (toByteString bad)+      len = T.length (E.decodeUtf8With onErr (toByteString bad))+  l <- Exception.try (Exception.evaluate len)+  pure $ case l of+    Left (err :: Exception.SomeException) -> counterexample (show err) $+      length (show err) >= 0+    Right _  -> counterexample (show (decoded, l)) $ de /= Strict++t_utf8_err' :: B.ByteString -> Bool+t_utf8_err' bs = case E.decodeUtf8' bs of+  Left err -> length (show err) >= 0+  Right t  -> T.length t >= 0++genInvalidUTF8 :: Gen B.ByteString+genInvalidUTF8 = B.pack <$> oneof [+    -- invalid leading byte of a 2-byte sequence+    (:) <$> choose (0xC0, 0xC1) <*> upTo 1 contByte+    -- invalid leading byte of a 4-byte sequence+  , (:) <$> choose (0xF5, 0xFF) <*> upTo 3 contByte+    -- 4-byte sequence greater than U+10FFFF+  , do k <- choose (0x11, 0x13)+       let w0 = 0xF0 + (k `Bits.shiftR` 2)+           w1 = 0x80 + ((k .&. 3) `Bits.shiftL` 4)+       ([w0,w1]++) <$> vectorOf 2 contByte+    -- continuation bytes without a start byte+  , listOf1 contByte+    -- short 2-byte sequence+  , (:[]) <$> choose (0xC2, 0xDF)+    -- short 3-byte sequence+  , (:) <$> choose (0xE0, 0xEF) <*> upTo 1 contByte+    -- short 4-byte sequence+  , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte+    -- overlong encoding+  , do k <- choose (0 :: Int, 0xFFFF)+       case k of+         _ | k < 0x80   -> elements [ord2_ k, ord3_ k, ord4_ k]+           | k < 0x7FF  -> elements [ord3_ k, ord4_ k]+           | otherwise  -> return (ord4_ k)+  ]+  where+    contByte = (0x80 +) <$> choose (0, 0x3f)+    upTo n gen = do+      k <- choose (0,n)+      vectorOf k gen+    -- Data.Text.Internal.Encoding.Utf8.ord{2,3,4} without sanity checks+    ord2_ n = map fromIntegral [(n `shiftR` 6) + 0xC0, (n .&. 0x3F) + 0x80]+    ord3_ n = map fromIntegral [(n `shiftR` 12) + 0xE0, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80]+    ord4_ n = map fromIntegral [(n `shiftR` 18) + 0xF0, ((n `shiftR` 12) .&. 0x3F) + 0x80, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80]++decodeLL :: BL.ByteString -> TL.Text+decodeLL = EL.decodeUtf8With E.lenientDecode++decodeL :: B.ByteString -> T.Text+decodeL = E.decodeUtf8With E.lenientDecode++-- The lenient decoding of lazy bytestrings should not depend on how they are chunked,+-- and it should behave the same as decoding of strict bytestrings.+t_decode_utf8_lenient :: Property+t_decode_utf8_lenient = forAllShrinkShow arbitrary shrink (show . BL.toChunks) $ \bs ->+    decodeLL bs === (TL.fromStrict . decodeL . B.concat . BL.toChunks) bs++-- See http://unicode.org/faq/utf_bom.html#gen8+-- A sequence such as <110xxxxx2 0xxxxxxx2> is illegal ...+-- When faced with this illegal byte sequence ... a UTF-8 conformant process+-- must treat the first byte 110xxxxx2 as an illegal termination error+-- (e.g. filter it out or replace by 0xFFFD) ...+-- ... and continue processing at the second byte 0xxxxxxx2+t_decode_with_error2 =+  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) === "xa"+t_decode_with_error3 =+  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xE0, 97, 97]) === "xaa"+t_decode_with_error4 =+  E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"++t_decode_with_error1' = do+  E.Some x1 bs1 f1 <- pure $ E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xc2])+  x1 @?= ""+  bs1 @?= B.pack [0xc2]+  E.Some x2 bs2 _ <- pure $ f1 $ B.pack [0x80, 0x80]+  x2 @?= "\x80x"+  bs2 @?= mempty+t_decode_with_error2' =+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of+    E.Some x _ _ -> x @?= "xa"+t_decode_with_error3' =+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of+    E.Some x _ _ -> x @?= "xaa"+t_decode_with_error4' =+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of+    E.Some x _ _ -> x @?= "xaaa"+t_decode_with_error5' = do+  ret <- Exception.try $ Exception.evaluate $ E.streamDecodeUtf8 (B.pack [0x81])+  case ret of+    Left (_ :: E.UnicodeException) -> pure ()+    Right{} -> assertFailure "Unexpected success"++testDecodeUtf8With :: (Maybe E.Utf8State -> IO r) -> E.Utf8State -> [Word8] -> T.Text -> IO r+testDecodeUtf8With k s xs expected =+  let xs' = B.pack xs in+  case E.decodeUtf8More s xs' of+    (prefix, bs, s') -> do+      let txt = E.strictBuilderToText prefix+      txt @?= expected+      if T.null txt then+        bs @?= xs'+      else+        E.encodeUtf8 txt `B.append` bs @?= E.getPartialUtf8 s `B.append` xs'+      k s'++testDecodeUtf8 :: E.Utf8State -> [Word8] -> T.Text -> IO E.Utf8State+testDecodeUtf8 = testDecodeUtf8With (\ms -> case ms of+  Just s -> pure s+  Nothing -> assertFailure "Unexpected failure")++testDecodeUtf8Fail :: E.Utf8State -> [Word8] -> T.Text -> IO ()+testDecodeUtf8Fail = testDecodeUtf8With (\ms -> case ms of+  Just _ -> assertFailure "Unexpected failure"+  Nothing -> pure ())++t_decode_chunk1 = do+  s1 <- testDecodeUtf8 E.startUtf8State [0xc2] ""+  B.length (E.getPartialUtf8 s1) @?= 1+  testDecodeUtf8Fail s1 [0x80, 0x80] "\128"++t_decode_chunk2 = do+  s1 <- testDecodeUtf8 E.startUtf8State [0xf0] ""+  s2 <- testDecodeUtf8 s1 [0x90, 0x80] ""+  _  <- testDecodeUtf8 s2 [0x80, 0x41] "\65536A"+  pure ()++t_infix_concat bs1 text bs2 =+  forAll (Blind <$> genDecodeErr Replace) $ \(Blind onErr) ->+  text `T.isInfixOf`+    E.decodeUtf8With onErr (B.concat [bs1, E.encodeUtf8 text, bs2])++t_textToStrictBuilder =+  (E.strictBuilderToText . E.textToStrictBuilder) `eq` id++-- decodeUtf8Chunk splits the input bytestring+t_decodeUtf8Chunk_split chunk =+  let (pre, suf, _ms) = E.decodeUtf8Chunk chunk+  in s2b pre `B.append` suf === chunk++-- decodeUtf8More mostly splits the input bytestring,+-- also inserting bytes from the partial code point in s.+--+-- This is wrapped by t_decodeUtf8More_split to have more+-- likely valid chunks.+t_decodeUtf8More_split' s chunk =+  let (pre, suf, _ms) = E.decodeUtf8More s chunk+  in if B.length chunk > B.length suf+  then s2b pre `B.append` suf === p2b s `B.append` chunk+  else suf === chunk++-- The output state of decodeUtf8More contains the suffix.+--+-- Precondition (valid chunk): ms = Just s'+pre_decodeUtf8More_suffix s chunk =+  let (_pre, suf, ms) = E.decodeUtf8More s chunk+  in case ms of+    Nothing -> discard+    Just s' -> if B.length chunk > B.length suf+      then p2b s' === suf+      else p2b s' === p2b s `B.append` suf++-- Decoding chunks separately is equivalent to decoding their concatenation.+pre_decodeUtf8More_append s chunk1 chunk2 =+  let (pre1, _, ms1) = E.decodeUtf8More s chunk1 in+  case ms1 of+    Nothing -> discard+    Just s1 ->+      let (pre2, _, ms2) = E.decodeUtf8More s1 chunk2 in+      let (pre3, _, ms3) = E.decodeUtf8More s (chunk1 `B.append` chunk2) in+      (s2b (pre1 <> pre2), ms2) === (s2b pre3, ms3)++-- Properties for any chunk+-- (but do try to generate valid chunks often enough)+t_decodeUtf8More1 = property $ do+  cex@(s, chunk, _, _) <- randomMoreChunk+  pure $ counterexample (show cex) $+    t_decodeUtf8More_split' s chunk++-- Properties that require valid chunks+t_decodeUtf8More2 = property $ do+  cex@(s, chunk, _, _, chunk2) <- validMoreChunks+  pure $ counterexample (show cex) $+    pre_decodeUtf8More_suffix s chunk .&&.+    pre_decodeUtf8More_append s chunk chunk2++s2b = E.encodeUtf8 . E.strictBuilderToText+p2b = E.getPartialUtf8++testTranscoding :: TestTree+testTranscoding =+  testGroup "transcoding" [+    testProperty "t_ascii" t_ascii,+    testProperty "tl_ascii" tl_ascii,+    testProperty "t_latin1" t_latin1,+    testProperty "tl_latin1" tl_latin1,+    testProperty "t_utf8" t_utf8,+    testProperty "t_utf8'" t_utf8',+    testProperty "t_utf8_undecoded" t_utf8_undecoded,+    testProperty "t_utf8_incr" t_utf8_incr,+    testProperty "tl_utf8" tl_utf8,+    testProperty "tl_utf8'" tl_utf8',+    testProperty "t_utf16LE" t_utf16LE,+    testProperty "tl_utf16LE" tl_utf16LE,+    testProperty "t_utf16BE" t_utf16BE,+    testProperty "tl_utf16BE" tl_utf16BE,+    testProperty "t_utf32LE" t_utf32LE,+    testProperty "tl_utf32LE" tl_utf32LE,+    testProperty "t_utf32BE" t_utf32BE,+    testProperty "tl_utf32BE" tl_utf32BE,+    testGroup "builder" [+      testProperty "t_encodeUtf8Builder" t_encodeUtf8Builder,+      testProperty "t_encodeUtf8Builder_nonZeroOffset" t_encodeUtf8Builder_nonZeroOffset,+      testProperty "t_encodeUtf8BuilderEscaped" t_encodeUtf8BuilderEscaped,+      testProperty "t_encodeUtf8BuilderEscaped_nonZeroOffset" t_encodeUtf8BuilderEscaped_nonZeroOffset,+      testProperty "t_encodeUtf8Builder_sanity" t_encodeUtf8Builder_sanity+    ],+    testGroup "errors" [+      testProperty "t_utf8_err" t_utf8_err,+      testProperty "t_utf8_err'" t_utf8_err'+    ],+    testGroup "error recovery" [+      testProperty "t_decode_utf8_lenient" t_decode_utf8_lenient,+      testProperty "t_decode_with_error2" t_decode_with_error2,+      testProperty "t_decode_with_error3" t_decode_with_error3,+      testProperty "t_decode_with_error4" t_decode_with_error4,+      testCase "t_decode_with_error1'" t_decode_with_error1',+      testCase "t_decode_with_error2'" t_decode_with_error2',+      testCase "t_decode_with_error3'" t_decode_with_error3',+      testCase "t_decode_with_error4'" t_decode_with_error4',+      testCase "t_decode_with_error5'" t_decode_with_error5',+      testProperty "t_infix_concat" t_infix_concat+    ],+    testGroup "validate" [+      testProperty "t_validateUtf8More_validPrefix" t_validateUtf8More_validPrefix,+      testProperty "t_validateUtf8More_maximalPrefix" t_validateUtf8More_maximalPrefix,+      testProperty "t_validateUtf8More_valid" t_validateUtf8More_valid+    ],+    testGroup "streaming" [+      testProperty "t_utf8_c" t_utf8_c,+      testCase "t_p_utf8_1" t_p_utf8_1,+      testCase "t_p_utf8_2" t_p_utf8_2,+      testCase "t_p_utf8_3" t_p_utf8_3,+      testCase "t_p_utf8_4" t_p_utf8_4,+      testCase "t_p_utf8_5" t_p_utf8_5,+      testCase "t_p_utf8_6" t_p_utf8_6,+      testCase "t_p_utf8_7" t_p_utf8_7,+      testCase "t_p_utf8_8" t_p_utf8_8,+      testCase "t_p_utf8_9" t_p_utf8_9,+      testCase "t_p_utf8_0" t_p_utf8_0,+      testCase "t_pn_utf8_1" t_pn_utf8_1,+      testCase "t_pn_utf8_2" t_pn_utf8_2,+      testCase "t_pn_utf8_3" t_pn_utf8_3,+      testCase "t_decode_chunk1" t_decode_chunk1,+      testCase "t_decode_chunk2" t_decode_chunk2,+      testProperty "t_decodeUtf8Chunk_split" t_decodeUtf8Chunk_split,+      testProperty "t_decodeUtf8More1" t_decodeUtf8More1,+      testProperty "t_decodeUtf8More2" t_decodeUtf8More2+    ],+    testGroup "strictBuilder" [+      testProperty "textToStrictBuilder" t_textToStrictBuilder+    ]+  ]
+ tests/Tests/Properties/Validate.hs view
@@ -0,0 +1,50 @@+{-# LANGUAGE CPP #-}+module Tests.Properties.Validate (testValidate) where++import Data.Array.Byte (ByteArray)+import Data.ByteString (ByteString)+import qualified Data.ByteString as B+import Data.ByteString.Short (toShort)+import Data.Either (isRight)+import Data.Text.Encoding (decodeUtf8', encodeUtf8)+import Data.Text.Internal.Validate (isValidUtf8ByteString, isValidUtf8ByteArray)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck ((===), Gen, Property,+  testProperty, arbitrary, forAllShrink, oneof, shrink)+import Tests.QuickCheckUtils ()+#if MIN_VERSION_bytestring(0,12,0)+import Data.ByteString.Short (unShortByteString)+#else+#if MIN_VERSION_bytestring(0,11,1)+import Data.ByteString.Short (ShortByteString(SBS))+#else+import Data.ByteString.Short.Internal (ShortByteString(SBS))+#endif+import Data.Array.Byte (ByteArray(ByteArray))++unShortByteString :: ShortByteString -> ByteArray+unShortByteString (SBS ba) = ByteArray ba+#endif++testValidate :: TestTree+testValidate = testGroup "validate"+  [ testProperty "bytestring" $ forAllShrink genByteString shrink $ \bs ->+      isValidUtf8ByteString bs === isRight (decodeUtf8' bs)+  , testProperty "bytearray" $ forAllByteArray $ \ba off len bs ->+      isValidUtf8ByteArray ba off len === isRight (decodeUtf8' bs)+  ]++genByteString :: Gen ByteString+genByteString = oneof+  [ arbitrary+  , encodeUtf8 <$> arbitrary+  ]++-- | We want to test 'isValidUtf8ByteArray' with various offsets, so we insert a random+-- prefix and remember its length.+forAllByteArray :: (ByteArray -> Int -> Int -> ByteString -> Property) -> Property+forAllByteArray prop =+  forAllShrink genByteString shrink $ \mainSlice ->+  forAllShrink arbitrary shrink $ \prefix ->+  let bs2ba = unShortByteString . toShort in+  prop (bs2ba (prefix `B.append` mainSlice)) (B.length prefix) (B.length mainSlice) mainSlice
tests/Tests/QuickCheckUtils.hs view
@@ -1,363 +1,326 @@--- | This module provides quickcheck utilities, e.g. arbitrary and show--- instances, and comparison functions, so we can focus on the actual properties--- in the 'Tests.Properties' module.----{-# LANGUAGE CPP, FlexibleInstances, TypeSynonymInstances #-}-{-# OPTIONS_GHC -fno-warn-orphans #-}-module Tests.QuickCheckUtils-    (-      genUnicode-    , unsquare-    , smallArbitrary--    , BigBounded(..)-    , BigInt(..)-    , NotEmpty(..)--    , Small(..)-    , small--    , Precision(..)-    , precision--    , integralRandomR--    , DecodeErr(..)-    , genDecodeErr--    , Stringy(..)-    , eq-    , eqP--    , Encoding(..)--    , write_read-    ) where--import Control.Applicative ((<$>))-import Control.Arrow (first, (***))-import Control.DeepSeq (NFData (..), deepseq)-import Control.Exception (bracket)-import Data.String (IsString, fromString)-import Data.Text.Foreign (I16)-import Data.Text.Lazy.Builder.RealFloat (FPFormat(..))-import Data.Word (Word8, Word16)-import Debug.Trace (trace)-import System.Random (Random(..), RandomGen)-import Test.QuickCheck hiding (Fixed(..), Small (..), (.&.))-import Test.QuickCheck.Monadic (assert, monadicIO, run)-import Test.QuickCheck.Unicode (string)-import Tests.Utils-import qualified Data.ByteString as B-import qualified Data.Text as T-import qualified Data.Text.Encoding.Error as T-import qualified Data.Text.Internal.Fusion as TF-import qualified Data.Text.Internal.Fusion.Common as TF-import qualified Data.Text.Internal.Lazy as TL-import qualified Data.Text.Internal.Lazy.Fusion as TLF-import qualified Data.Text.Lazy as TL-import qualified System.IO as IO--#if !MIN_VERSION_base(4,4,0)-import Data.Int (Int64)-import Data.Word (Word, Word64)-#endif--genUnicode :: IsString a => Gen a-genUnicode = fromString <$> string--instance Random I16 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary I16 where-    arbitrary     = arbitrarySizedIntegral-    shrink        = shrinkIntegral--instance Arbitrary B.ByteString where-    arbitrary     = B.pack `fmap` arbitrary-    shrink        = map B.pack . shrink . B.unpack--#if !MIN_VERSION_base(4,4,0)-instance Random Int64 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Random Word where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Random Word8 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Random Word64 where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)-#endif---- For tests that have O(n^2) running times or input sizes, resize--- their inputs to the square root of the originals.-unsquare :: (Arbitrary a, Show a, Testable b) => (a -> b) -> Property-unsquare = forAll smallArbitrary--smallArbitrary :: (Arbitrary a, Show a) => Gen a-smallArbitrary = sized $ \n -> resize (smallish n) arbitrary-  where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs--instance Arbitrary T.Text where-    arbitrary = T.pack `fmap` string-    shrink = map T.pack . shrink . T.unpack--instance Arbitrary TL.Text where-    arbitrary = (TL.fromChunks . map notEmpty) `fmap` smallArbitrary-    shrink = map TL.pack . shrink . TL.unpack--newtype BigInt = Big Integer-               deriving (Eq, Show)--instance Arbitrary BigInt where-    arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e)-    shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l]-      where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer--newtype BigBounded a = BigBounded a-                     deriving (Eq, Show)--instance (Bounded a, Random a, Arbitrary a) => Arbitrary (BigBounded a) where-    arbitrary = BigBounded <$> choose (minBound, maxBound)--newtype NotEmpty a = NotEmpty { notEmpty :: a }-    deriving (Eq, Ord)--instance Show a => Show (NotEmpty a) where-    show (NotEmpty a) = show a--instance Functor NotEmpty where-    fmap f (NotEmpty a) = NotEmpty (f a)--instance Arbitrary a => Arbitrary (NotEmpty [a]) where-    arbitrary   = sized (\n -> NotEmpty `fmap` (choose (1,n+1) >>= vector))-    shrink      = shrinkNotEmpty null--instance Arbitrary (NotEmpty T.Text) where-    arbitrary   = (fmap T.pack) `fmap` arbitrary-    shrink      = shrinkNotEmpty T.null--instance Arbitrary (NotEmpty TL.Text) where-    arbitrary   = (fmap TL.pack) `fmap` arbitrary-    shrink      = shrinkNotEmpty TL.null--instance Arbitrary (NotEmpty B.ByteString) where-    arbitrary   = (fmap B.pack) `fmap` arbitrary-    shrink      = shrinkNotEmpty B.null--shrinkNotEmpty :: Arbitrary a => (a -> Bool) -> NotEmpty a -> [NotEmpty a]-shrinkNotEmpty isNull (NotEmpty xs) =-  [ NotEmpty xs' | xs' <- shrink xs, not (isNull xs') ]--data Small = S0  | S1  | S2  | S3  | S4  | S5  | S6  | S7-           | S8  | S9  | S10 | S11 | S12 | S13 | S14 | S15-           | S16 | S17 | S18 | S19 | S20 | S21 | S22 | S23-           | S24 | S25 | S26 | S27 | S28 | S29 | S30 | S31-    deriving (Eq, Ord, Enum, Bounded)--small :: Integral a => Small -> a-small = fromIntegral . fromEnum--intf :: (Int -> Int -> Int) -> Small -> Small -> Small-intf f a b = toEnum ((fromEnum a `f` fromEnum b) `mod` 32)--instance Show Small where-    show = show . fromEnum--instance Read Small where-    readsPrec n = map (first toEnum) . readsPrec n--instance Num Small where-    fromInteger = toEnum . fromIntegral-    signum _ = 1-    abs = id-    (+) = intf (+)-    (-) = intf (-)-    (*) = intf (*)--instance Real Small where-    toRational = toRational . fromEnum--instance Integral Small where-    toInteger = toInteger . fromEnum-    quotRem a b = (toEnum x, toEnum y)-        where (x, y) = fromEnum a `quotRem` fromEnum b--instance Random Small where-    randomR = integralRandomR-    random  = randomR (minBound,maxBound)--instance Arbitrary Small where-    arbitrary     = choose (minBound, maxBound)-    shrink        = shrinkIntegral--integralRandomR :: (Integral a, RandomGen g) => (a,a) -> g -> (a,g)-integralRandomR  (a,b) g = case randomR (fromIntegral a :: Integer,-                                         fromIntegral b :: Integer) g of-                            (x,h) -> (fromIntegral x, h)--data DecodeErr = Lenient | Ignore | Strict | Replace-               deriving (Show, Eq)--genDecodeErr :: DecodeErr -> Gen T.OnDecodeError-genDecodeErr Lenient = return T.lenientDecode-genDecodeErr Ignore  = return T.ignore-genDecodeErr Strict  = return T.strictDecode-genDecodeErr Replace = (\c _ _ -> c) <$> frequency-  [ (1, return Nothing)-  , (50, Just <$> choose ('\x1', '\xffff'))-  ]--instance Arbitrary DecodeErr where-    arbitrary = elements [Lenient, Ignore, Strict, Replace]--class Stringy s where-    packS    :: String -> s-    unpackS  :: s -> String-    splitAtS :: Int -> s -> (s,s)-    packSChunkSize :: Int -> String -> s-    packSChunkSize _ = packS--instance Stringy String where-    packS    = id-    unpackS  = id-    splitAtS = splitAt--instance Stringy (TF.Stream Char) where-    packS        = TF.streamList-    unpackS      = TF.unstreamList-    splitAtS n s = (TF.take n s, TF.drop n s)--instance Stringy T.Text where-    packS    = T.pack-    unpackS  = T.unpack-    splitAtS = T.splitAt--instance Stringy TL.Text where-    packSChunkSize k = TLF.unstreamChunks k . TF.streamList-    packS    = TL.pack-    unpackS  = TL.unpack-    splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) .-               TL.splitAt . fromIntegral---- Do two functions give the same answer?-eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Bool-eq a b s  = a s =^= b s---- What about with the RHS packed?-eqP :: (Eq a, Show a, Stringy s) =>-       (String -> a) -> (s -> a) -> String -> Word8 -> Bool-eqP f g s w  = eql "orig" (f s) (g t) &&-               eql "mini" (f s) (g mini) &&-               eql "head" (f sa) (g ta) &&-               eql "tail" (f sb) (g tb)-    where t             = packS s-          mini          = packSChunkSize 10 s-          (sa,sb)       = splitAt m s-          (ta,tb)       = splitAtS m t-          l             = length s-          m | l == 0    = n-            | otherwise = n `mod` l-          n             = fromIntegral w-          eql d a b-            | a =^= b   = True-            | otherwise = trace (d ++ ": " ++ show a ++ " /= " ++ show b) False--instance Arbitrary FPFormat where-    arbitrary = elements [Exponent, Fixed, Generic]--newtype Precision a = Precision (Maybe Int)-                    deriving (Eq, Show)--precision :: a -> Precision a -> Maybe Int-precision _ (Precision prec) = prec--arbitraryPrecision :: Int -> Gen (Precision a)-arbitraryPrecision maxDigits = Precision <$> do-  n <- choose (-1,maxDigits)-  return $ if n == -1-           then Nothing-           else Just n--instance Arbitrary (Precision Float) where-    arbitrary = arbitraryPrecision 11-    shrink    = map Precision . shrink . precision undefined--instance Arbitrary (Precision Double) where-    arbitrary = arbitraryPrecision 22-    shrink    = map Precision . shrink . precision undefined---- Work around lack of Show instance for TextEncoding.-data Encoding = E String IO.TextEncoding--instance Show Encoding where show (E n _) = "utf" ++ n--instance Arbitrary Encoding where-    arbitrary = oneof . map return $-      [ E "8" IO.utf8, E "8_bom" IO.utf8_bom, E "16" IO.utf16-      , E "16le" IO.utf16le, E "16be" IO.utf16be, E "32" IO.utf32-      , E "32le" IO.utf32le, E "32be" IO.utf32be-      ]--windowsNewlineMode :: IO.NewlineMode-windowsNewlineMode = IO.NewlineMode-    { IO.inputNL = IO.CRLF, IO.outputNL = IO.CRLF-    }--instance Arbitrary IO.NewlineMode where-    arbitrary = oneof . map return $-      [ IO.noNewlineTranslation, IO.universalNewlineMode, IO.nativeNewlineMode-      , windowsNewlineMode-      ]--instance Arbitrary IO.BufferMode where-    arbitrary = oneof [ return IO.NoBuffering,-                        return IO.LineBuffering,-                        return (IO.BlockBuffering Nothing),-                        (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap`-                        (arbitrary :: Gen Word16) ]---- This test harness is complex!  What property are we checking?------ Reading after writing a multi-line file should give the same--- results as were written.------ What do we vary while checking this property?--- * The lines themselves, scrubbed to contain neither CR nor LF.  (By---   working with a list of lines, we ensure that the data will---   sometimes contain line endings.)--- * Encoding.--- * Newline translation mode.--- * Buffering.-write_read :: (NFData a, Eq a)-           => ([b] -> a)-           -> ((Char -> Bool) -> a -> b)-           -> (IO.Handle -> a -> IO ())-           -> (IO.Handle -> IO a)-           -> Encoding-           -> IO.NewlineMode-           -> IO.BufferMode-           -> [a]-           -> Property-write_read unline filt writer reader (E _ _) nl buf ts =-    monadicIO $ assert . (==t) =<< run act-  where t = unline . map (filt (not . (`elem` "\r\n"))) $ ts-        act = withTempFile $ \path h -> do-                -- hSetEncoding h enc-                IO.hSetNewlineMode h nl-                IO.hSetBuffering h buf-                () <- writer h t-                IO.hClose h-                bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do-                  -- hSetEncoding h' enc-                  IO.hSetNewlineMode h' nl-                  IO.hSetBuffering h' buf-                  r <- reader h'-                  r `deepseq` return r+-- | This module provides quickcheck utilities, e.g. arbitrary and show
+-- instances, and comparison functions, so we can focus on the actual properties
+-- in the 'Tests.Properties' module.
+--
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE DeriveFunctor #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
+module Tests.QuickCheckUtils
+    ( BigInt(..)
+    , NotEmpty(..)
+    , Sqrt(..)
+    , SpacyString(..)
+    , SkewedBool(..)
+
+    , Precision(..)
+    , precision
+
+    , DecodeErr(..)
+    , genDecodeErr
+
+    , Stringy(..)
+    , unpack2
+    , eq
+    , eqP
+    , eqPSqrt
+
+    , write_read
+    ) where
+
+import Control.Arrow ((***))
+import Control.Monad (when)
+import Data.Char (isSpace)
+import Data.IORef (writeIORef)
+import Data.Text.Foreign (I8)
+import Data.Text.Lazy.Builder.RealFloat (FPFormat(..))
+import Data.Word (Word8)
+#if !MIN_VERSION_QuickCheck(2,17,0)
+import Data.Word (Word16)
+#endif
+import qualified GHC.IO.Buffer as GIO
+import qualified GHC.IO.Handle.Internals as GIO
+import qualified GHC.IO.Handle.Types as GIO
+import GHC.IO.Encoding.Types (TextEncoding(textEncodingName))
+import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, counterexample, scale, (.&&.), NonEmptyList(..), forAllShrink)
+import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized)
+import Tests.Utils
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as BL
+import qualified Data.Text as T
+import qualified Data.Text.Encoding.Error as T
+import qualified Data.Text.Internal.Fusion as TF
+import qualified Data.Text.Internal.Fusion.Common as TF
+import qualified Data.Text.Internal.Lazy as TL
+import qualified Data.Text.Internal.Lazy.Fusion as TLF
+import qualified Data.Text.Lazy as TL
+import qualified System.IO as IO
+
+genWord8 :: Gen Word8
+genWord8 = chooseAny
+
+instance Arbitrary I8 where
+    arbitrary     = arbitrarySizedIntegral
+    shrink        = shrinkIntegral
+
+instance Arbitrary B.ByteString where
+    arbitrary     = B.pack `fmap` listOf genWord8
+    shrink        = map B.pack . shrink . B.unpack
+
+instance Arbitrary BL.ByteString where
+    arbitrary = oneof
+      [ BL.fromChunks <$> arbitrary
+      -- so that a single utf8 code point could appear split over up to 4 chunks
+      , BL.fromChunks . map B.singleton <$> listOf genWord8
+      -- so that a code point with 4 byte long utf8 representation
+      -- could appear split over 3 non-singleton chunks
+      , (\a b c -> BL.fromChunks [a, b, c])
+        <$> arbitrary
+        <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8)
+        <*> arbitrary
+      ]
+    shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs)
+
+-- | For tests that have O(n^2) running times or input sizes, resize
+-- their inputs to the square root of the originals.
+newtype Sqrt a = Sqrt { unSqrt :: a }
+    deriving (Eq, Show)
+
+instance Arbitrary a => Arbitrary (Sqrt a) where
+    arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary
+        where
+            smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs
+    shrink = map Sqrt . shrink . unSqrt
+
+instance Arbitrary T.Text where
+    arbitrary = do
+        t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary
+        -- Generate chunks that start in the middle of their buffers.
+        (\i -> T.drop i t) <$> choose (0, T.length t)
+    shrink = map T.pack . shrink . T.unpack
+
+instance Arbitrary TL.Text where
+    arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary
+    shrink = map TL.pack . shrink . TL.unpack
+
+newtype BigInt = Big Integer
+               deriving (Eq, Show)
+
+instance Arbitrary BigInt where
+    arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e)
+    shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l]
+      where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer
+
+newtype NotEmpty a = NotEmpty { notEmpty :: a }
+    deriving (Eq, Ord, Show)
+
+instance Arbitrary (NotEmpty T.Text) where
+    arbitrary   = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary
+    shrink      = fmap (NotEmpty . T.pack . getNonEmpty)
+                . shrink . NonEmpty . T.unpack . notEmpty
+
+instance Arbitrary (NotEmpty TL.Text) where
+    arbitrary   = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary
+    shrink      = fmap (NotEmpty . TL.pack . getNonEmpty)
+                . shrink . NonEmpty . TL.unpack . notEmpty
+
+data DecodeErr = Lenient | Ignore | Strict | Replace
+               deriving (Show, Eq, Bounded, Enum)
+
+genDecodeErr :: DecodeErr -> Gen T.OnDecodeError
+genDecodeErr Lenient = return T.lenientDecode
+genDecodeErr Ignore  = return T.ignore
+genDecodeErr Strict  = return T.strictDecode
+genDecodeErr Replace = (\c _ _ -> c) <$> frequency
+  [ (1, return Nothing)
+  , (50, Just <$> arbitraryUnicodeChar)
+  ]
+
+instance Arbitrary DecodeErr where
+    arbitrary = arbitraryBoundedEnum
+
+class Stringy s where
+    packS    :: String -> s
+    unpackS  :: s -> String
+    splitAtS :: Int -> s -> (s,s)
+    packSChunkSize :: Int -> String -> s
+    packSChunkSize _ = packS
+
+instance Stringy String where
+    packS    = id
+    unpackS  = id
+    splitAtS = splitAt
+
+instance Stringy (TF.Stream Char) where
+    packS        = TF.streamList
+    unpackS      = TF.unstreamList
+    splitAtS n s = (TF.take n s, TF.drop n s)
+
+instance Stringy T.Text where
+    packS    = T.pack
+    unpackS  = T.unpack
+    splitAtS = T.splitAt
+
+instance Stringy TL.Text where
+    packSChunkSize k = TLF.unstreamChunks k . TF.streamList
+    packS    = TL.pack
+    unpackS  = TL.unpack
+    splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) .
+               TL.splitAt . fromIntegral
+
+unpack2 :: (Stringy s) => (s,s) -> (String,String)
+unpack2 = unpackS *** unpackS
+
+-- Do two functions give the same answer?
+eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property
+eq a b s  = a s =^= b s
+
+-- What about with the RHS packed?
+eqP :: (Eq a, Show a, Stringy s) =>
+       (String -> a) -> (s -> a) -> String -> Word8 -> Property
+eqP f g s w  = counterexample "orig" (f s =^= g t) .&&.
+               counterexample "mini" (f s =^= g mini) .&&.
+               counterexample "head" (f sa =^= g ta) .&&.
+               counterexample "tail" (f sb =^= g tb)
+    where t             = packS s
+          mini          = packSChunkSize 10 s
+          (sa,sb)       = splitAt m s
+          (ta,tb)       = splitAtS m t
+          l             = length s
+          m | l == 0    = n
+            | otherwise = n `mod` l
+          n             = fromIntegral w
+
+eqPSqrt :: (Eq a, Show a, Stringy s) =>
+       (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property
+eqPSqrt f g s = eqP f g (unSqrt s)
+
+instance Arbitrary FPFormat where
+    arbitrary = arbitraryBoundedEnum
+
+newtype Precision a = Precision (Maybe Int)
+                    deriving (Eq, Show)
+
+precision :: a -> Precision a -> Maybe Int
+precision _ (Precision prec) = prec
+
+arbitraryPrecision :: Int -> Gen (Precision a)
+arbitraryPrecision maxDigits = Precision <$> do
+  n <- choose (-1,maxDigits)
+  return $ if n == -1
+           then Nothing
+           else Just n
+
+instance Arbitrary (Precision Float) where
+    arbitrary = arbitraryPrecision 11
+    shrink    = map Precision . shrink . precision undefined
+
+instance Arbitrary (Precision Double) where
+    arbitrary = arbitraryPrecision 22
+    shrink    = map Precision . shrink . precision undefined
+
+#if !MIN_VERSION_QuickCheck(2,14,3)
+instance Arbitrary IO.Newline where
+    arbitrary = oneof [return IO.LF, return IO.CRLF]
+
+instance Arbitrary IO.NewlineMode where
+    arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary
+#endif
+
+#if !MIN_VERSION_QuickCheck(2,17,0)
+instance Arbitrary IO.BufferMode where
+    arbitrary = oneof [ return IO.NoBuffering,
+                        return IO.LineBuffering,
+                        return (IO.BlockBuffering Nothing),
+                        (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap`
+                        genWord16 ]
+
+genWord16 :: Gen Word16
+genWord16 = chooseAny
+#endif
+
+-- This test harness is complex!  What property are we checking?
+--
+-- Reading after writing a multi-line file should give the same
+-- results as were written.
+--
+-- What do we vary while checking this property?
+-- * The lines themselves, scrubbed to contain neither CR nor LF.  (By
+--   working with a list of lines, we ensure that the data will
+--   sometimes contain line endings.)
+-- * Newline translation mode.
+-- * Buffering.
+write_read :: forall a.
+  (Eq a, Show a)
+  => Gen a
+  -> (a -> [a])
+  -> (a -> a) -- ^ replace '\n' with '\r\n' (for multiline tests) or append '\r' (for single-line tests)
+  -> (IO.Handle -> a -> IO ())
+  -> (IO.Handle -> IO a)
+  -> Property
+write_read genTxt shrinkTxt expandNl writer reader
+  = forAllShrink genEncoding shrinkEncoding propTest
+  where
+  propTest :: TextEncoding -> IO.BufferMode -> Property
+  propTest enc mode = forAllShrink genTxt shrinkTxt $ \txt -> ioProperty $ do
+    file <- emptyTempFile
+    let with nl k = IO.withFile file IO.ReadWriteMode $ \h -> do
+          IO.hSetEncoding h enc
+          IO.hSetBuffering h mode
+          IO.hSetNewlineMode h nl
+          setSmallBuffer h
+          k h
+        -- Put a very small buffer in Handle to easily test boundary conditions in `writeBlocks`
+        setSmallBuffer h = GIO.withHandle_ "setSmallBuffer" h $ \h_ -> do
+          buf <- GIO.newCharBuffer 9 GIO.WriteBuffer
+          writeIORef (GIO.haCharBuffer h_) buf
+        readExpecting h txt' msg = do
+          out <- reader h
+          when (txt' /= out) $ error (show txt' ++ " /= " ++ show out ++ msg)
+    -- 'reader' may be 'hGetContents', which closes the handle
+    -- So we reopen a new file every time.
+
+    -- Test with CRLF encoding
+    with (IO.NewlineMode IO.CRLF IO.CRLF) $ \h -> do
+      writer h txt
+      IO.hSeek h IO.AbsoluteSeek 0
+      readExpecting h txt " (at location 1)"
+
+    -- Re-read without CRLF decoding to check that we did encode CRLF correctly
+    with (IO.NewlineMode IO.LF IO.LF) $ \h -> do
+      readExpecting h (expandNl txt) " (at location 2)"
+
+    -- Test without CRLF encoding
+    with (IO.NewlineMode IO.LF IO.LF) $ \h -> do
+      IO.hSetFileSize h 0
+      writer h txt
+      IO.hSeek h IO.AbsoluteSeek 0
+      readExpecting h txt " (at location 3)"
+
+  genEncoding = elements [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be]
+  shrinkEncoding enc = if textEncodingName enc == textEncodingName IO.utf8 then [] else [IO.utf8]
+
+-- Generate various Unicode space characters with high probability
+arbitrarySpacyChar :: Gen Char
+arbitrarySpacyChar = oneof
+  [ arbitraryUnicodeChar
+  , elements $ filter isSpace [minBound..maxBound]
+  ]
+
+newtype SpacyString = SpacyString { getSpacyString :: String }
+  deriving (Eq, Ord, Show, Read)
+
+instance Arbitrary SpacyString where
+  arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar
+  shrink (SpacyString xs) = SpacyString `fmap` shrink xs
+
+newtype SkewedBool = Skewed { getSkewed :: Bool }
+  deriving Show
+
+instance Arbitrary SkewedBool where
+  arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)]
+ tests/Tests/RebindableSyntaxTest.hs view
@@ -0,0 +1,18 @@+{-# LANGUAGE CPP, RebindableSyntax, TemplateHaskell #-}
+
+module Tests.RebindableSyntaxTest where
+
+import qualified Data.Text as Text
+#if __GLASGOW_HASKELL__ >= 914
+import Language.Haskell.TH.Lift (lift)
+#else
+import Language.Haskell.TH.Syntax (lift)
+#endif
+import Test.Tasty.HUnit (testCase, assertEqual)
+import Test.Tasty (TestTree, testGroup)
+import Prelude (($))
+
+tests :: TestTree
+tests = testGroup "RebindableSyntax"
+  [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a")
+  ]
tests/Tests/Regressions.hs view
@@ -1,108 +1,236 @@--- | Regression tests for specific bugs.----{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-}-module Tests.Regressions-    (-      tests-    ) where--import Control.Exception (SomeException, handle)-import Data.Char (isLetter)-import System.IO-import Test.HUnit (assertBool, assertEqual, assertFailure)-import qualified Data.ByteString as B-import Data.ByteString.Char8 ()-import qualified Data.ByteString.Lazy as LB-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as LT-import qualified Data.Text.Lazy.Encoding as LE-import qualified Data.Text.Unsafe as T-import qualified Test.Framework as F-import qualified Test.Framework.Providers.HUnit as F--import Tests.Utils (withTempFile)---- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring--- caused either a segfault or attempt to allocate a negative number--- of bytes.-lazy_encode_crash :: IO ()-lazy_encode_crash = withTempFile $ \ _ h ->-   LB.hPut h . LE.encodeUtf8 . LT.pack . replicate 100000 $ 'a'---- Reported by Pieter Laeremans: attempting to read an incorrectly--- encoded file can result in a crash in the RTS (i.e. not merely an--- exception).-hGetContents_crash :: IO ()-hGetContents_crash = withTempFile $ \ path h -> do-  B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) >> hClose h-  h' <- openFile path ReadMode-  hSetEncoding h' utf8-  handle (\(_::SomeException) -> return ()) $-    T.hGetContents h' >> assertFailure "T.hGetContents should crash"---- Reported by Ian Lynagh: attempting to allocate a sufficiently large--- string (via either Array.new or Text.replicate) could result in an--- integer overflow.-replicate_crash :: IO ()-replicate_crash = handle (\(_::SomeException) -> return ()) $-                  T.replicate (2^power) "0123456789abcdef" `seq`-                  assertFailure "T.replicate should crash"-  where-    power | maxBound == (2147483647::Int) = 28-          | otherwise                     = 60 :: Int---- Reported by John Millikin: a UTF-8 decode error handler could--- return a bogus substitution character, which we would write without--- checking.-utf8_decode_unsafe :: IO ()-utf8_decode_unsafe = do-  let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80"-  assertBool "broken error recovery shouldn't break us" (t == "\xfffd")---- Reported by Eric Seidel: we mishandled mapping Chars that fit in a--- single Word16 to Chars that require two.-mapAccumL_resize :: IO ()-mapAccumL_resize = do-  let f a _ = (a, '\65536')-      count = 5-      val   = T.mapAccumL f (0::Int) (T.replicate count "a")-  assertEqual "mapAccumL should correctly fill buffers for two-word results"-             (0, T.replicate count "\65536") val-  assertEqual "mapAccumL should correctly size buffers for two-word results"-             (count * 2) (T.lengthWord16 (snd val))---- See GitHub #197-t197 :: IO ()-t197 =-    assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")-  where-    currencyParser x = cond == 1-      where-        cond = length fltr-        fltr = filter (== ',') x--t221 :: IO ()-t221 =-    assertEqual "toLower of large input shouldn't crash"-                (T.toLower (T.replicate 200000 "0") `seq` ())-                ()--t227 :: IO ()-t227 =-    assertEqual "take (-3) shouldn't crash with overflow"-                (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?")-                0--tests :: F.Test-tests = F.testGroup "Regressions"-    [ F.testCase "hGetContents_crash" hGetContents_crash-    , F.testCase "lazy_encode_crash" lazy_encode_crash-    , F.testCase "mapAccumL_resize" mapAccumL_resize-    , F.testCase "replicate_crash" replicate_crash-    , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe-    , F.testCase "t197" t197-    , F.testCase "t221" t221-    , F.testCase "t227" t227-    ]+-- | Regression tests for specific bugs.
+--
+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE MagicHash #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Tests.Regressions
+    (
+      tests
+    ) where
+
+import Control.Exception (ErrorCall, SomeException, handle, evaluate, displayException, try)
+import Data.Char (isLetter, chr)
+import GHC.Exts (Int(..), sizeofByteArray#)
+import System.IO
+import System.IO.Temp (withSystemTempFile)
+import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure, (@?=))
+import qualified Data.ByteString as B
+import Data.ByteString.Char8 ()
+import qualified Data.ByteString.Lazy as LB
+import Data.Semigroup (stimes)
+import qualified Data.Text as T
+import qualified Data.Text.Array as TA
+import qualified Data.Text.Encoding as TE
+import qualified Data.Text.Encoding.Error as E
+import qualified Data.Text.Internal as T
+import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E
+import qualified Data.Text.Internal.Lazy.Fusion as LF
+import qualified Data.Text.IO as T
+import qualified Data.Text.Lazy as LT
+import qualified Data.Text.Lazy.Builder as TB
+import qualified Data.Text.Lazy.Encoding as LE
+import qualified Data.Text.Unsafe as T
+import qualified Test.Tasty as F
+import qualified Test.Tasty.HUnit as F
+import Tests.Utils (withTempFile)
+import System.IO.Error (isFullError)
+
+-- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring
+-- caused either a segfault or attempt to allocate a negative number
+-- of bytes.
+lazy_encode_crash :: IO ()
+lazy_encode_crash = withTempFile $ \ _ h -> do
+  putRes <- try $ LB.hPut h $ LE.encodeUtf8 $ LT.pack $ replicate 100000 'a'
+  case putRes of
+    Left e
+      -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it
+      | isFullError e -> pure ()
+      | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e
+    Right () -> pure ()
+
+-- Reported by Pieter Laeremans: attempting to read an incorrectly
+-- encoded file can result in a crash in the RTS (i.e. not merely an
+-- exception).
+hGetContents_crash :: IO ()
+hGetContents_crash = withSystemTempFile "crashy.txt" $ \path h -> do
+  putRes <- try $ B.hPut h (B.pack [0x78, 0xc4 ,0x0a])
+  case putRes of
+    Left e
+      -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it
+      | isFullError e -> pure ()
+      | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e
+    Right () -> do
+      hClose h
+      h' <- openFile path ReadMode
+      hSetEncoding h' utf8
+      handle (\(_::SomeException) -> pure ()) $
+        T.hGetContents h' >> assertFailure "T.hGetContents should crash"
+      hClose h'
+
+-- Reported by Ian Lynagh: attempting to allocate a sufficiently large
+-- string (via either Array.new or Text.replicate) could result in an
+-- integer overflow.
+replicate_crash :: IO ()
+replicate_crash = handle (\(_::SomeException) -> return ()) $
+                  T.replicate (2^power) "0123456789abcdef" `seq`
+                  assertFailure "T.replicate should crash"
+  where
+    power | maxBound == (2147483647::Int) = 28
+          | otherwise                     = 60 :: Int
+
+-- Reported by John Millikin: a UTF-8 decode error handler could
+-- return a bogus substitution character, which we would write without
+-- checking.
+utf8_decode_unsafe :: IO ()
+utf8_decode_unsafe = do
+  let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80"
+  assertBool "broken error recovery shouldn't break us" (t == "\xfffd")
+
+-- Reported by Eric Seidel: we mishandled mapping Chars that fit in a
+-- single Word16 to Chars that require two.
+mapAccumL_resize :: IO ()
+mapAccumL_resize = do
+  let f a _ = (a, '\65536')
+      count = 5
+      val   = T.mapAccumL f (0::Int) (T.replicate count "a")
+  assertEqual "mapAccumL should correctly fill buffers for four-byte results"
+             (0, T.replicate count "\65536") val
+  assertEqual "mapAccumL should correctly size buffers for four-byte results"
+             (count * 4) (T.lengthWord8 (snd val))
+
+-- See GitHub #197
+t197 :: IO ()
+t197 =
+    assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")
+  where
+    currencyParser x = cond == 1
+      where
+        cond = length fltr
+        fltr = filter (== ',') x
+
+t221 :: IO ()
+t221 =
+    assertEqual "toLower of large input shouldn't crash"
+                (T.toLower (T.replicate 200000 "0") `seq` ())
+                ()
+
+t227 :: IO ()
+t227 =
+    assertEqual "take (-3) shouldn't crash with overflow"
+                (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?")
+                0
+
+t280_fromString :: IO ()
+t280_fromString =
+    assertEqual "TB.fromString performs replacement on invalid scalar values"
+                (TB.toLazyText (TB.fromString "\xD800"))
+                (LT.pack "\xFFFD")
+
+t280_singleton :: IO ()
+t280_singleton =
+    assertEqual "TB.singleton performs replacement on invalid scalar values"
+                (TB.toLazyText (TB.singleton '\xD800'))
+                (LT.pack "\xFFFD")
+
+-- See GitHub issue #301
+-- This tests whether the "TEXT take . drop -> unfused" rule is applied to the
+-- slice function. When the slice function is fused, a new array will be
+-- constructed that is shorter than the original array. Without fusion the
+-- array remains unmodified.
+t301 :: IO ()
+t301 = do
+    assertEqual "The length of the array remains the same despite slicing"
+                (I# (sizeofByteArray# originalArr))
+                (I# (sizeofByteArray# newArr))
+
+    assertEqual "The new array still contains the original value"
+                (T.Text (TA.ByteArray newArr) originalOff originalLen)
+                original
+  where
+    !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890"
+    !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original
+
+t330 :: IO ()
+t330 = do
+  let decodeL = LE.decodeUtf8With E.lenientDecode
+  assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked"
+    (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]]))
+    (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]]))
+
+-- Stream decoders should not loop on incomplete code points
+t525 :: IO ()
+t525 = do
+    let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs)
+    decodeUtf8With E.lenientDecode "\xC0" @?= "\65533"
+    LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533"
+    LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533"
+    LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533"
+    LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533"
+
+-- Stream decoders skip one invalid byte at a time
+t528 :: IO ()
+t528 = do
+    let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs)
+    decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536"
+    LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536"
+    LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536"
+    LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0"
+    LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280"
+
+t529 :: IO ()
+t529 = do
+  let decode = TE.decodeUtf8With E.lenientDecode
+  -- https://github.com/haskell/bytestring/issues/575
+  assertEqual "Data.ByteString.isValidUtf8 should work correctly"
+    (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0]))
+    (decode (B.pack (33 : replicate 31 0 ++ [128, 0])))
+
+-- See Github #559
+-- filter/filter fusion rules should apply predicates in the right order.
+t559 :: IO ()
+t559 = do
+  T.filter undefined (T.filter (const False) "a") @?= ""
+  LT.filter undefined (LT.filter (const False) "a") @?= ""
+
+-- Github #633
+-- stimes checked for an `a` to `Int` to `a` roundtrip, but the `a` and `Int` values could represent different integers.
+t633 :: IO ()
+t633 =
+  handle (\(_ :: ErrorCall) -> return ()) $ do
+    _ <- evaluate (stimes (maxBound :: Word) "a" :: T.Text)
+    assertFailure "should fail"
+
+t648 :: IO ()
+t648 = withTempFile $ \_ h -> do
+  hSetEncoding h utf8
+  hSetNewlineMode h (NewlineMode LF CRLF)
+  hSetBuffering h (BlockBuffering $ Just 4)
+  let line = T.replicate 2047 "_"
+  T.hPutStrLn h line
+  hSeek h AbsoluteSeek 0
+  line' <- T.hGetLine h
+  T.append line "\r" @?= line'
+
+tests :: F.TestTree
+tests = F.testGroup "Regressions"
+    [ F.testCase "hGetContents_crash" hGetContents_crash
+    , F.testCase "lazy_encode_crash" lazy_encode_crash
+    , F.testCase "mapAccumL_resize" mapAccumL_resize
+    , F.testCase "replicate_crash" replicate_crash
+    , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe
+    , F.testCase "t197" t197
+    , F.testCase "t221" t221
+    , F.testCase "t227" t227
+    , F.testCase "t280/fromString" t280_fromString
+    , F.testCase "t280/singleton" t280_singleton
+    , F.testCase "t301" t301
+    , F.testCase "t330" t330
+    , F.testCase "t525" t525
+    , F.testCase "t528" t528
+    , F.testCase "t529" t529
+    , F.testCase "t559" t559
+    , F.testCase "t633" t633
+    , F.testCase "t648" t648
+    ]
+ tests/Tests/ShareEmpty.hs view
@@ -0,0 +1,138 @@+{-# LANGUAGE CPP #-}
+{-# LANGUAGE MagicHash #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE BangPatterns #-}
+
+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}
+{-# OPTIONS_GHC -Wno-x-partial #-}
+
+module Tests.ShareEmpty
+  ( tests
+  ) where
+
+import Control.Exception (evaluate)
+import Data.Text
+#if __GLASGOW_HASKELL__ >= 914
+import Language.Haskell.TH.Lift (lift)
+#else
+import Language.Haskell.TH.Syntax (lift)
+#endif
+import Test.Tasty.HUnit (testCase, assertFailure, assertEqual)
+import Test.Tasty (TestTree, testGroup)
+import GHC.Exts
+import GHC.Stack
+import qualified Data.List as L
+import qualified Data.List.NonEmpty as NonEmptyList
+import qualified Data.Text as T
+
+
+-- | assert that a text value is represented by the same pointer
+-- as the 'empty' value.
+assertPtrEqEmpty :: HasCallStack => Text -> IO ()
+assertPtrEqEmpty t = do 
+    t' <- evaluate t
+    empty' <- evaluate empty
+    assertEqual "" empty' t'
+    case reallyUnsafePtrEquality# empty' t' of
+      1# -> pure ()
+      _ -> assertFailure "Pointers are not equal"
+{-# NOINLINE assertPtrEqEmpty #-}
+
+tests :: TestTree
+tests = testGroup "empty Text values are shared"
+  [ testCase "empty = empty" $ assertPtrEqEmpty T.empty
+  , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack ""
+  , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString ""
+  , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack ""))
+  , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a"
+  , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b"
+  , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty
+  , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " []
+  , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' ""
+  , testCase "reverse empty = empty" $ assertPtrEqEmpty $
+      T.reverse empty
+  , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $
+      T.replace "needle" "replacement" empty
+  , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold ""
+  , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower ""
+  , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper ""
+  , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle ""
+  , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $
+      justifyLeft 0 ' ' empty
+  , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $
+      justifyRight 0 ' ' empty
+  , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $
+      T.center 0 ' ' empty
+  , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $
+      T.transpose [empty]
+  , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat []
+  , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty]
+  , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x"
+  , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty
+  , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $
+      T.unfoldr (const Nothing) ()
+  , testCase "take 0 _ = empty" $ assertPtrEqEmpty $
+      T.take 0 "xyz"
+  , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $
+      T.takeEnd 0 "xyz"
+  , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $
+      T.takeWhile (const False) "xyz"
+  , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $
+      T.takeWhileEnd (const False) "xyz"
+  , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $
+      T.drop 5 "xyz"
+  , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $
+      T.dropEnd 5 "xyz"
+  , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $
+      T.dropWhile (const True) "xyz"
+  , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $
+      dropWhileEnd (const True) "xyz"
+  , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $
+      dropAround (const True) empty
+  , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty
+  , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty
+  , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty
+  , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123"
+  , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $
+      snd $ T.splitAt 5 "123"
+  , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $
+      fst $ T.span (const False) "123"
+  , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $
+      snd $ T.span (const True) "123"
+  , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $
+      fst $ T.span (const False) "123"
+  , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $
+      snd $ T.span (const True) "123"
+  , testCase "fst (spanM (const $ pure False) _) = empty" $
+      assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123"
+  , testCase "snd (spanM (const $ pure True) _) = empty" $
+      assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123"
+  , testCase "fst (spanEndM (const $ pure True) _) = empty" $
+      assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123"
+  , testCase "snd (spanEndM (const $ pure False) _) = empty" $
+      assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123"
+  , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty
+  , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty
+  , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty
+  , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123"
+  , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123"
+  , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty
+  , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty
+  , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123"
+  , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123"
+  , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') ""
+  , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234"
+  , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" ""
+  , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines []
+  , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords []
+  , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $
+      T.stripPrefix empty empty
+  , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $
+      T.stripSuffix empty empty
+  , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $
+      mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123"
+  , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $
+      mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz"
+  , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy ""
+  ]
tests/Tests/Utils.hs view
@@ -1,52 +1,50 @@--- | Miscellaneous testing utilities----{-# LANGUAGE ScopedTypeVariables #-}-module Tests.Utils-    (-      (=^=)-    , withRedirect-    , withTempFile-    ) where--import Control.Exception (SomeException, bracket, bracket_, evaluate, try)-import Control.Monad (when)-import Debug.Trace (trace)-import GHC.IO.Handle.Internals (withHandle)-import System.Directory (removeFile)-import System.IO (Handle, hClose, hFlush, hIsOpen, hIsWritable, openTempFile)-import System.IO.Unsafe (unsafePerformIO)---- Ensure that two potentially bottom values (in the sense of crashing--- for some inputs, not looping infinitely) either both crash, or both--- give comparable results for some input.-(=^=) :: (Eq a, Show a) => a -> a -> Bool-i =^= j = unsafePerformIO $ do-  x <- try (evaluate i)-  y <- try (evaluate j)-  case (x,y) of-    (Left (_ :: SomeException), Left (_ :: SomeException))-                       -> return True-    (Right a, Right b) -> return (a == b)-    e                  -> trace ("*** Divergence: " ++ show e) return False-infix 4 =^=-{-# NOINLINE (=^=) #-}--withTempFile :: (FilePath -> Handle -> IO a) -> IO a-withTempFile = bracket (openTempFile "." "crashy.txt") cleanupTemp . uncurry-  where-    cleanupTemp (path,h) = do-      open <- hIsOpen h-      when open (hClose h)-      removeFile path--withRedirect :: Handle -> Handle -> IO a -> IO a-withRedirect tmp h = bracket_ swap swap-  where-    whenM p a = p >>= (`when` a)-    swap = do-      whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp-      whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h-      withHandle "spam" tmp $ \tmph -> do-        hh <- withHandle "spam" h $ \hh ->-          return (tmph,hh)-        return (hh,())+-- | Miscellaneous testing utilities
+--
+{-# LANGUAGE ScopedTypeVariables #-}
+module Tests.Utils
+    (
+      (=^=)
+    , withRedirect
+    , withTempFile
+    , emptyTempFile
+    ) where
+
+import Control.Exception (SomeException, bracket_, evaluate, try)
+import Control.Monad (when)
+import System.IO.Temp (withSystemTempFile, emptySystemTempFile)
+import GHC.IO.Handle.Internals (withHandle)
+import System.IO (Handle, hFlush, hIsOpen, hIsWritable)
+import Test.QuickCheck (Property, ioProperty, property, (===), counterexample)
+
+-- Ensure that two potentially bottom values (in the sense of crashing
+-- for some inputs, not looping infinitely) either both crash, or both
+-- give comparable results for some input.
+(=^=) :: (Eq a, Show a) => a -> a -> Property
+i =^= j = ioProperty $ do
+  x <- try (evaluate i)
+  y <- try (evaluate j)
+  return $ case (x, y) of
+    (Left (_ :: SomeException), Left (_ :: SomeException))
+                       -> property True
+    (Right a, Right b) -> a === b
+    e                  -> counterexample ("Divergence: " ++ show e) $ property False
+infix 4 =^=
+{-# NOINLINE (=^=) #-}
+
+withTempFile :: (FilePath -> Handle -> IO a) -> IO a
+withTempFile = withSystemTempFile "crashy.txt"
+
+emptyTempFile :: IO FilePath
+emptyTempFile = emptySystemTempFile "crashy.txt"
+
+withRedirect :: Handle -> Handle -> IO a -> IO a
+withRedirect tmp h = bracket_ swap swap
+  where
+    whenM p a = p >>= (`when` a)
+    swap = do
+      whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp
+      whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h
+      withHandle "spam" tmp $ \tmph -> do
+        hh <- withHandle "spam" h $ \hh ->
+          return (tmph,hh)
+        return (hh,())
− tests/cabal.config 
@@ -1,6 +0,0 @@--- These flags help to speed up building the test suite.--documentation: False-executable-stripping: False-flags: developer-library-profiling: False
+ tests/literal-rule-test.sh view
@@ -0,0 +1,29 @@+#!/bin/bash -e++failed=0++function check_firings() {+    rule=$1+    expected=$2+    build="ghc -O -ddump-rule-firings LiteralRuleTest.hs"+    build="$build -i.. -I../include"+    touch LiteralRuleTest.hs+    echo -n "Want to see $expected firings of rule $rule... " >&2+    firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l)+    rm -f LiteralRuleTest.{o.hi}++    if [ $firings != $expected ]; then+        echo "failed, saw $firings" >&2+        failed=1+    else+        echo "pass" >&2+    fi+}++check_firings "TEXT literal" 8+check_firings "TEXT literal UTF8" 7+check_firings "TEXT empty literal" 4+# This is broken at the moment. "TEXT literal" rule fires instead.+#check_firings "TEXT singleton literal" 5++exit $failed
− tests/scripts/cover-stdio.sh 
@@ -1,62 +0,0 @@-#!/bin/bash--if [[ $# < 1 ]]; then-    echo "Usage: $0 <exe>"-    exit 1-fi--exe=$1--rm -f $exe.tix--f=$(mktemp stdio-f.XXXXXX)-g=$(mktemp stdio-g.XXXXXX)--for t in T TL; do-    echo $t.readFile > $f-    $exe $t.readFile $f > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.readFile 1>&2-    fi--    $exe $t.writeFile $f $t.writeFile-    echo -n $t.writeFile > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.writeFile 1>&2-    fi--    echo -n quux > $f-    $exe $t.appendFile $f $t.appendFile-    echo -n quux$t.appendFile > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.appendFile 1>&2-    fi--    echo $t.interact | $exe $t.interact > $f-    echo $t.interact > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.interact 1>&2-    fi--    echo $t.getContents | $exe $t.getContents > $f-    echo $t.getContents > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.getContents 1>&2-    fi--    echo $t.getLine | $exe $t.getLine > $f-    echo $t.getLine > $g-    if ! diff -u $f $g; then-	errs=$((errs+1))-	echo FAIL: $t.getLine 1>&2-    fi-done--rm -f $f $g--exit $errs
− tests/text-tests.cabal 
@@ -1,155 +0,0 @@-name:          text-tests-version:       0.0.0.0-synopsis:      Functional tests for the text package-description:   Functional tests for the text package-homepage:      https://github.com/bos/text-license:       BSD2-license-file:  ../LICENSE-author:        Jasper Van der Jeugt <jaspervdj@gmail.com>,-               Bryan O'Sullivan <bos@serpentine.com>,-               Tom Harper <rtomharper@googlemail.com>,-               Duncan Coutts <duncan@haskell.org>-maintainer:    Bryan O'Sullivan <bos@serpentine.com>-category:      Text-build-type:    Simple--cabal-version: >=1.8--flag hpc-  description: Enable HPC to generate coverage reports-  default:     False-  manual:      True--flag bytestring-builder-  description: Depend on the bytestring-builder package for backwards compatibility.-  default: False-  manual: False--executable text-tests-  main-is: Tests.hs--  other-modules:-    Tests.Properties-    Tests.Properties.Mul-    Tests.QuickCheckUtils-    Tests.Regressions-    Tests.SlowFunctions-    Tests.Utils--  ghc-options:-    -Wall -threaded -O0 -rtsopts--  if flag(hpc)-    ghc-options:-      -fhpc--  cpp-options:-    -DTEST_SUITE-    -DASSERTS--  build-depends:-    HUnit >= 1.2,-    QuickCheck >= 2.7,-    base == 4.*,-    deepseq,-    directory,-    quickcheck-unicode >= 1.0.1.0,-    random,-    test-framework >= 0.4,-    test-framework-hunit >= 0.2,-    test-framework-quickcheck2 >= 0.2,-    text-tests--  if flag(bytestring-builder)-    build-depends: bytestring         >= 0.9    && < 0.10.4,-                   bytestring-builder >= 0.10.4-  else-    build-depends: bytestring         >= 0.10.4--executable text-tests-stdio-  main-is:        Tests/IO.hs--  ghc-options:-    -Wall -threaded -rtsopts--  -- Optional HPC support-  if flag(hpc)-    ghc-options:-      -fhpc--  build-depends:-    text-tests,-    base >= 4 && < 5--library-  hs-source-dirs: ..-  c-sources: ../cbits/cbits.c-  include-dirs: ../include-  ghc-options: -Wall-  exposed-modules:-    Data.Text-    Data.Text.Array-    Data.Text.Encoding-    Data.Text.Encoding.Error-    Data.Text.Internal.Encoding.Fusion-    Data.Text.Internal.Encoding.Fusion.Common-    Data.Text.Internal.Encoding.Utf16-    Data.Text.Internal.Encoding.Utf32-    Data.Text.Internal.Encoding.Utf8-    Data.Text.Foreign-    Data.Text.Internal.Fusion-    Data.Text.Internal.Fusion.CaseMapping-    Data.Text.Internal.Fusion.Common-    Data.Text.Internal.Fusion.Size-    Data.Text.Internal.Fusion.Types-    Data.Text.IO-    Data.Text.Internal.IO-    Data.Text.Internal-    Data.Text.Lazy-    Data.Text.Lazy.Builder-    Data.Text.Internal.Builder.Functions-    Data.Text.Lazy.Builder.Int-    Data.Text.Internal.Builder.Int.Digits-    Data.Text.Internal.Builder-    Data.Text.Lazy.Builder.RealFloat-    Data.Text.Internal.Builder.RealFloat.Functions-    Data.Text.Lazy.Encoding-    Data.Text.Internal.Lazy.Encoding.Fusion-    Data.Text.Internal.Lazy.Fusion-    Data.Text.Lazy.IO-    Data.Text.Internal.Lazy-    Data.Text.Lazy.Read-    Data.Text.Internal.Lazy.Search-    Data.Text.Internal.Private-    Data.Text.Read-    Data.Text.Show-    Data.Text.Internal.Read-    Data.Text.Internal.Search-    Data.Text.Unsafe-    Data.Text.Internal.Unsafe-    Data.Text.Internal.Unsafe.Char-    Data.Text.Internal.Unsafe.Shift-    Data.Text.Internal.Functions--  if flag(hpc)-    ghc-options:-      -fhpc--  cpp-options:-    -DTEST_SUITE-    -DASSERTS-    -DINTEGER_GMP--  build-depends:-    array,-    base == 4.*,-    binary,-    deepseq,-    ghc-prim,-    integer-gmp--  if flag(bytestring-builder)-    build-depends: bytestring         >= 0.9    && < 0.10.4,-                   bytestring-builder >= 0.10.4-  else-    build-depends: bytestring         >= 0.10.4
text.cabal view
@@ -1,286 +1,385 @@-cabal-version:  1.12-name:           text-version:        1.2.3.2--homepage:       https://github.com/haskell/text-bug-reports:    https://github.com/haskell/text/issues-synopsis:       An efficient packed Unicode text type.-description:-    .-    An efficient packed, immutable Unicode text type (both strict and-    lazy), with a powerful loop fusion optimization framework.-    .-    The 'Text' type represents Unicode character strings, in a time and-    space-efficient manner. This package provides text processing-    capabilities that are optimized for performance critical use, both-    in terms of large data quantities and high speed.-    .-    The 'Text' type provides character-encoding, type-safe case-    conversion via whole-string case conversion functions (see "Data.Text").-    It also provides a range of functions for converting 'Text' values to-    and from 'ByteStrings', using several standard encodings-    (see "Data.Text.Encoding").-    .-    Efficient locale-sensitive support for text IO is also supported-    (see "Data.Text.IO").-    .-    These modules are intended to be imported qualified, to avoid name-    clashes with Prelude functions, e.g.-    .-    > import qualified Data.Text as T-    .-    To use an extended and very rich family of functions for working-    with Unicode text (including normalization, regular expressions,-    non-standard encodings, text breaking, and locales), see-    the [text-icu package](https://hackage.haskell.org/package/text-icu)-    based on the well-respected and liberally-    licensed [ICU library](http://site.icu-project.org/).--license:        BSD2-license-file:   LICENSE-author:         Bryan O'Sullivan <bos@serpentine.com>-maintainer:     Bryan O'Sullivan <bos@serpentine.com>, Herbert Valerio Riedel <hvr@gnu.org>-copyright:      2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper-category:       Data, Text-build-type:     Simple-tested-with:    GHC==8.10.1-extra-source-files:-    -- scripts/CaseFolding.txt-    -- scripts/SpecialCasing.txt-    README.markdown-    benchmarks/Setup.hs-    benchmarks/cbits/*.c-    benchmarks/haskell/*.hs-    benchmarks/haskell/Benchmarks/*.hs-    benchmarks/haskell/Benchmarks/Programs/*.hs-    benchmarks/python/*.py-    benchmarks/ruby/*.rb-    benchmarks/text-benchmarks.cabal-    changelog.md-    include/*.h-    scripts/*.hs-    tests-and-benchmarks.markdown-    tests/*.hs-    tests/.ghci-    tests/Makefile-    tests/Tests/*.hs-    tests/Tests/Properties/*.hs-    tests/cabal.config-    tests/scripts/*.sh-    tests/text-tests.cabal--flag developer-  description: operate in developer mode-  default: False-  manual: True--flag integer-simple-  description:-    Use the [simple integer library](http://hackage.haskell.org/package/integer-simple)-    instead of [integer-gmp](http://hackage.haskell.org/package/integer-gmp)-  default: False-  manual: False--library-  c-sources:    cbits/cbits.c-  include-dirs: include--  exposed-modules:-    Data.Text-    Data.Text.Array-    Data.Text.Encoding-    Data.Text.Encoding.Error-    Data.Text.Foreign-    Data.Text.IO-    Data.Text.Internal-    Data.Text.Internal.Builder-    Data.Text.Internal.Builder.Functions-    Data.Text.Internal.Builder.Int.Digits-    Data.Text.Internal.Builder.RealFloat.Functions-    Data.Text.Internal.Encoding.Fusion-    Data.Text.Internal.Encoding.Fusion.Common-    Data.Text.Internal.Encoding.Utf16-    Data.Text.Internal.Encoding.Utf32-    Data.Text.Internal.Encoding.Utf8-    Data.Text.Internal.Functions-    Data.Text.Internal.Fusion-    Data.Text.Internal.Fusion.CaseMapping-    Data.Text.Internal.Fusion.Common-    Data.Text.Internal.Fusion.Size-    Data.Text.Internal.Fusion.Types-    Data.Text.Internal.IO-    Data.Text.Internal.Lazy-    Data.Text.Internal.Lazy.Encoding.Fusion-    Data.Text.Internal.Lazy.Fusion-    Data.Text.Internal.Lazy.Search-    Data.Text.Internal.Private-    Data.Text.Internal.Read-    Data.Text.Internal.Search-    Data.Text.Internal.Unsafe-    Data.Text.Internal.Unsafe.Char-    Data.Text.Internal.Unsafe.Shift-    Data.Text.Lazy-    Data.Text.Lazy.Builder-    Data.Text.Lazy.Builder.Int-    Data.Text.Lazy.Builder.RealFloat-    Data.Text.Lazy.Encoding-    Data.Text.Lazy.IO-    Data.Text.Lazy.Internal-    Data.Text.Lazy.Read-    Data.Text.Read-    Data.Text.Unsafe--  other-modules:-    Data.Text.Show--  build-depends:-    array      == 0.5.*,-    base       == 4.14.*,-    binary     == 0.8.*,-    deepseq    >= 1.1.0.0 && < 1.5,-    ghc-prim   == 0.6.*,-    bytestring >= 0.10.4 && < 0.11--  ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2-  if flag(developer)-    ghc-prof-options: -auto-all-    ghc-options: -Werror-    cpp-options: -DASSERTS--  if flag(integer-simple)-    cpp-options: -DINTEGER_SIMPLE-    build-depends: integer-simple >= 0.1 && < 0.5-  else-    cpp-options: -DINTEGER_GMP-    build-depends: integer-gmp >= 0.2 && < 1.1--  -- compiler specification-  default-language: Haskell2010-  default-extensions:-    NondecreasingIndentation-  other-extensions:-    BangPatterns-    CPP-    DeriveDataTypeable-    ExistentialQuantification-    ForeignFunctionInterface-    GeneralizedNewtypeDeriving-    MagicHash-    OverloadedStrings-    Rank2Types-    RankNTypes-    RecordWildCards-    ScopedTypeVariables-    TypeFamilies-    UnboxedTuples-    UnliftedFFITypes--test-suite tests-  type:           exitcode-stdio-1.0-  c-sources:      cbits/cbits.c-  include-dirs:   include--  ghc-options:-    -Wall -threaded -rtsopts--  cpp-options:-    -DASSERTS -DTEST_SUITE--  -- modules specific to test-suite-  hs-source-dirs: tests-  main-is:        Tests.hs-  other-modules:-    Tests.Properties-    Tests.Properties.Mul-    Tests.QuickCheckUtils-    Tests.Regressions-    Tests.SlowFunctions-    Tests.Utils--  -- Same as in `library` stanza; this is needed by cabal for accurate-  -- file-monitoring as well as to avoid `-Wmissing-home-modules`-  -- warnings We can't use an inter-package library dependency because-  -- of different `ghc-options`/`cpp-options` (as a side-benefitt,-  -- this enables per-component build parallelism in `cabal-  -- new-build`!); We could, however, use cabal-version:2.2's `common`-  -- blocks at some point in the future to reduce the duplication.-  hs-source-dirs: .-  other-modules:-    Data.Text-    Data.Text.Array-    Data.Text.Encoding-    Data.Text.Encoding.Error-    Data.Text.Foreign-    Data.Text.IO-    Data.Text.Internal-    Data.Text.Internal.Builder-    Data.Text.Internal.Builder.Functions-    Data.Text.Internal.Builder.Int.Digits-    Data.Text.Internal.Builder.RealFloat.Functions-    Data.Text.Internal.Encoding.Fusion-    Data.Text.Internal.Encoding.Fusion.Common-    Data.Text.Internal.Encoding.Utf16-    Data.Text.Internal.Encoding.Utf32-    Data.Text.Internal.Encoding.Utf8-    Data.Text.Internal.Functions-    Data.Text.Internal.Fusion-    Data.Text.Internal.Fusion.CaseMapping-    Data.Text.Internal.Fusion.Common-    Data.Text.Internal.Fusion.Size-    Data.Text.Internal.Fusion.Types-    Data.Text.Internal.IO-    Data.Text.Internal.Lazy-    Data.Text.Internal.Lazy.Encoding.Fusion-    Data.Text.Internal.Lazy.Fusion-    Data.Text.Internal.Lazy.Search-    Data.Text.Internal.Private-    Data.Text.Internal.Read-    Data.Text.Internal.Search-    Data.Text.Internal.Unsafe-    Data.Text.Internal.Unsafe.Char-    Data.Text.Internal.Unsafe.Shift-    Data.Text.Lazy-    Data.Text.Lazy.Builder-    Data.Text.Lazy.Builder.Int-    Data.Text.Lazy.Builder.RealFloat-    Data.Text.Lazy.Encoding-    Data.Text.Lazy.IO-    Data.Text.Lazy.Internal-    Data.Text.Lazy.Read-    Data.Text.Read-    Data.Text.Unsafe-    Data.Text.Show--  build-depends:-    HUnit >= 1.2,-    QuickCheck >= 2.7 && < 2.11,-    array,-    base,-    binary,-    deepseq,-    directory,-    ghc-prim,-    quickcheck-unicode >= 1.0.1.0,-    random,-    test-framework >= 0.4,-    test-framework-hunit >= 0.2,-    test-framework-quickcheck2 >= 0.2,-    bytestring >= 0.10.4--  if flag(integer-simple)-    cpp-options: -DINTEGER_SIMPLE-    build-depends: integer-simple >= 0.1 && < 0.5-  else-    cpp-options: -DINTEGER_GMP-    build-depends: integer-gmp >= 0.2--  default-language: Haskell2010-  default-extensions: NondecreasingIndentation--source-repository head-  type:     git-  location: https://github.com/haskell/text--source-repository head-  type:     mercurial-  location: https://bitbucket.org/bos/text+cabal-version:  2.2
+name:           text
+version:        2.1.4
+
+homepage:       https://github.com/haskell/text
+bug-reports:    https://github.com/haskell/text/issues
+synopsis:       An efficient packed Unicode text type.
+description:
+    .
+    An efficient packed, immutable Unicode text type (both strict and
+    lazy).
+    .
+    The 'Text' type represents Unicode character strings, in a time and
+    space-efficient manner. This package provides text processing
+    capabilities that are optimized for performance critical use, both
+    in terms of large data quantities and high speed.
+    .
+    The 'Text' type provides character-encoding, type-safe case
+    conversion via whole-string case conversion functions (see "Data.Text").
+    It also provides a range of functions for converting 'Text' values to
+    and from 'ByteStrings', using several standard encodings
+    (see "Data.Text.Encoding").
+    .
+    Efficient locale-sensitive support for text IO is also supported
+    (see "Data.Text.IO").
+    .
+    These modules are intended to be imported qualified, to avoid name
+    clashes with Prelude functions, e.g.
+    .
+    > import qualified Data.Text as T
+    .
+    == ICU Support
+    .
+    To use an extended and very rich family of functions for working
+    with Unicode text (including normalization, regular expressions,
+    non-standard encodings, text breaking, and locales), see
+    the [text-icu package](https://hackage.haskell.org/package/text-icu)
+    based on the well-respected and liberally
+    licensed [ICU library](http://site.icu-project.org/).
+
+license:        BSD-2-Clause
+license-file:   LICENSE
+author:         Bryan O'Sullivan <bos@serpentine.com>
+maintainer:     Haskell Text Team <andrew.lelechenko@gmail.com>, Core Libraries Committee
+copyright:      2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko
+category:       Data, Text
+build-type:     Simple
+tested-with:
+    GHC == 8.4.4
+    GHC == 8.6.5
+    GHC == 8.8.4
+    GHC == 8.10.7
+    GHC == 9.0.2
+    GHC == 9.2.8
+    GHC == 9.4.8
+    GHC == 9.6.7
+    GHC == 9.8.4
+    GHC == 9.10.1
+    GHC == 9.12.2
+
+extra-source-files:
+    -- scripts/CaseFolding.txt
+    -- scripts/SpecialCasing.txt
+    scripts/*.hs
+    simdutf/LICENSE-APACHE
+    simdutf/LICENSE-MIT
+    simdutf/simdutf_c.h
+    simdutf/simdutf.h
+    tests/literal-rule-test.sh
+    tests/LiteralRuleTest.hs
+extra-doc-files:
+    README.md
+    changelog.md
+
+flag developer
+  description: operate in developer mode
+  default: False
+  manual: True
+
+flag simdutf
+  description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed
+  default: True
+  manual: True
+
+flag pure-haskell
+  description: Don't use text's standard C routines
+    NB: This feature is not fully implemented. Several C routines are still in
+    use.
+
+    When this flag is true, text will use pure Haskell variants of the
+    routines. This is not recommended except for use with GHC's JavaScript
+    backend.
+
+    This flag also disables simdutf.
+
+  default: False
+  manual: True
+
+flag ExtendedBenchmarks
+  description: Runs extra benchmarks which can be very slow.
+  default: False
+  manual: True
+
+library
+  if arch(javascript) || flag(pure-haskell)
+    cpp-options: -DPURE_HASKELL
+  else
+    c-sources:  cbits/is_ascii.c
+                cbits/reverse.c
+                cbits/utils.c
+    if (arch(aarch64))
+      c-sources: cbits/aarch64/measure_off.c
+    else
+      c-sources: cbits/measure_off.c
+
+  hs-source-dirs: src
+
+  if flag(simdutf) && !(arch(javascript) || flag(pure-haskell))
+    exposed-modules: Data.Text.Internal.Validate.Simd
+    include-dirs: simdutf
+    c-sources: simdutf/hs_simdutf.c
+    cxx-sources: simdutf/simdutf.cpp
+    cxx-options: -std=c++17
+    cpp-options: -DSIMDUTF
+    if impl(ghc >= 9.4)
+      build-depends: system-cxx-std-lib == 1.0
+    elif os(darwin) || os(freebsd)
+      extra-libraries: c++
+    elif os(openbsd)
+      extra-libraries: c++ c++abi pthread
+    elif os(windows)
+      -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later
+      if impl(ghc < 9.3)
+        extra-libraries: stdc++
+      else
+        extra-libraries: c++ c++abi
+    elif arch(wasm32)
+      cpp-options: -DSIMDUTF_NO_THREADS
+      cxx-options: -fno-exceptions
+      extra-libraries: c++ c++abi
+    else
+      extra-libraries: stdc++
+
+  -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++.
+  -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417
+  if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1)
+    build-depends: base < 0
+
+  -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker.
+  if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1)
+    build-depends: base < 0
+
+  -- GHC 8.10 has linking issues (probably TH-related) on ARM.
+  if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*)
+    build-depends: base < 0
+
+  -- Subword primitives in GHC 9.2.1 are broken on ARM platforms.
+  if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1)
+    build-depends: base < 0
+
+  -- NetBSD + GHC 9.2.1 + TH + C++ does not work together.
+  -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577
+  if flag(simdutf) && os(netbsd) && impl(ghc < 9.4)
+    build-depends: base < 0
+
+  exposed-modules:
+    Data.Text
+    Data.Text.Array
+    Data.Text.Encoding
+    Data.Text.Encoding.Error
+    Data.Text.Foreign
+    Data.Text.IO
+    Data.Text.IO.Utf8
+    Data.Text.Internal
+    Data.Text.Internal.ArrayUtils
+    Data.Text.Internal.Builder
+    Data.Text.Internal.Builder.Functions
+    Data.Text.Internal.Builder.Int.Digits
+    Data.Text.Internal.Builder.RealFloat.Functions
+    Data.Text.Internal.ByteStringCompat
+    Data.Text.Internal.PrimCompat
+    Data.Text.Internal.Encoding
+    Data.Text.Internal.Encoding.Fusion
+    Data.Text.Internal.Encoding.Fusion.Common
+    Data.Text.Internal.Encoding.Utf16
+    Data.Text.Internal.Encoding.Utf32
+    Data.Text.Internal.Encoding.Utf8
+    Data.Text.Internal.Fusion
+    Data.Text.Internal.Fusion.CaseMapping
+    Data.Text.Internal.Fusion.Common
+    Data.Text.Internal.Fusion.Size
+    Data.Text.Internal.Fusion.Types
+    Data.Text.Internal.IO
+    Data.Text.Internal.Lazy
+    Data.Text.Internal.Lazy.Encoding.Fusion
+    Data.Text.Internal.Lazy.Fusion
+    Data.Text.Internal.Lazy.Search
+    Data.Text.Internal.Private
+    Data.Text.Internal.Read
+    Data.Text.Internal.Search
+    Data.Text.Internal.StrictBuilder
+    Data.Text.Internal.Unsafe
+    Data.Text.Internal.Unsafe.Char
+    Data.Text.Internal.Validate
+    Data.Text.Internal.Validate.Native
+    Data.Text.Lazy
+    Data.Text.Lazy.Builder
+    Data.Text.Lazy.Builder.Int
+    Data.Text.Lazy.Builder.RealFloat
+    Data.Text.Lazy.Encoding
+    Data.Text.Lazy.IO
+    Data.Text.Lazy.Internal
+    Data.Text.Lazy.Read
+    Data.Text.Read
+    Data.Text.Unsafe
+
+  other-modules:
+    Data.Text.Show
+    Data.Text.Internal.Measure
+    Data.Text.Internal.Reverse
+    Data.Text.Internal.Transformation
+    Data.Text.Internal.IsAscii
+
+  build-depends:
+    array            >= 0.3 && < 0.6,
+    base             >= 4.11 && < 5,
+    binary           >= 0.8.3 && < 0.9,
+    bytestring       >= 0.10.4 && < 0.13,
+    deepseq          >= 1.1 && < 1.6,
+    ghc-prim         >= 0.2 && < 0.15,
+
+  -- template-haskell-lift was added as a boot library in GHC-9.14
+  -- once we no longer wish to backport releases to older major releases of GHC,
+  -- this conditional can be dropped
+  if impl(ghc < 9.14)
+    build-depends: template-haskell >= 2.5 && < 3
+  else
+    build-depends: template-haskell-lift >= 0.1 && <0.2
+
+  if impl(ghc < 9.4)
+    build-depends: data-array-byte >= 0.1 && < 0.2
+
+  ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2
+  if flag(developer)
+    ghc-options: -fno-ignore-asserts
+    cpp-options: -DASSERTS
+    if impl(ghc >= 9.2.2)
+      ghc-options: -fcheck-prim-bounds
+
+  default-language: Haskell2010
+  default-extensions:
+    NondecreasingIndentation
+  other-extensions:
+    BangPatterns
+    CPP
+    DeriveDataTypeable
+    ExistentialQuantification
+    ForeignFunctionInterface
+    GeneralizedNewtypeDeriving
+    MagicHash
+    OverloadedStrings
+    Rank2Types
+    RankNTypes
+    RecordWildCards
+    Safe
+    ScopedTypeVariables
+    TemplateHaskellQuotes
+    Trustworthy
+    TypeFamilies
+    UnboxedTuples
+    UnliftedFFITypes
+
+source-repository head
+  type:     git
+  location: https://github.com/haskell/text
+
+test-suite tests
+  type:           exitcode-stdio-1.0
+  ghc-options:
+    -Wall -threaded -rtsopts -with-rtsopts=-N
+
+  hs-source-dirs: tests
+  main-is:        Tests.hs
+  other-modules:
+    Tests.Lift
+    Tests.Properties
+    Tests.Properties.Basics
+    Tests.Properties.Builder
+    Tests.Properties.Folds
+    Tests.Properties.Instances
+    Tests.Properties.LowLevel
+    Tests.Properties.Read
+    Tests.Properties.Substrings
+    Tests.Properties.Text
+    Tests.Properties.Transcoding
+    Tests.Properties.CornerCases
+    Tests.Properties.Validate
+    Tests.QuickCheckUtils
+    Tests.RebindableSyntaxTest
+    Tests.Regressions
+    Tests.SlowFunctions
+    Tests.ShareEmpty
+    Tests.Utils
+
+  build-depends:
+    QuickCheck >= 2.12.6 && < 2.18,
+    base <5,
+    binary,
+    bytestring,
+    deepseq,
+    ghc-prim,
+    tasty,
+    tasty-hunit,
+    tasty-quickcheck,
+    temporary,
+    transformers,
+    text
+  if impl(ghc < 9.4)
+    build-depends: data-array-byte >= 0.1 && < 0.2
+  if impl(ghc < 9.14)
+    build-depends: template-haskell
+  else
+    build-depends: template-haskell-lift
+
+  -- Plugin infrastructure does not work properly in 8.6.1, and
+  -- ghc-9.2.1 library depends on parsec, which causes a circular dependency.
+  if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2)
+    build-depends: tasty-inspection-testing
+
+  -- https://github.com/haskellari/splitmix/issues/101
+  if os(openbsd)
+    build-depends: splitmix < 0.1.3 || > 0.1.3.1
+
+  default-language: Haskell2010
+  default-extensions: NondecreasingIndentation
+
+benchmark text-benchmarks
+  type:           exitcode-stdio-1.0
+
+  ghc-options:    -Wall -O2 -rtsopts "-with-rtsopts=-A32m"
+  if impl(ghc >= 8.6)
+    ghc-options:  -fproc-alignment=64
+  if flag(ExtendedBenchmarks)
+    cpp-options: -DExtendedBenchmarks
+
+  build-depends:  base,
+                  bytestring >= 0.10.4,
+                  containers,
+                  deepseq,
+                  directory,
+                  filepath,
+                  tasty-bench >= 0.2,
+                  temporary,
+                  text,
+                  transformers
+
+  hs-source-dirs: benchmarks/haskell
+  main-is:        Benchmarks.hs
+  other-modules:
+    Benchmarks.Builder
+    Benchmarks.Concat
+    Benchmarks.DecodeUtf8
+    Benchmarks.EncodeUtf8
+    Benchmarks.Equality
+    Benchmarks.FileRead
+    Benchmarks.FileWrite
+    Benchmarks.FoldLines
+    Benchmarks.Micro
+    Benchmarks.Multilang
+    Benchmarks.Programs.BigTable
+    Benchmarks.Programs.Cut
+    Benchmarks.Programs.Fold
+    Benchmarks.Programs.Sort
+    Benchmarks.Programs.StripTags
+    Benchmarks.Programs.Throughput
+    Benchmarks.Pure
+    Benchmarks.ReadNumbers
+    Benchmarks.Replace
+    Benchmarks.Search
+    Benchmarks.Stream
+    Benchmarks.WordFrequencies
+
+  default-language: Haskell2010
+  default-extensions: NondecreasingIndentation
+  other-extensions: DeriveGeneric