text 2.0 → 2.1.4
raw patch · 87 files changed
Files
- README.markdown +0/−30
- README.md +31/−0
- benchmarks/haskell/Benchmarks.hs +11/−2
- benchmarks/haskell/Benchmarks/FileWrite.hs +135/−0
- benchmarks/haskell/Benchmarks/Micro.hs +33/−0
- benchmarks/haskell/Benchmarks/Programs/Fold.hs +5/−4
- benchmarks/haskell/Benchmarks/Pure.hs +27/−1
- benchmarks/haskell/Benchmarks/ReadNumbers.hs +2/−1
- benchmarks/haskell/Benchmarks/Search.hs +2/−2
- benchmarks/haskell/Benchmarks/WordFrequencies.hs +3/−2
- cbits/aarch64/measure_off.c +109/−0
- cbits/is_ascii.c +7/−0
- cbits/measure_off.c +231/−158
- cbits/validate_utf8.cpp +0/−6
- changelog.md +440/−272
- scripts/Arsec.hs +48/−44
- scripts/CaseFolding.hs +47/−53
- scripts/CaseMapping.hs +66/−41
- scripts/SpecialCasing.hs +61/−63
- scripts/UnicodeData.hs +51/−0
- simdutf/hs_simdutf.c +9/−0
- simdutf/simdutf.cpp too large to diff
- simdutf/simdutf.h +13573/−1084
- simdutf/simdutf_c.h +339/−0
- src/Data/Text.hs +2276/−2031
- src/Data/Text/Array.hs +361/−330
- src/Data/Text/Encoding.hs +588/−603
- src/Data/Text/Encoding/Error.hs +119/−120
- src/Data/Text/Foreign.hs +212/−163
- src/Data/Text/IO.hs +211/−309
- src/Data/Text/IO/Utf8.hs +94/−0
- src/Data/Text/Internal.hs +274/−233
- src/Data/Text/Internal/ArrayUtils.hs +33/−0
- src/Data/Text/Internal/Builder.hs +350/−332
- src/Data/Text/Internal/Builder/Int/Digits.hs +7/−8
- src/Data/Text/Internal/ByteStringCompat.hs +0/−28
- src/Data/Text/Internal/Encoding.hs +535/−0
- src/Data/Text/Internal/Encoding/Fusion.hs +4/−3
- src/Data/Text/Internal/Encoding/Utf8.hs +318/−292
- src/Data/Text/Internal/Fusion.hs +28/−6
- src/Data/Text/Internal/Fusion/CaseMapping.hs +7984/−6420
- src/Data/Text/Internal/Fusion/Common.hs +30/−6
- src/Data/Text/Internal/IO.hs +308/−166
- src/Data/Text/Internal/IsAscii.hs +94/−0
- src/Data/Text/Internal/Lazy.hs +151/−135
- src/Data/Text/Internal/Lazy/Encoding/Fusion.hs +14/−13
- src/Data/Text/Internal/Lazy/Fusion.hs +25/−3
- src/Data/Text/Internal/Lazy/Search.hs +5/−18
- src/Data/Text/Internal/Measure.hs +53/−0
- src/Data/Text/Internal/PrimCompat.hs +3/−5
- src/Data/Text/Internal/Private.hs +1/−1
- src/Data/Text/Internal/Reverse.hs +108/−0
- src/Data/Text/Internal/Search.hs +5/−17
- src/Data/Text/Internal/StrictBuilder.hs +129/−0
- src/Data/Text/Internal/Transformation.hs +340/−0
- src/Data/Text/Internal/Validate.hs +107/−0
- src/Data/Text/Internal/Validate/Native.hs +60/−0
- src/Data/Text/Internal/Validate/Simd.hs +42/−0
- src/Data/Text/Lazy.hs +1911/−1668
- src/Data/Text/Lazy/Builder.hs +2/−1
- src/Data/Text/Lazy/Builder/RealFloat.hs +3/−0
- src/Data/Text/Lazy/Encoding.hs +254/−236
- src/Data/Text/Lazy/IO.hs +5/−5
- src/Data/Text/Lazy/Internal.hs +0/−1
- src/Data/Text/Lazy/Read.hs +16/−10
- src/Data/Text/Read.hs +16/−10
- src/Data/Text/Show.hs +53/−40
- src/Data/Text/Unsafe.hs +5/−1
- tests/Tests.hs +4/−0
- tests/Tests/Lift.hs +39/−28
- tests/Tests/Properties.hs +5/−1
- tests/Tests/Properties/Basics.hs +5/−1
- tests/Tests/Properties/CornerCases.hs +39/−0
- tests/Tests/Properties/Folds.hs +357/−263
- tests/Tests/Properties/Instances.hs +23/−1
- tests/Tests/Properties/LowLevel.hs +176/−146
- tests/Tests/Properties/Read.hs +61/−20
- tests/Tests/Properties/Substrings.hs +56/−8
- tests/Tests/Properties/Text.hs +491/−401
- tests/Tests/Properties/Transcoding.hs +320/−16
- tests/Tests/Properties/Validate.hs +50/−0
- tests/Tests/QuickCheckUtils.hs +326/−279
- tests/Tests/RebindableSyntaxTest.hs +18/−0
- tests/Tests/Regressions.hs +236/−162
- tests/Tests/ShareEmpty.hs +138/−0
- tests/Tests/Utils.hs +50/−51
- text.cabal +385/−287
− README.markdown
@@ -1,30 +0,0 @@-# text [](https://hackage.haskell.org/package/text) [](http://stackage.org/lts/package/text) [](http://stackage.org/nightly/package/text)--Haskell library for space- and time-efficient operations over Unicode text.--# Get involved!--Please report bugs via the-[github issue tracker](https://github.com/haskell/text/issues).--The main repo:--```bash-git clone https://github.com/haskell/text-```--To run benchmarks please clone and unpack test files:--```bash-git clone https://github.com/haskell/text-test-data benchmarks/text-test-data-make -Cbenchmarks/text-test-data-```--# Authors--The base code for this library was originally written by Tom Harper,-based on the stream fusion framework developed by Roman Leshchinskiy,-Duncan Coutts, and Don Stewart.--The core library was fleshed out, debugged, and tested by Bryan-O'Sullivan. Transition from UTF-16 to UTF-8 is by Andrew Lelechenko.
+ README.md view
@@ -0,0 +1,31 @@+# text [](https://hackage.haskell.org/package/text) [](http://stackage.org/lts/package/text) [](http://stackage.org/nightly/package/text)++Haskell library for space- and time-efficient operations over Unicode text.++# Get involved!++Please report bugs via the+[github issue tracker](https://github.com/haskell/text/issues).++The main repo:++```bash+git clone https://github.com/haskell/text+```++To run benchmarks please clone and unpack test files:++```bash+cd text+git clone https://github.com/haskell/text-test-data benchmarks/text-test-data+make -Cbenchmarks/text-test-data+```++# Authors++The base code for this library was originally written by Tom Harper,+based on the stream fusion framework developed by Roman Leshchinskiy,+Duncan Coutts, and Don Stewart.++The core library was fleshed out, debugged, and tested by Bryan+O'Sullivan. Transition from UTF-16 to UTF-8 is by Andrew Lelechenko.
benchmarks/haskell/Benchmarks.hs view
@@ -10,6 +10,7 @@ import System.IO #ifdef mingw32_HOST_OS+import System.IO.Temp (emptySystemTempFile) import System.Directory (removeFile) #endif @@ -19,7 +20,9 @@ import qualified Benchmarks.EncodeUtf8 as EncodeUtf8 import qualified Benchmarks.Equality as Equality import qualified Benchmarks.FileRead as FileRead+import qualified Benchmarks.FileWrite as FileWrite import qualified Benchmarks.FoldLines as FoldLines+import qualified Benchmarks.Micro as Micro import qualified Benchmarks.Multilang as Multilang import qualified Benchmarks.Pure as Pure import qualified Benchmarks.ReadNumbers as ReadNumbers@@ -38,11 +41,11 @@ mkSink :: IO (FilePath, Handle) mkSink = do #ifdef mingw32_HOST_OS- (sinkFn, sink) <- openTempFile "." "dev.null"+ sinkFn <- emptySystemTempFile "dev.null" #else let sinkFn = "/dev/null"- sink <- openFile sinkFn WriteMode #endif+ sink <- openFile sinkFn WriteMode hSetEncoding sink utf8 pure (sinkFn, sink) @@ -58,9 +61,13 @@ let tf = ("benchmarks/text-test-data" </>) -- Cannot use envWithCleanup, because there is no instance NFData Handle (sinkFn, sink) <- mkSink+ (fileWriteBenchmarks, fileWriteCleanup) <- FileWrite.mkFileWriteBenchmarks $ do+ (fp, h) <- mkSink+ return (h, rmSink fp) defaultMain [ Builder.benchmark , Concat.benchmark+ , Micro.benchmark , bgroup "DecodeUtf8" [ env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html") , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml")@@ -75,6 +82,7 @@ ] , env (Equality.initEnv (tf "japanese.txt")) Equality.benchmark , FileRead.benchmark (tf "russian.txt")+ , fileWriteBenchmarks , FoldLines.benchmark (tf "russian.txt") , Multilang.benchmark , bgroup "Pure"@@ -100,3 +108,4 @@ ] ] rmSink sinkFn+ fileWriteCleanup
+ benchmarks/haskell/Benchmarks/FileWrite.hs view
@@ -0,0 +1,135 @@+-- | Benchmarks simple file writing+--+-- Tested in this benchmark:+--+-- * Writing a file to the disk+--++{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE CPP #-}++module Benchmarks.FileWrite+ ( mkFileWriteBenchmarks+ ) where++import Control.DeepSeq (NFData, deepseq)+import Data.Bifunctor (first)+import Data.List (intercalate, intersperse)+import Data.String (fromString)+import Data.Text (StrictText)+import Data.Text.Internal.Lazy (LazyText, defaultChunkSize)+import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(..), hSetBuffering, hSetNewlineMode)+import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfAppIO)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Data.Text.IO.Utf8 as Utf8+import qualified Data.Text.Lazy as L+import qualified Data.Text.Lazy.IO as L++mkFileWriteBenchmarks :: IO (Handle, IO ()) -> IO (Benchmark, IO ())+mkFileWriteBenchmarks mkSinkNRemove = do+ let writeData = L.cycle $ fromString [minBound..maxBound]++#ifdef ExtendedBenchmarks+ lengths = [0..5] <> [10,20..100] <> [1000,3000,10000,100000]+#else+ lengths = [0,1,100,3000,10000,100000]+#endif++ testGroup :: NFData text => (Handle -> text -> IO ()) -> ((String, StrictText -> text)) -> Newline -> BufferMode -> IO (Benchmark, IO ())+ testGroup hPutStr (textCharacteristics, select) nl mode = do+ (h, removeFile) <- mkSinkNRemove+ hSetBuffering h mode+ hSetNewlineMode h $ NewlineMode nl nl+ pure+ ( bgroup (intercalate " " [textCharacteristics, show nl, show mode]) $+ lengths <&> \n -> let+ t = select $ L.toStrict $ L.take n writeData+ in bench ("length " <> show n)+ $ deepseq t+ $ whnfAppIO (hPutStr h) t+ , removeFile+ )++ sequenceGroup "FileWrite hPutStr"+#ifdef ExtendedBenchmarks+ [ testGroup T.hPutStr strict LF NoBuffering+ , testGroup L.hPutStr lazy LF NoBuffering++ , testGroup T.hPutStr strict LF LineBuffering+ , testGroup T.hPutStr strict CRLF LineBuffering+ , testGroup T.hPutStr strictNewlines LF LineBuffering+ , testGroup T.hPutStr strictNewlines CRLF LineBuffering++ , testGroup L.hPutStr lazy LF LineBuffering+ , testGroup L.hPutStr lazy CRLF LineBuffering+ , testGroup L.hPutStr lazySmallChunks LF LineBuffering+ , testGroup L.hPutStr lazySmallChunks CRLF LineBuffering+ , testGroup L.hPutStr lazyNewlines LF LineBuffering+ , testGroup L.hPutStr lazyNewlines CRLF LineBuffering+ , testGroup L.hPutStr lazySmallChunksNewlines LF LineBuffering+ , testGroup L.hPutStr lazySmallChunksNewlines CRLF LineBuffering++ , testGroup T.hPutStr strict LF (BlockBuffering Nothing)+ , testGroup T.hPutStr strict CRLF (BlockBuffering Nothing)+ , testGroup T.hPutStr strictNewlines LF (BlockBuffering Nothing)+ , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing)++ , testGroup L.hPutStr lazy LF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazy CRLF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazySmallChunks LF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazySmallChunks CRLF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazyNewlines LF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazySmallChunksNewlines LF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazySmallChunksNewlines CRLF (BlockBuffering Nothing)++ , sequenceGroup "UTF-8"+ [ testGroup Utf8.hPutStr strict LF NoBuffering+ , testGroup Utf8.hPutStr strict LF LineBuffering+ , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing)+ ]+ ]+#else+ [ testGroup T.hPutStr strictNewlines LF LineBuffering+ , testGroup T.hPutStr strictNewlines CRLF LineBuffering++ , testGroup T.hPutStr strict LF (BlockBuffering Nothing)+ , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing)++ , testGroup L.hPutStr lazyNewlines LF LineBuffering+ , testGroup L.hPutStr lazyNewlines CRLF LineBuffering++ , testGroup L.hPutStr lazy LF (BlockBuffering Nothing)+ , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing)++ , sequenceGroup "UTF-8"+ [ testGroup Utf8.hPutStr strict LF LineBuffering+ , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing)+ ]+ ]+#endif++ where+ lazy, lazyNewlines :: (String, StrictText -> LazyText)+ lazy = ("lazy", L.fromChunks . T.chunksOf defaultChunkSize)+ lazyNewlines = ("lazy many newlines", snd lazy . snd strictNewlines)++#ifdef ExtendedBenchmarks+ lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText)+ lazySmallChunks = ("lazy small chunks", L.fromChunks . T.chunksOf 10)+ lazySmallChunksNewlines = ("lazy small chunks many newlines", snd lazySmallChunks . snd strictNewlines)+#endif++ strict, strictNewlines :: (String, StrictText -> StrictText)+ strict = ("strict", id)+ strictNewlines = ("strict many newlines", mconcat . intersperse "\n" . T.chunksOf 5)++ sequenceGroup groupName tgs+ = first (bgroup groupName)+ . foldr (\(b,r) (bs,rs) -> (b:bs,r>>rs)) ([], return ())+ <$> sequence tgs++(<&>) :: Functor f => f a -> (a -> b) -> f b+(<&>) = flip fmap+
+ benchmarks/haskell/Benchmarks/Micro.hs view
@@ -0,0 +1,33 @@+-- | Benchmarks on artificial data. ++module Benchmarks.Micro (benchmark) where++import qualified Data.List.NonEmpty as NE+import qualified Data.Text.Lazy as TL+import qualified Data.Text as T+import Test.Tasty.Bench (Benchmark, Benchmarkable, bgroup, bcompareWithin, bench, nf)++benchmark :: Benchmark+benchmark = bgroup "Micro"+ [ blinear "lazy-inits--last" 500000 2 0.1 $ \len ->+ nf (NE.last . TL.initsNE) (chunks len)+ , blinear "lazy-inits--map-take1" 500000 2 0.1 $ \len ->+ nf (map (TL.take 1) . TL.inits) (chunks len)+ ]++chunks :: Int -> TL.Text+chunks n = TL.fromChunks (replicate n (T.pack "a"))++-- Check that running an action with input length (m * baseLen)+-- runs m times slower than the same action with input length baseLen.+blinear :: String -- ^ Name (must be globally unique!)+ -> Int -- ^ Base length+ -> Int -- ^ Multiplier m+ -> Double -- ^ Slack s+ -> (Int -> Benchmarkable) -- ^ Action to measure, parameterized by input length+ -> Benchmark+blinear name baseLen m s run = bgroup name+ [ bench "baseline" $ run baseLen+ , bcompareWithin (fromIntegral m * (1 - s)) (fromIntegral m * (1 + s)) (name ++ ".baseline") $+ bench ("x" ++ show m) $ run (m * baseLen)+ ]
benchmarks/haskell/Benchmarks/Programs/Fold.hs view
@@ -17,8 +17,9 @@ ( benchmark ) where -import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.List (intersperse)+import Prelude hiding (Foldable(..)) import System.IO (Handle) import Test.Tasty.Bench (Benchmark, bench, whnfIO) import qualified Data.Text as T@@ -29,7 +30,7 @@ benchmark :: FilePath -> Handle -> Benchmark benchmark i o =- bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80+ bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . foldText 80 -- | We represent a paragraph by a word list --@@ -37,8 +38,8 @@ -- | Fold a text ---fold :: Int -> T.Text -> TL.Text-fold maxWidth = TLB.toLazyText . mconcat .+foldText :: Int -> T.Text -> TL.Text+foldText maxWidth = TLB.toLazyText . mconcat . intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs -- | Fold a paragraph
benchmarks/haskell/Benchmarks/Pure.hs view
@@ -26,6 +26,8 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import qualified Data.Text.Lazy.Encoding as TL+import Data.Semigroup+import Data.List.NonEmpty (NonEmpty((:|))) data Env = Env { bsa :: !BS.ByteString@@ -83,6 +85,14 @@ [ benchT $ nf T.concat tl , benchTL $ nf TL.concat tll ]+ , bgroup "sconcat"+ [ benchT $ nf sconcat (T.empty :| tl)+ , benchTL $ nf sconcat (TL.empty :| tll)+ ]+ , bgroup "stimes"+ [ benchT $ nf (stimes (10 :: Int)) ta+ , benchTL $ nf (stimes (10 :: Int)) tla+ ] , bgroup "cons" [ benchT $ nf (T.cons c) ta , benchTL $ nf (TL.cons c) tla@@ -192,6 +202,10 @@ [ benchT $ nf T.toUpper ta , benchTL $ nf TL.toUpper tla ]+ , bgroup "toTitle"+ [ benchT $ nf T.toTitle ta+ , benchTL $ nf TL.toTitle tla+ ] , bgroup "uncons" [ benchT $ nf T.uncons ta , benchTL $ nf TL.uncons tla@@ -204,6 +218,14 @@ [ benchT $ nf (T.zipWith min tb) ta , benchTL $ nf (TL.zipWith min tlb) tla ]+ , bgroup "length . unpack" -- length should fuse with unpack+ [ benchT $ nf (L.length . T.unpack) ta+ , benchTL $ nf (L.length . TL.unpack) tla+ ]+ , bgroup "length . drop 1 . unpack" -- no list fusion because of drop 1+ [ benchT $ nf (L.length . L.drop 1 . T.unpack) ta+ , benchTL $ nf (L.length . L.drop 1 . TL.unpack) tla+ ] , bgroup "length" [ bgroup "cons" [ benchT $ nf (T.length . T.cons c) ta@@ -269,6 +291,10 @@ [ benchT $ nf (T.length . T.toUpper) ta , benchTL $ nf (TL.length . TL.toUpper) tla ]+ , bgroup "toTitle"+ [ benchT $ nf (T.length . T.toTitle) ta+ , benchTL $ nf (TL.length . TL.toTitle) tla+ ] , bgroup "words" [ benchT $ nf (L.length . T.words) ta , benchTL $ nf (L.length . TL.words) tla@@ -330,7 +356,7 @@ | i < n = TB.singleton c `mappend` go (i+1) | otherwise = mempty --- | Gives more opportunity for inlining and elimination of unnecesary+-- | Gives more opportunity for inlining and elimination of unnecessary -- bounds checks. -- mappend8Char :: Char -> TB.Builder
benchmarks/haskell/Benchmarks/ReadNumbers.hs view
@@ -21,8 +21,9 @@ , benchmark ) where +import Data.Foldable (Foldable(..))+import Prelude hiding (Foldable(..)) import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf)-import Data.List (foldl') import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL
benchmarks/haskell/Benchmarks/Search.hs view
@@ -1,8 +1,8 @@--- | Search for a pattern in a file, find the number of occurences+-- | Search for a pattern in a file, find the number of occurrences -- -- Tested in this benchmark: ----- * Searching all occurences of a pattern using library routines+-- * Searching all occurrences of a pattern using library routines -- module Benchmarks.Search ( initEnv
benchmarks/haskell/Benchmarks/WordFrequencies.hs view
@@ -13,9 +13,10 @@ , benchmark ) where -import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf)-import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.Map (Map)+import Prelude hiding (Foldable(..))+import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T
+ cbits/aarch64/measure_off.c view
@@ -0,0 +1,109 @@+/*+ * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>+ */++#include <string.h>+#include <sys/types.h>+#include <arm_neon.h>++/*+ measure_off_naive / measure_off_neon+ take a UTF-8 sequence between src and srcend, and a number of characters cnt.+ If the sequence is long enough to contain cnt characters, then return how many bytes+ remained unconsumed. Otherwise, if the sequence is shorter, return+ negated count of lacking characters. Cf. _hs_text_measure_off below.+*/++static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt)+{+ // Count leading bytes in 8 byte sequence+ while (src < srcend - 7){+ uint64_t w64;+ memcpy(&w64, src, sizeof(uint64_t));+ // find leading bytes by finding every byte that is not a continuation+ // byte. The bit twiddle only results in a 0 if the original byte starts+ // with 0b11...+ w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL;+ // compute the popcount of w64 with two bit shifts and a multiplication+ size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1})+ * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i+ // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k+ // and each i satisfies this equation for exactly one such j+ // So each byte of the result contains the sum we want.+ ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j+ if (cnt < leads) break;+ cnt-= leads;+ src+= 8;+ }++ // Skip until next leading byte+ while (src < srcend){+ uint8_t w8 = *src;+ if ((int8_t)w8 >= -0x40) break;+ src++;+ }++ // Finish up with tail+ while (src < srcend && cnt > 0){+ uint8_t leadByte = *src++;+ cnt--;+ src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0);+ }++ return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt);+}++static inline const ssize_t measure_off_neon(const uint8_t *src, const uint8_t *srcend, size_t cnt)+{+ while (src < srcend - 63){+ uint8x16_t w128[4] = {vld1q_u8(src), vld1q_u8(src + 16), vld1q_u8(src + 32), vld1q_u8(src + 48)};+ // Which bytes are either < 128 or >= 192?+ uint8x16_t mask0 = vcgtq_s8((int8x16_t)w128[0], vdupq_n_s8(0xBF));+ uint8x16_t mask1 = vcgtq_s8((int8x16_t)w128[1], vdupq_n_s8(0xBF));+ uint8x16_t mask2 = vcgtq_s8((int8x16_t)w128[2], vdupq_n_s8(0xBF));+ uint8x16_t mask3 = vcgtq_s8((int8x16_t)w128[3], vdupq_n_s8(0xBF));++ uint8x16_t mask01 = vaddq_u8(mask0, mask1);+ uint8x16_t mask23 = vaddq_u8(mask2, mask3);+ uint8x16_t mask = vaddq_u8(mask01, mask23);++ size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask));++ if (cnt < leads) break;+ cnt-= leads;+ src+= 64;+ }++ while (src < srcend - 15){+ uint8x16_t w128 = vld1q_u8(src);+ // Which bytes are either < 128 or >= 192?+ uint8x16_t mask = vcgtq_s8((int8x16_t)w128, vdupq_n_s8(0xBF));+ size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask));+ if (cnt < leads) break;+ cnt-= leads;+ src+= 16;+ }++ return measure_off_naive(src, srcend, cnt);+}++/*+ _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len),+ and a number of code points (aka characters) cnt. If the buffer is long enough+ to contain cnt characters, then _hs_text_measure_off returns a non-negative number,+ measuring their size in code units (aka bytes). If the buffer is shorter,+ _hs_text_measure_off returns a non-positive number, which is a negated total count+ of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0+ as well.++ This scheme allows us to implement both take/drop and length with the same C function.++ The input buffer (src, off, len) must be a valid UTF-8 sequence,+ this condition is not checked.+*/+ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) {+ ssize_t ret = measure_off_neon(src + off, src + off + len, cnt);+ return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret));+}
cbits/is_ascii.c view
@@ -45,3 +45,10 @@ return src - src0; }++/*+ _hs_text_is_ascii_offset is a helper for calling _hs_text_is_ascii on Texts.+*/+const size_t _hs_text_is_ascii_offset(const uint8_t *arr, size_t off, size_t len){+ return _hs_text_is_ascii(arr + off, arr + off + len);+}
cbits/measure_off.c view
@@ -1,158 +1,231 @@-/*- * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>- */--#include <string.h>-#include <stdint.h>-#include <sys/types.h>-#ifdef __x86_64__-#include <emmintrin.h>-#include <xmmintrin.h>-#include <immintrin.h>-#include <cpuid.h>-#endif-#include <stdbool.h>--#ifndef __STDC_NO_ATOMICS__-#include <stdatomic.h>-#endif--bool has_avx512_vl_bw() {-#ifdef __x86_64__- uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;- __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);- // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features- const bool has_avx512_bw = ebx & (1 << 30);- const bool has_avx512_vl = ebx & (1 << 31);- // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl);- return has_avx512_bw && has_avx512_vl;-#else- return false;-#endif-}--/*- measure_off_naive / measure_off_avx / measure_off_sse- take a UTF-8 sequence between src and srcend, and a number of characters cnt.- If the sequence is long enough to contain cnt characters, then return how many bytes- remained unconsumed. Otherwise, if the sequence is shorter, return- negated count of lacking characters. Cf. _hs_text_measure_off below.-*/--static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{- // Count leading bytes in 8 byte sequence- while (src < srcend - 7){- uint64_t w64;- memcpy(&w64, src, sizeof(uint64_t));- size_t leads = __builtin_popcountll(((w64 << 1) | ~w64) & 0x8080808080808080ULL);- if (cnt < leads) break;- cnt-= leads;- src+= 8;- }-- // Skip until next leading byte- while (src < srcend){- uint8_t w8 = *src;- if ((int8_t)w8 >= -0x40) break;- src++;- }-- // Finish up with tail- while (src < srcend && cnt > 0){- uint8_t leadByte = *src++;- cnt--;- src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0);- }-- return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt);-}--#ifdef __x86_64__-__attribute__((target("avx512vl,avx512bw")))-static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{- while (src < srcend - 63){- __m512i w512 = _mm512_loadu_si512((__m512i *)src);- // Which bytes are either < 128 or >= 192?- uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF));- size_t leads = __builtin_popcountll(mask);- if (cnt < leads) break;- cnt-= leads;- src+= 64;- }-- // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties- // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html-- if (src < srcend - 31){- __m256i w256 = _mm256_loadu_si256((__m256i *)src);- uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF));- size_t leads = __builtin_popcountl(mask);- if (cnt >= leads){- cnt-= leads;- src+= 32;- }- }-- if (src < srcend - 15){- __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16- uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8- size_t leads = __builtin_popcountl(mask);- if (cnt >= leads){- cnt-= leads;- src+= 16;- }- }-- return measure_off_naive(src, srcend, cnt);-}-#endif--static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{-#ifdef __x86_64__- while (src < srcend - 15){- __m128i w128 = _mm_loadu_si128((__m128i *)src);- // Which bytes are either < 128 or >= 192?- uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF)));- size_t leads = __builtin_popcount(mask);- if (cnt < leads) break;- cnt-= leads;- src+= 16;- }-#endif-- return measure_off_naive(src, srcend, cnt);-}--typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t);--/*- _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len),- and a number of code points (aka characters) cnt. If the buffer is long enough- to contain cnt characters, then _hs_text_measure_off returns a non-negative number,- measuring their size in code units (aka bytes). If the buffer is shorter,- _hs_text_measure_off returns a non-positive number, which is a negated total count- of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0- as well.-- This scheme allows us to implement both take/drop and length with the same C function.-- The input buffer (src, off, len) must be a valid UTF-8 sequence,- this condition is not checked.-*/-ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) {- static _Atomic measure_off_t s_impl = (measure_off_t)NULL;- measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed);- if (!impl) {-#ifdef __x86_64__- impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse;-#else- impl = measure_off_sse;-#endif- atomic_store_explicit(&s_impl, impl, memory_order_relaxed);- }- ssize_t ret = (*impl)(src + off, src + off + len, cnt);- return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret));-}+/* + * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com> + */ + +#include <string.h> +#include <stdint.h> +#include <sys/types.h> +#ifdef __x86_64__ +#include <emmintrin.h> +#include <xmmintrin.h> +#include <immintrin.h> +#include <cpuid.h> +#endif +#include <stdbool.h> + +// stdatomic.h has been introduces in gcc 4.9 +#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__)) +#ifndef __STDC_NO_ATOMICS__ +#define __STDC_NO_ATOMICS__ +#endif +#endif + +#ifndef __STDC_NO_ATOMICS__ +#include <stdatomic.h> +#endif + +/* + Clang-6 does not enable proper -march flags for assembly modules + which leads to "error: instruction requires: AVX-512 ISA" + at the assembler phase. + + Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6 + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions + and it's the latest available version on macOS 10.13. + + Disable AVX-512 instructions as they are most likely not supported + on the hardware running clang-6. +*/ +#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \ + || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__) +#define COMPILER_SUPPORTS_AVX512 +#endif + + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +bool has_avx512_vl_bw() { +#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__) + uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; + __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); + // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features + const bool has_avx512_bw = ebx & (1 << 30); + const bool has_avx512_vl = ebx & (1 << 31); + // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); + return has_avx512_bw && has_avx512_vl; +#else + return false; +#endif +} +#endif + +/* + measure_off_naive / measure_off_avx / measure_off_sse + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +__attribute__((target("avx512vl,avx512bw"))) +static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + __m512i w512 = _mm512_loadu_si512((__m512i *)src); + // Which bytes are either < 128 or >= 192? + uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); + size_t leads = __builtin_popcountll(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties + // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html + + if (src < srcend - 31){ + __m256i w256 = _mm256_loadu_si256((__m256i *)src); + uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 32; + } + } + + if (src < srcend - 15){ + __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 + uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 16; + } + } + + return measure_off_naive(src, srcend, cnt); +} +#endif + +/* Count the number of bits set in the argument + * + This is a temporary workaround for the fact that + the GHC RTS linker does not recognize the `__builtin_popcountll` + symbol. + + See https://gitlab.haskell.org/ghc/ghc/-/issues/21787 + https://gitlab.haskell.org/ghc/ghc/-/issues/19900 + https://github.com/haskell/text/issues/450 + + It can be removed and the usages of popcount64 replaced with + `__builtin_popcountll` once affected versions of the compiler + are no longer in widespread use. + + Once GHC learns to recognize the symbol, this can be gated + by CPP to call __builtin_popcountll when using the appropriate + version of GHC. +*/ +static inline const size_t popcount16(uint16_t x) { + + // Taken from https://en.wikipedia.org/wiki/Hamming_weight + const uint16_t m1 = 0x5555; //binary: 0101... + const uint16_t m2 = 0x3333; //binary: 00110011.. + const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ... + x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits + x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits + return (x >> 8) + (x & 0x00FF); +} + +static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ +#ifdef __x86_64__ + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are either < 128 or >= 192? + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); + size_t leads = popcount16(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } +#endif + + return measure_off_naive(src, srcend, cnt); +} + +typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { +#ifndef __STDC_NO_ATOMICS__ + static _Atomic measure_off_t s_impl = (measure_off_t)NULL; + measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); + if (!impl) { +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) + impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; +#else + impl = measure_off_sse; +#endif + atomic_store_explicit(&s_impl, impl, memory_order_relaxed); + } +#else + measure_off_t impl = measure_off_sse; +#endif + ssize_t ret = (*impl)(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +}
− cbits/validate_utf8.cpp
@@ -1,6 +0,0 @@-#include "simdutf.h"--extern "C"-int _hs_text_is_valid_utf8(const char* str, size_t len){- return simdutf::validate_utf8(str, len);-}
changelog.md view
@@ -1,272 +1,440 @@-### 2.0--* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365):- * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`.- * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`.- * Rename functions and types in `Data.Text.Foreign` to reflect switch- from `Word16` to `Word8`.- * Rename slicing functions in `Data.Text.Unsafe` to reflect switch- from `Word16` to `Word8`.- * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`.- * Change semantics and order of arguments of `Data.Text.Array.copyI`:- pass length, not end offset.- * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines.- * Extend interface of `Data.Text.Array` with more utility functions.- * Add `instance Show Data.Text.Unsafe.Iter`.- * Add `Data.Text.measureOff`.- * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`.- * Export `Data.Text.Internal.Lazy.equal`.- * Export `Data.Text.Internal.append`.- * Add `Data.Text.Internal.Private.spanAscii_`.- * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane.-* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348)-* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342)-* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343)-* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354)-* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355)-* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355)-* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388)--### 1.2.5.0--* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305)-* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320)-* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274)-* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281)-* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325)-* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302)-* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334)-* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333)--### 1.2.4.1--* Support `template-haskell-2.17.0.0`-* Support `bytestring-0.11`-* Add `take . drop` related RULE--### 1.2.4.0--* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232)--* Update Haddock documentation to better reflect fusion eligibility; improve fusion- rules for `takeWhileEnd` and `length` (gh-241, ghc-202)--* Optimise `Data.Text.replicate` from `O(n)` to `O(log n)` (gh-209)--* Support `base-4.13.0.0`--### 1.2.3.1--* Make `decodeUtf8With` fail explicitly for unsupported non-BMP- replacement characters instead silent undefined behaviour (gh-213)--* Fix termination condition for file reads via `Data.Text.IO`- operations (gh-223)--* A serious correctness issue affecting uses of `take` and `drop` with- negative counts has been fixed (gh-227)--* A bug in the case-mapping functions resulting in unreasonably large- allocations with large arguments has been fixed (gh-221)--### 1.2.3.0--* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec- (updated from 8.0).--* Bug fix: the lazy `takeWhileEnd` function violated the- [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)- (gh-184).--* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).--* New function: `unsnoc` (gh-173).--* Reduce memory overhead in `encodeUTF8` (gh-194).--* Improve UTF-8 decoder error-recovery (gh-182).--* Minor documentation improvements (`@since` annotations, more- examples, clarifications).--#### 1.2.2.2--* The `toTitle` function now correctly handles letters that- immediately follow punctuation. Before, `"there's"` would turn into- `"There'S"`. Now, it becomes `"There's"`.--* The implementation of unstreaming is faster, resulting in operations- such as `map` and `intersperse` speeding up by up to 30%, with- smaller code generated.--* The optimised length comparison function is now more likely to be- used after some rewrite rule tweaking.--* Bug fix: an off-by-one bug in `takeEnd` is fixed.--* Bug fix: a logic error in `takeWord16` is fixed.--#### 1.2.2.1--* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.- The build flag has been renamed accordingly. Your army of diligent- maintainers apologizes for the churn.--* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec- (updated from 7.0)--* An STG lint error has been fixed--### 1.2.2.0--* The `integer-simple` package, upon which this package optionally- depended, has been replaced with `integer-pure`. The build flag has- been renamed accordingly.--* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a- `get`, the error is propagated via `fail` instead of an uncatchable- crash.--* New function: `takeWhileEnd`--* New instances for the `Text` types:- * if `base` >= 4.7: `PrintfArg`- * if `base` >= 4.9: `Semigroup`--#### 1.2.1.3--* Bug fix: As it turns out, moving the literal rewrite rules to simplifier- phase 2 does not prevent competition with the `unpack` rule, which is- also active in this phase. Unfortunately this was hidden due to a silly- test environment mistake. Moving literal rules back to phase 1 finally- fixes GHC Trac #10528 correctly.--#### 1.2.1.2--* Bug fix: Run literal rewrite rules in simplifier phase 2.- The behavior of the simplifier changed in GHC 7.10.2,- causing these rules to fail to fire, leading to poor code generation- and long compilation times. See- [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).--#### 1.2.1.1--* Expose unpackCString#, which you should never use.--### 1.2.1.0--* Added Binary instances for both Text types. (If you have previously- been using the text-binary package to get a Binary instance, it is- now obsolete.)--#### 1.2.0.6--* Fixed a space leak in UTF-8 decoding--#### 1.2.0.5--* Feature parity: repeat, cycle, iterate are now implemented for lazy- Text, and the Data instance is more complete--* Build speed: an inliner space explosion has been fixed with toCaseFold--* Bug fix: encoding Int to a Builder would infinite-loop if the- integer-simple package was used--* Deprecation: OnEncodeError and EncodeError are deprecated, as they- are never used--* Internals: some types that are used internally in fusion-related- functions have moved around, been renamed, or been deleted (we don't- bump the major version if .Internal modules change)--* Spec compliance: toCaseFold now follows the Unicode 7.0 spec- (updated from 6.3)--#### 1.2.0.4--* Fixed an incompatibility with base < 4.5--#### 1.2.0.3--* Update formatRealFloat to correspond to the definition in versions- of base newer than 4.5 (https://github.com/bos/text/issues/105)--#### 1.2.0.2--* Bumped lower bound on deepseq to 1.4 for compatibility with the- upcoming GHC 7.10--#### 1.2.0.1--* Fixed a buffer overflow in rendering of large Integers- (https://github.com/bos/text/issues/99)--## 1.2.0.0--* Fixed an integer overflow in the replace function- (https://github.com/bos/text/issues/81)--* Fixed a hang in lazy decodeUtf8With- (https://github.com/bos/text/issues/87)--* Reduced codegen bloat caused by use of empty and single-character- literals--* Added an instance of IsList for GHC 7.8 and above--### 1.1.1.0--* The Data.Data instance now allows gunfold to work, via a virtual- pack constructor--* dropEnd, takeEnd: new functions--* Comparing the length of a Text against a number can now- short-circuit in more cases--#### 1.1.0.1--* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes- in single-byte chunks--## 1.1.0.0--* encodeUtf8: Performance is improved by up to 4x.--* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions,- available only if bytestring >= 0.10.4.0 is installed, that allow- very fast and flexible encoding of a Text value to a bytestring- Builder.-- As an example of the performance gain to be had, the- encodeUtf8BuilderEscaped function helps to double the speed of JSON- encoding in the latest version of aeson! (Note: if all you need is a- plain ByteString, encodeUtf8 is still the faster way to go.)--* All of the internal module hierarchy is now publicly exposed. If a- module is in the .Internal hierarchy, or is documented as internal,- use at your own risk - there are no API stability guarantees for- internal modules!--#### 1.0.0.1--* decodeUtf8: Fixed a regression that caused us to incorrectly- identify truncated UTF-8 as valid (gh-61)--# 1.0.0.0--* Added support for Unicode 6.3.0 to case conversion functions--* New function toTitle converts words in a string to title case--* New functions peekCStringLen and withCStringLen simplify- interoperability with C functionns--* Added support for decoding UTF-8 in stream-friendly fashion--* Fixed a bug in mapAccumL--* Added trusted Haskell support--* Removed support for GHC 6.10 (released in 2008) and older+### 2.1.4 - 2026-01-27 + +* [Upgrade to Unicode 17.0](https://github.com/haskell/text/pull/658) + [Fix `CaseMapping` generation script to not depend on GHC's Unicode data](https://github.com/haskell/text/pull/687) + +* [simdutf: update to 8.0.0](https://github.com/haskell/text/pull/685) + +* [Add `decodeUtf8Lenient` for lazy `Text`](https://github.com/haskell/text/pull/690) + +* [`scanl`/`scanr` should replace invalid `Char` in the initial value](https://github.com/haskell/text/pull/669) + +* [Shave off redundant field of `Text.Internal.Buffer`](https://github.com/haskell/text/pull/659) + +* [Switch from template-haskell to template-haskell-lift](https://github.com/haskell/text/pull/661) + +#### Minor changes + +* [Avoid calling `length` on chunks in lazy `splitAt`](https://github.com/haskell/text/pull/676) + +* [Check for zero length in internal `isSingleton`](https://github.com/haskell/text/pull/675) + +* [Implement folds directly, without resorting to streaming framework](https://github.com/haskell/text/pull/667) + +* [Implement `cons`, `snoc`, `head`, `isSingleton`, `isPrefixOf` directly, without resorting to streaming framework](https://github.com/haskell/text/pull/666) + +* [Mark `caseConvert` (the underlying implementation of `toUpper` / `toLower` / `toTitle`) as `INLINABLE`, not `INLINE`](https://github.com/haskell/text/pull/664) + +* [Express `index` via `measureOff` instead of going through fusion framework](https://github.com/haskell/text/pull/663) + +* [Guard `#define __STDC_NO_ATOMICS__` by `#ifndef`](https://github.com/haskell/text/pull/657) + +* [Support QuickCheck-2.17](https://github.com/haskell/text/pull/662) + +* [Bump lower bound of binary to >= 0.8.3](https://github.com/haskell/text/pull/673) + +#### Documentation + +* [A bit more documentation for `Data.Text.Internal.Encoding.Utf8`](https://github.com/haskell/text/pull/691) + +* [Clarify documentation of `Data.Text.Foreign`](https://github.com/haskell/text/pull/681) + +* [Haddocks: Hyperlink some identifiers and modules](https://github.com/haskell/text/pull/677) + +* [`since` pragmas for type synonyms](https://github.com/haskell/text/pull/671) + +* [Improve documentation for `streamDecodeUtf8With`](https://github.com/haskell/text/pull/665) + +* [Add comprehensive documentation for `hGetChunk`](https://github.com/haskell/text/pull/655) + +### 2.1.3 - 2025-08-01 + +* [Fix CRLF handling in IO functions](https://github.com/haskell/text/pull/649) + +* [Change `utf8LengthByLeader` to a branching implementation](https://github.com/haskell/text/pull/635) + +* [Define `stimes 0` for lazy text](https://github.com/haskell/text/pull/641) + +* [Add implementation of `sconcat` and `stimes` for strict `Text`](https://github.com/haskell/text/pull/580) and [Fix `stimes` for strict text when size wraps around `Int`](https://github.com/haskell/text/pull/639) + +* [Allow list fusion for `unpack` over both strict and lazy `Text`](https://github.com/haskell/text/pull/629) + +### 2.1.2 + +* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618) + +* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581) + +* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558) + +* [Add `foldlM'`](https://github.com/haskell/text/pull/543) + +* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599) + +* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608) + +* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619) + +* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565) + +* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579) + +* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600) + +* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589) + +* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617) + +* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575) + +* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572) + +* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611) + +* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code. + +### 2.1.1 + +* Add pure Haskell implementations as an alternative to C-based ones, + suitable for JavaScript backend. + +* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547) + +* [Share empty `Text` values](https://github.com/haskell/text/pull/493) + +* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553) + +* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551) + +* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560) + +### 2.1 + +* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474) + +* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503) + +* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483) + +* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527) + +* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528) + +* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534) + +### 2.0.2 + +* [Add decoding functions in `Data.Text.Encoding` that allow + more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge! + * `decodeASCIIPrefix` + * `decodeUtf8Chunk` + * `decodeUtf8More` + * `Utf8ValidState` + * `startUtf8ValidState` + * `StrictBuilder` + * `strictBuilderToText` + * `textToStrictBuilder` + * `validateUtf8Chunk` + * `validateUtf8More` + +* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495) + +* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497) + +* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499) + +* Add internal module `Data.Text.Internal.StrictBuilder` + +* Add internal module `Data.Text.Internal.Encoding` + +* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module. + +* [Speed up case conversions](https://github.com/haskell/text/pull/460) + +* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468) + +* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485) + +### 2.0.1 + +* Improve portability of C and C++ code. +* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413) +* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402) +* [Add `fromPtr0`](https://github.com/haskell/text/pull/423) +* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436) +* [Add `withCString`](https://github.com/haskell/text/pull/431) +* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437) + +### 2.0 + +* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): + * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. + * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. + * Rename functions and types in `Data.Text.Foreign` to reflect switch + from `Word16` to `Word8`. + * Rename slicing functions in `Data.Text.Unsafe` to reflect switch + from `Word16` to `Word8`. + * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. + * Change semantics and order of arguments of `Data.Text.Array.copyI`: + pass length, not end offset. + * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. + * Extend interface of `Data.Text.Array` with more utility functions. + * Add `instance Show Data.Text.Unsafe.Iter`. + * Add `Data.Text.measureOff`. + * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. + * Export `Data.Text.Internal.Lazy.equal`. + * Export `Data.Text.Internal.append`. + * Add `Data.Text.Internal.Private.spanAscii_`. + * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. +* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) +* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) +* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) +* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) +* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) +* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) +* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) + +### 1.2.5.0 + +* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) +* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320) +* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274) +* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281) +* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325) +* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302) +* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334) +* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333) + +### 1.2.4.1 + +* Support `template-haskell-2.17.0.0` +* Support `bytestring-0.11` +* Add `take . drop` related RULE + +### 1.2.4.0 + +* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232) + +* Update Haddock documentation to better reflect fusion eligibility; improve fusion + rules for `takeWhileEnd` and `length` (gh-241, ghc-202) + +* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times, + call it only `O(log n)` times on chunks of increasing size. The total + asymptotic complexity remains `O(nm)`. (gh-209) + +* Support `base-4.13.0.0` + +### 1.2.3.1 + +* Make `decodeUtf8With` fail explicitly for unsupported non-BMP + replacement characters instead silent undefined behaviour (gh-213) + +* Fix termination condition for file reads via `Data.Text.IO` + operations (gh-223) + +* A serious correctness issue affecting uses of `take` and `drop` with + negative counts has been fixed (gh-227) + +* A bug in the case-mapping functions resulting in unreasonably large + allocations with large arguments has been fixed (gh-221) + +### 1.2.3.0 + +* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec + (updated from 8.0). + +* Bug fix: the lazy `takeWhileEnd` function violated the + [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51) + (gh-184). + +* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197). + +* New function: `unsnoc` (gh-173). + +* Reduce memory overhead in `encodeUTF8` (gh-194). + +* Improve UTF-8 decoder error-recovery (gh-182). + +* Minor documentation improvements (`@since` annotations, more + examples, clarifications). + +#### 1.2.2.2 + +* The `toTitle` function now correctly handles letters that + immediately follow punctuation. Before, `"there's"` would turn into + `"There'S"`. Now, it becomes `"There's"`. + +* The implementation of unstreaming is faster, resulting in operations + such as `map` and `intersperse` speeding up by up to 30%, with + smaller code generated. + +* The optimised length comparison function is now more likely to be + used after some rewrite rule tweaking. + +* Bug fix: an off-by-one bug in `takeEnd` is fixed. + +* Bug fix: a logic error in `takeWord16` is fixed. + +#### 1.2.2.1 + +* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. + The build flag has been renamed accordingly. Your army of diligent + maintainers apologizes for the churn. + +* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec + (updated from 7.0) + +* An STG lint error has been fixed + +### 1.2.2.0 + +* The `integer-simple` package, upon which this package optionally + depended, has been replaced with `integer-pure`. The build flag has + been renamed accordingly. + +* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a + `get`, the error is propagated via `fail` instead of an uncatchable + crash. + +* New function: `takeWhileEnd` + +* New instances for the `Text` types: + * if `base` >= 4.7: `PrintfArg` + * if `base` >= 4.9: `Semigroup` + +#### 1.2.1.3 + +* Bug fix: As it turns out, moving the literal rewrite rules to simplifier + phase 2 does not prevent competition with the `unpack` rule, which is + also active in this phase. Unfortunately this was hidden due to a silly + test environment mistake. Moving literal rules back to phase 1 finally + fixes GHC Trac #10528 correctly. + +#### 1.2.1.2 + +* Bug fix: Run literal rewrite rules in simplifier phase 2. + The behavior of the simplifier changed in GHC 7.10.2, + causing these rules to fail to fire, leading to poor code generation + and long compilation times. See + [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). + +#### 1.2.1.1 + +* Expose unpackCString#, which you should never use. + +### 1.2.1.0 + +* Added Binary instances for both Text types. (If you have previously + been using the text-binary package to get a Binary instance, it is + now obsolete.) + +#### 1.2.0.6 + +* Fixed a space leak in UTF-8 decoding + +#### 1.2.0.5 + +* Feature parity: repeat, cycle, iterate are now implemented for lazy + Text, and the Data instance is more complete + +* Build speed: an inliner space explosion has been fixed with toCaseFold + +* Bug fix: encoding Int to a Builder would infinite-loop if the + integer-simple package was used + +* Deprecation: OnEncodeError and EncodeError are deprecated, as they + are never used + +* Internals: some types that are used internally in fusion-related + functions have moved around, been renamed, or been deleted (we don't + bump the major version if .Internal modules change) + +* Spec compliance: toCaseFold now follows the Unicode 7.0 spec + (updated from 6.3) + +#### 1.2.0.4 + +* Fixed an incompatibility with base < 4.5 + +#### 1.2.0.3 + +* Update formatRealFloat to correspond to the definition in versions + of base newer than 4.5 (https://github.com/bos/text/issues/105) + +#### 1.2.0.2 + +* Bumped lower bound on deepseq to 1.4 for compatibility with the + upcoming GHC 7.10 + +#### 1.2.0.1 + +* Fixed a buffer overflow in rendering of large Integers + (https://github.com/bos/text/issues/99) + +## 1.2.0.0 + +* Fixed an integer overflow in the replace function + (https://github.com/bos/text/issues/81) + +* Fixed a hang in lazy decodeUtf8With + (https://github.com/bos/text/issues/87) + +* Reduced codegen bloat caused by use of empty and single-character + literals + +* Added an instance of IsList for GHC 7.8 and above + +### 1.1.1.0 + +* The Data.Data instance now allows gunfold to work, via a virtual + pack constructor + +* dropEnd, takeEnd: new functions + +* Comparing the length of a Text against a number can now + short-circuit in more cases + +#### 1.1.0.1 + +* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes + in single-byte chunks + +## 1.1.0.0 + +* encodeUtf8: Performance is improved by up to 4x. + +* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, + available only if bytestring >= 0.10.4.0 is installed, that allow + very fast and flexible encoding of a Text value to a bytestring + Builder. + + As an example of the performance gain to be had, the + encodeUtf8BuilderEscaped function helps to double the speed of JSON + encoding in the latest version of aeson! (Note: if all you need is a + plain ByteString, encodeUtf8 is still the faster way to go.) + +* All of the internal module hierarchy is now publicly exposed. If a + module is in the .Internal hierarchy, or is documented as internal, + use at your own risk - there are no API stability guarantees for + internal modules! + +#### 1.0.0.1 + +* decodeUtf8: Fixed a regression that caused us to incorrectly + identify truncated UTF-8 as valid (gh-61) + +# 1.0.0.0 + +* Added support for Unicode 6.3.0 to case conversion functions + +* New function toTitle converts words in a string to title case + +* New functions peekCStringLen and withCStringLen simplify + interoperability with C functions + +* Added support for decoding UTF-8 in stream-friendly fashion + +* Fixed a bug in mapAccumL + +* Added trusted Haskell support + +* Removed support for GHC 6.10 (released in 2008) and older
scripts/Arsec.hs view
@@ -1,44 +1,48 @@-module Arsec- (- Comment- , comment- , semi- , showC- , unichar- , unichars- , module Control.Applicative- , module Control.Monad- , module Data.Char- , module Text.ParserCombinators.Parsec.Char- , module Text.ParserCombinators.Parsec.Combinator- , module Text.ParserCombinators.Parsec.Error- , module Text.ParserCombinators.Parsec.Prim- ) where--import Control.Monad-import Control.Applicative-import Data.Char-import Numeric-import Text.ParserCombinators.Parsec.Char hiding (lower, upper)-import Text.ParserCombinators.Parsec.Combinator hiding (optional)-import Text.ParserCombinators.Parsec.Error-import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many)--type Comment = String--unichar :: Parser Char-unichar = chr . fst . head . readHex <$> many1 hexDigit--unichars :: Parser [Char]-unichars = manyTill (unichar <* spaces) semi--semi :: Parser ()-semi = char ';' *> spaces *> pure ()--comment :: Parser Comment-comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n"--showC :: Char -> String-showC c = "'\\x" ++ d ++ "'"- where h = showHex (ord c) ""- d = replicate (4 - length h) '0' ++ h+module Arsec + ( + Comment + , comment + , semi + , showC + , unichar + , unichars + , module Control.Applicative + , module Control.Monad + , module Text.ParserCombinators.Parsec.Char + , module Text.ParserCombinators.Parsec.Combinator + , module Text.ParserCombinators.Parsec.Error + , module Text.ParserCombinators.Parsec.Prim + ) where + +import Prelude hiding (head, tail) +import Control.Monad +import Control.Applicative +import Data.Char +import Numeric (readHex, showHex) +import Text.ParserCombinators.Parsec.Char hiding (lower, upper) +import Text.ParserCombinators.Parsec.Combinator hiding (optional) +import Text.ParserCombinators.Parsec.Error +import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) + +type Comment = String + +unichar :: Parser Char +unichar = do + digits <- many1 hexDigit + case readHex digits of + [] -> error "unichar: cannot parse hex digits" + (hd, _) : _ -> pure $ chr hd + +unichars :: Parser [Char] +unichars = manyTill (unichar <* spaces) semi + +semi :: Parser () +semi = char ';' *> spaces *> pure () + +comment :: Parser Comment +comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" + +showC :: Char -> String +showC c = "'\\x" ++ d ++ "'" + where h = showHex (ord c) "" + d = replicate (4 - length h) '0' ++ h
scripts/CaseFolding.hs view
@@ -1,53 +1,47 @@--- This script processes the following source file:------ http://unicode.org/Public/UNIDATA/CaseFolding.txt--module CaseFolding- (- CaseFolding(..)- , Fold(..)- , parseCF- , mapCF- ) where--import Arsec-import Data.Bits--data Fold = Fold {- code :: Char- , status :: Char- , mapping :: [Char]- , name :: String- } deriving (Eq, Ord, Show)--data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] }- deriving (Show)--entries :: Parser CaseFolding-entries = CF <$> many comment <*> many (entry <* many comment)- where- entry = Fold <$> unichar <* semi- <*> oneOf "CFST" <* semi- <*> unichars- <*> (string "# " *> manyTill anyToken (char '\n'))--parseCF :: FilePath -> IO (Either ParseError CaseFolding)-parseCF name = parse entries name <$> readFile name--mapCF :: CaseFolding -> [String]-mapCF (CF _ ms) = typ ++ map printUnusual ms' ++ map printUsual usual ++ [last]- where- ms' = filter p ms- p f = status f `elem` "CF" &&- mapping f /= [toLower (code f)]- unusual = map code ms'- usual = filter (\c -> toLower c /= c && c `notElem` unusual) [minBound..maxBound]-- typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}"- ,"{-# NOINLINE foldMapping #-}"- ,"foldMapping = \\case"]- last = " _ -> unI64 0"- printUnusual c = " -- " ++ name c ++ "\n" ++- " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))- where x:y:z:_ = mapping c ++ repeat '\0'- printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (toLower c))+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/CaseFolding.txt + +module CaseFolding + ( + CaseFolding(..) + , Fold(..) + , parseCF + , mapCF + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data Fold = Fold { + code :: Char + , status :: Char + , mapping :: [Char] + , name :: String + } deriving (Eq, Ord, Show) + +data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } + deriving (Show) + +entries :: Parser CaseFolding +entries = CF <$> many comment <*> many (entry <* many comment) + where + entry = Fold <$> unichar <* semi + <*> oneOf "CFST" <* semi + <*> unichars + <*> (string "# " *> manyTill anyToken (char '\n')) + +parseCF :: FilePath -> IO (Either ParseError CaseFolding) +parseCF name = parse entries name <$> readFile name + +mapCF :: CaseFolding -> [String] +mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last] + where + typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE foldMapping #-}" + ,"foldMapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = mapping c ++ repeat '\0'
scripts/CaseMapping.hs view
@@ -1,41 +1,66 @@-import System.Environment-import System.IO--import Arsec-import CaseFolding-import SpecialCasing--main = do- args <- getArgs- let oname = case args of- [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs"- [o] -> o- psc <- parseSC "SpecialCasing.txt"- pcf <- parseCF "CaseFolding.txt"- scs <- case psc of- Left err -> print err >> return undefined- Right ms -> return ms- cfs <- case pcf of- Left err -> print err >> return undefined- Right ms -> return ms- h <- openFile oname WriteMode- let comments = map ("--" ++) $- take 2 (cfComments cfs) ++ take 2 (scComments scs)- mapM_ (hPutStrLn h) $- ["-- AUTOMATICALLY GENERATED - DO NOT EDIT"- ,"-- Generated by scripts/CaseMapping.hs"] ++- comments ++- [""- ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}"- ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}"- ,"module Data.Text.Internal.Fusion.CaseMapping where"- ,"import GHC.Int"- ,"import GHC.Exts"- ,"unI64 :: Int64 -> _ {- unboxed Int64 -}"- ,"unI64 (I64# n) = n"- ,""]- mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs)- mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs)- mapM_ (hPutStrLn h) (mapSC "title" title toTitle scs)- mapM_ (hPutStrLn h) (mapCF cfs)- hClose h+import Data.Char (isDigit) +import Data.Foldable (toList) +import Data.List (stripPrefix) +import Data.Maybe (fromJust) +import System.Environment +import System.IO + +import Arsec +import CaseFolding +import SpecialCasing +import UnicodeData + +main = do + args <- getArgs + let oname = case args of + [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs" + [o] -> o + psc <- parseSC "SpecialCasing.txt" + pcf <- parseCF "CaseFolding.txt" + ud <- parseUD "UnicodeData.txt" + scs <- case psc of + Left err -> print err >> return undefined + Right ms -> return ms + cfs <- case pcf of + Left err -> print err >> return undefined + Right ms -> return ms + ud <- case ud of + Left err -> print err >> return undefined + Right ms -> return ms + h <- openFile oname WriteMode + let comments = map ("--" ++) $ + take 2 (cfComments cfs) ++ take 2 (scComments scs) + version = parseVersion (cfComments cfs) + mapM_ (hPutStrLn h) $ + ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ,"-- Generated by scripts/CaseMapping.hs"] ++ + comments ++ + ["" + ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" + ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" + ,"module Data.Text.Internal.Fusion.CaseMapping where" + ,"import GHC.Int" + ,"import GHC.Exts" + ,"import Data.Version (Version, makeVersion)" + ,"unicodeVersion :: Version" + ,"unicodeVersion = makeVersion " ++ version + ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" + ,"unI64 (I64# n) = n" + ,""] + let get f = [(k, d) | c <- toList ud, Just d <- [f c], let k = charUD c, k /= d] + mapM_ (hPutStrLn h) (mapSC "upper" upper (get toUpperUD) scs) + mapM_ (hPutStrLn h) (mapSC "lower" lower (get toLowerUD) scs) + mapM_ (hPutStrLn h) (mapSC "title" title (get toTitleUD) scs) + mapM_ (hPutStrLn h) (mapCF cfs) + hClose h + +-- Parse version from CaseFolding comments +-- and render it as a list (an argument of makeVersion) +parseVersion :: [String] -> String +parseVersion comments = fromJust $ do + line' : _ <- pure comments + line'' <- stripPrefix " CaseFolding-" line' + let (v1, line1) = span isDigit line'' + (v2, line2) = span isDigit (drop 1 line1) + (v3, _) = span isDigit (drop 1 line2) + pure $ "[" ++ v1 ++ ", " ++ v2 ++ ", " ++ v3 ++ "]"
scripts/SpecialCasing.hs view
@@ -1,63 +1,61 @@--- This script processes the following source file:------ http://unicode.org/Public/UNIDATA/SpecialCasing.txt--module SpecialCasing- (- SpecialCasing(..)- , Case(..)- , parseSC- , mapSC- ) where--import Arsec-import Data.Bits--data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] }- deriving (Show)--data Case = Case {- code :: Char- , lower :: [Char]- , title :: [Char]- , upper :: [Char]- , conditions :: String- , name :: String- } deriving (Eq, Ord, Show)--entries :: Parser SpecialCasing-entries = SC <$> many comment <*> many (entry <* many comment)- where- entry = Case <$> unichar <* semi- <*> unichars- <*> unichars- <*> unichars- <*> manyTill anyToken (string "# ")- <*> manyTill anyToken (char '\n')--parseSC :: FilePath -> IO (Either ParseError SpecialCasing)-parseSC name = parse entries name <$> readFile name--mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing- -> [String]-mapSC which access twiddle (SC _ ms) =- typ ++ map printUnusual ms' ++ map printUsual usual ++ [last]- where- ms' = filter p ms- p c = [k] /= a && a /= [twiddle k] && null (conditions c)- where a = access c- k = code c- unusual = map code ms'- usual = filter (\c -> twiddle c /= c && c `notElem` unusual) [minBound..maxBound]-- typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}"- ,"{-# NOINLINE " ++ which ++ "Mapping #-}"- ,which ++ "Mapping = \\case"]- last = " _ -> unI64 0"- printUnusual c = " -- " ++ name c ++ "\n" ++- " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))- where x:y:z:_ = access c ++ repeat '\0'- printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (twiddle c))--ucFirst (c:cs) = toUpper c : cs-ucFirst [] = []+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/SpecialCasing.txt + +module SpecialCasing + ( + SpecialCasing(..) + , Case(..) + , parseSC + , mapSC + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } + deriving (Show) + +data Case = Case { + code :: Char + , lower :: [Char] + , title :: [Char] + , upper :: [Char] + , conditions :: String + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser SpecialCasing +entries = SC <$> many comment <*> many (entry <* many comment) + where + entry = Case <$> unichar <* semi + <*> unichars + <*> unichars + <*> unichars + <*> manyTill anyToken (string "# ") + <*> manyTill anyToken (char '\n') + +parseSC :: FilePath -> IO (Either ParseError SpecialCasing) +parseSC name = parse entries name <$> readFile name + +mapSC :: String -> (Case -> String) -> [(Char, Char)] -> SpecialCasing + -> [String] +mapSC which access twiddle (SC _ ms) = + typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] + where + ms' = filter p ms + p c = [k] /= a && null (conditions c) + where a = access c + k = code c + unusual = map code ms' + usual = filter (\(c, _) -> c `notElem` unusual) twiddle + + typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE " ++ which ++ "Mapping #-}" + ,which ++ "Mapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = access c ++ repeat '\0' + printUsual (c, c') = " " ++ showC c ++ "# -> unI64 " ++ show (ord c')
+ scripts/UnicodeData.hs view
@@ -0,0 +1,51 @@+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/UnicodeData.txt +-- +-- Format description: https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt + +module UnicodeData + ( UnicodeData + , Data(..) + , toTitleUD + , parseUD + ) where + +import Debug.Trace +import Arsec hiding (semi) +import Data.Array +import Data.Functor (void) +import Data.List (sort) +import Data.Maybe (fromMaybe) + +type UnicodeData = Array Int Data + +-- "Simple_Titlecase_Mapping: If this field is null, then the Simple_Titlecase_Mapping +-- is the same as the Simple_Uppercase_Mapping for this character." +-- -- https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt +toTitleUD :: Data -> Maybe Char +toTitleUD d = toTitleUD_ d <|> toUpperUD d + +data Data = Data { + charUD :: {-# UNPACK #-} !Char + , toUpperUD :: {-# UNPACK #-} !(Maybe Char) + , toLowerUD :: {-# UNPACK #-} !(Maybe Char) + , toTitleUD_ :: {-# UNPACK #-} !(Maybe Char) + } deriving (Eq, Ord, Show) + +-- I'm pretty sure UnicodeData.txt is sorted but still sort it to be 100% certain. +entries :: Parser UnicodeData +entries = (\xs -> listArray (0, length xs - 1) xs) <$> many entry <* eof + where + entry = Data <$> unichar <* semi + <* replicateM_ 11 (ignoreField <* semi) + <*> optional unichar <* semi + <*> optional unichar <* semi + <*> optional unichar <* char '\n' + semi = char ';' + +ignoreField :: Parser () +ignoreField = void (many (satisfy (\c -> c /= ';'))) + +parseUD :: FilePath -> IO (Either ParseError UnicodeData) +parseUD name = parse entries name <$> readFile name
+ simdutf/hs_simdutf.c view
@@ -0,0 +1,9 @@+#include "simdutf_c.h" + +int _hs_text_is_valid_utf8(const char *buf, size_t len) { + return simdutf_validate_utf8(buf, len); +} + +int _hs_text_is_valid_utf8_offset(const char *buf, size_t off, size_t len) { + return simdutf_validate_utf8(buf + off, len); +}
simdutf/simdutf.cpp view
file too large to diff
simdutf/simdutf.h view
@@ -1,1084 +1,13573 @@-/* auto-generated on 2021-07-29 10:43:28 -0400. Do not edit! */-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf.h-/* begin file include/simdutf.h */-#ifndef SIMDUTF_H-#define SIMDUTF_H-#include <string>-#include <cstring>-#include <atomic>-#include <vector>--// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/compiler_check.h-/* begin file include/simdutf/compiler_check.h */-#ifndef SIMDUTF_COMPILER_CHECK_H-#define SIMDUTF_COMPILER_CHECK_H--#ifndef __cplusplus-#error simdutf requires a C++ compiler-#endif--#ifndef SIMDUTF_CPLUSPLUS-#if defined(_MSVC_LANG) && !defined(__clang__)-#define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)-#else-#define SIMDUTF_CPLUSPLUS __cplusplus-#endif-#endif--// C++ 17-#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L)-#define SIMDUTF_CPLUSPLUS17 1-#endif--// C++ 14-#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L)-#define SIMDUTF_CPLUSPLUS14 1-#endif--// C++ 11-#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L)-#define SIMDUTF_CPLUSPLUS11 1-#endif--#ifndef SIMDUTF_CPLUSPLUS11-#error simdutf requires a compiler compliant with the C++11 standard-#endif--#endif // SIMDUTF_COMPILER_CHECK_H-/* end file include/simdutf/compiler_check.h */-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/common_defs.h-/* begin file include/simdutf/common_defs.h */-#ifndef SIMDUTF_COMMON_DEFS_H-#define SIMDUTF_COMMON_DEFS_H--#include <cassert>-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/portability.h-/* begin file include/simdutf/portability.h */-#ifndef SIMDUTF_PORTABILITY_H-#define SIMDUTF_PORTABILITY_H--#include <cstddef>-#include <cstdint>-#include <cstdlib>-#include <cfloat>-#include <cassert>-#ifndef _WIN32-// strcasecmp, strncasecmp-#include <strings.h>-#endif--#ifdef _MSC_VER-#define SIMDUTF_VISUAL_STUDIO 1-/**- * We want to differentiate carefully between- * clang under visual studio and regular visual- * studio.- *- * Under clang for Windows, we enable:- * * target pragmas so that part and only part of the- * code gets compiled for advanced instructions.- *- */-#ifdef __clang__-// clang under visual studio-#define SIMDUTF_CLANG_VISUAL_STUDIO 1-#else-// just regular visual studio (best guess)-#define SIMDUTF_REGULAR_VISUAL_STUDIO 1-#endif // __clang__-#endif // _MSC_VER--#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO-// https://en.wikipedia.org/wiki/C_alternative_tokens-// This header should have no effect, except maybe-// under Visual Studio.-#include <iso646.h>-#endif--#if defined(__x86_64__) || defined(_M_AMD64)-#define SIMDUTF_IS_X86_64 1-#elif defined(__aarch64__) || defined(_M_ARM64)-#define SIMDUTF_IS_ARM64 1-#elif defined(__PPC64__) || defined(_M_PPC64)-//#define SIMDUTF_IS_PPC64 1-#pragma message("The simdutf library does yet support SIMD acceleration under\-POWER processors. Please see https://github.com/lemire/simdutf/issues/51")-#else-// The simdutf library is designed-// for 64-bit processors and it seems that you are not-// compiling for a known 64-bit platform. Please-// use a 64-bit target such as x64 or 64-bit ARM for best performance.-#define SIMDUTF_IS_32BITS 1--// We do not support 32-bit platforms, but it can be-// handy to identify them.-#if defined(_M_IX86) || defined(__i386__)-#define SIMDUTF_IS_X86_32BITS 1-#elif defined(__arm__) || defined(_M_ARM)-#define SIMDUTF_IS_ARM_32BITS 1-#elif defined(__PPC__) || defined(_M_PPC)-#define SIMDUTF_IS_PPC_32BITS 1-#endif--#endif // defined(__x86_64__) || defined(_M_AMD64)--#ifdef SIMDUTF_IS_32BITS-#ifndef SIMDUTF_NO_PORTABILITY_WARNING-#pragma message("The simdutf library is designed \-for 64-bit processors and it seems that you are not \-compiling for a known 64-bit platform. All fast kernels \-will be disabled and performance may be poor. Please \-use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")-#endif // SIMDUTF_NO_PORTABILITY_WARNING-#endif // SIMDUTF_IS_32BITS--// this is almost standard?-#undef STRINGIFY_IMPLEMENTATION_-#undef STRINGIFY-#define STRINGIFY_IMPLEMENTATION_(a) #a-#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a)--// Our fast kernels require 64-bit systems.-//-// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions.-// Furthermore, the number of SIMD registers is reduced.-//-// On 32-bit ARM, we would have smaller registers.-//-// The simdutf users should still have the fallback kernel. It is-// slower, but it should run everywhere.--//-// Enable valid runtime implementations, and select SIMDUTF_BUILTIN_IMPLEMENTATION-//--// We are going to use runtime dispatch.-#ifdef SIMDUTF_IS_X86_64-#ifdef __clang__-// clang does not have GCC push pop-// warning: clang attribute push can't be used within a namespace in clang up-// til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be *outside* of a-// namespace.-#define SIMDUTF_TARGET_REGION(T) \- _Pragma(STRINGIFY( \- clang attribute push(__attribute__((target(T))), apply_to = function)))-#define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop")-#elif defined(__GNUC__)-// GCC is easier-#define SIMDUTF_TARGET_REGION(T) \- _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))-#define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options")-#endif // clang then gcc--#endif // x86--// Default target region macros don't do anything.-#ifndef SIMDUTF_TARGET_REGION-#define SIMDUTF_TARGET_REGION(T)-#define SIMDUTF_UNTARGET_REGION-#endif--// Is threading enabled?-#if defined(_REENTRANT) || defined(_MT)-#ifndef SIMDUTF_THREADS_ENABLED-#define SIMDUTF_THREADS_ENABLED-#endif-#endif--// workaround for large stack sizes under -O0.-// https://github.com/simdutf/simdutf/issues/691-#ifdef __APPLE__-#ifndef __OPTIMIZE__-// Apple systems have small stack sizes in secondary threads.-// Lack of compiler optimization may generate high stack usage.-// Users may want to disable threads for safety, but only when-// in debug mode which we detect by the fact that the __OPTIMIZE__-// macro is not defined.-#undef SIMDUTF_THREADS_ENABLED-#endif-#endif---#if defined(__clang__)-#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))-#elif defined(__GNUC__)-#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))-#else-#define NO_SANITIZE_UNDEFINED-#endif--#ifdef SIMDUTF_VISUAL_STUDIO-// This is one case where we do not distinguish between-// regular visual studio and clang under visual studio.-// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has)-#define simdutf_strcasecmp _stricmp-#define simdutf_strncasecmp _strnicmp-#else-// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8).-// So they are only useful for ASCII in our context.-// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings-#define simdutf_strcasecmp strcasecmp-#define simdutf_strncasecmp strncasecmp-#endif--#ifdef NDEBUG--#ifdef SIMDUTF_VISUAL_STUDIO-#define SIMDUTF_UNREACHABLE() __assume(0)-#define SIMDUTF_ASSUME(COND) __assume(COND)-#else-#define SIMDUTF_UNREACHABLE() __builtin_unreachable();-#define SIMDUTF_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0)-#endif--#else // NDEBUG--#define SIMDUTF_UNREACHABLE() assert(0);-#define SIMDUTF_ASSUME(COND) assert(COND)--#endif--#endif // SIMDUTF_PORTABILITY_H-/* end file include/simdutf/portability.h */---#if defined(__GNUC__)- // Marks a block with a name so that MCA analysis can see it.- #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);- #define SIMDUTF_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);- #define SIMDUTF_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);-#else- #define SIMDUTF_BEGIN_DEBUG_BLOCK(name)- #define SIMDUTF_END_DEBUG_BLOCK(name)- #define SIMDUTF_DEBUG_BLOCK(name, block)-#endif--// Align to N-byte boundary-#define SIMDUTF_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))-#define SIMDUTF_ROUNDDOWN_N(a, n) ((a) & ~((n)-1))--#define SIMDUTF_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)--#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO)-- #define simdutf_really_inline __forceinline- #define simdutf_never_inline __declspec(noinline)-- #define simdutf_unused- #define simdutf_warn_unused-- #ifndef simdutf_likely- #define simdutf_likely(x) x- #endif- #ifndef simdutf_unlikely- #define simdutf_unlikely(x) x- #endif-- #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning( push ))- #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 ))- #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER ))- // Get rid of Intellisense-only warnings (Code Analysis)- // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910).- #ifdef __has_include- #if __has_include(<CppCoreCheck\Warnings.h>)- #include <CppCoreCheck\Warnings.h>- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS)- #endif- #endif-- #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #endif-- #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996)- #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING- #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning( pop ))--#else // SIMDUTF_REGULAR_VISUAL_STUDIO-- #define simdutf_really_inline inline __attribute__((always_inline))- #define simdutf_never_inline inline __attribute__((noinline))-- #define simdutf_unused __attribute__((unused))- #define simdutf_warn_unused __attribute__((warn_unused_result))-- #ifndef simdutf_likely- #define simdutf_likely(x) __builtin_expect(!!(x), 1)- #endif- #ifndef simdutf_unlikely- #define simdutf_unlikely(x) __builtin_expect(!!(x), 0)- #endif-- #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")- // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary- #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS SIMDUTF_PUSH_DISABLE_WARNINGS \- SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \- SIMDUTF_DISABLE_GCC_WARNING(-Wall) \- SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \- SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \- SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \- SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \- SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \- SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \- SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \- SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \- SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable)- #define SIMDUTF_PRAGMA(P) _Pragma(#P)- #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING)- #if defined(SIMDUTF_CLANG_VISUAL_STUDIO)- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include)- #else- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #endif- #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations)- #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow)- #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")----#endif // MSC_VER--#if defined(SIMDUTF_VISUAL_STUDIO)- /**- * It does not matter here whether you are using- * the regular visual studio or clang under visual- * studio.- */- #if SIMDUTF_USING_LIBRARY- #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport)- #else- #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport)- #endif-#else- #define SIMDUTF_DLLIMPORTEXPORT-#endif--/// If EXPR is an error, returns it.-#define SIMDUTF_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } }---#endif // SIMDUTF_COMMON_DEFS_H-/* end file include/simdutf/common_defs.h */-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/encoding_types.h-/* begin file include/simdutf/encoding_types.h */-#include <string>--namespace simdutf {--enum encoding_type {- UTF16_LE, // BOM 0xff 0xfe- UTF16_BE, // BOM 0xfe 0xff- UTF32_LE, // BOM 0xff 0xfe 0x00 0x00- UTF32_BE, // BOM 0x00 0x00 0xfe 0xff- UTF8, // BOM 0xef 0xbb 0xbf- unspecified-};--std::string to_string(encoding_type bom);--// Note that BOM for UTF8 is discouraged.-namespace BOM {--/**- * Checks for a BOM. If not, returns unspecified- * @param input the string to process- * @param length the length of the string in words- * @return the corresponding encoding- */--encoding_type check_bom(const uint8_t* byte, size_t length);-encoding_type check_bom(const char* byte, size_t length);-/**- * Returns the size, in bytes, of the BOM for a given encoding type.- * Note that UTF8 BOM are discouraged.- * @param bom the encoding type- * @return the size in bytes of the corresponding BOM- */-size_t bom_byte_size(encoding_type bom);--} // BOM namespace-} // simdutf namespace-/* end file include/simdutf/encoding_types.h */--SIMDUTF_PUSH_DISABLE_WARNINGS-SIMDUTF_DISABLE_UNDESIRED_WARNINGS--// Public API-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/simdutf_version.h-/* begin file include/simdutf/simdutf_version.h */-// /include/simdutf/simdutf_version.h automatically generated by release.py,-// do not change by hand-#ifndef SIMDUTF_SIMDUTF_VERSION_H-#define SIMDUTF_SIMDUTF_VERSION_H--/** The version of simdutf being used (major.minor.revision) */-#define SIMDUTF_VERSION 0.1.0--namespace simdutf {-enum {- /**- * The major version (MAJOR.minor.revision) of simdutf being used.- */- SIMDUTF_VERSION_MAJOR = 0,- /**- * The minor version (major.MINOR.revision) of simdutf being used.- */- SIMDUTF_VERSION_MINOR = 1,- /**- * The revision (major.minor.REVISION) of simdutf being used.- */- SIMDUTF_VERSION_REVISION = 0-};-} // namespace simdutf--#endif // SIMDUTF_SIMDUTF_VERSION_H-/* end file include/simdutf/simdutf_version.h */-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/implementation.h-/* begin file include/simdutf/implementation.h */-#ifndef SIMDUTF_IMPLEMENTATION_H-#define SIMDUTF_IMPLEMENTATION_H-#include <string>-#include <atomic>-#include <vector>-// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/internal/isadetection.h-/* begin file include/simdutf/internal/isadetection.h */-/* From-https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h-Highly modified.--Copyright (c) 2016- Facebook, Inc (Adam Paszke)-Copyright (c) 2014- Facebook, Inc (Soumith Chintala)-Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)-Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)-Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)-Copyright (c) 2011-2013 NYU (Clement Farabet)-Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,-Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute-(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,-Samy Bengio, Johnny Mariethoz)--All rights reserved.--Redistribution and use in source and binary forms, with or without-modification, are permitted provided that the following conditions are met:--1. Redistributions of source code must retain the above copyright- notice, this list of conditions and the following disclaimer.--2. Redistributions in binary form must reproduce the above copyright- notice, this list of conditions and the following disclaimer in the- documentation and/or other materials provided with the distribution.--3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories-America and IDIAP Research Institute nor the names of its contributors may be- used to endorse or promote products derived from this software without- specific prior written permission.--THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE-POSSIBILITY OF SUCH DAMAGE.-*/--#ifndef SIMDutf_INTERNAL_ISADETECTION_H-#define SIMDutf_INTERNAL_ISADETECTION_H--#include <cstdint>-#include <cstdlib>-#if defined(_MSC_VER)-#include <intrin.h>-#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)-#include <cpuid.h>-#endif--namespace simdutf {-namespace internal {---enum instruction_set {- DEFAULT = 0x0,- NEON = 0x1,- AVX2 = 0x4,- SSE42 = 0x8,- PCLMULQDQ = 0x10,- BMI1 = 0x20,- BMI2 = 0x40,- ALTIVEC = 0x80-};--#if defined(__PPC64__)--static inline uint32_t detect_supported_architectures() {- return instruction_set::ALTIVEC;-}--#elif defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64--#if defined(__ARM_NEON)--static inline uint32_t detect_supported_architectures() {- return instruction_set::NEON;-}--#else // ARM without NEON--static inline uint32_t detect_supported_architectures() {- return instruction_set::DEFAULT;-}--#endif--#elif defined(__x86_64__) || defined(_M_AMD64) // x64---namespace {-// Can be found on Intel ISA Reference for CPUID-constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7-constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7-constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7-constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1-constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1-}----static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,- uint32_t *edx) {-#if defined(_MSC_VER)- int cpu_info[4];- __cpuid(cpu_info, *eax);- *eax = cpu_info[0];- *ebx = cpu_info[1];- *ecx = cpu_info[2];- *edx = cpu_info[3];-#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)- uint32_t level = *eax;- __get_cpuid(level, eax, ebx, ecx, edx);-#else- uint32_t a = *eax, b, c = *ecx, d;- asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));- *eax = a;- *ebx = b;- *ecx = c;- *edx = d;-#endif-}--static inline uint32_t detect_supported_architectures() {- uint32_t eax, ebx, ecx, edx;- uint32_t host_isa = 0x0;-- // ECX for EAX=0x7- eax = 0x7;- ecx = 0x0;- cpuid(&eax, &ebx, &ecx, &edx);- if (ebx & cpuid_avx2_bit) {- host_isa |= instruction_set::AVX2;- }- if (ebx & cpuid_bmi1_bit) {- host_isa |= instruction_set::BMI1;- }-- if (ebx & cpuid_bmi2_bit) {- host_isa |= instruction_set::BMI2;- }-- // EBX for EAX=0x1- eax = 0x1;- cpuid(&eax, &ebx, &ecx, &edx);-- if (ecx & cpuid_sse42_bit) {- host_isa |= instruction_set::SSE42;- }-- if (ecx & cpuid_pclmulqdq_bit) {- host_isa |= instruction_set::PCLMULQDQ;- }-- return host_isa;-}-#else // fallback---static inline uint32_t detect_supported_architectures() {- return instruction_set::DEFAULT;-}---#endif // end SIMD extension detection code--} // namespace internal-} // namespace simdutf--#endif // SIMDutf_INTERNAL_ISADETECTION_H-/* end file include/simdutf/internal/isadetection.h */---namespace simdutf {--/**- * Autodetect the encoding of the input.- *- * @param input the string to analyze.- * @param length the length of the string in bytes.- * @return the detected encoding type- */-simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * input, size_t length) noexcept;-simdutf_really_inline simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const uint8_t * input, size_t length) noexcept {- return autodetect_encoding(reinterpret_cast<const char *>(input), length);-}---/**- * Validate the UTF-8 string.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid UTF-8.- */-simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept;--/**- * Validate the UTF-16LE string.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte words (char16_t).- * @return true if and only if the string is valid UTF-16LE.- */-simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */-simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) noexcept;--/**- * Convert valid UTF-8 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */-simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE- */-simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept;--/**- * Convert possibly broken UTF-16LE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written words; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert valid UTF-16LE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written words; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to process- * @param length the length of the string in 2-byte words (char16_t)- * @return number of code points- */-simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return number of code points- */-simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept;--/**- * An implementation of simdutf for a particular CPU architecture.- *- * Also used to maintain the currently active implementation. The active implementation is- * automatically initialized on first use to the most advanced implementation supported by the host.- */-class implementation {-public:-- /**- * The name of this implementation.- *- * const implementation *impl = simdutf::active_implementation;- * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;- *- * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"- */- virtual const std::string &name() const { return _name; }-- /**- * The description of this implementation.- *- * const implementation *impl = simdutf::active_implementation;- * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;- *- * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"- */- virtual const std::string &description() const { return _description; }-- /**- * The instruction sets this implementation is compiled against- * and the current CPU match. This function may poll the current CPU/system- * and should therefore not be called too often if performance is a concern.- *- *- * @return true if the implementation can be safely used on the current system (determined at runtime)- */- bool supported_by_runtime_system() const;-- /**- * This function will try to detect the encoding- * @param input the string to identify- * @param length the length of the string in bytes.- * @return the encoding type detected- */- virtual encoding_type autodetect_encoding(const char * input, size_t length) const noexcept;-- /**- * @private For internal implementation use- *- * The instruction sets this implementation is compiled against.- *- * @return a mask of all required `internal::instruction_set::` values- */- virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; };--- /**- * Validate the UTF-8 string.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid UTF-8.- */- simdutf_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-16LE string.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte words (char16_t).- * @return true if and only if the string is valid UTF-16LE.- */- simdutf_warn_unused virtual bool validate_utf16(const char16_t *buf, size_t len) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused virtual size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) const noexcept = 0;-- /**- * Convert valid UTF-8 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */- simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format.- *- * This function does not validate the input.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE- */- simdutf_warn_unused virtual size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written words; 0 if input is not a valid UTF-16LE string- */- simdutf_warn_unused virtual size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16LE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written words; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte words (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-8- */- simdutf_warn_unused virtual size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to process- * @param length the length of the string in 2-byte words (char16_t)- * @return number of code points- */- simdutf_warn_unused virtual size_t count_utf16(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return number of code points- */- simdutf_warn_unused virtual size_t count_utf8(const char * input, size_t length) const noexcept = 0;----protected:- /** @private Construct an implementation with the given name and description. For subclasses. */- simdutf_really_inline implementation(- std::string name,- std::string description,- uint32_t required_instruction_sets- ) :- _name(name),- _description(description),- _required_instruction_sets(required_instruction_sets)- {- }- virtual ~implementation()=default;--private:- /**- * The name of this implementation.- */- const std::string _name;-- /**- * The description of this implementation.- */- const std::string _description;-- /**- * Instruction sets required for this implementation.- */- const uint32_t _required_instruction_sets;-};--/** @private */-namespace internal {--/**- * The list of available implementations compiled into simdutf.- */-class available_implementation_list {-public:- /** Get the list of available implementations compiled into simdutf */- simdutf_really_inline available_implementation_list() {}- /** Number of implementations */- size_t size() const noexcept;- /** STL const begin() iterator */- const implementation * const *begin() const noexcept;- /** STL const end() iterator */- const implementation * const *end() const noexcept;-- /**- * Get the implementation with the given name.- *- * Case sensitive.- *- * const implementation *impl = simdutf::available_implementations["westmere"];- * if (!impl) { exit(1); }- * if (!imp->supported_by_runtime_system()) { exit(1); }- * simdutf::active_implementation = impl;- *- * @param name the implementation to find, e.g. "westmere", "haswell", "arm64"- * @return the implementation, or nullptr if the parse failed.- */- const implementation * operator[](const std::string &name) const noexcept {- for (const implementation * impl : *this) {- if (impl->name() == name) { return impl; }- }- return nullptr;- }-- /**- * Detect the most advanced implementation supported by the current host.- *- * This is used to initialize the implementation on startup.- *- * const implementation *impl = simdutf::available_implementation::detect_best_supported();- * simdutf::active_implementation = impl;- *- * @return the most advanced supported implementation for the current host, or an- * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported- * implementation. Will never return nullptr.- */- const implementation *detect_best_supported() const noexcept;-};--template<typename T>-class atomic_ptr {-public:- atomic_ptr(T *_ptr) : ptr{_ptr} {}-- operator const T*() const { return ptr.load(); }- const T& operator*() const { return *ptr; }- const T* operator->() const { return ptr.load(); }-- operator T*() { return ptr.load(); }- T& operator*() { return *ptr; }- T* operator->() { return ptr.load(); }- atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; }--private:- std::atomic<T*> ptr;-};--} // namespace internal--/**- * The list of available implementations compiled into simdutf.- */-extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations;--/**- * The active implementation.- *- * Automatically initialized on first use to the most advanced implementation supported by this hardware.- */-extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation;--} // namespace simdutf--#endif // SIMDUTF_IMPLEMENTATION_H-/* end file include/simdutf/implementation.h */---// Implementation-internal files (must be included before the implementations themselves, to keep-// amalgamation working--otherwise, the first time a file is included, it might be put inside the-// #ifdef SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other implementations can't-// compile unless that implementation is turned on).---SIMDUTF_POP_DISABLE_WARNINGS--#endif // SIMDUTF_H-/* end file include/simdutf.h */+/* auto-generated on 2026-01-13 09:03:21 +0100. Do not edit! */ +/* begin file include/simdutf.h */ +#ifndef SIMDUTF_H +#define SIMDUTF_H +#include <cstring> + +/* begin file include/simdutf/compiler_check.h */ +#ifndef SIMDUTF_COMPILER_CHECK_H +#define SIMDUTF_COMPILER_CHECK_H + +#ifndef __cplusplus + #error simdutf requires a C++ compiler +#endif + +#ifndef SIMDUTF_CPLUSPLUS + #if defined(_MSVC_LANG) && !defined(__clang__) + #define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) + #else + #define SIMDUTF_CPLUSPLUS __cplusplus + #endif +#endif + +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + +// C++ 23 +#if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) + #define SIMDUTF_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDUTF_CPLUSPLUS20) && (SIMDUTF_CPLUSPLUS >= 202002L) + #define SIMDUTF_CPLUSPLUS20 1 +#endif + +// C++ 17 +#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) + #define SIMDUTF_CPLUSPLUS17 1 +#endif + +// C++ 14 +#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) + #define SIMDUTF_CPLUSPLUS14 1 +#endif + +// C++ 11 +#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) + #define SIMDUTF_CPLUSPLUS11 1 +#endif + +#ifndef SIMDUTF_CPLUSPLUS11 + #error simdutf requires a compiler compliant with the C++11 standard +#endif + +#endif // SIMDUTF_COMPILER_CHECK_H +/* end file include/simdutf/compiler_check.h */ +/* begin file include/simdutf/common_defs.h */ +#ifndef SIMDUTF_COMMON_DEFS_H +#define SIMDUTF_COMMON_DEFS_H + +/* begin file include/simdutf/portability.h */ +#ifndef SIMDUTF_PORTABILITY_H +#define SIMDUTF_PORTABILITY_H + + +#include <cfloat> +#include <cstddef> +#include <cstdint> +#include <cstdlib> +#ifndef _WIN32 + // strcasecmp, strncasecmp + #include <strings.h> +#endif + +#if defined(__apple_build_version__) + #if __apple_build_version__ < 14000000 + #define SIMDUTF_SPAN_DISABLED \ + 1 // apple-clang/13 doesn't support std::convertible_to + #endif +#endif + +#if SIMDUTF_CPLUSPLUS20 + #include <version> + #if __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L && \ + !defined(SIMDUTF_SPAN_DISABLED) + #define SIMDUTF_SPAN 1 + #endif // __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L + #if __cpp_lib_atomic_ref >= 201806L + #define SIMDUTF_ATOMIC_REF 1 + #endif // __cpp_lib_atomic_ref + #if __has_cpp_attribute(maybe_unused) >= 201603L + #define SIMDUTF_MAYBE_UNUSED_AVAILABLE 1 + #endif // __has_cpp_attribute(maybe_unused) >= 201603L +#endif + +/** + * We want to check that it is actually a little endian system at + * compile-time. + */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) + #define SIMDUTF_IS_BIG_ENDIAN 0 +#else + #if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ + #include <machine/endian.h> + #elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) + #include <sys/byteorder.h> + #else // defined(__APPLE__) || defined(__FreeBSD__) + + #ifdef __has_include + #if __has_include(<endian.h>) + #include <endian.h> + #endif //__has_include(<endian.h>) + #endif //__has_include + + #endif // defined(__APPLE__) || defined(__FreeBSD__) + + #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN 0 + #endif + + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 0 + #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 1 + #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ + +/** + * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined. + */ + +#ifdef _MSC_VER + #define SIMDUTF_VISUAL_STUDIO 1 + /** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + * + * Under clang for Windows, we enable: + * * target pragmas so that part and only part of the + * code gets compiled for advanced instructions. + * + */ + #ifdef __clang__ + // clang under visual studio + #define SIMDUTF_CLANG_VISUAL_STUDIO 1 + #else + // just regular visual studio (best guess) + #define SIMDUTF_REGULAR_VISUAL_STUDIO 1 + #endif // __clang__ +#endif // _MSC_VER + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + // https://en.wikipedia.org/wiki/C_alternative_tokens + // This header should have no effect, except maybe + // under Visual Studio. + #include <iso646.h> +#endif + +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) + #define SIMDUTF_IS_X86_64 1 +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + #define SIMDUTF_IS_ARM64 1 +#elif defined(__PPC64__) || defined(_M_PPC64) + #if defined(__VEC__) && defined(__ALTIVEC__) + #define SIMDUTF_IS_PPC64 1 + #endif +#elif defined(__s390__) +// s390 IBM system. Big endian. +#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64 + // RISC-V 64-bit + #define SIMDUTF_IS_RISCV64 1 + + // #if __riscv_v_intrinsic >= 1000000 + // #define SIMDUTF_HAS_RVV_INTRINSICS 1 + // #define SIMDUTF_HAS_RVV_TARGET_REGION 1 + // #elif ... + // Check for special compiler versions that implement pre v1.0 intrinsics + #if __riscv_v_intrinsic >= 11000 + #define SIMDUTF_HAS_RVV_INTRINSICS 1 + #endif + + #define SIMDUTF_HAS_ZVBB_INTRINSICS \ + 0 // there is currently no way to detect this + + #if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector && \ + __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 + // RISC-V V extension + #define SIMDUTF_IS_RVV 1 + #if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000 + // RISC-V Vector Basic Bit-manipulation + #define SIMDUTF_IS_ZVBB 1 + #endif + #endif + +#elif defined(__loongarch_lp64) + #if defined(__loongarch_sx) && defined(__loongarch_asx) + #define SIMDUTF_IS_LSX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both + #elif defined(__loongarch_sx) + #define SIMDUTF_IS_LSX 1 + #endif +#else + // The simdutf library is designed + // for 64-bit processors and it seems that you are not + // compiling for a known 64-bit platform. Please + // use a 64-bit target such as x64 or 64-bit ARM for best performance. + #define SIMDUTF_IS_32BITS 1 + + // We do not support 32-bit platforms, but it can be + // handy to identify them. + #if defined(_M_IX86) || defined(__i386__) + #define SIMDUTF_IS_X86_32BITS 1 + #elif defined(__arm__) || defined(_M_ARM) + #define SIMDUTF_IS_ARM_32BITS 1 + #elif defined(__PPC__) || defined(_M_PPC) + #define SIMDUTF_IS_PPC_32BITS 1 + #endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) + +#ifdef SIMDUTF_IS_32BITS + #ifndef SIMDUTF_NO_PORTABILITY_WARNING + // In the future, we may want to warn users of 32-bit systems that + // the simdutf does not support accelerated kernels for such systems. + #endif // SIMDUTF_NO_PORTABILITY_WARNING +#endif // SIMDUTF_IS_32BITS + +// this is almost standard? +#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a +#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The simdutf users should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select +// SIMDUTF_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + #ifdef __clang__ + // clang does not have GCC push pop + // warning: clang attribute push can't be used within a namespace in clang + // up til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be + // *outside* of a namespace. + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma(SIMDUTF_STRINGIFY(clang attribute push( \ + __attribute__((target(T))), apply_to = function))) + #define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") + #elif defined(__GNUC__) + // GCC is easier + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T))) + #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") + #endif // clang then gcc + +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + +// Default target region macros don't do anything. +#ifndef SIMDUTF_TARGET_REGION + #define SIMDUTF_TARGET_REGION(T) + #define SIMDUTF_UNTARGET_REGION +#endif + +// Is threading enabled? +#if defined(_REENTRANT) || defined(_MT) + #ifndef SIMDUTF_THREADS_ENABLED + #define SIMDUTF_THREADS_ENABLED + #endif +#endif + +// workaround for large stack sizes under -O0. +// https://github.com/simdutf/simdutf/issues/691 +#ifdef __APPLE__ + #ifndef __OPTIMIZE__ + // Apple systems have small stack sizes in secondary threads. + // Lack of compiler optimization may generate high stack usage. + // Users may want to disable threads for safety, but only when + // in debug mode which we detect by the fact that the __OPTIMIZE__ + // macro is not defined. + #undef SIMDUTF_THREADS_ENABLED + #endif +#endif + +#ifdef SIMDUTF_VISUAL_STUDIO + // This is one case where we do not distinguish between + // regular visual studio and clang under visual studio. + // clang under Windows has _stricmp (like visual studio) but not strcasecmp + // (as clang normally has) + #define simdutf_strcasecmp _stricmp + #define simdutf_strncasecmp _strnicmp +#else + // The strcasecmp, strncasecmp, and strcasestr functions do not work with + // multibyte strings (e.g. UTF-8). So they are only useful for ASCII in our + // context. + // https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings + #define simdutf_strcasecmp strcasecmp + #define simdutf_strncasecmp strncasecmp +#endif + +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ >= 11 + #define SIMDUTF_GCC11ORMORE 1 + #endif // __GNUC__ >= 11 + #if __GNUC__ == 10 + #define SIMDUTF_GCC10 1 + #endif // __GNUC__ == 10 + #if __GNUC__ < 10 + #define SIMDUTF_GCC9OROLDER 1 + #endif // __GNUC__ == 10 +#endif // defined(__GNUC__) && !defined(__clang__) + +#endif // SIMDUTF_PORTABILITY_H +/* end file include/simdutf/portability.h */ +/* begin file include/simdutf/avx512.h */ +#ifndef SIMDUTF_AVX512_H_ +#define SIMDUTF_AVX512_H_ + +/* + It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS. + + All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`, + where a feature is a code name for extensions. + + Please see the listing below to find which are supported. +*/ + +#ifndef SIMDUTF_HAS_AVX512F + #if defined(__AVX512F__) && __AVX512F__ == 1 + #define SIMDUTF_HAS_AVX512F 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512DQ + #if defined(__AVX512DQ__) && __AVX512DQ__ == 1 + #define SIMDUTF_HAS_AVX512DQ 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512IFMA + #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1 + #define SIMDUTF_HAS_AVX512IFMA 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512CD + #if defined(__AVX512CD__) && __AVX512CD__ == 1 + #define SIMDUTF_HAS_AVX512CD 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BW + #if defined(__AVX512BW__) && __AVX512BW__ == 1 + #define SIMDUTF_HAS_AVX512BW 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VL + #if defined(__AVX512VL__) && __AVX512VL__ == 1 + #define SIMDUTF_HAS_AVX512VL 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI + #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1 + #define SIMDUTF_HAS_AVX512VBMI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI2 + #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1 + #define SIMDUTF_HAS_AVX512VBMI2 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VNNI + #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1 + #define SIMDUTF_HAS_AVX512VNNI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BITALG + #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1 + #define SIMDUTF_HAS_AVX512BITALG 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ + #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1 + #define SIMDUTF_HAS_AVX512VPOPCNTDQ 1 + #endif +#endif + +#endif // SIMDUTF_AVX512_H_ +/* end file include/simdutf/avx512.h */ + +// Sometimes logging is useful, but we want it disabled by default +// and free of any logging code in release builds. +#ifdef SIMDUTF_LOGGING + #include <iostream> + #define simdutf_log(msg) \ + std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; + #define simdutf_log_assert(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cerr << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; \ + std::abort(); \ + } \ + } while (0) +#else + #define simdutf_log(msg) + #define simdutf_log_assert(cond, msg) +#endif + +#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) + #define SIMDUTF_DEPRECATED __declspec(deprecated) + + #define simdutf_really_inline __forceinline // really inline in release mode + #define simdutf_always_inline __forceinline // always inline, no matter what + #define simdutf_never_inline __declspec(noinline) + + #define simdutf_unused + #define simdutf_warn_unused + + #ifndef simdutf_likely + #define simdutf_likely(x) x + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) x + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning(push)) + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0)) + #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) \ + __pragma(warning(disable : WARNING_NUMBER)) + // Get rid of Intellisense-only warnings (Code Analysis) + // Though __has_include is C++17, it is supported in Visual Studio 2017 or + // better (_MSC_VER>=1910). + #ifdef __has_include + #if __has_include(<CppCoreCheck\Warnings.h>) + #include <CppCoreCheck\Warnings.h> + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) + #endif + #endif + + #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING + #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning(pop)) + #define SIMDUTF_DISABLE_UNUSED_WARNING +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + #if defined(__OPTIMIZE__) || defined(NDEBUG) + #define simdutf_really_inline inline __attribute__((always_inline)) + #else + #define simdutf_really_inline inline + #endif + #define simdutf_always_inline \ + inline __attribute__((always_inline)) // always inline, no matter what + #define SIMDUTF_DEPRECATED __attribute__((deprecated)) + #define simdutf_never_inline inline __attribute__((noinline)) + + #define simdutf_unused __attribute__((unused)) + #define simdutf_warn_unused __attribute__((warn_unused_result)) + + #ifndef simdutf_likely + #define simdutf_likely(x) __builtin_expect(!!(x), 1) + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) + #endif + // clang-format off + #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") + // gcc doesn't seem to disable all warnings with all and extra, add warnings + // here as necessary + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) + #define SIMDUTF_PRAGMA(P) _Pragma(#P) + #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) \ + SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) + #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) + #else + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + #define SIMDUTF_DISABLE_DEPRECATED_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) + #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") + #define SIMDUTF_DISABLE_UNUSED_WARNING \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-function) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-const-variable) + // clang-format on + +#endif // MSC_VER + +// Conditional constexpr macro: expands to constexpr for C++17+, empty otherwise +#if SIMDUTF_CPLUSPLUS17 + #define simdutf_constexpr constexpr +#else + #define simdutf_constexpr +#endif + +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + +#ifndef SIMDUTF_DLLIMPORTEXPORT + #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio + /** + * Windows users need to do some extra work when building + * or using a dynamic library (DLL). When building, we need + * to set SIMDUTF_DLLIMPORTEXPORT to __declspec(dllexport). + * When *using* the DLL, the user needs to set + * SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport). + * + * Static libraries not need require such work. + * + * It does not matter here whether you are using + * the regular visual studio or clang under visual + * studio, you still need to handle these issues. + * + * Non-Windows systems do not have this complexity. + */ + #if SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY + + // We set SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL + // under Windows. It should never happen that both + // SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY and + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY are set. + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) + #elif SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY + // Windows user who call a dynamic library should set + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY to 1. + + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) + #else + // We assume by default static linkage + #define SIMDUTF_DLLIMPORTEXPORT + #endif + #else // defined(SIMDUTF_VISUAL_STUDIO) + // Non-Windows systems do not have this complexity. + #define SIMDUTF_DLLIMPORTEXPORT + #endif // defined(SIMDUTF_VISUAL_STUDIO) +#endif + +#if SIMDUTF_MAYBE_UNUSED_AVAILABLE + #define simdutf_maybe_unused [[maybe_unused]] +#else + #define simdutf_maybe_unused +#endif + +#endif // SIMDUTF_COMMON_DEFS_H +/* end file include/simdutf/common_defs.h */ +/* begin file include/simdutf/encoding_types.h */ +#ifndef SIMDUTF_ENCODING_TYPES_H +#define SIMDUTF_ENCODING_TYPES_H +#include <string> + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include <text_encoding> +#endif + +namespace simdutf { + +enum encoding_type { + UTF8 = 1, // BOM 0xef 0xbb 0xbf + UTF16_LE = 2, // BOM 0xff 0xfe + UTF16_BE = 4, // BOM 0xfe 0xff + UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00 + UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff + Latin1 = 32, + + unspecified = 0 +}; + +#ifndef SIMDUTF_IS_BIG_ENDIAN + #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." +#endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = +#if SIMDUTF_IS_BIG_ENDIAN + BIG +#else + LITTLE +#endif +}; + +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; +} + +simdutf_warn_unused std::string to_string(encoding_type bom); + +// Note that BOM for UTF8 is discouraged. +namespace BOM { + +/** + * Checks for a BOM. If not, returns unspecified + * @param input the string to process + * @param length the length of the string in code units + * @return the corresponding encoding + */ + +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); +/** + * Returns the size, in bytes, of the BOM for a given encoding type. + * Note that UTF8 BOM are discouraged. + * @param bom the encoding type + * @return the size in bytes of the corresponding BOM + */ +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); + +} // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + +} // namespace simdutf +#endif +/* end file include/simdutf/encoding_types.h */ +/* begin file include/simdutf/error.h */ +#ifndef SIMDUTF_ERROR_H +#define SIMDUTF_ERROR_H +namespace simdutf { + +enum error_code { + SUCCESS = 0, + HEADER_BITS, // Any byte must have fewer than 5 header bits. + TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, + // where N is the UTF-8 character length This is also the error + // when the input is truncated. + TOO_LONG, // We either have too many consecutive continuation bytes or the + // string starts with a continuation byte. + OVERLONG, // The decoded character must be above U+7F for two-byte characters, + // U+7FF for three-byte characters, and U+FFFF for four-byte + // characters. + TOO_LARGE, // The decoded character must be less than or equal to + // U+10FFFF,less than or equal than U+7F for ASCII OR less than + // equal than U+FF for Latin1 + SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or + // UTF-32) + // OR + // a high surrogate must be followed by a low surrogate + // and a low surrogate must be preceded by a high surrogate + // (UTF-16) + // OR + // there must be no surrogate at all and one is + // found (Latin1 functions) + // OR + // *specifically* for the function + // utf8_length_from_utf16_with_replacement, a surrogate (whether + // in error or not) has been found (I.e., whether we are in the + // Basic Multilingual Plane or not). + INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid + // base64 string. This may include a misplaced + // padding character ('='). + BASE64_INPUT_REMAINDER, // The base64 input terminates with a single + // character, excluding padding (=). It is also used + // in strict mode when padding is not adequate. + BASE64_EXTRA_BITS, // The base64 input terminates with non-zero + // padding bits. + OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. + OTHER // Not related to validation/transcoding. +}; +#if SIMDUTF_CPLUSPLUS17 +inline std::string_view error_to_string(error_code code) noexcept { + switch (code) { + case SUCCESS: + return "SUCCESS"; + case HEADER_BITS: + return "HEADER_BITS"; + case TOO_SHORT: + return "TOO_SHORT"; + case TOO_LONG: + return "TOO_LONG"; + case OVERLONG: + return "OVERLONG"; + case TOO_LARGE: + return "TOO_LARGE"; + case SURROGATE: + return "SURROGATE"; + case INVALID_BASE64_CHARACTER: + return "INVALID_BASE64_CHARACTER"; + case BASE64_INPUT_REMAINDER: + return "BASE64_INPUT_REMAINDER"; + case BASE64_EXTRA_BITS: + return "BASE64_EXTRA_BITS"; + case OUTPUT_BUFFER_TOO_SMALL: + return "OUTPUT_BUFFER_TOO_SMALL"; + default: + return "OTHER"; + } +} +#endif + +struct result { + error_code error; + size_t count; // In case of error, indicates the position of the error. In + // case of success, indicates the number of code units + // validated/written. + + simdutf_really_inline simdutf_constexpr23 result() noexcept + : error{error_code::SUCCESS}, count{0} {} + + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept + : error{err}, count{pos} {} + + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { + return error == error_code::SUCCESS; + } + + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { + return error != error_code::SUCCESS; + } +}; + +struct full_result { + error_code error; + size_t input_count; + size_t output_count; + bool padding_error = false; // true if the error is due to padding, only + // meaningful when error is not SUCCESS + + simdutf_really_inline simdutf_constexpr23 full_result() noexcept + : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} + + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out} {} + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out}, + padding_error{padding_err} {} + + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { + if (error == error_code::SUCCESS) { + return result{error, output_count}; + } else { + return result{error, input_count}; + } + } +}; + +} // namespace simdutf +#endif +/* end file include/simdutf/error.h */ + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + +// Public API +/* begin file include/simdutf/simdutf_version.h */ +// /include/simdutf/simdutf_version.h automatically generated by release.py, +// do not change by hand +#ifndef SIMDUTF_SIMDUTF_VERSION_H +#define SIMDUTF_SIMDUTF_VERSION_H + +/** The version of simdutf being used (major.minor.revision) */ +#define SIMDUTF_VERSION "8.0.0" + +namespace simdutf { +enum { + /** + * The major version (MAJOR.minor.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MAJOR = 8, + /** + * The minor version (major.MINOR.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MINOR = 0, + /** + * The revision (major.minor.REVISION) of simdutf being used. + */ + SIMDUTF_VERSION_REVISION = 0 +}; +} // namespace simdutf + +#endif // SIMDUTF_SIMDUTF_VERSION_H +/* end file include/simdutf/simdutf_version.h */ +/* begin file include/simdutf/implementation.h */ +#ifndef SIMDUTF_IMPLEMENTATION_H +#define SIMDUTF_IMPLEMENTATION_H +#if !defined(SIMDUTF_NO_THREADS) + #include <atomic> +#endif +#include <string> +#ifdef SIMDUTF_INTERNAL_TESTS + #include <vector> +#endif +/* begin file include/simdutf/internal/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDutf_INTERNAL_ISADETECTION_H +#define SIMDutf_INTERNAL_ISADETECTION_H + +#include <cstdint> +#include <cstdlib> +#if defined(_MSC_VER) + #include <intrin.h> +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + #include <cpuid.h> +#endif + + +// RISC-V ISA detection utilities +#if SIMDUTF_IS_RISCV64 && defined(__linux__) + #include <unistd.h> // for syscall +// We define these ourselves, for backwards compatibility +struct simdutf_riscv_hwprobe { + int64_t key; + uint64_t value; +}; + #define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__) + #define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4 + #define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2) + #define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17) +#endif // SIMDUTF_IS_RISCV64 && defined(__linux__) + +#if defined(__loongarch__) && defined(__linux__) + #include <sys/auxv.h> +// bits/hwcap.h +// #define HWCAP_LOONGARCH_LSX (1 << 4) +// #define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +namespace simdutf { +namespace internal { + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40, + ALTIVEC = 0x80, + AVX512F = 0x100, + AVX512DQ = 0x200, + AVX512IFMA = 0x400, + AVX512PF = 0x800, + AVX512ER = 0x1000, + AVX512CD = 0x2000, + AVX512BW = 0x4000, + AVX512VL = 0x8000, + AVX512VBMI2 = 0x10000, + AVX512VPOPCNTDQ = 0x2000, + RVV = 0x4000, + ZVBB = 0x8000, + LSX = 0x40000, + LASX = 0x80000, +}; + +#if defined(__PPC64__) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::ALTIVEC; +} + +#elif SIMDUTF_IS_RISCV64 + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if SIMDUTF_IS_RVV + host_isa |= instruction_set::RVV; + #endif + #if SIMDUTF_IS_ZVBB + host_isa |= instruction_set::ZVBB; + #endif + #if defined(__linux__) + simdutf_riscv_hwprobe probes[] = {{SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0}}; + long ret = simdutf_riscv_hwprobe(&probes, sizeof probes / sizeof *probes, 0, + nullptr, 0); + if (ret == 0) { + uint64_t extensions = probes[0].value; + if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V) + host_isa |= instruction_set::RVV; + if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB) + host_isa |= instruction_set::ZVBB; + } + #endif + #if defined(RUN_IN_SPIKE_SIMULATOR) + // Proxy Kernel does not implement yet hwprobe syscall + host_isa |= instruction_set::RVV; + #endif + return host_isa; +} + +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#elif defined(__x86_64__) || defined(_M_AMD64) // x64 + +namespace { +namespace cpuid_bit { +// Can be found on Intel ISA Reference for CPUID + +// EAX = 0x01 +constexpr uint32_t pclmulqdq = uint32_t(1) + << 1; ///< @private bit 1 of ECX for EAX=0x1 +constexpr uint32_t sse42 = uint32_t(1) + << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t osxsave = + (uint32_t(1) << 26) | + (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + +// EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf) +// See: "Table 3-8. Information Returned by CPUID Instruction" +namespace ebx { +constexpr uint32_t bmi1 = uint32_t(1) << 3; +constexpr uint32_t avx2 = uint32_t(1) << 5; +constexpr uint32_t bmi2 = uint32_t(1) << 8; +constexpr uint32_t avx512f = uint32_t(1) << 16; +constexpr uint32_t avx512dq = uint32_t(1) << 17; +constexpr uint32_t avx512ifma = uint32_t(1) << 21; +constexpr uint32_t avx512cd = uint32_t(1) << 28; +constexpr uint32_t avx512bw = uint32_t(1) << 30; +constexpr uint32_t avx512vl = uint32_t(1) << 31; +} // namespace ebx + +namespace ecx { +constexpr uint32_t avx512vbmi = uint32_t(1) << 1; +constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6; +constexpr uint32_t avx512vnni = uint32_t(1) << 11; +constexpr uint32_t avx512bitalg = uint32_t(1) << 12; +constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14; +} // namespace ecx +namespace edx { +constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8; +} +namespace xcr0_bit { +constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX +constexpr uint64_t avx512_saved = + uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM +} // namespace xcr0_bit +} // namespace cpuid_bit +} // namespace + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { + #if defined(_MSC_VER) + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; + #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); + #else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; + #endif +} + +static inline uint64_t xgetbv() { + #if defined(_MSC_VER) + return _xgetbv(0); + #else + uint32_t xcr0_lo, xcr0_hi; + asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); + #endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax; + uint32_t ebx = 0; + uint32_t ecx = 0; + uint32_t edx = 0; + uint32_t host_isa = 0x0; + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_bit::sse42) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_bit::pclmulqdq) { + host_isa |= instruction_set::PCLMULQDQ; + } + + if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) { + return host_isa; + } + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; // Sub-leaf = 0 + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_bit::ebx::avx2) { + host_isa |= instruction_set::AVX2; + } + if (ebx & cpuid_bit::ebx::bmi1) { + host_isa |= instruction_set::BMI1; + } + if (ebx & cpuid_bit::ebx::bmi2) { + host_isa |= instruction_set::BMI2; + } + if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == + cpuid_bit::xcr0_bit::avx512_saved)) { + return host_isa; + } + if (ebx & cpuid_bit::ebx::avx512f) { + host_isa |= instruction_set::AVX512F; + } + if (ebx & cpuid_bit::ebx::avx512bw) { + host_isa |= instruction_set::AVX512BW; + } + if (ebx & cpuid_bit::ebx::avx512cd) { + host_isa |= instruction_set::AVX512CD; + } + if (ebx & cpuid_bit::ebx::avx512dq) { + host_isa |= instruction_set::AVX512DQ; + } + if (ebx & cpuid_bit::ebx::avx512vl) { + host_isa |= instruction_set::AVX512VL; + } + if (ecx & cpuid_bit::ecx::avx512vbmi2) { + host_isa |= instruction_set::AVX512VBMI2; + } + if (ecx & cpuid_bit::ecx::avx512vpopcnt) { + host_isa |= instruction_set::AVX512VPOPCNTDQ; + } + return host_isa; +} +#elif defined(__loongarch__) + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if defined(__linux__) + uint64_t hwcap = 0; + hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_LOONGARCH_LSX) { + host_isa |= instruction_set::LSX; + } + if (hwcap & HWCAP_LOONGARCH_LASX) { + host_isa |= instruction_set::LASX; + } + #endif + return host_isa; +} +#else // fallback + +// includes 32-bit ARM. +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif // end SIMD extension detection code + +} // namespace internal +} // namespace simdutf + +#endif // SIMDutf_INTERNAL_ISADETECTION_H +/* end file include/simdutf/internal/isadetection.h */ + +#if SIMDUTF_SPAN + #include <concepts> + #include <type_traits> + #include <span> + #include <tuple> +#endif +#if SIMDUTF_CPLUSPLUS17 + #include <string_view> +#endif +// The following defines are conditionally enabled/disabled during amalgamation. +// By default all features are enabled, regular code shouldn't check them. Only +// when user code really relies of a selected subset, it's good to verify these +// flags, like: +// +// #if !SIMDUTF_FEATURE_UTF16 +// # error("Please amalgamate simdutf with UTF-16 support") +// #endif +// +#define SIMDUTF_FEATURE_DETECT_ENCODING 1 +#define SIMDUTF_FEATURE_ASCII 1 +#define SIMDUTF_FEATURE_LATIN1 1 +#define SIMDUTF_FEATURE_UTF8 1 +#define SIMDUTF_FEATURE_UTF16 1 +#define SIMDUTF_FEATURE_UTF32 1 +#define SIMDUTF_FEATURE_BASE64 1 + +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include <cstddef> + +namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template <typename to, typename from> + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; + + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast<to>(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast<to>(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template <typename to, typename from> +constexpr constexpr_ptr<to, from> constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr<to, from>{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template <typename SrcType, typename TargetType> +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast<TargetType>(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template <typename SrcType, typename TargetType> struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy<SrcType, TargetType> operator*() const { + return constexpr_write_ptr_proxy<SrcType, TargetType>{p}; + } + + constexpr constexpr_write_ptr_proxy<SrcType, TargetType> + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy<SrcType, TargetType>{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template <typename SrcType, typename TargetType> +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr<SrcType, TargetType>{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif + +#if SIMDUTF_SPAN +/// helpers placed in namespace detail are not a part of the public API +namespace simdutf { +namespace detail { +/** + * matches a byte, in the many ways C++ allows. note that these + * are all distinct types. + */ +template <typename T> +concept byte_like = std::is_same_v<T, std::byte> || // + std::is_same_v<T, char> || // + std::is_same_v<T, signed char> || // + std::is_same_v<T, unsigned char> || // + std::is_same_v<T, char8_t>; + +template <typename T> +concept is_byte_like = byte_like<std::remove_cvref_t<T>>; + +template <typename T> +concept is_pointer = std::is_pointer_v<T>; + +/** + * matches anything that behaves like std::span and points to character-like + * data such as: std::byte, char, unsigned char, signed char, std::int8_t, + * std::uint8_t + */ +template <typename T> +concept input_span_of_byte_like = requires(const T &t) { + { t.size() } noexcept -> std::convertible_to<std::size_t>; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; +}; + +template <typename T> +concept is_mutable = !std::is_const_v<std::remove_reference_t<T>>; + +/** + * like span_of_byte_like, but for an output span (intended to be written to) + */ +template <typename T> +concept output_span_of_byte_like = requires(T &t) { + { t.size() } noexcept -> std::convertible_to<std::size_t>; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; + { *t.data() } noexcept -> is_mutable; +}; + +/** + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array<char,10> + * invalid examples: int*, std::array<int,10> + */ +template <class InputPtr> +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> simdutf::detail::byte_like; +}; +template <class InputPtr> +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<char16_t>; +}; +template <class InputPtr> +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<char32_t>; +}; + +template <class InputPtr> +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; + +/** + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* + */ +template <class InputPtr> +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<std::uint32_t>; +}; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN + +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); +} + +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 +} + +namespace utf32 { +template <endianness big_endian> constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; +} +} // namespace utf32 + +namespace utf16 { +template <endianness big_endian> constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; +} +} // namespace utf16 + +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template <class InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template <class InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include <atomic> +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast<char *>(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref<char>(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast<std::uintptr_t>(src) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast<uint64_t *>(const_cast<char *>(src)); + const auto dst_value = + std::atomic_ref<uint64_t>(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref<char>(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast<std::uintptr_t>(dst) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast<uint64_t *>(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref<uint64_t>(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast<const uint8_t *>(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template <endianness big_endian> +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast<const unsigned char *>(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast<const unsigned char *>(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast<const unsigned char *>(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast<uint8_t>(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template <endianness big_endian> +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template <endianness big_endian> +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template <endianness big_endian> +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template <endianness big_endian> +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template <endianness big_endian> +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter++; // ASCII + counter += static_cast<size_t>( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast<size_t>((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template <endianness big_endian> +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template <endianness big_endian> +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed<big_endian>(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template <endianness big_endian> +simdutf_constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed<big_endian>(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template <endianness big_endian> +simdutf_constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed<big_endian>(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +simdutf_really_inline constexpr bool low_surrogate(char16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +template <endianness big_endian> +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate<big_endian>(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate<big_endian>(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate<big_endian>(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast<size_t>(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast<size_t>(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template <endianness big_endian> constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template <endianness big_endian> +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement<big_endian>(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate<big_endian>(c); + low_surrogate = is_low_surrogate<big_endian>(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include <cstring> // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if simdutf_constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template <endianness big_endian, class InputIterator, class OutputIterator> +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template <endianness big_endian> +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl<big_endian>(reinterpret_cast<const uint16_t *>(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template <endianness big_endian> +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16<InputPtr> +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template <endianness big_endian, bool check_output = false, typename InputPtr, + typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template <endianness big_endian> +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors<big_endian, false>(buf, len, utf8_output, 0); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast<const uint32_t *>(buf), len); +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast<const uint32_t *>(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast<size_t>(p[i] > 0x7F); // two-byte + counter += static_cast<size_t>(p[i] > 0x7FF); // three-byte + counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast<size_t>(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template <typename ReadPtr, typename WritePtr> +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast<const uint32_t *>(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template <endianness big_endian> +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template <class BytePtr> +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast<const uint8_t *>(buf), len); +} + +template <class BytePtr> +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; + return res; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::indexes_into_byte_like<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template <endianness endian> +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors<endian>(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + +#if SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const char *input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const uint8_t *input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast<const char *>(input), length); +} + #if SIMDUTF_SPAN +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. can be a anything span-like that has a + * data() and size() that points to character data: std::string, + * std::string_view, std::vector<char>, std::span<const std::byte> etc. + * @return the detected encoding type + */ +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding( + const detail::input_span_of_byte_like auto &input) noexcept { + return autodetect_encoding(reinterpret_cast<const char *>(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN + +/** + * Autodetect the possible encodings of the input in one pass. + * E.g., if the input might be UTF-16LE or UTF-8, this function returns + * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). + * + * Overridden by each implementation. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused int detect_encodings(const char *input, + size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const uint8_t *input, size_t length) noexcept { + return detect_encodings(reinterpret_cast<const char *>(input), length); +} + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { + return detect_encodings(reinterpret_cast<const char *>(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-8 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf8_with_errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool +validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf8(reinterpret_cast<const char *>(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 +/** + * Validate the UTF-8 string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf8_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +validate_utf8_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate_with_errors( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf8_with_errors( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate( + detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_ascii(reinterpret_cast<const char *>(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string and stop on error. It might be faster than + * validate_utf8 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_ascii_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_ascii_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate_with_errors( + detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_ascii_with_errors( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string as a UTF-16 sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::NATIVE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::BIG>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16le_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::LITTLE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Validate the UTF-16 string. + * This function may be best when you expect the input to be almost always + * valid. Otherwise, consider using validate_utf16_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::NATIVE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-16LE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ +simdutf_warn_unused bool validate_utf16le(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool +validate_utf16le(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::LITTLE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 +/** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ +simdutf_warn_unused bool validate_utf16be(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::BIG>(input.data(), input.size()); + } else + #endif + { + return validate_utf16be(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Validate the UTF-16 string and stop on error. + * It might be faster than validate_utf16 when an error is expected to occur + * early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::NATIVE>( + input.data(), input.size()); + } else + #endif + { + return validate_utf16_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16le_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::LITTLE>( + input.data(), input.size()); + } else + #endif + { + return validate_utf16le_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16be_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::BIG>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16le(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::LITTLE>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16le(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16be(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::BIG>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16be(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. If input and output points to different + * memory areas, the procedure copies string, and it's expected that output + * memory is at least as big as the input. It's also possible to set input equal + * output, that makes replacements an in-place operation. + * + * @param input the UTF-16 string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-32 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf32_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ +simdutf_warn_unused bool validate_utf32(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf32(std::span<const char32_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate( + detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf32(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 +/** + * Validate the UTF-32 string and stop on error. It might be faster than + * validate_utf32 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf32_with_errors(std::span<const char32_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate_with_errors( + detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf32_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input, + size_t length, + char *utf8_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8( + const detail::input_span_of_byte_like auto &latin1_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert( + detail::constexpr_cast_ptr<char>(latin1_input.data()), + latin1_input.size(), + detail::constexpr_cast_writeptr<char>(utf8_output.data())); + } else + #endif + { + return convert_latin1_to_utf8( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-8 string with output limit. + * + * This function is suitable to work with inputs from untrusted sources. + * + * We write as many characters as possible. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t +convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert_safe_constexpr( + input.data(), input.size(), utf8_output.data(), utf8_output.size()); + } else + #endif + { + return convert_latin1_to_utf8_safe( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16le( + const detail::input_span_of_byte_like auto &latin1_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::LITTLE>( + latin1_input.data(), latin1_input.size(), utf16_output.data()); + } else + #endif + { + return convert_latin1_to_utf16le( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::BIG>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16be( + reinterpret_cast<const char *>(input.data()), input.size(), + output.data()); + } +} + #endif // SIMDUTF_SPAN +/** + * Compute the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in Latin1 code units (char) required to + * encode the UTF-16 string as Latin1 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf16(size_t length) noexcept { + return length; +} + +/** + * Compute the number of code units that this Latin1 string would require in + * UTF-16 format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 2-byte code units (char16_t) required to + * encode the Latin1 string as UTF-16 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf32( + const detail::input_span_of_byte_like auto &latin1_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf32::convert( + latin1_input.data(), latin1_input.size(), utf32_output.data()); + } else + #endif + { + return convert_latin1_to_utf32( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input, + size_t length, + char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_latin1( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert(input.data(), input.size(), + output.data()); + } else + #endif + { + return convert_utf8_to_latin1(reinterpret_cast<const char *>(input.data()), + input.size(), + reinterpret_cast<char *>(output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_utf8_to_utf16(reinterpret_cast<const char *>(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format even when the UTF-16LE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +utf8_length_from_utf16le_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format even when the UTF-16BE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16be_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert a Latin1 string into a UTF-16 string. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t. + */ +simdutf_warn_unused size_t convert_latin1_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16(reinterpret_cast<const char *>(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::LITTLE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::BIG>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_latin1_with_errors( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_latin1_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_with_errors( + utf8_input.data(), utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf8_to_latin1_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::NATIVE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16le_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::LITTLE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16be_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::BIG>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf32_with_errors( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf32_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_valid_utf8_to_latin1( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_latin1( + const detail::input_span_of_byte_like auto &valid_utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_valid_utf8_to_latin1( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), latin1_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert valid UTF-8 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::NATIVE>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16le( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::LITTLE>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16le( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16be( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::BIG>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16be( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf32( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf32( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_latin1(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_latin1( + const detail::input_span_of_byte_like auto &latin1_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(), + latin1_input.size()); + } else + #endif + { + return utf8_length_from_latin1( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ +simdutf_warn_unused size_t latin1_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return latin1_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Compute the number of 2-byte code units that this UTF-8 string would require + * in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 string + * as UTF-16LE + */ +simdutf_warn_unused size_t utf16_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf16_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of 4-byte code units that this UTF-8 string would require + * in UTF-32 format. + * + * This function is equivalent to count_utf8 + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 string + * as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf32_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string with output limit. + * + * We write as many characters as possible into the output buffer, + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 16-bit code units (char16_t) + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input, + size_t length, + char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_safe( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + const full_result r = + scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE, true>( + utf16_input.data(), utf16_input.size(), utf8_output.data(), + utf8_output.size()); + if (r.error != error_code::SUCCESS && + r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + return 0; + } + return r.output_count; + } else + #endif + { + return convert_utf16_to_utf8_safe( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert UTF-16 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::NATIVE>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16LE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16le_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::LITTLE>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16le_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16BE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16be_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::BIG>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16be_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-32 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into + * UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf32_with_errors(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf32_with_errors( + std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf32_with_errors( + std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; compute the number of bytes that this UTF-16 + * string would require in UTF-8 format even when the UTF-16LE content contains + * mismatched surrogates that have to be replaced by the replacement character + * (0xFFFD). + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf8( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf8_with_errors( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf8_with_errors( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf8( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf8( + std::span<const char32_t> valid_utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf8( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-32 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::NATIVE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16le(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::LITTLE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_latin1( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_latin1_with_errors( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_latin1_with_errors( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_with_errors( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf32_to_latin1( + std::span<const char32_t> valid_utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_valid( + detail::constexpr_cast_ptr<uint32_t>(valid_utf32_input.data()), + valid_utf32_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } + #endif + { + return convert_valid_utf32_to_latin1( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +latin1_length_from_utf32(size_t length) noexcept { + return length; +} + +/** + * Compute the number of bytes that this Latin1 string would require in UTF-32 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 4-byte code units (char32_t) required to + * encode the Latin1 string as UTF-32 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +utf32_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16be(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::BIG>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-32 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16_with_errors(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::NATIVE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16le_with_errors( + std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::LITTLE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16be_with_errors( + std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::BIG>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-32 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::NATIVE>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16le(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::LITTLE>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16le(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16be(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::BIG>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16be(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ +void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(std::span<const char16_t> utf16_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::change_endianness_utf16( + utf16_input.data(), utf16_input.size(), utf16_output.data()); + } else + #endif + { + return change_endianness_utf16(utf16_input.data(), utf16_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ +simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-32 format. + * + * This function is equivalent to count_utf16. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16le( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16be( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). It is acceptable to pass invalid UTF-16 strings but in such + * cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ +simdutf_warn_unused size_t count_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return count_utf8(reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-8 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so + * that the short UTF-8 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-8, but possibly + * truncated. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes + */ +simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return trim_partial_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Given a valid UTF-16BE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16BE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16BE, but possibly + * truncated. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16LE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16LE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16LE, but possibly + * truncated. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16, but possibly + * truncated. We use the native endianness. + * + * @param input the UTF-16 string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || \ + SIMDUTF_FEATURE_DETECT_ENCODING + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif +#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_BASE64 +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include <cstdint> + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <iostream> + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template <class char_type> bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template <class char_type> simdutf_constexpr23 bool is_eight_byte(char_type c) { + if simdutf_constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template <class char_type> +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template <class char_type> +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template <class char_type> +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template <class char_type> +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template <class char_type> +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template <bool check_capacity, class char_type> +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>(x & 0xFF); + *dst++ = static_cast<char>((x >> 8) & 0xFF); + *dst++ = static_cast<char>((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + *dst++ = static_cast<char>((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + *dst++ = static_cast<char>((triple >> 8) & 0xFF); + *dst++ = static_cast<char>(triple & 0xFF); + } +} + +template <class char_type> +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl<false>(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template <class char_type> +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl<true>(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template <bool use_lines = false> +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if simdutf_constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if simdutf_constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src, + size_t srclen, + base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template <class InputPtr> +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +template <typename char_type> +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template <typename char_type> +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template <typename char_type> +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen +} + +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(base64_options options) { + switch (options) { + case base64_default: + return "base64_default"; + case base64_url: + return "base64_url"; + case base64_reverse_padding: + return "base64_reverse_padding"; + case base64_url_with_padding: + return "base64_url_with_padding"; + case base64_default_accept_garbage: + return "base64_default_accept_garbage"; + case base64_url_accept_garbage: + return "base64_url_accept_garbage"; + case base64_default_or_url: + return "base64_default_or_url"; + case base64_default_or_url_accept_garbage: + return "base64_default_or_url_accept_garbage"; + } + return "<unknown>"; +} + #endif // SIMDUTF_CPLUSPLUS17 + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(last_chunk_handling_options options) { + switch (options) { + case loose: + return "loose"; + case strict: + return "strict"; + case stop_before_partial: + return "stop_before_partial"; + case only_full_chunks: + return "only_full_chunks"; + } + return "<unknown>"; +} + #endif + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximum number of binary bytes + */ +simdutf_warn_unused size_t +maximal_binary_length_from_base64(const char *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ +simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, usually base64_default or + * base64_url, and base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if successful. + */ +simdutf_warn_unused result base64_to_binary( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast<const char *>(input.data()), + input.size(), + reinterpret_cast<char *>(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} + +/** + * Provide the base64 length in bytes given the length of a binary input, + * taking into account line breaks. + * + * @param length the length of the input in bytes + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( + size_t length, base64_options options = base64_default, + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} + +/** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a binary input to a base64 output with line breaks. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options) + */ +size_t +binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl<true>( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), line_length, options); + } +} + #endif // SIMDUTF_SPAN + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a binary input to a base64 output, using atomic accesses. + * This function comes with a potentially significant performance + * penalty, but it may be useful in some cases where the input + * buffers are shared between threads, to avoid undefined + * behavior in case of data races. + * + * The function is for advanced users. Its main use case is when + * to silence sanitizer warnings. We have no documented use case + * where this function is actually necessary in terms of practical correctness. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @brief atomic_binary_to_base64 + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t +atomic_binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused size_t +atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + return atomic_binary_to_base64( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), options); +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail + * to provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of bytes written if successful. + */ +simdutf_warn_unused result +base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span<const char16_t> input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast<char *>(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Check if a character is an ignorable base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is an ignorable base64 character, false + * otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} + +/** + * Check if a character is a valid base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * Note that padding characters are not considered valid base64 characters in + * this context, nor are spaces. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} + +/** + * Check if a character is a valid base64 character or the padding character + * ('='). Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are three possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer + * is too small (OUTPUT_BUFFER_TOO_SMALL). + * + * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written + * and the number of units processed, see description of the parameters and + * returned value. + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected + * to discard the output unless the parameter decode_up_to_bad_char is set to + * true. In that case, the function will decode up to the first invalid + * character. Extra padding characters ('=') are considered invalid characters. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 8-bit + * or 16-bit units + * @param length the length of the string in 8-bit or 16-bit units. + * @param output the pointer to a buffer that can hold the conversion + * result. + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of units processed if successful. + */ +simdutf_warn_unused result +base64_to_binary_safe(const char *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +// the span overload has moved to the bottom of the file + +simdutf_warn_unused result +base64_to_binary_safe(const char16_t *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; + // span overload moved to bottom of file + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a base64 input to a binary output with a size limit and using atomic + * operations. + * + * Like `base64_to_binary_safe` but using atomic operations, this function is + * thread-safe for concurrent memory access, allowing the output + * buffers to be shared between threads without undefined behavior in case of + * data races. + * + * This function comes with a potentially significant performance penalty, but + * is useful when thread safety is needed during base64 decoding. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @param input the base64 input to decode + * @param length the length of the input in bytes + * @param output the pointer to buffer that can hold the conversion + * result + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use (default, url, etc.) + * @param last_chunk_options the last chunk handling options (loose, strict, + * stop_before_partial) + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result struct with an error code and count indicating error + * position or success + */ +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char16_t *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept; + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline simdutf_warn_unused std::tuple<result, std::size_t> +atomic_base64_to_binary_safe( + const detail::input_span_of_byte_like auto &binary_input, + detail::output_span_of_byte_like auto &&output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = output.size(); + auto ret = atomic_base64_to_binary_safe( + reinterpret_cast<const char *>(binary_input.data()), binary_input.size(), + reinterpret_cast<char *>(output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_warn_unused std::tuple<result, std::size_t> +atomic_base64_to_binary_safe( + std::span<const char16_t> base64_input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + auto ret = atomic_base64_to_binary_safe( + base64_input.data(), base64_input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +#endif // SIMDUTF_FEATURE_BASE64 + +/** + * An implementation of simdutf for a particular CPU architecture. + * + * Also used to maintain the currently active implementation. The active + * implementation is automatically initialized on first use to the most advanced + * implementation supported by the host. + */ +class implementation { +public: + /** + * The name of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string name() const { return std::string(_name); } + + /** + * The description of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string description() const { return std::string(_description); } + + /** + * The instruction sets this implementation is compiled against + * and the current CPU match. This function may poll the current CPU/system + * and should therefore not be called too often if performance is a concern. + * + * + * @return true if the implementation can be safely used on the current system + * (determined at runtime) + */ + bool supported_by_runtime_system() const; + +#if SIMDUTF_FEATURE_DETECT_ENCODING + /** + * This function will try to detect the encoding + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual encoding_type autodetect_encoding(const char *input, + size_t length) const noexcept; + + /** + * This function will try to detect the possible encodings in one pass + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual int detect_encodings(const char *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + + /** + * @private For internal implementation use + * + * The instruction sets this implementation is compiled against. + * + * @return a mask of all required `internal::instruction_set::` values + */ + virtual uint32_t required_instruction_sets() const { + return _required_instruction_sets; + } + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ + simdutf_warn_unused virtual bool validate_utf8(const char *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 + /** + * Validate the UTF-8 string and stop on errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_ascii(const char *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-16LE string.This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ + simdutf_warn_unused virtual bool + validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 + /** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ + simdutf_warn_unused virtual bool + validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16le_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + + /** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16be_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + /** + * Copies the UTF-16LE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; + /** + * Copies the UTF-16BE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-32 string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ + simdutf_warn_unused virtual bool + validate_utf32(const char32_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 + /** + * Validate the UTF-32 string and stop on error. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf32_with_errors(const char32_t *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf8(const char *input, size_t length, + char *utf8_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_latin1_with_errors(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format even when the UTF-16LE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format even when the UTF-16BE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_utf32_with_errors(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of 2-byte code units that this UTF-8 string would + * require in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 + * string as UTF-16LE + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of 4-byte code units that this UTF-8 string would + * require in UTF-32 format. + * + * This function is equivalent to count_utf8. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function does not validate the input. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 + * string as UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-L16LE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF16-BE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32BE string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_latin1_with_errors(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_utf8_with_errors(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ + virtual void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_latin1(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf32(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-16LE/BE string would require in + * Latin1 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf16(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16le(const char16_t *input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16be(const char16_t *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_BASE64 + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char *input, size_t length) const noexcept; + + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) const noexcept; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if + * successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you + * fail to provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process, in ASCII stored as + * 16-bit units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the + * number of bytes written if successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of base64 bytes + */ + simdutf_warn_unused size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) const noexcept; + + /** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ + virtual size_t + binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) const noexcept = 0; + + /** + * Convert a binary input to a base64 output with lines of given length. + * Lines are separated by a single linefeed character. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of each line, values smaller than 4 are + * interpreted as 4 + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options, line_length) + */ + virtual size_t binary_to_base64_with_lines( + const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) const noexcept = 0; + + /** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ + virtual const char *find(const char *start, const char *end, + char character) const noexcept = 0; + virtual const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept = 0; +#endif // SIMDUTF_FEATURE_BASE64 + +#ifdef SIMDUTF_INTERNAL_TESTS + // This method is exported only in developer mode, its purpose + // is to expose some internal test procedures from the given + // implementation and then use them through our standard test + // framework. + // + // Regular users should not use it, the tests of the public + // API are enough. + + struct TestProcedure { + // display name + std::string name; + + // procedure should return whether given test pass or not + void (*procedure)(const implementation &); + }; + + virtual std::vector<TestProcedure> internal_tests() const; +#endif + +protected: + /** @private Construct an implementation with the given name and description. + * For subclasses. */ + simdutf_really_inline implementation(const char *name, + const char *description, + uint32_t required_instruction_sets) + : _name(name), _description(description), + _required_instruction_sets(required_instruction_sets) {} + +protected: + ~implementation() = default; + +private: + /** + * The name of this implementation. + */ + const char *_name; + + /** + * The description of this implementation. + */ + const char *_description; + + /** + * Instruction sets required for this implementation. + */ + const uint32_t _required_instruction_sets; +}; + +/** @private */ +namespace internal { + +/** + * The list of available implementations compiled into simdutf. + */ +class available_implementation_list { +public: + /** Get the list of available implementations compiled into simdutf */ + simdutf_really_inline available_implementation_list() {} + /** Number of implementations */ + size_t size() const noexcept; + /** STL const begin() iterator */ + const implementation *const *begin() const noexcept; + /** STL const end() iterator */ + const implementation *const *end() const noexcept; + + /** + * Get the implementation with the given name. + * + * Case sensitive. + * + * const implementation *impl = + * simdutf::available_implementations["westmere"]; if (!impl) { exit(1); } if + * (!imp->supported_by_runtime_system()) { exit(1); } + * simdutf::active_implementation = impl; + * + * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" + * @return the implementation, or nullptr if the parse failed. + */ + const implementation *operator[](const std::string &name) const noexcept { + for (const implementation *impl : *this) { + if (impl->name() == name) { + return impl; + } + } + return nullptr; + } + + /** + * Detect the most advanced implementation supported by the current host. + * + * This is used to initialize the implementation on startup. + * + * const implementation *impl = + * simdutf::available_implementation::detect_best_supported(); + * simdutf::active_implementation = impl; + * + * @return the most advanced supported implementation for the current host, or + * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no + * supported implementation. Will never return nullptr. + */ + const implementation *detect_best_supported() const noexcept; +}; + +template <typename T> class atomic_ptr { +public: + atomic_ptr(T *_ptr) : ptr{_ptr} {} + +#if defined(SIMDUTF_NO_THREADS) + operator const T *() const { return ptr; } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr; } + + operator T *() { return ptr; } + T &operator*() { return *ptr; } + T *operator->() { return ptr; } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#else + operator const T *() const { return ptr.load(); } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr.load(); } + + operator T *() { return ptr.load(); } + T &operator*() { return *ptr; } + T *operator->() { return ptr.load(); } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#endif + +private: +#if defined(SIMDUTF_NO_THREADS) + T *ptr; +#else + std::atomic<T *> ptr; +#endif +}; + +class detect_best_supported_implementation_on_first_use; + +} // namespace internal + +/** + * The list of available implementations compiled into simdutf. + */ +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & +get_available_implementations(); + +/** + * The active implementation. + * + * Automatically initialized on first use to the most advanced implementation + * supported by this hardware. + */ +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> & +get_active_implementation(); + +} // namespace simdutf + +#if SIMDUTF_FEATURE_BASE64 + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +namespace simdutf { + +template <typename chartype> +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template <typename chartype> +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same<chartype, char>::value || + std::is_same<chartype, char16_t>::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = (std::min)( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple<result, std::size_t> + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t<decltype(*input.data())>; + static_assert(std::is_same_v<CInput, char>, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t<decltype(*binary_output.data())>; + static_assert(std::is_same_v<COutput, char>, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl<char>( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple<result, std::size_t> + base64_to_binary_safe( + std::span<const char16_t> input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#endif // SIMDUTF_FEATURE_BASE64 + +#endif // SIMDUTF_IMPLEMENTATION_H +/* end file include/simdutf/implementation.h */ + +// Implementation-internal files (must be included before the implementations +// themselves, to keep amalgamation working--otherwise, the first time a file is +// included, it might be put inside the #ifdef +// SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other +// implementations can't compile unless that implementation is turned on). + +SIMDUTF_POP_DISABLE_WARNINGS + +#endif // SIMDUTF_H +/* end file include/simdutf.h */
+ simdutf/simdutf_c.h view
@@ -0,0 +1,339 @@+/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include <stddef.h> +#include <stdbool.h> +#include <stdint.h> + +#ifdef __has_include + #if __has_include(<uchar.h>) + #include <uchar.h> + #else // __has_include(<uchar.h>) + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include(<uchar.h>) +#else // __has_include(<uchar.h>) + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { +#endif + +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */
src/Data/Text.hs view
@@ -1,2031 +1,2276 @@-{-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types, UnboxedTuples, TypeFamilies #-}-{-# LANGUAGE TemplateHaskellQuotes #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE UnliftedFFITypes #-}-{-# LANGUAGE ScopedTypeVariables #-}--{-# OPTIONS_GHC -fno-warn-orphans #-}---- |--- Module : Data.Text--- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts,--- (c) 2008, 2009 Tom Harper--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text.--- Suitable for performance critical use, both in terms of large data--- quantities and high speed.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text as T------ To use an extended and very rich family of functions for working--- with Unicode text (including normalization, regular expressions,--- non-standard encodings, text breaking, and locales), see the--- <http://hackage.haskell.org/package/text-icu text-icu package >.-----module Data.Text- (- -- * Strict vs lazy types- -- $strict-- -- * Acceptable data- -- $replacement-- -- * Definition of character- -- $character_definition-- -- * Fusion- -- $fusion-- -- * Types- Text-- -- * Creation and elimination- , pack- , unpack- , singleton- , empty-- -- * Basic interface- , cons- , snoc- , append- , uncons- , unsnoc- , head- , last- , tail- , init- , null- , length- , compareLength-- -- * Transformations- , map- , intercalate- , intersperse- , transpose- , reverse- , replace-- -- ** Case conversion- -- $case- , toCaseFold- , toLower- , toUpper- , toTitle-- -- ** Justification- , justifyLeft- , justifyRight- , center-- -- * Folds- , foldl- , foldl'- , foldl1- , foldl1'- , foldr- , foldr1-- -- ** Special folds- , concat- , concatMap- , any- , all- , maximum- , minimum-- -- * Construction-- -- ** Scans- , scanl- , scanl1- , scanr- , scanr1-- -- ** Accumulating maps- , mapAccumL- , mapAccumR-- -- ** Generation and unfolding- , replicate- , unfoldr- , unfoldrN-- -- * Substrings-- -- ** Breaking strings- , take- , takeEnd- , drop- , dropEnd- , takeWhile- , takeWhileEnd- , dropWhile- , dropWhileEnd- , dropAround- , strip- , stripStart- , stripEnd- , splitAt- , breakOn- , breakOnEnd- , break- , span- , group- , groupBy- , inits- , tails-- -- ** Breaking into many substrings- -- $split- , splitOn- , split- , chunksOf-- -- ** Breaking into lines and words- , lines- --, lines'- , words- , unlines- , unwords-- -- * Predicates- , isPrefixOf- , isSuffixOf- , isInfixOf-- -- ** View patterns- , stripPrefix- , stripSuffix- , commonPrefixes-- -- * Searching- , filter- , breakOnAll- , find- , elem- , partition-- -- , findSubstring-- -- * Indexing- -- $index- , index- , findIndex- , count-- -- * Zipping- , zip- , zipWith-- -- -* Ordered text- -- , sort-- -- * Low level operations- , copy- , unpackCString#- , unpackCStringAscii#-- , measureOff- ) where--import Prelude (Char, Bool(..), Int, Maybe(..), String,- Eq, (==), (/=), Ord(..), Ordering(..), (++),- Read(..),- (&&), (||), (+), (-), (.), ($), ($!), (>>),- not, return, otherwise, quot, IO)-import Control.DeepSeq (NFData(rnf))-#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Bits ((.&.))-import Data.Char (isSpace, isAscii, ord)-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,- Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Control.Monad (foldM)-import Control.Monad.ST (ST, runST)-import Control.Monad.ST.Unsafe (unsafeIOToST)-import qualified Data.Text.Array as A-import qualified Data.List as L-import Data.Binary (Binary(get, put))-import Data.Int (Int8)-import Data.Monoid (Monoid(..))-import Data.Semigroup (Semigroup(..))-import Data.String (IsString(..))-import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr2, chr3, chr4, ord2, ord3, ord4)-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import Data.Text.Encoding (decodeUtf8', encodeUtf8)-import Data.Text.Internal.Fusion (stream, reverseStream, unstream)-import Data.Text.Internal.Private (span_)-import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text, append)-import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8)-import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#)-import qualified Prelude as P-import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter,- reverseIter_, unsafeHead, unsafeTail, unsafeDupablePerformIO, iterArray, reverseIterArray)-import Data.Text.Internal.Search (indices)-#if defined(__HADDOCK__)-import Data.ByteString (ByteString)-import qualified Data.Text.Lazy as L-import Data.Int (Int64)-#endif-import Data.Word (Word8)-import Foreign.C.Types-import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt, ByteArray#)-import qualified GHC.Exts as Exts-import GHC.Stack (HasCallStack)-import qualified Language.Haskell.TH.Lib as TH-import qualified Language.Haskell.TH.Syntax as TH-import Text.Printf (PrintfArg, formatArg, formatString)-import System.Posix.Types (CSsize(..))---- $setup--- >>> import Data.Text--- >>> import qualified Data.Text as T--- >>> :seti -XOverloadedStrings---- $character_definition------ This package uses the term /character/ to denote Unicode /code points/.------ Note that this is not the same thing as a grapheme (e.g. a--- composition of code points that form one visual symbol). For--- instance, consider the grapheme \"ä\". This symbol has two--- Unicode representations: a single code-point representation--- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),--- and a two code point representation @U+0061@ (the \"@A@\" code--- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).---- $strict------ This package provides both strict and lazy 'Text' types. The--- strict type is provided by the "Data.Text" module, while the lazy--- type is provided by the "Data.Text.Lazy" module. Internally, the--- lazy @Text@ type consists of a list of strict chunks.------ The strict 'Text' type requires that an entire string fit into--- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of--- streaming strings that are larger than memory using a small memory--- footprint. In many cases, the overhead of chunked streaming makes--- the lazy 'Data.Text.Lazy.Text' type slower than its strict--- counterpart, but this is not always the case. Sometimes, the time--- complexity of a function in one module may be different from the--- other, due to their differing internal structures.------ Each module provides an almost identical API, with the main--- difference being that the strict module uses 'Int' values for--- lengths and counts, while the lazy module uses 'Data.Int.Int64'--- lengths.---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values--- (corresponding to 'Data.Char.Surrogate' category) that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"�\" (U+FFFD). Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\". The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)---- $fusion------ Starting from @text-1.3@ fusion is no longer implicit,--- and pipelines of transormations usually allocate intermediate 'Text' values.--- Users, who observe significant changes to performances,--- are encouraged to use fusion framework explicitly, employing--- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".--instance Eq Text where- Text arrA offA lenA == Text arrB offB lenB- | lenA == lenB = A.equal arrA offA arrB offB lenA- | otherwise = False- {-# INLINE (==) #-}--instance Ord Text where- compare = compareText--instance Read Text where- readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]---- | @since 1.2.2.0-instance Semigroup Text where- (<>) = append--instance Monoid Text where- mempty = empty- mappend = (<>)- mconcat = concat--instance IsString Text where- fromString = pack---- | @since 1.2.0.0-instance Exts.IsList Text where- type Item Text = Char- fromList = pack- toList = unpack--instance NFData Text where rnf !_ = ()---- | @since 1.2.1.0-instance Binary Text where- put t = put (encodeUtf8 t)- get = do- bs <- get- case decodeUtf8' bs of- P.Left exn -> P.fail (P.show exn)- P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you--- feel a mistake has been made, please feel free to submit--- improvements.------ The original discussion is archived here:--- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? >------ The followup discussion that changed the behavior of 'Data.Set.Set'--- and 'Data.Map.Map' is archived here:--- <http://markmail.org/message/trovdc6zkphyi3cr#query:+page:1+mid:a46der3iacwjcf6n+state:results Proposal: Allow gunfold for Data.Map, ... >--instance Data Text where- gfoldl f z txt = z pack `f` (unpack txt)- toConstr _ = packConstr- gunfold k z c = case constrIndex c of- 1 -> k (z pack)- _ -> P.error "gunfold"- dataTypeOf _ = textDataType---- | This instance has similar considerations to the 'Data' instance:--- it preserves abstraction at the cost of inefficiency.------ @since 1.2.4.0-instance TH.Lift Text where- lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack-#if MIN_VERSION_template_haskell(2,17,0)- liftTyped = TH.unsafeCodeCoerce . TH.lift-#elif MIN_VERSION_template_haskell(2,16,0)- liftTyped = TH.unsafeTExpCoerce . TH.lift-#endif---- | @since 1.2.2.0-instance PrintfArg Text where- formatArg txt = formatString $ unpack txt--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Text" [packConstr]---- | /O(n)/ Compare two 'Text' values lexicographically.-compareText :: Text -> Text -> Ordering-compareText (Text arrA offA lenA) (Text arrB offB lenB) =- A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB--- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),--- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering--- of underlying bytearrays, no decoding is needed.---- -------------------------------------------------------------------------------- * Conversion to/from 'Text'---- | /O(n)/ Convert a 'String' into a 'Text'.--- Performs replacement on invalid scalar values.-pack :: String -> Text-pack = unstream . S.map safe . S.streamList-{-# INLINE [1] pack #-}---- -------------------------------------------------------------------------------- * Basic functions---- | /O(n)/ Adds a character to the front of a 'Text'. This function--- is more costly than its 'List' counterpart because it requires--- copying a new array. Performs replacement on--- invalid scalar values.-cons :: Char -> Text -> Text-cons c = unstream . S.cons (safe c) . stream-{-# INLINE [1] cons #-}--infixr 5 `cons`---- | /O(n)/ Adds a character to the end of a 'Text'. This copies the--- entire array in the process.--- Performs replacement on invalid scalar values.-snoc :: Text -> Char -> Text-snoc t c = unstream (S.snoc (stream t) (safe c))-{-# INLINE snoc #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'uncons' instead.-head :: HasCallStack => Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty.-uncons :: Text -> Maybe (Char, Text)-uncons t@(Text arr off len)- | len <= 0 = Nothing- | otherwise = Just $ let !(Iter c d) = iter t 0- in (c, text arr (off+d) (len-d))-{-# INLINE [1] uncons #-}---- | /O(1)/ Returns the last character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'unsnoc' instead.-last :: HasCallStack => Text -> Char-last t@(Text _ _ len)- | len <= 0 = emptyError "last"- | otherwise = let Iter c _ = reverseIter t (len - 1) in c-{-# INLINE [1] last #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty. This is a partial function, consider using 'uncons' instead.-tail :: HasCallStack => Text -> Text-tail t@(Text arr off len)- | len <= 0 = emptyError "tail"- | otherwise = text arr (off+d) (len-d)- where d = iter_ t 0-{-# INLINE [1] tail #-}---- | /O(1)/ Returns all but the last character of a 'Text', which must--- be non-empty. This is a partial function, consider using 'unsnoc' instead.-init :: HasCallStack => Text -> Text-init t@(Text arr off len)- | len <= 0 = emptyError "init"- | otherwise = text arr off (len + reverseIter_ t (len - 1))-{-# INLINE [1] init #-}---- | /O(1)/ Returns all but the last character and the last character of a--- 'Text', or 'Nothing' if empty.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc t@(Text arr off len)- | len <= 0 = Nothing- | otherwise = Just (text arr off (len + d), c)- where- Iter c d = reverseIter t (len - 1)-{-# INLINE [1] unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.-null :: Text -> Bool-null (Text _arr _off len) =-#if defined(ASSERTS)- assert (len >= 0) $-#endif- len <= 0-{-# INLINE [1] null #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n)/ Returns the number of characters in a 'Text'.-length ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Int-length = P.negate . measureOff P.maxBound-{-# INLINE [1] length #-}--- length needs to be phased after the compareN/length rules otherwise--- it may inline before the rules have an opportunity to fire.--{-# RULES-"TEXT length/filter -> S.length/S.filter" forall p t.- length (filter p t) = S.length (S.filter p (stream t))-"TEXT length/unstream -> S.length" forall t.- length (unstream t) = S.length t-"TEXT length/pack -> P.length" forall t.- length (pack t) = P.length t-"TEXT length/map -> length" forall f t.- length (map f t) = length t-"TEXT length/zipWith -> length" forall f t1 t2.- length (zipWith f t1 t2) = min (length t1) (length t2)-"TEXT length/replicate -> n" forall n t.- length (replicate n t) = mul (max 0 n) (length t)-"TEXT length/cons -> length+1" forall c t.- length (cons c t) = 1 + length t-"TEXT length/intersperse -> 2*length-1" forall c t.- length (intersperse c t) = max 0 (mul 2 (length t) - 1)-"TEXT length/intercalate -> n*length" forall s ts.- length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)- #-}---- | /O(n)/ Compare the count of characters in a 'Text' to a number.------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int -> Ordering-compareLength t n = S.compareLengthI (stream t) n-{-# INLINE [1] compareLength #-}--{-# RULES-"TEXT compareN/length -> compareLength" [~1] forall t n.- compare (length t) n = compareLength t n- #-}--{-# RULES-"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.- eqInt (length t) n = compareLength t n == EQ- #-}--{-# RULES-"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.- neInt (length t) n = compareLength t n /= EQ- #-}--{-# RULES-"TEXT <N/length -> compareLength/==LT" [~1] forall t n.- ltInt (length t) n = compareLength t n == LT- #-}--{-# RULES-"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.- leInt (length t) n = compareLength t n /= GT- #-}--{-# RULES-"TEXT >N/length -> compareLength/==GT" [~1] forall t n.- gtInt (length t) n = compareLength t n == GT- #-}--{-# RULES-"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.- geInt (length t) n = compareLength t n /= LT- #-}---- -------------------------------------------------------------------------------- * Transformations--- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.------ Example:------ >>> let message = pack "I am not angry. Not at all."--- >>> T.map (\c -> if c == '.' then '!' else c) message--- "I am not angry! Not at all!"------ Performs replacement on invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + 4) o 0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text- outer !dst !dstLen = inner- where- inner !srcOff !dstOff- | srcOff >= l + o = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- | dstOff + 4 > dstLen = do- let !dstLen' = dstLen + (l + o) - srcOff + 4- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff- | otherwise = do- let !(Iter c d) = iterArray src srcOff- d' <- unsafeWrite dst dstOff (safe (f c))- inner (srcOff + d) (dstOff + d')-{-# INLINE [1] map #-}--{-# RULES-"TEXT map/map -> map" forall f g t.- map f (map g t) = map (f . safe . g) t-#-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.------ Example:------ >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]--- "WeNI!seekNI!theNI!HolyNI!Grail"-intercalate :: Text -> [Text] -> Text-intercalate t = concat . L.intersperse t-{-# INLINE [1] intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.------ Example:------ >>> T.intersperse '.' "SHIELD"--- "S.H.I.E.L.D"------ Performs replacement on invalid scalar values.-intersperse :: Char -> Text -> Text-intersperse c t@(Text src o l) = if l == 0 then mempty else runST $ do- let !cLen = utf8Length c- dstLen = l + length t P.* cLen-- dst <- A.new dstLen-- let writeSep = case cLen of- 1 -> \dstOff ->- A.unsafeWrite dst dstOff (ord8 c)- 2 -> let (c0, c1) = ord2 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- A.unsafeWrite dst (dstOff + 2) c2- _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- A.unsafeWrite dst (dstOff + 2) c2- A.unsafeWrite dst (dstOff + 3) c3- let go !srcOff !dstOff = if srcOff >= o + l then return () else do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0- case d of- 1 -> do- A.unsafeWrite dst dstOff m0- writeSep (dstOff + 1)- go (srcOff + 1) (dstOff + 1 + cLen)- 2 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- writeSep (dstOff + 2)- go (srcOff + 2) (dstOff + 2 + cLen)- 3 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- writeSep (dstOff + 3)- go (srcOff + 3) (dstOff + 3 + cLen)- _ -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- writeSep (dstOff + 4)- go (srcOff + 4) (dstOff + 4 + cLen)-- go o 0- arr <- A.unsafeFreeze dst- return (Text arr 0 (dstLen - cLen))-{-# INLINE [1] intersperse #-}---- | /O(n)/ Reverse the characters of a string.------ Example:------ >>> T.reverse "desrever"--- "reversed"-reverse ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Text-reverse (Text (A.ByteArray ba) off len) = runST $ do- marr@(A.MutableByteArray mba) <- A.new len- unsafeIOToST $ c_reverse mba ba (intToCSize off) (intToCSize len)- brr <- A.unsafeFreeze marr- return $ Text brr 0 len-{-# INLINE reverse #-}---- | The input buffer (src :: ByteArray#, off :: CSize, len :: CSize)--- must specify a valid UTF-8 sequence, this condition is not checked.-foreign import ccall unsafe "_hs_text_reverse" c_reverse- :: Exts.MutableByteArray# s -> ByteArray# -> CSize -> CSize -> IO ()---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =--- 'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once. So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ >>> replace "oo" "foo" "oo"--- "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ >>> replace "ofo" "bar" "ofofo"--- "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: HasCallStack- => Text- -- ^ @needle@ to search for. If this string is empty, an- -- error will occur.- -> Text- -- ^ @replacement@ to replace @needle@ with.- -> Text- -- ^ @haystack@ in which to search.- -> Text-replace needle@(Text _ _ neeLen)- (Text repArr repOff repLen)- haystack@(Text hayArr hayOff hayLen)- | neeLen == 0 = emptyError "replace"- | L.null ixs = haystack- | len > 0 = Text (A.run x) 0 len- | otherwise = empty- where- ixs = indices needle haystack- len = hayLen - (neeLen - repLen) `mul` L.length ixs- x :: ST s (A.MArray s)- x = do- marr <- A.new len- let loop (i:is) o d = do- let d0 = d + i - o- d1 = d0 + repLen- A.copyI (i - o) marr d hayArr (hayOff+o)- A.copyI repLen marr d0 repArr repOff- loop is (i + neeLen) d1- loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o)- loop ixs 0 0- return marr---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ When case converting 'Text' values, do not use combinators like--- @map toUpper@ to case convert each character of a string--- individually, as this gives incorrect results according to the--- rules of some writing systems. The whole-string case conversion--- functions from this module, such as @toUpper@, obey the correct--- case conversion rules. As a result, these functions may map one--- input character to two or three output characters. For examples,--- see the documentation of each function.------ /Note/: In some languages, case conversion is a locale- and--- context-dependent operation. The case conversion functions in this--- module are /not/ locale sensitive. Programs that require locale--- sensitivity should use appropriate versions of the--- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >.---- | /O(n)/ Convert a string to folded case.------ This function is mainly useful for performing caseless (also known--- as case insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string. For instance,--- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case--- folded to the sequence \"մ\" (men, U+0574) followed by--- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign,--- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC)--- instead of itself.-toCaseFold :: Text -> Text-toCaseFold t = unstream (S.toCaseFold (stream t))-{-# INLINE toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, \"İ\" (Latin capital letter I with dot above,--- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069)--- followed by \" ̇\" (combining dot above, U+0307).-toLower :: Text -> Text-toLower t = unstream (S.toLower (stream t))-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the German \"ß\" (eszett, U+00DF) maps to the--- two-letter sequence \"SS\".-toUpper :: Text -> Text-toUpper t = unstream (S.toUpper (stream t))-{-# INLINE toUpper #-}---- | /O(n)/ Convert a string to title case, using simple case--- conversion.------ The first letter of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature fl (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased—but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle t = unstream (S.toTitle (stream t))-{-# INLINE toTitle #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right.--- Performs replacement on invalid scalar values.------ Examples:------ >>> justifyLeft 7 'x' "foo"--- "fooxxxx"------ >>> justifyLeft 3 'x' "foobar"--- "foobar"-justifyLeft :: Int -> Char -> Text -> Text-justifyLeft k c t- | len >= k = t- | otherwise = t `append` replicateChar (k-len) c- where len = length t-{-# INLINE [1] justifyLeft #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Performs replacement on--- invalid scalar values.------ Examples:------ >>> justifyRight 7 'x' "bar"--- "xxxxbar"------ >>> justifyRight 3 'x' "foobar"--- "foobar"-justifyRight :: Int -> Char -> Text -> Text-justifyRight k c t- | len >= k = t- | otherwise = replicateChar (k-len) c `append` t- where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side. Performs replacement on invalid--- scalar values.------ Examples:------ >>> center 8 'x' "HS"--- "xxxHSxxx"-center :: Int -> Char -> Text -> Text-center k c t- | len >= k = t- | otherwise = replicateChar l c `append` t `append` replicateChar r c- where len = length t- d = k - len- r = d `quot` 2- l = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument. Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.------ Examples:------ >>> transpose ["green","orange"]--- ["go","rr","ea","en","ng","e"]------ >>> transpose ["blue","red"]--- ["br","le","ud","e"]-transpose :: [Text] -> [Text]-transpose ts = P.map pack (L.transpose (P.map unpack ts))---- -------------------------------------------------------------------------------- * Reducing 'Text's (folds)---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.-foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.-foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- -------------------------------------------------------------------------------- ** Special folds---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat ts = case ts' of- [] -> empty- [t] -> t- _ -> Text (A.run go) 0 len- where- ts' = L.filter (not . null) ts- len = sumP "concat" $ L.map lengthWord8 ts'- go :: ST s (A.MArray s)- go = do- arr <- A.new len- let step i (Text a o l) = A.copyI l arr i a o >> return (i + l)- foldM step 0 ts' >> return arr---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty.-maximum :: HasCallStack => Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty.-minimum :: HasCallStack => Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- -------------------------------------------------------------------------------- * Building 'Text's--- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ __Properties__------ @'head' ('scanl' f z xs) = z@--- --- @'last' ('scanl' f z xs) = 'foldl' f z xs@-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))- where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument. Performs replacement on invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t | null t = empty- | otherwise = scanl f (unsafeHead t) (unsafeTail t)-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f z = S.reverse . S.reverseScanr g z . reverseStream- where g a b = safe (f a b)-{-# INLINE scanr #-}---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument. Performs replacement on invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t = empty- | otherwise = scanr f (last t) (init t)-{-# INLINE scanr1 #-}---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'. Performs--- replacement on invalid scalar values.-mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)-mapAccumL f z0 = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + 4) o 0 z0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text)- outer !dst !dstLen = inner- where- inner !srcOff !dstOff !z- | srcOff >= l + o = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (z, Text arr 0 dstOff)- | dstOff + 4 > dstLen = do- let !dstLen' = dstLen + (l + o) - srcOff + 4- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff z- | otherwise = do- let !(Iter c d) = iterArray src srcOff- (z', c') = f z c- d' <- unsafeWrite dst dstOff (safe c')- inner (srcOff + d) (dstOff + d') z'-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'.--- Performs replacement on invalid scalar values.-mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)-mapAccumR f z0 = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + o - 1) (l + 4 - 1) z0- where- outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text)- outer !dst = inner- where- inner !srcOff !dstOff !z- | srcOff < o = do- dstLen <- A.getSizeofMArray dst- arr <- A.unsafeFreeze dst- return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1))- | dstOff < 3 = do- dstLen <- A.getSizeofMArray dst- let !dstLen' = dstLen + (srcOff - o) + 4- dst' <- A.new dstLen'- A.copyM dst' (dstLen' - dstLen) dst 0 dstLen- outer dst' srcOff (dstOff + dstLen' - dstLen) z- | otherwise = do- let !(Iter c d) = reverseIterArray src (srcOff)- (z', c') = f z c- c'' = safe c'- !d' = utf8Length c''- dstOff' = dstOff - d'- _ <- unsafeWrite dst (dstOff' + 1) c''- inner (srcOff + d) dstOff' z'-{-# INLINE mapAccumR #-}---- -------------------------------------------------------------------------------- ** Generating and unfolding 'Text's---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int -> Text -> Text-replicate n t@(Text a o l)- | n <= 0 || l <= 0 = empty- | n == 1 = t- | isSingleton t = replicateChar n (unsafeHead t)- | otherwise = runST $ do- let totalLen = n `mul` l- marr <- A.new totalLen- A.copyI l marr 0 a o- A.tile marr l- arr <- A.unsafeFreeze marr- return $ Text arr 0 totalLen-{-# INLINE [1] replicate #-}--{-# RULES-"TEXT replicate/singleton -> replicateChar" [~1] forall n c.- replicate n (singleton c) = replicateChar n c- #-}---- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the--- value of every element.-replicateChar :: Int -> Char -> Text-replicateChar !len !c'- | len <= 0 = empty- | isAscii c = runST $ do- marr <- A.newFilled len (ord c)- arr <- A.unsafeFreeze marr- return $ Text arr 0 len- | otherwise = runST $ do- let cLen = utf8Length c- totalLen = cLen P.* len- marr <- A.new totalLen- _ <- unsafeWrite marr 0 c- A.tile marr cLen- arr <- A.unsafeFreeze marr- return $ Text arr 0 totalLen- where- c = safe c'-{-# INLINE replicateChar #-}---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.--- Performs replacement on invalid scalar values.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'.--- Performs replacement on invalid scalar values.-unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- -------------------------------------------------------------------------------- * Substrings---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text.-take :: Int -> Text -> Text-take n t@(Text arr off len)- | n <= 0 = empty- | n >= len = t- | otherwise = let m = measureOff n t in if m >= 0 then text arr off m else t-{-# INLINE [1] take #-}---- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@--- returns a non-negative number, measuring their size in 'Word8'. Otherwise,--- if @t@ is shorter, return a non-positive number, which is a negated total count--- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0.------ This function is used to implement 'take', 'drop', 'splitAt' and 'length'--- and is useful on its own in streaming and parsing libraries.------ @since 2.0-measureOff :: Int -> Text -> Int-measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else- cSsizeToInt $ unsafeDupablePerformIO $- c_measure_off arr (intToCSize off) (intToCSize len) (intToCSize n)---- | The input buffer (arr :: ByteArray#, off :: CSize, len :: CSize)--- must specify a valid UTF-8 sequence, this condition is not checked.-foreign import ccall unsafe "_hs_text_measure_off" c_measure_off- :: ByteArray# -> CSize -> CSize -> CSize -> IO CSsize---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ >>> takeEnd 3 "foobar"--- "bar"------ @since 1.1.1.0-takeEnd :: Int -> Text -> Text-takeEnd n t@(Text arr off len)- | n <= 0 = empty- | n >= len = t- | otherwise = text arr (off+i) (len-i)- where i = iterNEnd n t--iterNEnd :: Int -> Text -> Int-iterNEnd n t@(Text _arr _off len) = loop (len-1) n- where loop i !m- | m <= 0 = i+1- | i <= 0 = 0- | otherwise = loop (i+d) (m-1)- where d = reverseIter_ t i---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'.-drop :: Int -> Text -> Text-drop n t@(Text arr off len)- | n <= 0 = t- | n >= len = empty- | otherwise = if m >= 0 then text arr (off+m) (len-m) else mempty- where m = measureOff n t-{-# INLINE [1] drop #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ >>> dropEnd 3 "foobar"--- "foo"------ @since 1.1.1.0-dropEnd :: Int -> Text -> Text-dropEnd n t@(Text arr off len)- | n <= 0 = t- | n >= len = empty- | otherwise = text arr off (iterNEnd n t)---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t@(Text arr off len) = loop 0- where loop !i | i >= len = t- | p c = loop (i+d)- | otherwise = text arr off i- where Iter c d = iter t i-{-# INLINE [1] takeWhile #-}---- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.--- Examples:------ >>> takeWhileEnd (=='o') "foo"--- "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p t@(Text arr off len) = loop (len-1) len- where loop !i !l | l <= 0 = t- | p c = loop (i+d) (l+d)- | otherwise = text arr (off+l) (len-l)- where Iter c d = reverseIter t i-{-# INLINE [1] takeWhileEnd #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t@(Text arr off len) = loop 0 0- where loop !i !l | l >= len = empty- | p c = loop (i+d) (l+d)- | otherwise = Text arr (off+i) (len-l)- where Iter c d = iter t i-{-# INLINE [1] dropWhile #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.------ Examples:------ >>> dropWhileEnd (=='.') "foo..."--- "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p t@(Text arr off len) = loop (len-1) len- where loop !i !l | l <= 0 = empty- | p c = loop (i+d) (l+d)- | otherwise = Text arr off l- where Iter c d = reverseIter t i-{-# INLINE [1] dropWhileEnd #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string. Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile isSpace-{-# INLINE stripStart #-}---- | /O(n)/ Remove trailing white space from a string. Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int -> Text -> (Text, Text)-splitAt n t@(Text arr off len)- | n <= 0 = (empty, t)- | n >= len = (t, empty)- | otherwise = let m = measureOff n t in- if m >= 0 then (text arr off m, text arr (off+m) (len-m)) else (t, mempty)---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the list.------ >>> T.span (=='0') "000AB"--- ("000","AB")-span :: (Char -> Bool) -> Text -> (Text, Text)-span p t = case span_ p t of- (# hd,tl #) -> (hd,tl)-{-# INLINE span #-}---- | /O(n)/ 'break' is like 'span', but the prefix returned is--- over elements that fail the predicate @p@.------ >>> T.break (=='c') "180cm"--- ("180","cm")-break :: (Char -> Bool) -> Text -> (Text, Text)-break p = span (not . p)-{-# INLINE break #-}---- | /O(n)/ Group characters in a string according to a predicate.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy p = loop- where- loop t@(Text arr off len)- | null t = []- | otherwise = text arr off n : loop (text arr (off+n) (len-n))- where Iter c d = iter t 0- n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d))---- | Returns the /array/ index (in units of 'Word8') at which a--- character may be found. This is /not/ the same as the logical--- index returned by e.g. 'findIndex'.-findAIndexOrEnd :: (Char -> Bool) -> Text -> Int-findAIndexOrEnd q t@(Text _arr _off len) = go 0- where go !i | i >= len || q c = i- | otherwise = go (i+d)- where Iter c d = iter t i---- | /O(n)/ Group characters in a string by equality.-group :: Text -> [Text]-group = groupBy (==)---- | /O(n)/ Return all initial segments of the given 'Text', shortest--- first.-inits :: Text -> [Text]-inits t@(Text arr off len) = loop 0- where loop i | i >= len = [t]- | otherwise = Text arr off i : loop (i + iter_ t i)---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails t | null t = [empty]- | otherwise = t : tails (unsafeTail t)---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be empty), consuming the delimiter. An empty--- delimiter is invalid, and will cause an error to be raised.------ Examples:------ >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"--- ["a","b","d","e"]------ >>> splitOn "aaa" "aaaXaaaXaaaXaaa"--- ["","X","X","X",""]------ >>> splitOn "x" "x"--- ["",""]------ and------ > intercalate s . splitOn s == id--- > splitOn (singleton c) == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: HasCallStack- => Text- -- ^ String to split on. If this string is empty, an error- -- will occur.- -> Text- -- ^ Input text.- -> [Text]-splitOn pat@(Text _ _ l) src@(Text arr off len)- | l <= 0 = emptyError "splitOn"- | isSingleton pat = split (== unsafeHead pat) src- | otherwise = go 0 (indices pat src)- where- go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs- go s _ = [text arr (s+off) (len-s)]-{-# INLINE [1] splitOn #-}--{-# RULES-"TEXT splitOn/singleton -> split/==" [~1] forall c t.- splitOn (singleton c) t = split (==c) t- #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element. The--- resulting components do not contain the separators. Two adjacent--- separators result in an empty component in the output. eg.------ >>> split (=='a') "aabbaca"--- ["","","bb","c",""]------ >>> split (=='a') ""--- [""]-split :: (Char -> Bool) -> Text -> [Text]-split _ t@(Text _off _arr 0) = [t]-split p t = loop t- where loop s | null s' = [l]- | otherwise = l : loop (unsafeTail s')- where (# l, s' #) = span_ (not . p) s-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@. The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ >>> chunksOf 3 "foobarbaz"--- ["foo","bar","baz"]------ >>> chunksOf 4 "haskell.org"--- ["hask","ell.","org"]-chunksOf :: Int -> Text -> [Text]-chunksOf k = go- where- go t = case splitAt k t of- (a,b) | null a -> []- | otherwise -> a : go b-{-# INLINE chunksOf #-}---- ------------------------------------------------------------------------------- * Searching------------------------------------------------------------------------------------ ** Searching with a predicate---- | /O(n)/ The 'elem' function takes a character and a 'Text', and--- returns 'True' if the element is found in the given 'Text', or--- 'False' otherwise.-elem :: Char -> Text -> Bool-elem c t = S.any (== c) (stream t)-{-# INLINE elem #-}---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element matching the predicate, or 'Nothing' if--- there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p = go- where- go (Text src o l) = runST $ do- -- It's tempting to allocate l elements at once and avoid resizing.- -- However, this can be unacceptable in scenarios where a huge array- -- is filtered with a rare predicate, resulting in a much shorter buffer.- let !dstLen = min l 64- dst <- A.new dstLen- outer dst dstLen o 0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text- outer !dst !dstLen = inner- where- inner !srcOff !dstOff- | srcOff >= o + l = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- | dstOff + 4 > dstLen = do- -- Double size of the buffer, unless it becomes longer than- -- source string. Ensure to extend it by least 4 bytes.- let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen)- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff- -- In case of success, filter writes exactly the same character- -- it just read (this is not a case for map, for example).- -- We leverage this fact below: no need to decode Char back into UTF8,- -- just copy bytes from input.- | otherwise = do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0- case d of- 1 -> do- let !c = unsafeChr8 m0- if not (p c) then inner (srcOff + 1) dstOff else do- A.unsafeWrite dst dstOff m0- inner (srcOff + 1) (dstOff + 1)- 2 -> do- let !c = chr2 m0 m1- if not (p c) then inner (srcOff + 2) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- inner (srcOff + 2) (dstOff + 2)- 3 -> do- let !c = chr3 m0 m1 m2- if not (p c) then inner (srcOff + 3) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- inner (srcOff + 3) (dstOff + 3)- _ -> do- let !c = chr4 m0 m1 m2 m3- if not (p c) then inner (srcOff + 4) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- inner (srcOff + 4) (dstOff + 4)-{-# INLINE [1] filter #-}--{-# RULES-"TEXT filter/filter -> filter" forall p q t.- filter p (filter q t) = filter (\c -> p c && q c) t-#-}---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@. The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched. The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ >>> breakOn "::" "a::b::c"--- ("a","::b::c")------ >>> breakOn "/" "foobar"--- ("foobar","")------ Laws:------ > append prefix match == haystack--- > where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: HasCallStack => Text -> Text -> (Text, Text)-breakOn pat src@(Text arr off len)- | null pat = emptyError "breakOn"- | otherwise = case indices pat src of- [] -> (src, empty)- (x:_) -> (text arr off x, text arr (off+x) (len-x))-{-# INLINE breakOn #-}---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the--- string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@. The second is the--- remainder of @haystack@, following the match.------ >>> breakOnEnd "::" "a::b::c"--- ("a::b::","c")-breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)-breakOnEnd pat src = (reverse b, reverse a)- where (a,b) = breakOn (reverse pat) (reverse src)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@. Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ >>> breakOnAll "::" ""--- []------ >>> breakOnAll "/" "a/b/c/"--- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: HasCallStack- => Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-breakOnAll pat src@(Text arr off slen)- | null pat = emptyError "breakOnAll"- | otherwise = L.map step (indices pat src)- where- step x = (chunk 0 x, chunk x (slen-x))- chunk !n !l = text arr (n+off) l-{-# INLINE breakOnAll #-}------------------------------------------------------------------------------------ ** Indexing 'Text's---- $index------ If you think of a 'Text' value as an array of 'Char' values (which--- it is not), you run the risk of writing inefficient code.------ An idiom that is common in some languages is to find the numeric--- offset of a character or substring, then use that number to split--- or trim the searched string. With a 'Text' value, this approach--- would require two /O(n)/ operations: one to perform the search, and--- one to operate from wherever the search ended.------ For example, suppose you have a string that you want to split on--- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of--- searching for the index of @\"::\"@ and taking the substrings--- before and after that index, you would instead use @breakOnAll \"::\"@.---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: HasCallStack => Text -> Int -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text'--- and returns the index of the first element in the 'Text' satisfying--- the predicate.-findIndex :: (Char -> Bool) -> Text -> Maybe Int-findIndex p t = S.findIndex p (stream t)-{-# INLINE findIndex #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: HasCallStack => Text -> Text -> Int-count pat- | null pat = emptyError "count"- | isSingleton pat = countChar (unsafeHead pat)- | otherwise = L.length . indices pat-{-# INLINE [1] count #-}--{-# RULES-"TEXT count/singleton -> countChar" [~1] forall c t.- count (singleton c) t = countChar c t- #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'.-countChar :: Char -> Text -> Int-countChar c t = S.countChar c (stream t)-{-# INLINE countChar #-}------------------------------------------------------------------------------------ * Zipping---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))- where g a b = safe (f a b)-{-# INLINE [1] zipWith #-}---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words (Text arr off len) = loop 0 0- where- loop !start !n- | n >= len = if start == n- then []- else [Text arr (start + off) (n - start)]- -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20- | isAsciiSpace w0 =- if start == n- then loop (n + 1) (n + 1)- else Text arr (start + off) (n - start) : loop (n + 1) (n + 1)- | w0 < 0x80 = loop start (n + 1)- -- or 2 bytes for 0xA0- | w0 == 0xC2, w1 == 0xA0 =- if start == n- then loop (n + 2) (n + 2)- else Text arr (start + off) (n - start) : loop (n + 2) (n + 2)- | w0 < 0xE0 = loop start (n + 2)- -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000- | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80- || w0 == 0xE2 && (w1 == 0x80 && isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F)- || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 =- if start == n- then loop (n + 3) (n + 3)- else Text arr (start + off) (n - start) : loop (n + 3) (n + 3)- | otherwise = loop start (n + utf8LengthByLeader w0)- where- w0 = A.unsafeIndex arr (off + n)- w1 = A.unsafeIndex arr (off + n + 1)- w2 = A.unsafeIndex arr (off + n + 2)-{-# INLINE words #-}---- Adapted from Data.ByteString.Internal.isSpaceWord8-isAsciiSpace :: Word8 -> Bool-isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5)-{-# INLINE isAsciiSpace #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters--- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.------ 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.-lines :: Text -> [Text]-lines (Text arr@(A.ByteArray arr#) off len) = go off- where- go !n- | n >= len + off = []- | delta < 0 = [Text arr n (len + off - n)]- | otherwise = Text arr n delta : go (n + delta + 1)- where- delta = cSsizeToInt $ unsafeDupablePerformIO $- memchr arr# (intToCSize n) (intToCSize (len + off - n)) 0x0A-{-# INLINE lines #-}--foreign import ccall unsafe "_hs_text_memchr" memchr- :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) =- alen <= blen && S.isPrefixOf (stream a) (stream b)-{-# INLINE [1] isPrefixOf #-}---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' iff the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf a@(Text _aarr _aoff alen) b@(Text barr boff blen) =- d >= 0 && a == b'- where d = blen - alen- b' | d == 0 = b- | otherwise = Text barr (boff+d) alen-{-# INLINE isSuffixOf #-}---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' iff the first is contained, wholly and intact, anywhere--- within the second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Text -> Bool-isInfixOf needle haystack- | null needle = True- | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack- | otherwise = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ >>> stripPrefix "foo" "foobar"--- Just "bar"------ >>> stripPrefix "" "baz"--- Just "baz"------ >>> stripPrefix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _ = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p@(Text _arr _off plen) t@(Text arr off len)- | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen)- | otherwise = Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ >>> commonPrefixes "foobar" "fooquux"--- Just ("foo","bar","quux")------ >>> commonPrefixes "veeble" "fetzer"--- Nothing------ >>> commonPrefixes "" "baz"--- Nothing-commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text)-commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1)- | len0 == 0 = Nothing- | len1 == 0 = Nothing- | otherwise = go 0 0- where- go !i !j- | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i))- | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty)- | a == b = go (i + 1) k- | k > 0 = Just (Text arr0 off0 k,- Text arr0 (off0 + k) (len0 - k),- Text arr1 (off1 + k) (len1 - k))- | otherwise = Nothing- where- a = A.unsafeIndex arr0 (off0 + i)- b = A.unsafeIndex arr1 (off1 + i)- isLeader = word8ToInt8 a >= -64- k = if isLeader then i else j-{-# INLINE commonPrefixes #-}---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ >>> stripSuffix "bar" "foobar"--- Just "foo"------ >>> stripSuffix "" "baz"--- Just "baz"------ >>> stripSuffix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _ = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p@(Text _arr _off plen) t@(Text arr off len)- | p `isSuffixOf` t = Just $! text arr off (len-plen)- | otherwise = Nothing---- | Add a list of non-negative numbers. Errors out on overflow.-sumP :: String -> [Int] -> Int-sumP fun = go 0- where go !a (x:xs)- | ax >= 0 = go ax xs- | otherwise = overflowError fun- where ax = a + x- go a _ = a--emptyError :: HasCallStack => String -> a-emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input"--overflowError :: HasCallStack => String -> a-overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow"---- | /O(n)/ Make a distinct copy of the given string, sharing no--- storage with the original string.------ As an example, suppose you read a large string, of which you need--- only a small portion. If you do not use 'copy', the entire original--- array will be kept alive in memory by the smaller string. Making a--- copy \"breaks the link\" to the original array, allowing it to be--- garbage collected if there are no other live references to it.-copy :: Text -> Text-copy (Text arr off len) = Text (A.run go) 0 len- where- go :: ST s (A.MArray s)- go = do- marr <- A.new len- A.copyI len marr 0 arr off- return marr--ord8 :: Char -> Word8-ord8 = P.fromIntegral . ord--intToCSize :: Int -> CSize-intToCSize = P.fromIntegral--cSsizeToInt :: CSsize -> Int-cSsizeToInt = P.fromIntegral--word8ToInt8 :: Word8 -> Int8-word8ToInt8 = P.fromIntegral------------------------------------------------------ NOTE: the named chunk below used by doctest;--- verify the doctests via `doctest -fobject-code Data/Text.hs`---- $setup--- >>> :set -XOverloadedStrings--- >>> import qualified Data.Text as T+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text +-- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text. +-- Suitable for performance critical use, both in terms of large data +-- quantities and high speed. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text as T +-- +-- To use an extended and very rich family of functions for working +-- with Unicode text (including normalization, regular expressions, +-- non-standard encodings, text breaking, and locales), see the +-- <http://hackage.haskell.org/package/text-icu text-icu package >. +-- + +module Data.Text + ( + -- * Strict vs lazy types + -- $strict + + -- * Acceptable data + -- $replacement + + -- * Definition of character + -- $character_definition + + -- * Fusion + -- $fusion + + -- * Types + Text + , StrictText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr' + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , replicate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , breakOn + , breakOnEnd + , break + , span + , spanM + , spanEndM + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + + -- ** Breaking into lines and words + , lines + --, lines' + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , breakOnAll + , find + , elem + , partition + + -- , findSubstring + + -- * Indexing + -- $index + , index + , findIndex + , count + + -- * Zipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + + -- * Low level operations + , copy + , unpackCString# + , unpackCStringAscii# + + , measureOff + ) where + +import Prelude (Char, Bool(..), Int, Maybe(..), String, + Eq, (==), (/=), Ord(..), Ordering(..), (++), + Monad(..), pure, Read(..), Show, + (&&), (||), (+), (-), (.), ($), ($!), (>>), + not, return, otherwise, quot) +import Control.DeepSeq (NFData(rnf)) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits ((.&.)) +import qualified Data.Char as Char +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Control.Monad (foldM) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import qualified Data.List as L hiding (head, tail) +import qualified Data.List.NonEmpty as NonEmptyList +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import Data.Text.Internal.ArrayUtils (memchr) +import Data.Text.Internal.IsAscii (isAscii) +import Data.Text.Internal.Reverse (reverse) +import Data.Text.Internal.Measure (measure_off) +import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, unstream) +import Data.Text.Internal.Private (span_) +import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, + reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray) +import Data.Text.Internal.Search (indices) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_) +#if defined(__HADDOCK__) +import Data.ByteString (ByteString) +import qualified Data.Text.Lazy as L +#endif +import Data.Word (Word8) +import Foreign.C.Types +import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) +import qualified GHC.Exts as Exts +import GHC.Int (Int8) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Lib as TH +import qualified Language.Haskell.TH.Syntax as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) +import System.Posix.Types (CSsize(..)) + +#if __GLASGOW_HASKELL__ >= 810 +import Data.Text.Foreign (asForeignPtr) +import System.IO.Unsafe (unsafePerformIO) +#endif + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +-- $character_definition +-- +-- This package uses the term /character/ to denote Unicode /code points/. +-- +-- Note that this is not the same thing as a grapheme (e.g. a +-- composition of code points that form one visual symbol). For +-- instance, consider the grapheme \"ä\". This symbol has two +-- Unicode representations: a single code-point representation +-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point), +-- and a two code point representation @U+0061@ (the \"@A@\" code +-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point). + +-- $strict +-- +-- This package provides both strict and lazy 'Text' types. The +-- strict type is provided by the "Data.Text" module, while the lazy +-- type is provided by the "Data.Text.Lazy" module. Internally, the +-- lazy @Text@ type consists of a list of strict chunks. +-- +-- The strict 'Text' type requires that an entire string fit into +-- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of +-- streaming strings that are larger than memory using a small memory +-- footprint. In many cases, the overhead of chunked streaming makes +-- the lazy 'Data.Text.Lazy.Text' type slower than its strict +-- counterpart, but this is not always the case. Sometimes, the time +-- complexity of a function in one module may be different from the +-- other, due to their differing internal structures. +-- +-- Each module provides an almost identical API, with the main +-- difference being that the strict module uses 'Int' values for +-- lengths and counts, while the lazy module uses 'Data.Int.Int64' +-- lengths. + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >. +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >) +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +instance Eq Text where + Text arrA offA lenA == Text arrB offB lenB + | lenA == lenB = A.equal arrA offA arrB offB lenA + | otherwise = False + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +-- +-- Beware: @stimes@ will crash if the given number does not fit into +-- an @Int@. +instance Semigroup Text where + (<>) = append + + stimes howManyTimes + | howManyTimes < 0 = P.error "Data.Text.stimes: given number is negative!" + | otherwise = + let howManyTimesInt = P.fromIntegral howManyTimes :: Int + in if P.fromIntegral howManyTimesInt == howManyTimes && howManyTimesInt >= 0 + then replicate howManyTimesInt + else P.error "Data.Text.stimes: given number does not fit into an Int!" + + sconcat = concat . NonEmptyList.toList + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where rnf !_ = () + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (lengthWord8 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you +-- feel a mistake has been made, please feel free to submit +-- improvements. +-- +-- The original discussion is archived here: +-- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? > +-- +-- The followup discussion that changed the behavior of 'Data.Set.Set' +-- and 'Data.Map.Map' is archived here: +-- <https://mail.haskell.org/pipermail/libraries/2012-August/018366.html Proposal: Allow gunfold for Data.Map, ... > + +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> P.error "gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 914 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> [| empty |] + _ -> + let + bytesQ = TH.liftAddrCompat ptr 0 (P.fromIntegral len) + lenQ = TH.liftIntCompat (P.fromIntegral len) + in [| unpackCStringLen# $bytesQ $lenQ |] +#elif __GLASGOW_HASKELL__ >= 810 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> TH.varE 'empty + _ -> + let + bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len) + lenQ = liftInt (P.fromIntegral len) + liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n))) + in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ +#else + lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +#if __GLASGOW_HASKELL__ >= 810 +unpackCStringLen# :: Exts.Addr# -> Int -> Text +unpackCStringLen# addr# l = Text ba 0 l + where + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Exts.Ptr addr#) l + A.unsafeFreeze marr +{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Text" [packConstr] + +-- | /O(n)/ Compare two 'Text' values lexicographically. +compareText :: Text -> Text -> Ordering +compareText (Text arrA offA lenA) (Text arrB offB lenB) = + A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(n)/ Adds a character to the front of a 'Text'. This function +-- is more costly than its 'List' counterpart because it requires +-- copying a new array. Performs replacement on +-- invalid scalar values. +cons :: Char -> Text -> Text +cons c (Text srcArr srcOff srcLen) = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = chLen + srcLen + marr <- A.new totalLen + _ <- unsafeWrite marr 0 ch + A.copyI srcLen marr chLen srcArr srcOff + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +-- Performs replacement on invalid scalar values. +snoc :: Text -> Char -> Text +snoc (Text srcArr srcOff srcLen) c = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = srcLen + chLen + marr <- A.new totalLen + A.copyI srcLen marr 0 srcArr srcOff + _ <- unsafeWrite marr srcLen ch + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE snoc #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t + | null t = emptyError "head" + | otherwise = let Iter c _ = iter t 0 in c +{-# INLINE head #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons t@(Text arr off len) + | len <= 0 = Nothing + | otherwise = Just $ let !(Iter c d) = iter t 0 + in (c, text arr (off+d) (len-d)) +{-# INLINE [1] uncons #-} + +-- | /O(1)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last t@(Text _ _ len) + | null t = emptyError "last" + | otherwise = let Iter c _ = reverseIter t (len - 1) in c +{-# INLINE [1] last #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail t@(Text arr off len) + | null t = emptyError "tail" + | otherwise = text arr (off+d) (len-d) + where d = iter_ t 0 +{-# INLINE [1] tail #-} + +-- | /O(1)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init t@(Text arr off len) + | null t = emptyError "init" + | otherwise = text arr off (len + reverseIter_ t (len - 1)) +{-# INLINE [1] init #-} + +-- | /O(1)/ Returns all but the last character and the last character of a +-- 'Text', or 'Nothing' if empty. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc t@(Text arr off len) + | null t = Nothing + | otherwise = Just (text arr off (len + d), c) + where + Iter c d = reverseIter t (len - 1) +{-# INLINE [1] unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null (Text _arr _off len) = +#if defined(ASSERTS) + assert (len >= 0) $ +#endif + len <= 0 +{-# INLINE [1] null #-} + +{-# RULES + "TEXT null/empty -> True" null empty = True +#-} + +-- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/), +-- to be used together with '(:<)' or '(:>)'. +-- +-- @since 2.1.2 +pattern Empty :: Text +pattern Empty <- (null -> True) where + Empty = empty + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton (Text arr off len) = + len /= 0 && len == utf8LengthByLeader (A.unsafeIndex arr off) +{-# INLINE isSingleton #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Int +length = P.negate . measureOff P.maxBound +{-# INLINE [1] length #-} +-- length needs to be phased after the compareN/length rules otherwise +-- it may inline before the rules have an opportunity to fire. + +{-# RULES +"TEXT length/filter -> S.length/S.filter" forall p t. + length (filter p t) = S.length (S.filter p (stream t)) +"TEXT length/unstream -> S.length" forall t. + length (unstream t) = S.length t +"TEXT length/pack -> P.length" forall t. + length (pack t) = P.length t +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = mul (max 0 n) (length t) +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (mul 2 (length t) - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) +"TEXT length/empty -> 0" + length empty = 0 + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +{-# RULES +"TEXT compareN/length -> compareLength" [~1] forall t n. + compare (length t) n = compareLength t n + #-} + +{-# RULES +"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. + eqInt (length t) n = compareLength t n == EQ + #-} + +{-# RULES +"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. + neInt (length t) n = compareLength t n /= EQ + #-} + +{-# RULES +"TEXT <N/length -> compareLength/==LT" [~1] forall t n. + ltInt (length t) n = compareLength t n == LT + #-} + +{-# RULES +"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. + leInt (length t) n = compareLength t n /= GT + #-} + +{-# RULES +"TEXT >N/length -> compareLength/==GT" [~1] forall t n. + gtInt (length t) n = compareLength t n == GT + #-} + +{-# RULES +"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. + geInt (length t) n = compareLength t n /= LT + #-} + +-- ----------------------------------------------------------------------------- +-- * Transformations +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- +-- Example: +-- +-- >>> let message = pack "I am not angry. Not at all." +-- >>> T.map (\c -> if c == '.' then '!' else c) message +-- "I am not angry! Not at all!" +-- +-- Performs replacement on invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = \t -> if null t then empty else mapNonEmpty f t +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +-- +-- Example: +-- +-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] +-- "WeNI!seekNI!theNI!HolyNI!Grail" +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. +-- +-- Example: +-- +-- >>> T.intersperse '.' "SHIELD" +-- "S.H.I.E.L.D" +-- +-- Performs replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t@(Text src o l) = if null t then empty else runST $ do + let !cLen = utf8Length c + dstLen = l + length t P.* cLen + + dst <- A.new dstLen + + let writeSep = case cLen of + 1 -> \dstOff -> + A.unsafeWrite dst dstOff (ord8 c) + 2 -> let (c0, c1) = ord2 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + A.unsafeWrite dst (dstOff + 3) c3 + let go !srcOff !dstOff = if srcOff >= o + l then return () else do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff m0 + writeSep (dstOff + 1) + go (srcOff + 1) (dstOff + 1 + cLen) + 2 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + writeSep (dstOff + 2) + go (srcOff + 2) (dstOff + 2 + cLen) + 3 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + writeSep (dstOff + 3) + go (srcOff + 3) (dstOff + 3 + cLen) + _ -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + writeSep (dstOff + 4) + go (srcOff + 4) (dstOff + 4 + cLen) + + go o 0 + arr <- A.unsafeFreeze dst + return (Text arr 0 (dstLen - cLen)) +{-# INLINE [1] intersperse #-} + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- >>> replace "oo" "foo" "oo" +-- "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- >>> replace "ofo" "bar" "ofofo" +-- "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace needle@(Text _ _ neeLen) + (Text repArr repOff repLen) + haystack@(Text hayArr hayOff hayLen) + | neeLen == 0 = emptyError "replace" + | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it + | L.null ixs = haystack + | otherwise = Text (A.run x) 0 len + where + ixs = indices needle haystack + len = hayLen - (neeLen - repLen) `mul` L.length ixs + x :: ST s (A.MArray s) + x = do + marr <- A.new len + let loop (i:is) o d = do + let d0 = d + i - o + d1 = d0 + repLen + A.copyI (i - o) marr d hayArr (hayOff+o) + A.copyI repLen marr d0 repArr repOff + loop is (i + neeLen) d1 + loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) + loop ixs 0 0 + return marr + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- When case converting 'Text' values, do not use combinators like +-- @map toUpper@ to case convert each character of a string +-- individually, as this gives incorrect results according to the +-- rules of some writing systems. The whole-string case conversion +-- functions from this module, such as @toUpper@, obey the correct +-- case conversion rules. As a result, these functions may map one +-- input character to two or three output characters. For examples, +-- see the documentation of each function. +-- +-- /Note/: In some languages, case conversion is a locale- and +-- context-dependent operation. The case conversion functions in this +-- module are /not/ locale sensitive. Programs that require locale +-- sensitivity should use appropriate versions of the +-- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (also known +-- as case insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case +-- folded to the sequence \"մ\" (men, U+0574) followed by +-- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, +-- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) +-- instead of itself. +toCaseFold :: Text -> Text +toCaseFold = \t -> + if null t then empty + else toCaseFoldNonEmpty t +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, \"İ\" (Latin capital letter I with dot above, +-- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) +-- followed by \" ̇\" (combining dot above, U+0307). +toLower :: Text -> Text +toLower = \t -> + if null t then empty + else toLowerNonEmpty t +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German \"ß\" (eszett, U+00DF) maps to the +-- two-letter sequence \"SS\". +toUpper :: Text -> Text +toUpper = \t -> + if null t then empty + else toUpperNonEmpty t +{-# INLINE toUpper #-} + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = \t -> + if null t then empty + else toTitleNonEmpty t +{-# INLINE toTitle #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. +-- Performs replacement on invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyLeft 7 'x' "foo" +-- "fooxxxx" +-- +-- >>> justifyLeft 3 'x' "foobar" +-- "foobar" +justifyLeft :: Int -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChar (k-len) c + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyRight 7 'x' "bar" +-- "xxxxbar" +-- +-- >>> justifyRight 3 'x' "foobar" +-- "foobar" +justifyRight :: Int -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChar (k-len) c `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- >>> center 8 'x' "HS" +-- "xxxHSxxx" +center :: Int -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChar l c `append` t `append` replicateChar r c + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +-- +-- Examples: +-- +-- >>> transpose ["green","orange"] +-- ["go","rr","ea","en","ng","e"] +-- +-- >>> transpose ["blue","red"] +-- ["br","le","ud","e"] +transpose :: [Text] -> [Text] +transpose ts = P.map pack (L.transpose (P.map unpack ts)) + +-- ----------------------------------------------------------------------------- +-- * Reducing 'Text's (folds) + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z (Text arr off len) = go (off + len - 1) + where + go !i + | i < off = z + | otherwise = let !(Iter c l) = reverseIterArray arr i in f (go (i + l)) c +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) (f acc c) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = case uncons t of + Nothing -> emptyError "foldl" + Just (c, t') -> foldl f c t' +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = case uncons t of + Nothing -> emptyError "foldl'" + Just (c, t') -> foldl' f c t' +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = pure acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) P.=<< f acc c +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- If the binary operator is strict in its second argument, use 'foldr'' +-- instead. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +-- +-- Searches from left to right with short-circuiting behavior can +-- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem'). +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = case unsnoc t of + Nothing -> emptyError "foldr1" + Just (t', c) -> foldr f c t' +{-# INLINE foldr1 #-} + +-- | /O(n)/ A strict version of 'foldr'. +-- +-- 'foldr'' evaluates as a right-to-left traversal using constant stack space. +-- +-- @since 2.0.1 +foldr' :: (Char -> a -> a) -> a -> Text -> a +foldr' f z (Text arr off len) = go (off + len - 1) z + where + go !i !acc + | i < off = acc + | otherwise = let !(Iter c l) = reverseIterArray arr i in go (i + l) (f c acc) +{-# INLINE foldr' #-} + +-- ----------------------------------------------------------------------------- +-- ** Special folds + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat ts = case ts of + [] -> empty + [t] -> t + _ | len == 0 -> empty + | otherwise -> Text (A.run go) 0 len + where + len = sumP "concat" $ L.map lengthWord8 ts + go :: ST s (A.MArray s) + go = do + arr <- A.new len + let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) + foldM step 0 ts >> return arr + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p = foldr (\c acc -> p c || acc) False +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p = foldr (\c acc -> p c && acc) True +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum = foldl1' max +-- This could be implemented faster: look for the longest +-- and largest UTF-8 sequence, then decode it to Char only once, +-- instead of decoding all characters, but I doubt anyone cares +-- about the performance of 'maximum' much. +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum = foldl1' min +-- This could be implemented faster, see the comment for 'maximum' above. +{-# INLINE minimum #-} + +-- ----------------------------------------------------------------------------- +-- * Building 'Text's +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + marr <- A.new l' + d' <- unsafeWrite marr 0 c0' + outer marr l' o d' c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Char -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !c + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + pure $ Text arr 0 dstOff + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff c + | otherwise = do + let !(Iter c' d) = iterArray src srcOff + c'' = safe $ f c c' + d' <- unsafeWrite dst dstOff c'' + inner (srcOff + d) (dstOff + d') c'' + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t | null t = empty + | otherwise = scanl f (unsafeHead t) (unsafeTail t) +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + !d' = utf8Length c0' + marr <- A.new l' + _ <- unsafeWrite marr (l' - d') c0' + outer marr (l + o - 1) (l' - d' - 1) c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Char -> ST s Text + outer !dst = inner + where + inner !srcOff !dstOff !c + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + pure $ Text arr (dstOff + 1) (dstLen - dstOff - 1) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) c + | otherwise = do + let !(Iter c' d) = reverseIterArray src srcOff + c'' = safe $ f c' c + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' c'' + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) +{-# INLINE scanr1 #-} + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumL f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !z + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (z, Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff z + | otherwise = do + let !(Iter c d) = iterArray src srcOff + (z', c') = f z c + d' <- unsafeWrite dst dstOff (safe c') + inner (srcOff + d) (dstOff + d') z' + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. +-- Performs replacement on invalid scalar values. +mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumR f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + o - 1) (l + 4 - 1) z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) + outer !dst = inner + where + inner !srcOff !dstOff !z + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) z + | otherwise = do + let !(Iter c d) = reverseIterArray src srcOff + (z', c') = f z c + c'' = safe c' + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' z' + +-- ----------------------------------------------------------------------------- +-- ** Generating and unfolding 'Text's + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int -> Text -> Text +replicate n t@(Text a o l) + | n <= 0 || l <= 0 = empty + | n == 1 = t + | isSingleton t = replicateChar n (unsafeHead t) + | otherwise = runST $ do + let totalLen = n `mul` l + marr <- A.new totalLen + A.copyI l marr 0 a o + A.tile marr l + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen +{-# INLINE [1] replicate #-} + +{-# RULES +"TEXT replicate/singleton -> replicateChar" [~1] forall n c. + replicate n (singleton c) = replicateChar n c + #-} + +-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the +-- value of every element. +replicateChar :: Int -> Char -> Text +replicateChar !len !c' + | len <= 0 = empty + | Char.isAscii c = runST $ do + marr <- A.newFilled len (Char.ord c) + arr <- A.unsafeFreeze marr + return $ Text arr 0 len + | otherwise = runST $ do + let cLen = utf8Length c + totalLen = cLen P.* len + marr <- A.new totalLen + _ <- unsafeWrite marr 0 c + A.tile marr cLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen + where + c = safe c' +{-# INLINE replicateChar #-} + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- ----------------------------------------------------------------------------- +-- * Substrings + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int -> Text -> Text +take n t@(Text arr off len) + | n <= 0 = empty + | n >= len || m >= len || m < 0 = t + | otherwise = Text arr off m + where + m = measureOff n t +{-# INLINE [1] take #-} + +-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ +-- returns a non-negative number, measuring their size in 'Word8'. Otherwise, +-- if @t@ is shorter, return a non-positive number, which is a negated total count +-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. +-- +-- This function is used to implement 'take', 'drop', 'splitAt' and 'length' +-- and is useful on its own in streaming and parsing libraries. +-- +-- @since 2.0 +measureOff :: Int -> Text -> Int +measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else + cSsizeToInt $ + measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> takeEnd 3 "foobar" +-- "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int -> Text -> Text +takeEnd n t@(Text arr off len) + | n <= 0 = empty + | n >= len = t + | otherwise = text arr (off+i) (len-i) + where i = iterNEnd n t + +iterNEnd :: Int -> Text -> Int +iterNEnd n t@(Text _arr _off len) = loop (len-1) n + where loop i !m + | m <= 0 = i+1 + | i <= 0 = 0 + | otherwise = loop (i+d) (m-1) + where d = reverseIter_ t i + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int -> Text -> Text +drop n t@(Text arr off len) + | n <= 0 = t + | n >= len || m >= len || m < 0 = empty + | otherwise = Text arr (off+m) (len-m) + where m = measureOff n t +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> dropEnd 3 "foobar" +-- "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int -> Text -> Text +dropEnd n t@(Text arr off len) + | n <= 0 = t + | n >= len = empty + | otherwise = text arr off (iterNEnd n t) + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t@(Text arr off len) = loop 0 + where loop !i | i >= len = t + | p c = loop (i+d) + | otherwise = text arr off i + where Iter c d = iter t i +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- >>> takeWhileEnd (=='o') "foo" +-- "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = t + | p c = loop (i+d) (l+d) + | otherwise = text arr (off+l) (len-l) + where Iter c d = reverseIter t i +{-# INLINE [1] takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t@(Text arr off len) = loop 0 0 + where loop !i !l | l >= len = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr (off+i) (len-l) + where Iter c d = iter t i +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- >>> dropWhileEnd (=='.') "foo..." +-- "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr off l + where Iter c d = reverseIter t i +{-# INLINE [1] dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile Char.isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd Char.isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround Char.isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int -> Text -> (Text, Text) +splitAt n t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len || m < 0 = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m = measureOff n t + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p t = case span_ p t of + (# hd,tl #) -> (hd,tl) +{-# INLINE span #-} + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is +-- over elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p = span (not . p) +{-# INLINE break #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t@(Text arr off len) = go 0 + where + go !i | i < len = case iterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off i, text arr (off+i) (len-i)) + go _ = pure (t, empty) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t@(Text arr off len) = go (len-1) + where + go !i | 0 <= i = case reverseIterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off (i+1), text arr (off+i+1) (len-i-1)) + go _ = pure (empty, t) +{-# INLINE spanEndM #-} + +-- | /O(n)/ Group characters in a string according to a predicate. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy p = loop + where + loop t@(Text arr off len) + | null t = [] + | otherwise = text arr off n : loop (text arr (off+n) (len-n)) + where Iter c d = iter t 0 + n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) + +-- | Returns the /array/ index (in units of 'Word8') at which a +-- character may be found. This is /not/ the same as the logical +-- index returned by e.g. 'findIndex'. +findAIndexOrEnd :: (Char -> Bool) -> Text -> Int +findAIndexOrEnd q t@(Text _arr _off len) = go 0 + where go !i | i >= len || q c = i + | otherwise = go (i+d) + where Iter c d = iter t i + +-- | /O(n)/ Group characters in a string by equality. +group :: Text -> [Text] +group = groupBy (==) + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +inits :: Text -> [Text] +inits = (NonEmptyList.toList $!) . initsNE + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmptyList.NonEmpty Text +initsNE t = empty NonEmptyList.:| case t of + Text arr off len -> + let loop i + | i >= len = [] + | otherwise = let !j = i + iter_ t i in Text arr off j : loop j + in loop 0 + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NonEmptyList.toList $!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmptyList.NonEmpty Text +tailsNE t + | null t = empty NonEmptyList.:| [] + | otherwise = t NonEmptyList.:| tails (unsafeTail t) + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be empty), consuming the delimiter. An empty +-- delimiter is invalid, and will cause an error to be raised. +-- +-- Examples: +-- +-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne" +-- ["a","b","d","e"] +-- +-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa" +-- ["","X","X","X",""] +-- +-- >>> splitOn "x" "x" +-- ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "splitOn" + | isSingleton pat = split (== unsafeHead pat) src + | otherwise = go 0 (indices pat src) + where + go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs + go s _ = [text arr (s+off) (len-s)] +{-# INLINE [1] splitOn #-} + +{-# RULES +"TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- >>> split (=='a') "aabbaca" +-- ["","","bb","c",""] +-- +-- >>> split (=='a') "" +-- [""] +split :: (Char -> Bool) -> Text -> [Text] +split p t + | null t = [empty] + | otherwise = loop t + where loop s | null s' = [l] + | otherwise = l : loop (unsafeTail s') + where (# l, s' #) = span_ (not . p) s +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- >>> chunksOf 3 "foobarbaz" +-- ["foo","bar","baz"] +-- +-- >>> chunksOf 4 "haskell.org" +-- ["hask","ell.","org"] +chunksOf :: Int -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- ---------------------------------------------------------------------------- +-- * Searching + +------------------------------------------------------------------------------- +-- ** Searching with a predicate + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem = any . (==) +-- TODO This can be implemented much faster: there is no need to decode +-- any UTF-8 sequences at all. +{-# INLINE elem #-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element matching the predicate, or 'Nothing' if +-- there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p = foldr (\c acc -> if p c then Just c else acc) Nothing +{-# INLINE find #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = filter_ text p +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- >>> breakOn "::" "a::b::c" +-- ("a","::b::c") +-- +-- >>> breakOn "/" "foobar" +-- ("foobar","") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src@(Text arr off len) + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> (text arr off x, text arr (off+x) (len-x)) +{-# INLINE breakOn #-} + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the +-- string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- >>> breakOnEnd "::" "a::b::c" +-- ("a::b::","c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = (reverse b, reverse a) + where (a,b) = breakOn (reverse pat) (reverse src) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- >>> breakOnAll "::" "" +-- [] +-- +-- >>> breakOnAll "/" "a/b/c/" +-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")] +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src@(Text arr off slen) + | null pat = emptyError "breakOnAll" + | otherwise = L.map step (indices pat src) + where + step x = (chunk 0 x, chunk x (slen-x)) + chunk !n !l = text arr (n+off) l +{-# INLINE breakOnAll #-} + +------------------------------------------------------------------------------- +-- ** Indexing 'Text's + +-- $index +-- +-- If you think of a 'Text' value as an array of 'Char' values (which +-- it is not), you run the risk of writing inefficient code. +-- +-- An idiom that is common in some languages is to find the numeric +-- offset of a character or substring, then use that number to split +-- or trim the searched string. With a 'Text' value, this approach +-- would require two /O(n)/ operations: one to perform the search, and +-- one to operate from wherever the search ended. +-- +-- For example, suppose you have a string that you want to split on +-- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of +-- searching for the index of @\"::\"@ and taking the substrings +-- before and after that index, you would instead use @breakOnAll \"::\"@. + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int -> Char +index t@(Text _ _ lenInBytes) ix + | ix < 0 + = P.error $ "Data.Text.index: negative index " ++ P.show ix + | off < 0 || off == lenInBytes + = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + | otherwise = ch + where + off = measureOff ix t + Iter ch _ = iter t off +{-# INLINE index #-} + +-- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' +-- and returns the index of the first element in the 'Text' satisfying +-- the predicate. +findIndex :: (Char -> Bool) -> Text -> Maybe Int +findIndex p t = S.findIndex p (stream t) +{-# INLINE findIndex #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int +count pat + | null pat = emptyError "count" + | isSingleton pat = countChar (unsafeHead pat) + | otherwise = L.length . indices pat +{-# INLINE [1] count #-} + +{-# RULES +"TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int +countChar c = foldl' (\acc c' -> if c == c' then acc + 1 else acc) 0 +{-# INLINE countChar #-} + +------------------------------------------------------------------------------- +-- * Zipping + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [1] zipWith #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words (Text arr off len) = loop 0 0 + where + loop !start !n + | n >= len = if start == n + then [] + else [Text arr (start + off) (n - start)] + -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 + | isAsciiSpace w0 = + if start == n + then loop (n + 1) (n + 1) + else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) + | w0 < 0x80 = loop start (n + 1) + -- or 2 bytes for 0xA0 + | w0 == 0xC2, w1 == 0xA0 = + if start == n + then loop (n + 2) (n + 2) + else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) + | w0 < 0xE0 = loop start (n + 2) + -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 + | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 + || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) + || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = + if start == n + then loop (n + 3) (n + 3) + else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) + | otherwise = loop start (n + utf8LengthByLeader w0) + where + w0 = A.unsafeIndex arr (off + n) + w1 = A.unsafeIndex arr (off + n + 1) + w2 = A.unsafeIndex arr (off + n + 2) +{-# INLINE words #-} + +-- Adapted from Data.ByteString.Internal.isSpaceWord8 +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) +{-# INLINE isAsciiSpace #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines (Text arr@(A.ByteArray arr#) off len) = go off + where + go !n + | n >= len + off = [] + | delta < 0 = [Text arr n (len + off - n)] + | otherwise = Text arr n delta : go (n + delta + 1) + where + delta = memchr arr# n (len + off - n) 0x0A +{-# INLINE lines #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr bOff aLen +{-# INLINE [1] isPrefixOf #-} + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr (bOff+d) aLen +{-# INLINE isSuffixOf #-} + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripPrefix "foo" "foobar" +-- Just "bar" +-- +-- >>> stripPrefix "" "baz" +-- Just "baz" +-- +-- >>> stripPrefix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p@(Text _arr _off plen) t@(Text arr off len) + | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) + | otherwise = Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- >>> commonPrefixes "foobar" "fooquux" +-- Just ("foo","bar","quux") +-- +-- >>> commonPrefixes "veeble" "fetzer" +-- Nothing +-- +-- >>> commonPrefixes "" "baz" +-- Nothing +commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) +commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) + | len0 == 0 = Nothing + | len1 == 0 = Nothing + | otherwise = go 0 0 + where + go !i !j + | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) + | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) + | a == b = go (i + 1) k + | k > 0 = Just (Text arr0 off0 k, + Text arr0 (off0 + k) (len0 - k), + Text arr1 (off1 + k) (len1 - k)) + | otherwise = Nothing + where + a = A.unsafeIndex arr0 (off0 + i) + b = A.unsafeIndex arr1 (off1 + i) + isLeader = word8ToInt8 a >= -64 + k = if isLeader then i else j +{-# INLINE commonPrefixes #-} + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripSuffix "bar" "foobar" +-- Just "foo" +-- +-- >>> stripSuffix "" "baz" +-- Just "baz" +-- +-- >>> stripSuffix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p@(Text _arr _off plen) t@(Text arr off len) + | p `isSuffixOf` t = Just $! text arr off (len-plen) + | otherwise = Nothing + +-- | Add a list of non-negative numbers. Errors out on overflow. +sumP :: String -> [Int] -> Int +sumP fun = L.foldl' add 0 + where add a x + | ax >= 0 = ax + | otherwise = overflowError fun + where ax = a + x +{-# INLINE sumP #-} -- Use foldl' and inline for fusion. + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" + +overflowError :: HasCallStack => String -> a +overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" + +-- | Convert a value to 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +-- | /O(n)/ Make a distinct copy of the given string, sharing no +-- storage with the original string. +-- +-- As an example, suppose you read a large string, of which you need +-- only a small portion. If you do not use 'copy', the entire original +-- array will be kept alive in memory by the smaller string. Making a +-- copy \"breaks the link\" to the original array, allowing it to be +-- garbage collected if there are no other live references to it. +copy :: Text -> Text +copy t@(Text arr off len) + | null t = empty + | otherwise = Text (A.run go) 0 len + where + go :: ST s (A.MArray s) + go = do + marr <- A.new len + A.copyI len marr 0 arr off + return marr + +ord8 :: Char -> Word8 +ord8 = P.fromIntegral . Char.ord + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = P.fromIntegral + +word8ToInt8 :: Word8 -> Int8 +word8ToInt8 = P.fromIntegral + +------------------------------------------------- +-- NOTE: the named chunk below used by doctest; +-- verify the doctests via `doctest -fobject-code Data/Text.hs` + +-- $setup +-- >>> :set -XOverloadedStrings +-- >>> import qualified Data.Text as T
src/Data/Text/Array.hs view
@@ -1,330 +1,361 @@-{-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types,- RecordWildCards, UnboxedTuples, UnliftedFFITypes #-}-{-# OPTIONS_GHC -fno-warn-unused-matches #-}--- |--- Module : Data.Text.Array--- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Packed, unboxed, heap-resident arrays. Suitable for performance--- critical use, both in terms of large data quantities and high--- speed.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text.Array as A------ The names in this module resemble those in the 'Data.Array' family--- of modules, but are shorter due to the assumption of qualified--- naming.-module Data.Text.Array- (- -- * Types- Array(..)- , MArray(..)- -- * Functions- , resizeM- , shrinkM- , copyM- , copyI- , copyFromPointer- , copyToPointer- , empty- , equal- , compare- , run- , run2- , toList- , unsafeFreeze- , unsafeIndex- , new- , newPinned- , newFilled- , unsafeWrite- , tile- , getSizeofMArray- ) where--#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif-#if !MIN_VERSION_base(4,11,0)-import Data.Text.Internal.Unsafe (inlinePerformIO)-import Foreign.C.Types (CInt(..))-#endif-import GHC.Exts hiding (toList)-import GHC.ST (ST(..), runST)-import GHC.Word (Word8(..))-import qualified Prelude-import Prelude hiding (length, read, compare)---- | Immutable array type.-data Array = ByteArray ByteArray#---- | Mutable array type, for use in the ST monad.-data MArray s = MutableByteArray (MutableByteArray# s)---- | Create an uninitialized mutable array.-new :: forall s. Int -> ST s (MArray s)-new (I# len#)-#if defined(ASSERTS)- | I# len# < 0 = error "Data.Text.Array.new: size overflow"-#endif- | otherwise = ST $ \s1# ->- case newByteArray# len# s1# of- (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)-{-# INLINE new #-}---- | Create an uninitialized mutable pinned array.------ @since 2.0-newPinned :: forall s. Int -> ST s (MArray s)-newPinned (I# len#)-#if defined(ASSERTS)- | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow"-#endif- | otherwise = ST $ \s1# ->- case newPinnedByteArray# len# s1# of- (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)-{-# INLINE newPinned #-}---- | @since 2.0-newFilled :: Int -> Int -> ST s (MArray s)-newFilled (I# len#) (I# c#) = ST $ \s1# ->- case newByteArray# len# s1# of- (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of- s3# -> (# s3#, MutableByteArray marr# #)-{-# INLINE newFilled #-}---- | @since 2.0-tile :: MArray s -> Int -> ST s ()-tile marr tileLen = do- totalLen <- getSizeofMArray marr- let go l- | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l)- | otherwise = copyM marr l marr 0 l >> go (2 * l)- go tileLen-{-# INLINE tile #-}---- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!-unsafeFreeze :: MArray s -> ST s Array-unsafeFreeze (MutableByteArray marr) = ST $ \s1# ->- case unsafeFreezeByteArray# marr s1# of- (# s2#, ba# #) -> (# s2#, ByteArray ba# #)-{-# INLINE unsafeFreeze #-}---- | Unchecked read of an immutable array. May return garbage or--- crash on an out-of-bounds access.-unsafeIndex ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Array -> Int -> Word8-unsafeIndex (ByteArray arr) i@(I# i#) =-#if defined(ASSERTS)- let word8len = I# (sizeofByteArray# arr) in- if i < 0 || i >= word8len- then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len)- else-#endif- case indexWord8Array# arr i# of r# -> (W8# r#)-{-# INLINE unsafeIndex #-}---- | @since 2.0-getSizeofMArray :: MArray s -> ST s Int-getSizeofMArray (MutableByteArray marr) = ST $ \s0# ->- -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is- -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'.- case getSizeofMutableByteArray# marr s0# of- (# s1#, word8len# #) -> (# s1#, I# word8len# #)--#if defined(ASSERTS)-checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s ()-checkBoundsM ma i elSize = do- len <- getSizeofMArray ma- if i < 0 || i + elSize > len- then error ("bounds error, offset " ++ show i ++ ", length " ++ show len)- else return ()-#endif---- | Unchecked write of a mutable array. May return garbage or crash--- on an out-of-bounds access.-unsafeWrite ::-#if defined(ASSERTS)- HasCallStack =>-#endif- MArray s -> Int -> Word8 -> ST s ()-unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) =-#if defined(ASSERTS)- checkBoundsM ma i 1 >>-#endif- (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of- s2# -> (# s2#, () #))-{-# INLINE unsafeWrite #-}---- | Convert an immutable array to a list.-toList :: Array -> Int -> Int -> [Word8]-toList ary off len = loop 0- where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1)- | otherwise = []---- | An empty immutable array.-empty :: Array-empty = runST (new 0 >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result.-run :: (forall s. ST s (MArray s)) -> Array-run k = runST (k >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result paired with whatever else the action returns.-run2 :: (forall s. ST s (MArray s, a)) -> (Array, a)-run2 k = runST (do- (marr,b) <- k- arr <- unsafeFreeze marr- return (arr,b))-{-# INLINE run2 #-}---- | @since 2.0-resizeM :: MArray s -> Int -> ST s (MArray s)-resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# ->- case resizeMutableByteArray# ma i# s1# of- (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #)-{-# INLINE resizeM #-}---- | @since 2.0-shrinkM ::-#if defined(ASSERTS)- HasCallStack =>-#endif- MArray s -> Int -> ST s ()-shrinkM (MutableByteArray marr) i@(I# newSize) = do-#if defined(ASSERTS)- oldSize <- getSizeofMArray (MutableByteArray marr)- if I# newSize > oldSize- then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize)- else return ()-#endif- ST $ \s1# ->- case shrinkMutableByteArray# marr newSize s1# of- s2# -> (# s2#, () #)-{-# INLINE shrinkM #-}---- | Copy some elements of a mutable array.-copyM :: MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> MArray s -- ^ Source- -> Int -- ^ Source offset- -> Int -- ^ Count- -> ST s ()-copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyM: count must be >= 0, but got " ++ show count-#endif- | otherwise = do-#if defined(ASSERTS)- srcLen <- getSizeofMArray src- dstLen <- getSizeofMArray dst- if srcOff + count > srcLen- then error "copyM: source is too short"- else return ()- if dstOff + count > dstLen- then error "copyM: destination is too short"- else return ()-#endif- ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyM #-}---- | Copy some elements of an immutable array.-copyI :: Int -- ^ Count- -> MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> Array -- ^ Source- -> Int -- ^ Source offset- -> ST s ()-copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#)-#if defined(ASSERTS)- | count < 0 = error $- "copyI: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyByteArray# src# srcOff# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyI #-}---- | Copy from pointer.------ @since 2.0-copyFromPointer- :: MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> Ptr Word8 -- ^ Source- -> Int -- ^ Count- -> ST s ()-copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyFromPointer: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyAddrToByteArray# src# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyFromPointer #-}---- | Copy to pointer.------ @since 2.0-copyToPointer- :: Array -- ^ Source- -> Int -- ^ Source offset- -> Ptr Word8 -- ^ Destination- -> Int -- ^ Count- -> ST s ()-copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyToPointer: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyByteArrayToAddr# src# srcOff# dst# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyToPointer #-}---- | Compare portions of two arrays for equality. No bounds checking--- is performed.-equal :: Array -> Int -> Array -> Int -> Int -> Bool-equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0-{-# INLINE equal #-}---- | Compare portions of two arrays. No bounds checking is performed.------ @since 2.0-compare :: Array -> Int -> Array -> Int -> Int -> Ordering-compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0-{-# INLINE compare #-}--compareInternal- :: Array -- ^ First- -> Int -- ^ Offset into first- -> Array -- ^ Second- -> Int -- ^ Offset into second- -> Int -- ^ Count- -> Int-compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i- where-#if MIN_VERSION_base(4,11,0)- i = I# (compareByteArrays# src1# off1# src2# off2# count#)-#else- i = fromIntegral (inlinePerformIO (memcmp src1# off1# src2# off2# count#))--foreign import ccall unsafe "_hs_text_memcmp2" memcmp- :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> IO CInt-#endif-{-# INLINE compareInternal #-}+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-unused-matches #-} +-- | +-- Module : Data.Text.Array +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Packed, unboxed, heap-resident arrays. Suitable for performance +-- critical use, both in terms of large data quantities and high +-- speed. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text.Array as A +-- +-- The names in this module resemble those in the 'Data.Array' family +-- of modules, but are shorter due to the assumption of qualified +-- naming. +module Data.Text.Array + ( + -- * Types + Array + , pattern ByteArray + , MArray + , pattern MutableByteArray + -- * Functions + , resizeM + , shrinkM + , copyM + , copyI + , copyFromPointer + , copyToPointer + , empty + , equal + , compare + , run + , run2 + , toList + , unsafeFreeze + , unsafeIndex + , new + , newPinned + , newFilled + , unsafeWrite + , tile + , getSizeofMArray + ) where + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif +#if !MIN_VERSION_base(4,11,0) +import Foreign.C.Types (CInt(..)) +#endif +import GHC.Exts hiding (toList) +import GHC.ST (ST(..), runST) +import GHC.Word (Word8(..)) +import qualified Prelude +import Prelude hiding (length, read, compare) +import Data.Array.Byte (ByteArray(..), MutableByteArray(..)) + +-- | Immutable array type. +type Array = ByteArray + +-- | Mutable array type, for use in the ST monad. +type MArray = MutableByteArray + +-- | Create an uninitialized mutable array. +new :: forall s. Int -> ST s (MArray s) +new (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.new: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE new #-} + +-- | Create an uninitialized mutable pinned array. +-- +-- @since 2.0 +newPinned :: forall s. Int -> ST s (MArray s) +newPinned (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newPinnedByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE newPinned #-} + +-- | @since 2.0 +newFilled :: Int -> Int -> ST s (MArray s) +newFilled (I# len#) (I# c#) = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of + s3# -> (# s3#, MutableByteArray marr# #) +{-# INLINE newFilled #-} + +-- | @since 2.0 +tile :: MArray s -> Int -> ST s () +tile marr tileLen = do + totalLen <- getSizeofMArray marr + let go l + | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) + | otherwise = copyM marr l marr 0 l >> go (2 * l) + go tileLen +{-# INLINE tile #-} + +-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! +unsafeFreeze :: MArray s -> ST s Array +unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> + case unsafeFreezeByteArray# marr s1# of + (# s2#, ba# #) -> (# s2#, ByteArray ba# #) +{-# INLINE unsafeFreeze #-} + +-- | Unchecked read of an immutable array. May return garbage or +-- crash on an out-of-bounds access. +unsafeIndex :: +#if defined(ASSERTS) + HasCallStack => +#endif + Array -> Int -> Word8 +unsafeIndex (ByteArray arr) i@(I# i#) = +#if defined(ASSERTS) + let word8len = I# (sizeofByteArray# arr) in + if i < 0 || i >= word8len + then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) + else +#endif + case indexWord8Array# arr i# of r# -> (W8# r#) +{-# INLINE unsafeIndex #-} + +-- | @since 2.0 +getSizeofMArray :: MArray s -> ST s Int +getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> + -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is + -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. + case getSizeofMutableByteArray# marr s0# of + (# s1#, word8len# #) -> (# s1#, I# word8len# #) + +#if defined(ASSERTS) +checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () +checkBoundsM ma i elSize = do + len <- getSizeofMArray ma + if i < 0 || i + elSize > len + then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) + else return () +#endif + +-- | Unchecked write of a mutable array. May return garbage or crash +-- on an out-of-bounds access. +unsafeWrite :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> Word8 -> ST s () +unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = +#if defined(ASSERTS) + checkBoundsM ma i 1 >> +#endif + (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of + s2# -> (# s2#, () #)) +{-# INLINE unsafeWrite #-} + +-- | Convert an immutable array to a list. +toList :: Array -> Int -> Int -> [Word8] +toList ary off len = loop 0 + where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) + | otherwise = [] + +-- | An empty immutable array. +empty :: Array +empty = runST (new 0 >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result. +run :: (forall s. ST s (MArray s)) -> Array +run k = runST (k >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result paired with whatever else the action returns. +run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) +run2 k = runST (do + (marr,b) <- k + arr <- unsafeFreeze marr + return (arr,b)) +{-# INLINE run2 #-} + +-- | @since 2.0 +resizeM :: MArray s -> Int -> ST s (MArray s) +resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> + case resizeMutableByteArray# ma i# s1# of + (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) +{-# INLINE resizeM #-} + +-- | @since 2.0 +shrinkM :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> ST s () +shrinkM (MutableByteArray marr) i@(I# newSize) = do +#if defined(ASSERTS) + oldSize <- getSizeofMArray (MutableByteArray marr) + if I# newSize > oldSize + then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) + else return () +#endif + ST $ \s1# -> + case shrinkMutableByteArray# marr newSize s1# of + s2# -> (# s2#, () #) +{-# INLINE shrinkM #-} + +-- | Copy some elements of a mutable array. +copyM :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> MArray s -- ^ Source + -> Int -- ^ Source offset + -> Int -- ^ Count + -> ST s () +copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyM: count must be >= 0, but got " ++ show count +#endif + | otherwise = do +#if defined(ASSERTS) + srcLen <- getSizeofMArray src + dstLen <- getSizeofMArray dst + if srcOff + count > srcLen + then error "copyM: source is too short" + else return () + if dstOff + count > dstLen + then error "copyM: destination is too short" + else return () +#endif + ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyM #-} + +-- | Copy some elements of an immutable array. +copyI :: Int -- ^ Count + -> MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Array -- ^ Source + -> Int -- ^ Source offset + -> ST s () +copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyI: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyI #-} + +-- | Copy from pointer. +-- +-- @since 2.0 +copyFromPointer + :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Ptr Word8 -- ^ Source + -> Int -- ^ Count + -> ST s () +copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyFromPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyAddrToByteArray# src# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyFromPointer #-} + +-- | Copy to pointer. +-- +-- @since 2.0 +copyToPointer + :: Array -- ^ Source + -> Int -- ^ Source offset + -> Ptr Word8 -- ^ Destination + -> Int -- ^ Count + -> ST s () +copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyToPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArrayToAddr# src# srcOff# dst# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyToPointer #-} + +-- | Compare portions of two arrays for equality. No bounds checking +-- is performed. +equal + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Bool +equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 +{-# INLINE equal #-} + +-- | Compare portions of two arrays. No bounds checking is performed. +-- +-- @since 2.0 +compare + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Ordering +compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 +{-# INLINE compare #-} + +compareInternal + :: Array -- ^ First + -> Int -- ^ Offset into first + -> Array -- ^ Second + -> Int -- ^ Offset into second + -> Int -- ^ Count + -> Int +compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i + where +#if MIN_VERSION_base(4,11,0) + i = I# (compareByteArrays# src1# off1# src2# off2# count#) +#else + i = fromIntegral (memcmp src1# off1# src2# off2# count#) + +foreign import ccall unsafe "_hs_text_memcmp2" memcmp + :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt +#endif +{-# INLINE compareInternal #-}
src/Data/Text/Encoding.hs view
@@ -1,603 +1,588 @@-{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash,- UnliftedFFITypes #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE ViewPatterns #-}--- |--- Module : Data.Text.Encoding--- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts,--- (c) 2008, 2009 Tom Harper--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Functions for converting 'Text' values to and from 'ByteString',--- using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Encoding- (- -- * Decoding ByteStrings to Text- -- $strict- decodeASCII- , decodeLatin1- , decodeUtf8- , decodeUtf16LE- , decodeUtf16BE- , decodeUtf32LE- , decodeUtf32BE-- -- ** Catchable failure- , decodeUtf8'-- -- ** Controllable error handling- , decodeUtf8With- , decodeUtf8Lenient- , decodeUtf16LEWith- , decodeUtf16BEWith- , decodeUtf32LEWith- , decodeUtf32BEWith-- -- ** Stream oriented decoding- -- $stream- , streamDecodeUtf8- , streamDecodeUtf8With- , Decoding(..)-- -- * Encoding Text to ByteStrings- , encodeUtf8- , encodeUtf16LE- , encodeUtf16BE- , encodeUtf32LE- , encodeUtf32BE-- -- * Encoding Text using ByteString Builders- , encodeUtf8Builder- , encodeUtf8BuilderEscaped- ) where--import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)--import Control.Exception (evaluate, try)-import Control.Monad.ST (runST, ST)-import Data.Bits (shiftR, (.&.))-import Data.ByteString (ByteString)-import qualified Data.ByteString as B-import qualified Data.ByteString.Internal as B-import qualified Data.ByteString.Short.Internal as SBS-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode)-import Data.Text.Internal (Text(..), safe, empty, append)-import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Data.Text.Show as T (singleton)-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Data.Word (Word8)-import Foreign.C.Types (CSize(..))-import Foreign.Ptr (Ptr, minusPtr, plusPtr)-import Foreign.Storable (poke, peekByteOff)-import GHC.Exts (byteArrayContents#, unsafeCoerce#)-import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr))-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Builder.Prim.Internal as BP-import Data.Text.Internal.Encoding.Utf8 (utf8DecodeStart, utf8DecodeContinue, DecoderResult(..))-import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Encoding.Fusion as E-import qualified Data.Text.Internal.Fusion as F-import Data.Text.Internal.ByteStringCompat-#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif--#ifdef SIMDUTF-import Foreign.C.Types (CInt(..))-#else-import qualified Data.ByteString.Unsafe as B-import Data.Text.Internal.Encoding.Utf8 (CodePoint(..))-#endif---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- | Decode a 'ByteString' containing 7-bit ASCII--- encoded text.------ This is a partial function: it checks that input does not contain--- anything except ASCII and copies buffer or throws an error otherwise.----decodeASCII :: ByteString -> Text-decodeASCII bs = withBS bs $ \fp len -> if len == 0 then empty else runST $ do- asciiPrefixLen <- fmap cSizeToInt $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- c_is_ascii src (src `plusPtr` len)- if asciiPrefixLen == len- then let !(SBS.SBS arr) = SBS.toShort bs in- return (Text (A.ByteArray arr) 0 len)- else error $ "decodeASCII: detected non-ASCII codepoint at " ++ show asciiPrefixLen---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.------ 'decodeLatin1' is semantically equivalent to--- @Data.Text.pack . Data.ByteString.Char8.unpack@------ This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII)--- characters to UTf-8 requires actual work and is not just buffer copying.----decodeLatin1 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Text-decodeLatin1 bs = withBS bs $ \fp len -> runST $ do- dst <- A.new (2 * len)- let inner srcOff dstOff = if srcOff >= len then return dstOff else do- asciiPrefixLen <- fmap cSizeToInt $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len)- if asciiPrefixLen == 0- then do- byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff- A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6))- A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F))- inner (srcOff + 1) (dstOff + 2)- else do- unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen- inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen)-- actualLen <- inner 0 0- dst' <- A.resizeM dst actualLen- arr <- A.unsafeFreeze dst'- return $ Text arr 0 actualLen--foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii- :: Ptr Word8 -> Ptr Word8 -> IO CSize--isValidBS :: ByteString -> Bool-#ifdef SIMDUTF-isValidBS bs = withBS bs $ \fp len -> unsafeDupablePerformIO $- unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> c_is_valid_utf8 ptr (fromIntegral len)-#else-#if MIN_VERSION_bytestring(0,11,2)-isValidBS = B.isValidUtf8-#else-isValidBS bs = start 0- where- start ix- | ix >= B.length bs = True- | otherwise = case utf8DecodeStart (B.unsafeIndex bs ix) of- Accept{} -> start (ix + 1)- Reject{} -> False- Incomplete st _ -> step (ix + 1) st- step ix st- | ix >= B.length bs = False- -- We do not use decoded code point, so passing a dummy value to save an argument.- | otherwise = case utf8DecodeContinue (B.unsafeIndex bs ix) st (CodePoint 0) of- Accept{} -> start (ix + 1)- Reject{} -> False- Incomplete st' _ -> step (ix + 1) st'-#endif-#endif---- | Decode a 'ByteString' containing UTF-8 encoded text.------ Surrogate code points in replacement character returned by 'OnDecodeError'--- will be automatically remapped to the replacement char @U+FFFD@.-decodeUtf8With ::-#if defined(ASSERTS)- HasCallStack =>-#endif- OnDecodeError -> ByteString -> Text-decodeUtf8With onErr bs- | isValidBS bs =- let !(SBS.SBS arr) = SBS.toShort bs in- (Text (A.ByteArray arr) 0 (B.length bs))- | B.null undecoded = txt- | otherwise = txt `append` (case onErr desc (Just (B.head undecoded)) of- Nothing -> txt'- Just c -> T.singleton c `append` txt')- where- (txt, undecoded) = decodeUtf8With2 onErr mempty bs- txt' = decodeUtf8With onErr (B.tail undecoded)- desc = "Data.Text.Internal.Encoding: Invalid UTF-8 stream"---- | Decode two consecutive bytestrings, returning Text and undecoded remainder.-decodeUtf8With2 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- OnDecodeError -> ByteString -> ByteString -> (Text, ByteString)-decodeUtf8With2 onErr bs1@(B.length -> len1) bs2@(B.length -> len2) = runST $ do- marr <- A.new len'- outer marr len' 0 0- where- len = len1 + len2- len' = len + 4-- index i- | i < len1 = B.index bs1 i- | otherwise = B.index bs2 (i - len1)-- -- We need Data.ByteString.findIndexEnd, but it is unavailable before bytestring-0.10.12.0- guessUtf8Boundary :: Int- guessUtf8Boundary- | len2 >= 1 && w0 < 0x80 = len2 -- last char is ASCII- | len2 >= 1 && w0 >= 0xC0 = len2 - 1 -- last char starts a code point- | len2 >= 2 && w1 >= 0xC0 = len2 - 2 -- pre-last char starts a code point- | len2 >= 3 && w2 >= 0xC0 = len2 - 3- | len2 >= 4 && w3 >= 0xC0 = len2 - 4- | otherwise = 0- where- w0 = B.index bs2 (len2 - 1)- w1 = B.index bs2 (len2 - 2)- w2 = B.index bs2 (len2 - 3)- w3 = B.index bs2 (len2 - 4)-- decodeFrom :: Int -> DecoderResult- decodeFrom off = step (off + 1) (utf8DecodeStart (index off))- where- step i (Incomplete a b)- | i < len = step (i + 1) (utf8DecodeContinue (index i) a b)- step _ st = st-- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s (Text, ByteString)- outer dst dstLen = inner- where- inner srcOff dstOff- | srcOff >= len = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff, mempty)-- | srcOff >= len1- , srcOff < len1 + guessUtf8Boundary- , dstOff + (len1 + guessUtf8Boundary - srcOff) <= dstLen- , bs <- B.drop (srcOff - len1) (B.take guessUtf8Boundary bs2)- , isValidBS bs = do- withBS bs $ \fp _ -> unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- unsafeSTToIO $ A.copyFromPointer dst dstOff src (len1 + guessUtf8Boundary - srcOff)- inner (len1 + guessUtf8Boundary) (dstOff + (len1 + guessUtf8Boundary - srcOff))-- | dstOff + 4 > dstLen = do- let dstLen' = dstLen + 4- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff-- | otherwise = case decodeFrom srcOff of- Accept c -> do- d <- unsafeWrite dst dstOff c- inner (srcOff + d) (dstOff + d)- Reject -> case onErr desc (Just (index srcOff)) of- Nothing -> inner (srcOff + 1) dstOff- Just c -> do- d <- unsafeWrite dst dstOff (safe c)- inner (srcOff + 1) (dstOff + d)- Incomplete{} -> do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- let bs = if srcOff >= len1- then B.drop (srcOff - len1) bs2- else B.drop srcOff (bs1 `B.append` bs2)- return (Text arr 0 dstOff, bs)-- desc = "Data.Text.Internal.Encoding: Invalid UTF-8 stream"---- $stream------ The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept--- a 'ByteString' that represents a possibly incomplete input (e.g. a--- packet from a network stream) that may not end on a UTF-8 boundary.------ 1. The maximal prefix of 'Text' that could be decoded from the--- given input.------ 2. The suffix of the 'ByteString' that could not be decoded due to--- insufficient input.------ 3. A function that accepts another 'ByteString'. That string will--- be assumed to directly follow the string that was passed as--- input to the original function, and it will in turn be decoded.------ To help understand the use of these functions, consider the Unicode--- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi--- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.------ Now suppose that we receive this encoded string as 3 packets that--- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",--- \"\\x83\"]@. We cannot decode the entire Unicode string until we--- have received all three packets, but we would like to make progress--- as we receive each one.------ @--- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\"--- ghci> s0--- 'Some' \"hi \" \"\\xe2\" _--- @------ We use the continuation @f0@ to decode our second packet.------ @--- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\"--- ghci> s1--- 'Some' \"\" \"\\xe2\\x98\"--- @------ We could not give @f0@ enough input to decode anything, so it--- returned an empty string. Once we feed our second continuation @f1@--- the last byte of input, it will make progress.------ @--- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\"--- ghci> s2--- 'Some' \"\\x2603\" \"\" _--- @------ If given invalid input, an exception will be thrown by the function--- or continuation where it is encountered.---- | A stream oriented decoding result.------ @since 1.0.0.0-data Decoding = Some !Text !ByteString (ByteString -> Decoding)--instance Show Decoding where- showsPrec d (Some t bs _) = showParen (d > prec) $- showString "Some " . showsPrec prec' t .- showChar ' ' . showsPrec prec' bs .- showString " _"- where prec = 10; prec' = prec + 1---- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8--- encoded text that is known to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown (either by this function or a continuation) that cannot be--- caught in pure code. For more control over the handling of invalid--- data, use 'streamDecodeUtf8With'.------ @since 1.0.0.0-streamDecodeUtf8 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Decoding-streamDecodeUtf8 = streamDecodeUtf8With strictDecode---- | Decode, in a stream oriented way, a lazy 'ByteString' containing UTF-8--- encoded text.------ @since 1.0.0.0-streamDecodeUtf8With ::-#if defined(ASSERTS)- HasCallStack =>-#endif- OnDecodeError -> ByteString -> Decoding-streamDecodeUtf8With onErr = go mempty- where- go bs1 bs2 = Some txt undecoded (go undecoded)- where- (txt, undecoded) = decodeUtf8With2 onErr bs1 bs2---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code. For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.------ This is a partial function: it checks that input is a well-formed--- UTF-8 sequence and copies buffer or throws an error otherwise.----decodeUtf8 :: ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}---- | Decode a 'ByteString' containing UTF-8 encoded text.------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.-decodeUtf8' ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Either UnicodeException Text-decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode-{-# INLINE decodeUtf8' #-}---- | Decode a 'ByteString' containing UTF-8 encoded text.------ Any invalid input bytes will be replaced with the Unicode replacement--- character U+FFFD.-decodeUtf8Lenient :: ByteString -> Text-decodeUtf8Lenient = decodeUtf8With lenientDecode---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder =- -- manual eta-expansion to ensure inlining works as expected- \txt -> B.builder (step txt)- where- step txt@(Text arr off len) !k br@(B.BufferRange op ope)- -- Ensure that the common case is not recursive and therefore yields- -- better code.- | op' <= ope = do- unsafeSTToIO $ A.copyToPointer arr off op len- k (B.BufferRange op' ope)- | otherwise = textCopyStep txt k br- where- op' = op `plusPtr` len-{-# INLINE encodeUtf8Builder #-}--textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a-textCopyStep (Text arr off len) k =- go off (off + len)- where- go !ip !ipe (B.BufferRange op ope)- | inpRemaining <= outRemaining = do- unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining- let !br = B.BufferRange (op `plusPtr` inpRemaining) ope- k br- | otherwise = do- unsafeSTToIO $ A.copyToPointer arr ip op outRemaining- let !ip' = ip + outRemaining- return $ B.bufferFull 1 ope (go ip' ipe)- where- outRemaining = ope `minusPtr` op- inpRemaining = ipe - ip---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}--- TODO: Extend documentation with references to source code in @blaze-html@--- or @aeson@ that uses this function.-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped be =- -- manual eta-expansion to ensure inlining works as expected- \txt -> B.builder (mkBuildstep txt)- where- bound = max 4 $ BP.sizeBound be-- mkBuildstep (Text arr off len) !k =- outerLoop off- where- iend = off + len-- outerLoop !i0 !br@(B.BufferRange op0 ope)- | i0 >= iend = k br- | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)- -- TODO: Use a loop with an integrated bound's check if outRemaining- -- is smaller than 8, as this will save on divisions.- | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)- where- outRemaining = (ope `minusPtr` op0) `quot` bound- inpRemaining = iend - i0-- goPartial !iendTmp = go i0 op0- where- go !i !op- | i < iendTmp = do- let w = A.unsafeIndex arr i- if w < 0x80- then BP.runB be w op >>= go (i + 1)- else poke op w >> go (i + 1) (op `plusPtr` 1)- | otherwise = outerLoop i (B.BufferRange op ope)---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> ByteString-encodeUtf8 (Text arr off len)- | len == 0 = B.empty- -- It would be easier to use Data.ByteString.Short.fromShort and slice later,- -- but this is undesirable when len is significantly smaller than length arr.- | otherwise = unsafeDupablePerformIO $ do- marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len- unsafeSTToIO $ A.copyI len marr 0 arr off- let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba))- (PlainPtr mba)- pure $ B.fromForeignPtr fp 0 len---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> ByteString-encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> ByteString-encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> ByteString-encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> ByteString-encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))-{-# INLINE encodeUtf32BE #-}--cSizeToInt :: CSize -> Int-cSizeToInt = fromIntegral--#ifdef SIMDUTF-foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8- :: Ptr Word8 -> CSize -> IO CInt-#endif+{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, + UnliftedFFITypes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +-- | +-- Module : Data.Text.Encoding +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting 'Text' values to and from 'ByteString', +-- using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- <http://hackage.haskell.org/package/text-icu text-icu package>. + +module Data.Text.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeASCIIPrefix + , decodeUtf8Lenient + , decodeUtf8' + , decodeASCII' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- *** Incremental UTF-8 decoding + -- $incremental + , decodeUtf8Chunk + , decodeUtf8More + , Utf8State + , startUtf8State + , StrictBuilder + , StrictTextBuilder + , strictBuilderToText + , textToStrictBuilder + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- ** Stream oriented decoding + -- $stream + , streamDecodeUtf8 + , streamDecodeUtf8With + , Decoding(..) + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + + -- * ByteString validation + -- $validation + , validateUtf8Chunk + , validateUtf8More + ) where + +import Control.Exception (evaluate, try) +import Data.Word (Word8) +import GHC.Exts (byteArrayContents#, unsafeCoerce#) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) +import Data.ByteString (ByteString) +#if defined(PURE_HASKELL) +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Char8 (unpack) +import Data.Text.Internal (pack) +import Foreign.Ptr (minusPtr, plusPtr) +import Foreign.Storable (poke) +#else +import Control.Monad.ST (runST) +import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) +import Data.Bits (shiftR, (.&.)) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Foreign.C.Types (CSize(..)) +import Foreign.Ptr (Ptr, minusPtr, plusPtr) +import Foreign.Storable (poke, peekByteOff) +#endif +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Encoding +import Data.Text.Internal.IsAscii (asciiPrefixLength) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Show () +import qualified Data.ByteString as B +import qualified Data.ByteString.Internal as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Builder.Prim.Internal as BP +import qualified Data.ByteString.Short.Internal as SBS +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Encoding.Fusion as E +import qualified Data.Text.Internal.Fusion as F +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +-- $validation +-- These functions are for validating 'ByteString's as encoded text. + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing ASCII text. +-- +-- This is a total function which returns a pair of the longest ASCII prefix +-- as 'Text', and the remaining suffix as 'ByteString'. +-- +-- Important note: the pair is lazy. This lets you check for errors by testing +-- whether the second component is empty, without forcing the first component +-- (which does a copy). +-- To drop references to the input bytestring, force the prefix +-- (using 'seq' or @BangPatterns@) and drop references to the suffix. +-- +-- === Properties +-- +-- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@. +-- - Either @suffix@ is empty, or @'B.head' suffix > 127@. +-- +-- @since 2.0.2 +decodeASCIIPrefix :: ByteString -> (Text, ByteString) +decodeASCIIPrefix bs = if B.null bs + then (empty, B.empty) + else + let len = asciiPrefixLength bs + prefix = + let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in + Text (A.ByteArray arr) 0 len + suffix = B.drop len bs in + (prefix, suffix) +{-# INLINE decodeASCIIPrefix #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a total function which returns either the 'ByteString' converted to a +-- 'Text' containing ASCII text, or 'Nothing'. +-- +-- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid +-- input instead of discarding it. +-- +-- @since 2.0.2 +decodeASCII' :: ByteString -> Maybe Text +decodeASCII' bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + if B.null suffix then Just prefix else Nothing +{-# INLINE decodeASCII' #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a partial function: it checks that input does not contain +-- anything except ASCII and copies buffer or throws an error otherwise. +decodeASCII :: ByteString -> Text +decodeASCII bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + case B.uncons suffix of + Nothing -> prefix + Just (word, _) -> + let !errPos = B.length bs - B.length suffix in + error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +-- +-- 'decodeLatin1' is semantically equivalent to +-- @'Data.Text.pack' . 'Data.ByteString.Char8.unpack'@ +-- +-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) +-- characters to UTf-8 requires actual work and is not just buffer copying. +-- +decodeLatin1 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Text +#if defined(PURE_HASKELL) +decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs) +#else +decodeLatin1 bs = withBS bs $ \fp len -> runST $ do + dst <- A.new (2 * len) + let inner srcOff dstOff = if srcOff >= len then return dstOff else do + asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) + if asciiPrefixLen == 0 + then do + byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff + A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) + A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) + inner (srcOff + 1) (dstOff + 2) + else do + unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen + inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) + actualLen <- inner 0 0 + dst' <- A.resizeM dst actualLen + arr <- A.unsafeFreeze dst' + return $ Text arr 0 actualLen +#endif + +#if !defined(PURE_HASKELL) +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize +#endif + +-- $stream +-- +-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept +-- a strict 'ByteString' that represents a possibly incomplete input (e.g. a +-- packet from a network stream) that may not end on a UTF-8 boundary +-- and return 'Decoding', which consists of: +-- +-- * The maximal prefix of 'Text' that could be decoded from the +-- given input. +-- +-- * The suffix of the 'ByteString' that could not be decoded due to +-- insufficient input. +-- +-- * A function that accepts another 'ByteString'. That string will +-- be assumed to directly follow the string that was passed as +-- input to the original function, and it will in turn be decoded. +-- +-- To help understand the use of these functions, consider the Unicode +-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi +-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. +-- +-- Now suppose that we receive this encoded string as 3 packets that +-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", +-- \"\\x83\"]@. We cannot decode the entire Unicode string until we +-- have received all three packets, but we would like to make progress +-- as we receive each one. +-- +-- @ +-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" +-- ghci> s0 +-- 'Some' \"hi \" \"\\xe2\" _ +-- @ +-- +-- We use the continuation @f0@ to decode our second packet. +-- +-- @ +-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" +-- ghci> s1 +-- 'Some' \"\" \"\\xe2\\x98\" +-- @ +-- +-- We could not give @f0@ enough input to decode anything, so it +-- returned an empty string. Once we feed our second continuation @f1@ +-- the last byte of input, it will make progress. +-- +-- @ +-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" +-- ghci> s2 +-- 'Some' \"\\x2603\" \"\" _ +-- @ +-- +-- If given invalid input, an exception will be thrown by the function +-- or continuation where it is encountered. + +-- | A stream-oriented decoding result (see 'streamDecodeUtf8' and 'streamDecodeUtf8With'). +-- +-- @since 1.0.0.0 +data Decoding = Some + !Text + -- ^ The maximal prefix that could be decoded from the given input. + !ByteString + -- ^ The remaining suffix of the input that could not be decoded + -- (usually because the input breaks in the middle of UTF-8 character) + (ByteString -> Decoding) + -- ^ The continuation call which should be fed with the next + -- chunk of the input. + +instance Show Decoding where + showsPrec d (Some t bs _) = showParen (d > prec) $ + showString "Some " . showsPrec prec' t . + showChar ' ' . showsPrec prec' bs . + showString " _" + where prec = 10; prec' = prec + 1 + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data that is known to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown (either by this function or a continuation) that cannot be +-- caught in pure code. For more control over the handling of invalid +-- data, use 'streamDecodeUtf8With'. +-- +-- @since 1.0.0.0 +streamDecodeUtf8 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Decoding +streamDecodeUtf8 = streamDecodeUtf8With strictDecode + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data. +-- +-- @since 1.0.0.0 +streamDecodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Decoding +streamDecodeUtf8With onErr = loop startUtf8State + where + loop s chunk = + let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk + in Some (strictBuilderToText builder) undecoded (loop s') + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Surrogate code points in replacement character returned by 'OnDecodeError' +-- will be automatically remapped to the replacement char @U+FFFD@. +decodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Text +decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg + +invalidUtf8Msg :: String +invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +-- +-- This is a partial function: it checks that input is a well-formed +-- UTF-8 sequence and copies buffer or throws an error otherwise. +-- +decodeUtf8 :: ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +decodeUtf8' :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Either UnicodeException Text +decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode +{-# INLINE decodeUtf8' #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.0 +decodeUtf8Lenient :: ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (step txt) + where + step txt@(Text arr off len) !k br@(B.BufferRange op ope) + -- Ensure that the common case is not recursive and therefore yields + -- better code. + | op' <= ope = do + unsafeSTToIO $ A.copyToPointer arr off op len + k (B.BufferRange op' ope) + | otherwise = textCopyStep txt k br + where + op' = op `plusPtr` len +{-# INLINE encodeUtf8Builder #-} + +textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a +textCopyStep (Text arr off len) k = + go off (off + len) + where + go !ip !ipe (B.BufferRange op ope) + | inpRemaining <= outRemaining = do + unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining + let !br = B.BufferRange (op `plusPtr` inpRemaining) ope + k br + | otherwise = do + unsafeSTToIO $ A.copyToPointer arr ip op outRemaining + let !ip' = ip + outRemaining + return $ B.bufferFull 1 ope (go ip' ipe) + where + outRemaining = ope `minusPtr` op + inpRemaining = ipe - ip + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +-- TODO: Extend documentation with references to source code in @blaze-html@ +-- or @aeson@ that uses this function. +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped be = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (mkBuildstep txt) + where + bound = max 4 $ BP.sizeBound be + + mkBuildstep (Text arr off len) !k = + outerLoop off + where + iend = off + len + + outerLoop !i0 !br@(B.BufferRange op0 ope) + | i0 >= iend = k br + | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) + -- TODO: Use a loop with an integrated bound's check if outRemaining + -- is smaller than 8, as this will save on divisions. + | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) + where + outRemaining = (ope `minusPtr` op0) `quot` bound + inpRemaining = iend - i0 + + goPartial !iendTmp = go i0 op0 + where + go !i !op + | i < iendTmp = do + let w = A.unsafeIndex arr i + if w < 0x80 + then BP.runB be w op >>= go (i + 1) + else poke op w >> go (i + 1) (op `plusPtr` 1) + | otherwise = outerLoop i (B.BufferRange op ope) + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> ByteString +encodeUtf8 (Text arr off len) + | len == 0 = B.empty + -- It would be easier to use Data.ByteString.Short.fromShort and slice later, + -- but this is undesirable when len is significantly smaller than length arr. + | otherwise = unsafeDupablePerformIO $ do + marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len + unsafeSTToIO $ A.copyI len marr 0 arr off + let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) + (PlainPtr mba) + pure $ B.fromForeignPtr fp 0 len + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> ByteString +encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> ByteString +encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> ByteString +encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> ByteString +encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) +{-# INLINE encodeUtf32BE #-} + +-- $incremental +-- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more +-- control for error-handling and streaming. +-- +-- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error. +-- For example this lets you insert or delete arbitrary text, or do some +-- stateful operations before resuming, such as keeping track of error locations. +-- In contrast, the older stream-oriented interface only lets you substitute +-- a single fixed 'Char' for each invalid byte in 'OnDecodeError'. +-- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks +-- before doing the copying work to construct a 'Text', or you can +-- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'. +-- +-- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'.
src/Data/Text/Encoding/Error.hs view
@@ -1,120 +1,119 @@-{-# LANGUAGE CPP, DeriveDataTypeable #-}-{-# LANGUAGE Safe #-}--- |--- Module : Data.Text.Encoding.Error--- Copyright : (c) Bryan O'Sullivan 2009------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Types and functions for dealing with encoding and decoding errors--- in Unicode text.------ The standard functions for encoding and decoding text are strict,--- which is to say that they throw exceptions on invalid input. This--- is often unhelpful on real world input, so alternative functions--- exist that accept custom handlers for dealing with invalid inputs.--- These 'OnError' handlers are normal Haskell functions. You can use--- one of the presupplied functions in this module, or you can write a--- custom handler of your own.--module Data.Text.Encoding.Error- (- -- * Error handling types- UnicodeException(..)- , OnError- , OnDecodeError- , OnEncodeError- -- * Useful error handling functions- , lenientDecode- , strictDecode- , strictEncode- , ignore- , replace- ) where--import Control.DeepSeq (NFData (..))-import Control.Exception (Exception, throw)-import Data.Typeable (Typeable)-import Data.Word (Word8)-import Numeric (showHex)---- | Function type for handling a coding error. It is supplied with--- two inputs:------ * A 'String' that describes the error.------ * The input value that caused the error. If the error arose--- because the end of input was reached or could not be identified--- precisely, this value will be 'Nothing'.------ If the handler returns a value wrapped with 'Just', that value will--- be used in the output as the replacement for the invalid input. If--- it returns 'Nothing', no value will be used in the output.------ Should the handler need to abort processing, it should use 'error'--- or 'throw' an exception (preferably a 'UnicodeException'). It may--- use the description provided to construct a more helpful error--- report.-type OnError a b = String -> Maybe a -> Maybe b---- | A handler for a decoding error.-type OnDecodeError = OnError Word8 Char---- | A handler for an encoding error.-{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-}-type OnEncodeError = OnError Char Word8---- | An exception type for representing Unicode encoding errors.-data UnicodeException =- DecodeError String (Maybe Word8)- -- ^ Could not decode a byte sequence because it was invalid under- -- the given encoding, or ran out of input in mid-decode.- | EncodeError String (Maybe Char)- -- ^ Tried to encode a character that could not be represented- -- under the given encoding, or ran out of input in mid-encode.- deriving (Eq, Typeable)--{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-}--showUnicodeException :: UnicodeException -> String-showUnicodeException (DecodeError desc (Just w))- = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)-showUnicodeException (DecodeError desc Nothing)- = "Cannot decode input: " ++ desc-showUnicodeException (EncodeError desc (Just c))- = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc)-showUnicodeException (EncodeError desc Nothing)- = "Cannot encode input: " ++ desc--instance Show UnicodeException where- show = showUnicodeException--instance Exception UnicodeException--instance NFData UnicodeException where- rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` ()- rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` ()---- | Throw a 'UnicodeException' if decoding fails.-strictDecode :: OnDecodeError-strictDecode desc c = throw (DecodeError desc c)---- | Replace an invalid input byte with the Unicode replacement--- character U+FFFD.-lenientDecode :: OnDecodeError-lenientDecode _ _ = Just '\xfffd'---- | Throw a 'UnicodeException' if encoding fails.-{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-}-strictEncode :: OnEncodeError-strictEncode desc c = throw (EncodeError desc c)---- | Ignore an invalid input, substituting nothing in the output.-ignore :: OnError a b-ignore _ _ = Nothing---- | Replace an invalid input with a valid output.-replace :: b -> OnError a b-replace c _ _ = Just c+{-# LANGUAGE CPP #-} +{-# LANGUAGE Safe #-} +-- | +-- Module : Data.Text.Encoding.Error +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Types and functions for dealing with encoding and decoding errors +-- in Unicode text. +-- +-- The standard functions for encoding and decoding text are strict, +-- which is to say that they throw exceptions on invalid input. This +-- is often unhelpful on real world input, so alternative functions +-- exist that accept custom handlers for dealing with invalid inputs. +-- These 'OnError' handlers are normal Haskell functions. You can use +-- one of the presupplied functions in this module, or you can write a +-- custom handler of your own. + +module Data.Text.Encoding.Error + ( + -- * Error handling types + UnicodeException(..) + , OnError + , OnDecodeError + , OnEncodeError + -- * Useful error handling functions + , lenientDecode + , strictDecode + , strictEncode + , ignore + , replace + ) where + +import Control.DeepSeq (NFData (..)) +import Control.Exception (Exception, throw) +import Data.Word (Word8) +import Numeric (showHex) + +-- | Function type for handling a coding error. It is supplied with +-- two inputs: +-- +-- * A 'String' that describes the error. +-- +-- * The input value that caused the error. If the error arose +-- because the end of input was reached or could not be identified +-- precisely, this value will be 'Nothing'. +-- +-- If the handler returns a value wrapped with 'Just', that value will +-- be used in the output as the replacement for the invalid input. If +-- it returns 'Nothing', no value will be used in the output. +-- +-- Should the handler need to abort processing, it should use 'error' +-- or 'throw' an exception (preferably a 'UnicodeException'). It may +-- use the description provided to construct a more helpful error +-- report. +type OnError a b = String -> Maybe a -> Maybe b + +-- | A handler for a decoding error. +type OnDecodeError = OnError Word8 Char + +-- | A handler for an encoding error. +{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} +type OnEncodeError = OnError Char Word8 + +-- | An exception type for representing Unicode encoding errors. +data UnicodeException = + DecodeError String (Maybe Word8) + -- ^ Could not decode a byte sequence because it was invalid under + -- the given encoding, or ran out of input in mid-decode. + | EncodeError String (Maybe Char) + -- ^ Tried to encode a character that could not be represented + -- under the given encoding, or ran out of input in mid-encode. + deriving (Eq) + +{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} + +showUnicodeException :: UnicodeException -> String +showUnicodeException (DecodeError desc (Just w)) + = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) +showUnicodeException (DecodeError desc Nothing) + = "Cannot decode input: " ++ desc +showUnicodeException (EncodeError desc (Just c)) + = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) +showUnicodeException (EncodeError desc Nothing) + = "Cannot encode input: " ++ desc + +instance Show UnicodeException where + show = showUnicodeException + +instance Exception UnicodeException + +instance NFData UnicodeException where + rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () + rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () + +-- | Throw a 'UnicodeException' if decoding fails. +strictDecode :: OnDecodeError +strictDecode desc c = throw (DecodeError desc c) + +-- | Replace an invalid input byte with the Unicode replacement +-- character U+FFFD. +lenientDecode :: OnDecodeError +lenientDecode _ _ = Just '\xfffd' + +-- | Throw a 'UnicodeException' if encoding fails. +{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} +strictEncode :: OnEncodeError +strictEncode desc c = throw (EncodeError desc c) + +-- | Ignore an invalid input, substituting nothing in the output. +ignore :: OnError a b +ignore _ _ = Nothing + +-- | Replace an invalid input with a valid output. +replace :: b -> OnError a b +replace c _ _ = Just c
src/Data/Text/Foreign.hs view
@@ -1,163 +1,212 @@-{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving #-}--- |--- Module : Data.Text.Foreign--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Support for using 'Text' data with native code via the Haskell--- foreign function interface.--module Data.Text.Foreign- (- -- * Interoperability with native code- -- $interop- I8- -- * Safe conversion functions- , fromPtr- , useAsPtr- , asForeignPtr- -- ** Encoding as UTF-8- , peekCStringLen- , withCStringLen- -- * Unsafe conversion code- , lengthWord8- , unsafeCopyToPtr- -- * Low-level manipulation- -- $lowlevel- , dropWord8- , takeWord8- ) where--import Control.Monad.ST.Unsafe (unsafeSTToIO)-import Data.ByteString.Unsafe (unsafePackCStringLen, unsafeUseAsCStringLen)-import Data.Text.Encoding (decodeUtf8, encodeUtf8)-import Data.Text.Internal (Text(..), empty)-import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)-import Data.Text.Unsafe (lengthWord8)-import Data.Word (Word8)-import Foreign.C.String (CStringLen)-import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray)-import Foreign.Marshal.Alloc (allocaBytes)-import Foreign.Ptr (Ptr, castPtr)-import qualified Data.Text.Array as A---- $interop------ The 'Text' type is implemented using arrays that are not guaranteed--- to have a fixed address in the Haskell heap. All communication with--- native code must thus occur by copying data back and forth.------ The 'Text' type's internal representation is UTF-8.--- To interoperate with native libraries that use different--- internal representations, such as UTF-16 or UTF-32, consider using--- the functions in the 'Data.Text.Encoding' module.---- | A type representing a number of UTF-8 code units.------ @since 2.0-newtype I8 = I8 Int- deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show)---- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the--- contents of the array.-fromPtr :: Ptr Word8 -- ^ source array- -> I8 -- ^ length of source array (in 'Word8' units)- -> IO Text-fromPtr ptr (I8 len) = unsafeSTToIO $ do- dst <- A.new len- A.copyFromPointer dst 0 ptr len- arr <- A.unsafeFreeze dst- return $! Text arr 0 len---- $lowlevel------ Foreign functions that use UTF-8 internally may return indices in--- units of 'Word8' instead of characters. These functions may--- safely be used with such indices, as they will adjust offsets if--- necessary to preserve the validity of a Unicode string.---- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in--- length.------ If @n@ would cause the 'Text' to end inside a code point, the--- end of the prefix will be advanced by several additional 'Word8' units--- to maintain its validity.------ @since 2.0-takeWord8 :: I8 -> Text -> Text-takeWord8 = (fst .) . splitAtWord8---- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units--- dropped from its beginning.------ If @n@ would cause the 'Text' to begin inside a code point, the--- beginning of the suffix will be advanced by several additional 'Word8'--- unit to maintain its validity.------ @since 2.0-dropWord8 :: I8 -> Text -> Text-dropWord8 = (snd .) . splitAtWord8--splitAtWord8 :: I8 -> Text -> (Text, Text)-splitAtWord8 (I8 n) t@(Text arr off len)- | n <= 0 = (empty, t)- | n >= len || m >= len = (t, empty)- | otherwise = (Text arr off m, Text arr (off+m) (len-m))- where- m | w0 < 0x80 = n -- last char is ASCII- | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence- | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence- | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence- | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence- | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence- | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence- | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence- | otherwise = n -- pre-pre-last char starts 3-byte sequence- w0 = A.unsafeIndex arr (off+n-1)- w1 = A.unsafeIndex arr (off+n-2)- w2 = A.unsafeIndex arr (off+n-3)---- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big--- enough to hold the contents of the entire 'Text'.-unsafeCopyToPtr :: Text -> Ptr Word8 -> IO ()-unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len---- | /O(n)/ Perform an action on a temporary, mutable copy of a--- 'Text'. The copy is freed as soon as the action returns.-useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a-useAsPtr t@(Text _arr _off len) action =- allocaBytes len $ \buf -> do- unsafeCopyToPtr t buf- action (castPtr buf) (I8 len)---- | /O(n)/ Make a mutable copy of a 'Text'.-asForeignPtr :: Text -> IO (ForeignPtr Word8, I8)-asForeignPtr t@(Text _arr _off len) = do- fp <- mallocForeignPtrArray len- unsafeWithForeignPtr fp $ unsafeCopyToPtr t- return (fp, I8 len)---- | /O(n)/ Decode a C string with explicit length, which is assumed--- to have been encoded as UTF-8. If decoding fails, a--- 'UnicodeException' is thrown.------ @since 1.0.0.0-peekCStringLen :: CStringLen -> IO Text-peekCStringLen cs = do- bs <- unsafePackCStringLen cs- return $! decodeUtf8 bs---- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary--- storage, with explicit length information. The encoded string may--- contain NUL bytes, and is not followed by a trailing NUL byte.------ The temporary storage is freed when the subcomputation terminates--- (either normally or via an exception), so the pointer to the--- temporary storage must /not/ be used after this function returns.------ @since 1.0.0.0-withCStringLen :: Text -> (CStringLen -> IO a) -> IO a-withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act+{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-} +-- | +-- Module : Data.Text.Foreign +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Support for using 'Text' data with native code via the Haskell +-- foreign function interface. + +module Data.Text.Foreign + ( + -- * Interoperability with native code + -- $interop + I8 + -- * Pointer conversion functions + , fromPtr + , fromPtr0 + , useAsPtr + , asForeignPtr + -- ** Encoding as UTF-8 + , peekCString + , withCString + , peekCStringLen + , withCStringLen + -- * Low-level manipulation + -- $lowlevel + , dropWord8 + , takeWord8 + , lengthWord8 + , unsafeCopyToPtr + ) where + +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Show (addrLen) +import Data.Text.Unsafe (lengthWord8) +import Data.Word (Word8) +import Foreign.C.String (CString, CStringLen) +import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) +import Foreign.Marshal.Alloc (allocaBytes) +import Foreign.Ptr (Ptr, castPtr) +import Foreign.Storable (pokeByteOff) +import GHC.Exts (Ptr(..)) +import qualified Data.Text.Array as A + +-- $interop +-- +-- The 'Text' type is implemented using arrays that are not guaranteed +-- to have a fixed address in the Haskell heap. All communication with +-- native code must thus occur by copying data back and forth. +-- +-- The 'Text' type's internal representation is UTF-8. +-- To interoperate with native libraries that use different +-- internal representations, such as UTF-16 or UTF-32, consider using +-- the functions in the 'Data.Text.Encoding' module. + +-- | A type representing a number of UTF-8 code units. +-- +-- @since 2.0 +newtype I8 = I8 Int + deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the array. +-- +-- __This function is unsafe.__ The source array must contain a valid +-- UTF-8 string of the given length. There are no guarantees about what +-- happens otherwise. +fromPtr :: Ptr Word8 -- ^ source array + -> I8 -- ^ length of source array (in 'Word8' units) + -> IO Text +fromPtr _ (I8 0) = pure empty +fromPtr ptr (I8 len) = unsafeSTToIO $ do + dst <- A.new len + A.copyFromPointer dst 0 ptr len + arr <- A.unsafeFreeze dst + return $! Text arr 0 len + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the NUL-terminated array. +-- +-- __This function is unsafe.__ The source array must contain a NULL-terminated +-- valid UTF-8 string. There are no guarantees about what happens otherwise. +-- +-- @since 2.0.1 +fromPtr0 :: Ptr Word8 -- ^ source array + -> IO Text +fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#)) + +-- $lowlevel +-- +-- Foreign functions that use UTF-8 internally may return indices in +-- units of 'Word8' instead of characters. These functions may +-- safely be used with such indices, as they will adjust offsets if +-- necessary to preserve the validity of a Unicode string. + +-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in +-- length. +-- +-- If @n@ would cause the 'Text' to end inside a code point, the +-- end of the prefix will be advanced by several additional 'Word8' units +-- to maintain its validity. +-- +-- @since 2.0 +takeWord8 :: I8 -> Text -> Text +takeWord8 = (fst .) . splitAtWord8 + +-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units +-- dropped from its beginning. +-- +-- If @n@ would cause the 'Text' to begin inside a code point, the +-- beginning of the suffix will be advanced by several additional 'Word8' +-- unit to maintain its validity. +-- +-- @since 2.0 +dropWord8 :: I8 -> Text -> Text +dropWord8 = (snd .) . splitAtWord8 + +splitAtWord8 :: I8 -> Text -> (Text, Text) +splitAtWord8 (I8 n) t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m | w0 < 0x80 = n -- last char is ASCII + | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence + | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence + | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence + | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence + | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence + | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence + | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence + | otherwise = n -- pre-pre-last char starts 3-byte sequence + w0 = A.unsafeIndex arr (off+n-1) + w1 = A.unsafeIndex arr (off+n-2) + w2 = A.unsafeIndex arr (off+n-3) + +-- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big +-- enough to hold the contents of the entire 'Text'. +unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () +unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len + +-- | /O(n)/ Perform an action on a temporary, mutable copy of a +-- 'Text'. The copy is freed as soon as the action returns. +useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a +useAsPtr t@(Text _arr _off len) action = + allocaBytes len $ \buf -> do + unsafeCopyToPtr t buf + action (castPtr buf) (I8 len) + +-- | /O(n)/ Make a mutable copy of a 'Text'. +asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) +asForeignPtr t@(Text _arr _off len) = do + fp <- mallocForeignPtrArray len + unsafeWithForeignPtr fp $ unsafeCopyToPtr t + return (fp, I8 len) + +-- | Marshal a 'Text' into a C string with a trailing NUL byte, +-- encoded as UTF-8 in temporary storage. +-- +-- The 'Text' itself must not contain any NUL bytes, this precondition +-- is not checked. Cf. 'withCStringLen'. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 2.0.1 +withCString :: Text -> (CString -> IO a) -> IO a +withCString t@(Text _arr _off len) action = + allocaBytes (len + 1) $ \buf -> do + unsafeCopyToPtr t buf + pokeByteOff buf len (0 :: Word8) + action (castPtr buf) + +-- | /O(n)/ Decode a C string with explicit length, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 1.0.0.0 +peekCStringLen :: CStringLen -> IO Text +peekCStringLen cs = do + bs <- unsafePackCStringLen cs + return $! decodeUtf8 bs + +-- | /O(n)/ Decode a null-terminated C string, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 2.1.2 +peekCString :: CString -> IO Text +peekCString cs = do + bs <- unsafePackCString cs + return $! decodeUtf8 bs + +-- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary +-- storage, with explicit length information. The encoded string may +-- contain NUL bytes, and is not followed by a trailing NUL byte. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 1.0.0.0 +withCStringLen :: Text -> (CStringLen -> IO a) -> IO a +withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act
src/Data/Text/IO.hs view
@@ -1,309 +1,211 @@-{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-}-{-# LANGUAGE Trustworthy #-}--- |--- Module : Data.Text.IO--- Copyright : (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Simon Marlow--- License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Efficient locale-sensitive support for text I\/O.------ The functions in this module obey the runtime system's locale,--- character set encoding, and line ending conversion settings.------ If you know in advance that you will be working with data that has--- a specific encoding (e.g. UTF-8), and your application is highly--- performance sensitive, you may find that it is faster to perform--- I\/O with bytestrings and to encode and decode yourself than to use--- the functions in this module.--module Data.Text.IO- (- -- * File-at-a-time operations- readFile- , writeFile- , appendFile- -- * Operations on handles- , hGetContents- , hGetChunk- , hGetLine- , hPutStr- , hPutStrLn- -- * Special cases for standard input and output- , interact- , getContents- , getLine- , putStr- , putStrLn- ) where--import Data.Text (Text)-import Prelude hiding (appendFile, getContents, getLine, interact,- putStr, putStrLn, readFile, writeFile)-import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout,- withFile)-import qualified Control.Exception as E-import Control.Monad (liftM2, when)-import Data.IORef (readIORef, writeIORef)-import qualified Data.Text as T-import Data.Text.Internal.Fusion (stream)-import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))-import Data.Text.Internal.IO (hGetLineWith, readChunk)-import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBufElem, CharBuffer,- RawCharBuffer, emptyBuffer, isEmptyBuffer, newCharBuffer,- writeCharBuf)-import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType))-import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle,- wantWritableHandle)-import GHC.IO.Handle.Text (commitBuffer')-import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..),- HandleType(..), Newline(..))-import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)-import System.IO.Error (isEOFError)---- | The 'readFile' function reads a file and returns the contents of--- the file as a string. The entire file is read strictly, as with--- 'getContents'.------ Beware that this function (similarly to 'Prelude.readFile') is locale-dependent.--- Unexpected system locale may cause your application to read corrupted data or--- throw runtime exceptions about "invalid argument (invalid byte sequence)"--- or "invalid argument (invalid character)". This is also slow, because GHC--- first converts an entire input to UTF-32, which is afterwards converted to UTF-8.------ If your data is UTF-8,--- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile'--- is a much faster and safer alternative.-readFile :: FilePath -> IO Text-readFile name = openFile name ReadMode >>= hGetContents---- | Write a string to a file. The file is truncated to zero length--- before writing begins.-writeFile :: FilePath -> Text -> IO ()-writeFile p = withFile p WriteMode . flip hPutStr---- | Write a string to the end of a file.-appendFile :: FilePath -> Text -> IO ()-appendFile p = withFile p AppendMode . flip hPutStr--catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool)-catchError caller h Handle__{..} err- | isEOFError err = do- buf <- readIORef haCharBuffer- return $ if isEmptyBuffer buf- then (T.empty, True)- else (T.singleton '\r', True)- | otherwise = E.throwIO (augmentIOError err caller h)---- | Wrap readChunk and return a value indicating if we're reached the EOF.--- This is needed because unpack_nl is unable to discern the difference--- between a buffer with just \r due to EOF or because not enough data was left--- for decoding. e.g. the final character decoded from the byte buffer was \r.-readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool)-readChunkEof hh buf = do t <- readChunk hh buf- return (t, False)---- | /Experimental./ Read a single chunk of strict text from a--- 'Handle'. The size of the chunk depends on the amount of input--- currently buffered.------ This function blocks only if there is no data available, and EOF--- has not yet been reached. Once EOF is reached, this function--- returns an empty string instead of throwing an exception.-hGetChunk :: Handle -> IO Text-hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk- where- readSingleChunk hh@Handle__{..} = do- buf <- readIORef haCharBuffer- (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh- return (hh, t)---- | Read the remaining contents of a 'Handle' as a string. The--- 'Handle' is closed once the contents have been read, or if an--- exception is thrown.------ Internally, this function reads a chunk at a time from the--- lower-level buffering abstraction, and concatenates the chunks into--- a single string once the entire file has been read.------ As a result, it requires approximately twice as much memory as its--- result to construct its result. For files more than a half of--- available RAM in size, this may result in memory exhaustion.-hGetContents :: Handle -> IO Text-hGetContents h = do- chooseGoodBuffering h- wantReadableHandle "hGetContents" h readAll- where- readAll hh@Handle__{..} = do- let readChunks = do- buf <- readIORef haCharBuffer- (t, eof) <- readChunkEof hh buf- `E.catch` catchError "hGetContents" h hh- if eof- then return [t]- else (t:) `fmap` readChunks- ts <- readChunks- (hh', _) <- hClose_help hh- return (hh'{haType=ClosedHandle}, T.concat ts)---- | Use a more efficient buffer size if we're reading in--- block-buffered mode with the default buffer size. When we can--- determine the size of the handle we're reading, set the buffer size--- to that, so that we can read the entire file in one chunk.--- Otherwise, use a buffer size of at least 16KB.-chooseGoodBuffering :: Handle -> IO ()-chooseGoodBuffering h = do- bufMode <- hGetBuffering h- case bufMode of- BlockBuffering Nothing -> do- d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) ->- if ioe_type e == InappropriateType- then return 16384 -- faster than the 2KB default- else E.throwIO e- when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d- _ -> return ()---- | Read a single line from a handle.-hGetLine :: Handle -> IO Text-hGetLine = hGetLineWith T.concat---- | Write a string to a handle.-hPutStr :: Handle -> Text -> IO ()--- This function is lifted almost verbatim from GHC.IO.Handle.Text.-hPutStr h t = do- (buffer_mode, nl) <-- wantWritableHandle "hPutStr" h $ \h_ -> do- bmode <- getSpareBuffer h_- return (bmode, haOutputNL h_)- let str = stream t- case buffer_mode of- (NoBuffering, _) -> hPutChars h str- (LineBuffering, buf) -> writeLines h nl buf str- (BlockBuffering _, buf)- | nl == CRLF -> writeBlocksCRLF h buf str- | otherwise -> writeBlocksRaw h buf str--hPutChars :: Handle -> Stream Char -> IO ()-hPutChars h (Stream next0 s0 _len) = loop s0- where- loop !s = case next0 s of- Done -> return ()- Skip s' -> loop s'- Yield x s' -> hPutChar h x >> loop s'---- The following functions are largely lifted from GHC.IO.Handle.Text,--- but adapted to a coinductive stream of data instead of an inductive--- list.------ We have several variations of more or less the same code for--- performance reasons. Splitting the original buffered write--- function into line- and block-oriented versions gave us a 2.1x--- performance improvement. Lifting out the raw/cooked newline--- handling gave a few more percent on top.--writeLines :: Handle -> Newline -> Buffer CharBufElem -> Stream Char -> IO ()-writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0- where- outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)- where- inner !s !n =- case next0 s of- Done -> commit n False{-no flush-} True{-release-} >> return ()- Skip s' -> inner s' n- Yield x s'- | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s- | x == '\n' -> do- n' <- if nl == CRLF- then do n1 <- writeCharBuf raw n '\r'- writeCharBuf raw n1 '\n'- else writeCharBuf raw n x- commit n' True{-needs flush-} False >>= outer s'- | otherwise -> writeCharBuf raw n x >>= inner s'- commit = commitBuffer h raw len--writeBlocksCRLF :: Handle -> Buffer CharBufElem -> Stream Char -> IO ()-writeBlocksCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0- where- outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)- where- inner !s !n =- case next0 s of- Done -> commit n False{-no flush-} True{-release-} >> return ()- Skip s' -> inner s' n- Yield x s'- | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s- | x == '\n' -> do n1 <- writeCharBuf raw n '\r'- writeCharBuf raw n1 '\n' >>= inner s'- | otherwise -> writeCharBuf raw n x >>= inner s'- commit = commitBuffer h raw len--writeBlocksRaw :: Handle -> Buffer CharBufElem -> Stream Char -> IO ()-writeBlocksRaw h buf0 (Stream next0 s0 _len) = outer s0 buf0- where- outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)- where- inner !s !n =- case next0 s of- Done -> commit n False{-no flush-} True{-release-} >> return ()- Skip s' -> inner s' n- Yield x s'- | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s- | otherwise -> writeCharBuf raw n x >>= inner s'- commit = commitBuffer h raw len---- This function is completely lifted from GHC.IO.Handle.Text.-getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer)-getSpareBuffer Handle__{haCharBuffer=ref,- haBuffers=spare_ref,- haBufferMode=mode}- = do- case mode of- NoBuffering -> return (mode, error "no buffer!")- _ -> do- bufs <- readIORef spare_ref- buf <- readIORef ref- case bufs of- BufferListCons b rest -> do- writeIORef spare_ref rest- return ( mode, emptyBuffer b (bufSize buf) WriteBuffer)- BufferListNil -> do- new_buf <- newCharBuffer (bufSize buf) WriteBuffer- return (mode, new_buf)----- This function is completely lifted from GHC.IO.Handle.Text.-commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool- -> IO CharBuffer-commitBuffer hdl !raw !sz !count flush release =- wantWritableHandle "commitAndReleaseBuffer" hdl $- commitBuffer' raw sz count flush release-{-# INLINE commitBuffer #-}---- | Write a string to a handle, followed by a newline.-hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h t = hPutStr h t >> hPutChar h '\n'---- | The 'interact' function takes a function of type @Text -> Text@--- as its argument. The entire input from the standard input device is--- passed to this function as its argument, and the resulting string--- is output on the standard output device.-interact :: (Text -> Text) -> IO ()-interact f = putStr . f =<< getContents---- | Read all user input on 'stdin' as a single string.-getContents :: IO Text-getContents = hGetContents stdin---- | Read a single line of user input from 'stdin'.-getLine :: IO Text-getLine = hGetLine stdin---- | Write a string to 'stdout'.-putStr :: Text -> IO ()-putStr = hPutStr stdout---- | Write a string to 'stdout', followed by a newline.-putStrLn :: Text -> IO ()-putStrLn = hPutStrLn stdout+{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +-- | +-- Module : Data.Text.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Efficient locale-sensitive support for text I\/O. +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you want to do I\/O using the UTF-8 encoding, use "Data.Text.IO.Utf8", +-- which is faster than this module. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding, and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. + +module Data.Text.IO + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetChunk + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Data.Text (Text) +import Prelude hiding (appendFile, getContents, getLine, interact, + putStr, putStrLn, readFile, writeFile) +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, + withFile) +import qualified Control.Exception as E +import Control.Monad (liftM2, when) +import Data.IORef (readIORef) +import qualified Data.Text as T +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn) +import GHC.IO.Buffer (CharBuffer, isEmptyBuffer) +import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) +import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle) +import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..)) +import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) +import System.IO.Error (isEOFError) + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' +-- is a much faster and safer alternative. +readFile :: FilePath -> IO Text +readFile name = openFile name ReadMode >>= hGetContents + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile p = withFile p WriteMode . flip hPutStr + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile p = withFile p AppendMode . flip hPutStr + +catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool) +catchError caller h Handle__{..} err + | isEOFError err = do + buf <- readIORef haCharBuffer + return $ if isEmptyBuffer buf + then (T.empty, True) + else (T.singleton '\r', True) + | otherwise = E.throwIO (augmentIOError err caller h) + +-- | Wrap readChunk and return a value indicating if we're reached the EOF. +-- This is needed because unpack_nl is unable to discern the difference +-- between a buffer with just \r due to EOF or because not enough data was left +-- for decoding. e.g. the final character decoded from the byte buffer was \r. +readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool) +readChunkEof hh buf = do t <- readChunk hh buf + return (t, False) + +-- | Read a single chunk of strict text from a +-- 'Handle'. The size of the chunk depends on the amount of input +-- currently buffered. +-- +-- This function blocks only if there is no data available, and EOF +-- has not yet been reached. Once EOF is reached, this function +-- returns an empty string instead of throwing an exception. +-- +-- === Behavior +-- +-- Unlike byte-oriented functions, 'hGetChunk' operates on complete UTF-8 +-- characters. Since UTF-8 characters can occupy 1 to 4 bytes, this function +-- cannot guarantee reading an exact number of bytes. Instead, it reads +-- complete characters up to the handle's internal buffer limit. +-- +-- === Buffer Size +-- +-- The maximum chunk size is determined by the handle's internal character +-- buffer, which is set to 8192 bytes (2048 characters) by the GHC runtime +-- constant @dEFAULT_CHAR_BUFFER_SIZE@. This buffer size cannot be modified +-- through any public API. +-- +-- === UTF-8 Considerations +-- +-- When working with UTF-8 encoded text: +-- +-- * The function will never return a partial character +-- * The actual number of bytes read may vary depending on the character +-- encoding (ASCII characters = 1 byte, other Unicode characters = 2-4 bytes) +hGetChunk :: Handle -> IO Text +hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk + where + readSingleChunk hh@Handle__{..} = do + buf <- readIORef haCharBuffer + (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh + return (hh, t) + +-- | Read the remaining contents of a 'Handle' as a string. The +-- 'Handle' is closed once the contents have been read, or if an +-- exception is thrown. +-- +-- Internally, this function reads a chunk at a time from the +-- lower-level buffering abstraction, and concatenates the chunks into +-- a single string once the entire file has been read. +-- +-- As a result, it requires approximately twice as much memory as its +-- result to construct its result. For files more than a half of +-- available RAM in size, this may result in memory exhaustion. +hGetContents :: Handle -> IO Text +hGetContents h = do + chooseGoodBuffering h + wantReadableHandle "hGetContents" h readAll + where + readAll hh@Handle__{..} = do + let readChunks = do + buf <- readIORef haCharBuffer + (t, eof) <- readChunkEof hh buf + `E.catch` catchError "hGetContents" h hh + if eof + then return [t] + else (t:) `fmap` readChunks + ts <- readChunks + (hh', _) <- hClose_help hh + return (hh'{haType=ClosedHandle}, T.concat ts) + +-- | Use a more efficient buffer size if we're reading in +-- block-buffered mode with the default buffer size. When we can +-- determine the size of the handle we're reading, set the buffer size +-- to that, so that we can read the entire file in one chunk. +-- Otherwise, use a buffer size of at least 16KB. +chooseGoodBuffering :: Handle -> IO () +chooseGoodBuffering h = do + bufMode <- hGetBuffering h + case bufMode of + BlockBuffering Nothing -> do + d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> + if ioe_type e == InappropriateType + then return 16384 -- faster than the 2KB default + else E.throwIO e + when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d + _ -> return () + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = hGetLineWith T.concat + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = hGetContents stdin + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = hGetLine stdin + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = hPutStr stdout + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = hPutStrLn stdout
+ src/Data/Text/IO/Utf8.hs view
@@ -0,0 +1,94 @@+-- | +-- Module : Data.Text.IO.Utf8 +-- License : BSD-style +-- Portability : GHC +-- +-- Efficient UTF-8 support for text I\/O. +-- Unlike "Data.Text.IO", these functions do not depend on the locale +-- and do not do line ending conversion. +module Data.Text.IO.Utf8 + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Prelude () +import Control.Exception (evaluate) +import Control.Monad ((<=<), (=<<)) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Function ((.)) +import Data.Text (Text) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import GHC.IO.Handle (Handle) +import System.IO (IO, FilePath) + +decodeUtf8IO :: ByteString -> IO Text +decodeUtf8IO = evaluate . decodeUtf8 + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +readFile :: FilePath -> IO Text +readFile = decodeUtf8IO <=< B.readFile + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile fp = B.writeFile fp . encodeUtf8 + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile fp = B.appendFile fp . encodeUtf8 + +-- | Read the remaining contents of a 'Handle' as a string. +hGetContents :: Handle -> IO Text +hGetContents = decodeUtf8IO <=< B.hGetContents + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = decodeUtf8IO <=< B.hGetLine + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h = B.hPutStr h . encodeUtf8 + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h = B.hPutStrLn h . encodeUtf8 + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = decodeUtf8IO =<< B.getContents + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = decodeUtf8IO =<< B.getLine + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = B.putStr . encodeUtf8 + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = B.putStrLn . encodeUtf8
src/Data/Text/Internal.hs view
@@ -1,233 +1,274 @@-{-# LANGUAGE CPP, DeriveDataTypeable, UnboxedTuples #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module : Data.Text.Internal--- Copyright : (c) 2008, 2009 Tom Harper,--- (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.------ You should not use this module unless you are determined to monkey--- with the internals, as the functions here do just about nothing to--- preserve data invariants. You have been warned!--module Data.Text.Internal- (- -- * Types- -- $internals- Text(..)- -- * Construction- , text- , textP- -- * Safety- , safe- -- * Code that must be here for accessibility- , empty- , empty_- , append- -- * Utilities- , firstf- -- * Checked multiplication- , mul- , mul32- , mul64- -- * Debugging- , showText- ) where--#if defined(ASSERTS)-import Control.Exception (assert)-import GHC.Stack (HasCallStack)-#endif-import Control.Monad.ST (ST)-import Data.Bits-import Data.Int (Int32, Int64)-import Data.Text.Internal.Unsafe.Char (ord)-import Data.Typeable (Typeable)-import qualified Data.Text.Array as A---- | A space efficient, packed, unboxed Unicode text type.-data Text = Text- {-# UNPACK #-} !A.Array -- bytearray encoded as UTF-8- {-# UNPACK #-} !Int -- offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- {-# UNPACK #-} !Int -- length in bytes (not in Char!), pointing to an end of UTF-8 sequence- deriving (Typeable)---- | Smart constructor.-text_ ::-#if defined(ASSERTS)- HasCallStack =>-#endif- A.Array -- ^ bytearray encoded as UTF-8- -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence- -> Text-text_ arr off len =-#if defined(ASSERTS)- let c = A.unsafeIndex arr off- in assert (len >= 0) .- assert (off >= 0) .- assert (len == 0 || c < 0x80 || c >= 0xC0) $-#endif- Text arr off len-{-# INLINE text_ #-}---- | /O(1)/ The empty 'Text'.-empty :: Text-empty = Text A.empty 0 0-{-# INLINE [1] empty #-}---- | A non-inlined version of 'empty'.-empty_ :: Text-empty_ = Text A.empty 0 0-{-# NOINLINE empty_ #-}---- | /O(n)/ Appends one 'Text' to the other by copying both of them--- into a new 'Text'.-append :: Text -> Text -> Text-append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2)- | len1 == 0 = b- | len2 == 0 = a- | len > 0 = Text (A.run x) 0 len- | otherwise = error $ "Data.Text.append: size overflow"- where- len = len1+len2- x :: ST s (A.MArray s)- x = do- arr <- A.new len- A.copyI len1 arr 0 arr1 off1- A.copyI len2 arr len1 arr2 off2- return arr-{-# NOINLINE append #-}---- | Construct a 'Text' without invisibly pinning its byte array in--- memory if its length has dwindled to zero.-text ::-#if defined(ASSERTS)- HasCallStack =>-#endif- A.Array -- ^ bytearray encoded as UTF-8- -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence- -> Text-text arr off len | len == 0 = empty- | otherwise = text_ arr off len-{-# INLINE text #-}--textP :: A.Array -> Int -> Int -> Text-{-# DEPRECATED textP "Use text instead" #-}-textP = text---- | A useful 'show'-like function for debugging purposes.-showText :: Text -> String-showText (Text arr off len) =- "Text " ++ show (A.toList arr off len) ++ ' ' :- show off ++ ' ' : show len---- | Map a 'Char' to a 'Text'-safe value.------ Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode--- scalar values, but are unfortunately admitted as valid 'Char'--- values by Haskell. They cannot be represented in a 'Text'. This--- function remaps those code points to the Unicode replacement--- character (U+FFFD, \'�\'), and leaves other code points--- unchanged.-safe :: Char -> Char-safe c- | ord c .&. 0x1ff800 /= 0xd800 = c- | otherwise = '\xfffd'-{-# INLINE [0] safe #-}---- | Apply a function to the first element of an optional pair.-firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)-firstf f (Just (a, b)) = Just (f a, b)-firstf _ Nothing = Nothing---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul :: Int -> Int -> Int-mul a b- | finiteBitSize (0 :: Word) == 64- = int64ToInt $ intToInt64 a `mul64` intToInt64 b- | otherwise- = int32ToInt $ intToInt32 a `mul32` intToInt32 b-{-# INLINE mul #-}-infixl 7 `mul`---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul64 :: Int64 -> Int64 -> Int64-mul64 a b- | a >= 0 && b >= 0 = mul64_ a b- | a >= 0 = -mul64_ a (-b)- | b >= 0 = -mul64_ (-a) b- | otherwise = mul64_ (-a) (-b)-{-# INLINE mul64 #-}-infixl 7 `mul64`--mul64_ :: Int64 -> Int64 -> Int64-mul64_ a b- | ahi > 0 && bhi > 0 = error "overflow"- | top > 0x7fffffff = error "overflow"- | total < 0 = error "overflow"- | otherwise = total- where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #)- (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #)- top = ahi * blo + alo * bhi- total = (top `shiftL` 32) + alo * blo-{-# INLINE mul64_ #-}---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul32 :: Int32 -> Int32 -> Int32-mul32 a b = case int32ToInt64 a * int32ToInt64 b of- ab | ab < min32 || ab > max32 -> error "overflow"- | otherwise -> int64ToInt32 ab- where min32 = -0x80000000 :: Int64- max32 = 0x7fffffff-{-# INLINE mul32 #-}-infixl 7 `mul32`--intToInt64 :: Int -> Int64-intToInt64 = fromIntegral--int64ToInt :: Int64 -> Int-int64ToInt = fromIntegral--intToInt32 :: Int -> Int32-intToInt32 = fromIntegral--int32ToInt :: Int32 -> Int-int32ToInt = fromIntegral--int32ToInt64 :: Int32 -> Int64-int32ToInt64 = fromIntegral--int64ToInt32 :: Int64 -> Int32-int64ToInt32 = fromIntegral---- $internals------ Internally, the 'Text' type is represented as an array of 'Word8'--- UTF-8 code units. The offset and length fields in the constructor--- are in these units, /not/ units of 'Char'.------ Invariants that all functions must maintain:------ * Since the 'Text' type uses UTF-8 internally, it cannot represent--- characters in the reserved surrogate code point range U+D800 to--- U+DFFF. To maintain this invariant, the 'safe' function maps--- 'Char' values in this range to the replacement character (U+FFFD,--- \'�\').------ * Offset and length must point to a valid UTF-8 sequence of bytes.--- Violation of this may cause memory access violation and divergence.+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE UnboxedTuples #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. +-- +-- You should not use this module unless you are determined to monkey +-- with the internals, as the functions here do just about nothing to +-- preserve data invariants. You have been warned! + +module Data.Text.Internal + ( + -- * Types + -- $internals + Text(..) + , StrictText + -- * Construction + , text + , textP + -- * Safety + , safe + -- * Code that must be here for accessibility + , empty + , append + -- * Utilities + , firstf + -- * Checked multiplication + , mul + , mul32 + , mul64 + -- * Debugging + , showText + -- * Conversions + , pack + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Control.Monad.ST (ST, runST) +import Data.Bits +import Data.Int (Int32, Int64) +import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) +import qualified Data.Text.Array as A + +-- | A space efficient, packed, unboxed Unicode text type. +data Text = Text + {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8 + {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + +-- | Type synonym for the strict flavour of 'Text'. +-- +-- @since 2.1.1 +type StrictText = Text + +-- | Smart constructor. +text_ :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text_ arr off len = +#if defined(ASSERTS) + let c = A.unsafeIndex arr off + in assert (len >= 0) . + assert (off >= 0) . + assert (len == 0 || c < 0x80 || c >= 0xC0) $ +#endif + Text arr off len +{-# INLINE text_ #-} + +-- | /O(1)/ The empty 'Text'. +empty :: Text +empty = Text A.empty 0 0 +{-# NOINLINE empty #-} + +-- | /O(n)/ Appends one 'Text' to the other by copying both of them +-- into a new 'Text'. +append :: Text -> Text -> Text +append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) + | len1 == 0 = b + | len2 == 0 = a + | len > 0 = Text (A.run x) 0 len + | otherwise = error $ "Data.Text.append: size overflow" + where + len = len1+len2 + x :: ST s (A.MArray s) + x = do + arr <- A.new len + A.copyI len1 arr 0 arr1 off1 + A.copyI len2 arr len1 arr2 off2 + return arr +{-# NOINLINE append #-} + +-- | Construct a 'Text' without invisibly pinning its byte array in +-- memory if its length has dwindled to zero. +-- It ensures that empty 'Text' values are shared. +text :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text arr off len | len == 0 = empty + | otherwise = text_ arr off len +{-# INLINE [0] text #-} + +textP :: A.Array -> Int -> Int -> Text +{-# DEPRECATED textP "Use text instead" #-} +textP = text + +-- | A useful 'show'-like function for debugging purposes. +showText :: Text -> String +showText (Text arr off len) = + "Text " ++ show (A.toList arr off len) ++ ' ' : + show off ++ ' ' : show len + +-- | Map a 'Char' to a 'Text'-safe value. +-- +-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode +-- scalar values, but are unfortunately admitted as valid 'Char' +-- values by Haskell. They cannot be represented in a 'Text'. This +-- function remaps those code points to the Unicode replacement +-- character (U+FFFD, \'�\'), and leaves other code points +-- unchanged. +safe :: Char -> Char +safe c + | ord c .&. 0x1ff800 /= 0xd800 = c + | otherwise = '\xfffd' +{-# INLINE [0] safe #-} + +-- | Apply a function to the first element of an optional pair. +firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) +firstf f (Just (a, b)) = Just (f a, b) +firstf _ Nothing = Nothing + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul :: Int -> Int -> Int +mul a b + | finiteBitSize (0 :: Word) == 64 + = int64ToInt $ intToInt64 a `mul64` intToInt64 b + | otherwise + = int32ToInt $ intToInt32 a `mul32` intToInt32 b +{-# INLINE mul #-} +infixl 7 `mul` + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul64 :: Int64 -> Int64 -> Int64 +mul64 a b + | a >= 0 && b >= 0 = mul64_ a b + | a >= 0 = -mul64_ a (-b) + | b >= 0 = -mul64_ (-a) b + | otherwise = mul64_ (-a) (-b) +{-# INLINE mul64 #-} +infixl 7 `mul64` + +mul64_ :: Int64 -> Int64 -> Int64 +mul64_ a b + | ahi > 0 && bhi > 0 = error "overflow" + | top > 0x7fffffff = error "overflow" + | total < 0 = error "overflow" + | otherwise = total + where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) + (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) + top = ahi * blo + alo * bhi + total = (top `shiftL` 32) + alo * blo +{-# INLINE mul64_ #-} + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul32 :: Int32 -> Int32 -> Int32 +mul32 a b = case int32ToInt64 a * int32ToInt64 b of + ab | ab < min32 || ab > max32 -> error "overflow" + | otherwise -> int64ToInt32 ab + where min32 = -0x80000000 :: Int64 + max32 = 0x7fffffff +{-# INLINE mul32 #-} +infixl 7 `mul32` + +intToInt64 :: Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Int +int64ToInt = fromIntegral + +intToInt32 :: Int -> Int32 +intToInt32 = fromIntegral + +int32ToInt :: Int32 -> Int +int32ToInt = fromIntegral + +int32ToInt64 :: Int32 -> Int64 +int32ToInt64 = fromIntegral + +int64ToInt32 :: Int64 -> Int32 +int64ToInt32 = fromIntegral + +-- $internals +-- +-- Internally, the 'Text' type is represented as an array of 'Word8' +-- UTF-8 code units. The offset and length fields in the constructor +-- are in these units, /not/ units of 'Char'. +-- +-- Invariants that all functions must maintain: +-- +-- * Since the 'Text' type uses UTF-8 internally, it cannot represent +-- characters in the reserved surrogate code point range U+D800 to +-- U+DFFF. To maintain this invariant, the 'safe' function maps +-- 'Char' values in this range to the replacement character (U+FFFD, +-- \'�\'). +-- +-- * Offset and length must point to a valid UTF-8 sequence of bytes. +-- Violation of this may cause memory access violation and divergence. + +-- ----------------------------------------------------------------------------- +-- * Conversion to/from 'Text' + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.unpack (pack "\55555") +-- "\65533" +pack :: String -> Text +pack [] = empty +pack xs = runST $ do + -- It's tempting to allocate a buffer of 4 * length xs bytes, + -- but not only it's wasteful for predominantly ASCII arguments, + -- the computation of length xs would force allocation of the entire xs at once. + let dstLen = 64 + dst <- A.new dstLen + outer dst dstLen 0 xs + where + outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text + outer !dst !dstLen = inner + where + inner !dstOff [] = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + inner !dstOff ccs@(c : cs) + -- Each 'Char' takes up to 4 bytes + | dstOff + 4 > dstLen = do + -- Double size of the buffer + let !dstLen' = dstLen * 2 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' dstOff ccs + | otherwise = do + d <- unsafeWrite dst dstOff (safe c) + inner (dstOff + d) cs +{-# NOINLINE [0] pack #-}
+ src/Data/Text/Internal/ArrayUtils.hs view
@@ -0,0 +1,33 @@+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}+{-# LANGUAGE CPP #-}++module Data.Text.Internal.ArrayUtils (memchr) where++#if defined(PURE_HASKELL)+import qualified Data.Text.Array as A+import Data.List (elemIndex)+#else+import Foreign.C.Types+import System.Posix.Types (CSsize(..))+#endif+import GHC.Exts (ByteArray#)+import Data.Word (Word8)++memchr :: ByteArray# -> Int -> Int -> Word8 -> Int+#if defined(PURE_HASKELL)+memchr arr# off len w =+ let tempBa = A.ByteArray arr#+ in case elemIndex w (A.toList tempBa off len) of+ Nothing -> -1+ Just i -> i+#else+memchr arr# off len w = fromIntegral $ c_memchr arr# (intToCSize off) (intToCSize len) w++intToCSize :: Int -> CSize+intToCSize = fromIntegral+++foreign import ccall unsafe "_hs_text_memchr" c_memchr+ :: ByteArray# -> CSize -> CSize -> Word8 -> CSsize+#endif
src/Data/Text/Internal/Builder.hs view
@@ -1,332 +1,350 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}-{-# OPTIONS_HADDOCK not-home #-}---------------------------------------------------------------------------------- |--- Module : Data.Text.Internal.Builder--- Copyright : (c) 2013 Bryan O'Sullivan--- (c) 2010 Johan Tibell--- License : BSD-style (see LICENSE)------ Maintainer : Johan Tibell <johan.tibell@gmail.com>--- Stability : experimental--- Portability : portable to Hugs and GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Efficient construction of lazy @Text@ values. The principal--- operations on a @Builder@ are @singleton@, @fromText@, and--- @fromLazyText@, which construct new builders, and 'mappend', which--- concatenates two builders.------ To get maximum performance when building lazy @Text@ values using a--- builder, associate @mappend@ calls to the right. For example,--- prefer------ > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')------ to------ > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'------ as the latter associates @mappend@ to the left.-----------------------------------------------------------------------------------module Data.Text.Internal.Builder- ( -- * Public API- -- ** The Builder type- Builder- , toLazyText- , toLazyTextWith-- -- ** Constructing Builders- , singleton- , fromText- , fromLazyText- , fromString-- -- ** Flushing the buffer state- , flush-- -- * Internal functions- , append'- , ensureFree- , writeN- ) where--import Control.Monad.ST (ST, runST)-import Data.Monoid (Monoid(..))-#if !MIN_VERSION_base(4,11,0)-import Data.Semigroup (Semigroup(..))-#endif-import Data.Text.Internal (Text(..), safe)-import Data.Text.Internal.Lazy (smallChunkSize)-import Data.Text.Unsafe (inlineInterleaveST)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Prelude hiding (map, putChar)--import qualified Data.String as String-import qualified Data.Text as S-import qualified Data.Text.Array as A-import qualified Data.Text.Lazy as L--#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif------------------------------------------------------------------------------ | A @Builder@ is an efficient way to build lazy @Text@ values.--- There are several functions for constructing builders, but only one--- to inspect them: to extract any data, you have to turn them into--- lazy @Text@ values using @toLazyText@.------ Internally, a builder constructs a lazy @Text@ by filling arrays--- piece by piece. As each buffer is filled, it is \'popped\' off, to--- become a new chunk of the resulting lazy @Text@. All this is--- hidden from the user of the @Builder@.-newtype Builder = Builder {- -- Invariant (from Data.Text.Lazy):- -- The lists include no null Texts.- runBuilder :: forall s. (Buffer s -> ST s [S.Text])- -> Buffer s- -> ST s [S.Text]- }--instance Semigroup Builder where- (<>) = append- {-# INLINE (<>) #-}--instance Monoid Builder where- mempty = empty- {-# INLINE mempty #-}- mappend = (<>)- {-# INLINE mappend #-}- mconcat = foldr mappend Data.Monoid.mempty- {-# INLINE mconcat #-}--instance String.IsString Builder where- fromString = fromString- {-# INLINE fromString #-}--instance Show Builder where- show = show . toLazyText--instance Eq Builder where- a == b = toLazyText a == toLazyText b--instance Ord Builder where- a <= b = toLazyText a <= toLazyText b------------------------------------------------------------------------------ | /O(1)./ The empty @Builder@, satisfying------ * @'toLazyText' 'empty' = 'L.empty'@----empty :: Builder-empty = Builder (\ k buf -> k buf)-{-# INLINE empty #-}---- | /O(1)./ A @Builder@ taking a single character, satisfying------ * @'toLazyText' ('singleton' c) = 'L.singleton' c@----singleton ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> Builder-singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c)-{-# INLINE singleton #-}------------------------------------------------------------------------------ | /O(1)./ The concatenation of two builders, an associative--- operation with identity 'empty', satisfying------ * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@----append :: Builder -> Builder -> Builder-append (Builder f) (Builder g) = Builder (f . g)-{-# INLINE [0] append #-}---- TODO: Experiment to find the right threshold.-copyLimit :: Int-copyLimit = 128---- This function attempts to merge small @Text@ values instead of--- treating each value as its own chunk. We may not always want this.---- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying------ * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@----fromText :: S.Text -> Builder-fromText t@(Text arr off l)- | S.null t = empty- | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off- | otherwise = flush `append` mapBuilder (t :)-{-# INLINE [1] fromText #-}--{-# RULES-"fromText/pack" forall s .- fromText (S.pack s) = fromString s- #-}---- | /O(1)./ A Builder taking a @String@, satisfying------ * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@----fromString :: String -> Builder-fromString str = Builder $ \k (Buffer p0 o0 u0 l0) ->- let loop !marr !o !u !l [] = k (Buffer marr o u l)- loop marr o u l s@(c:cs)- | l <= 3 = do- A.shrinkM marr (o + u)- arr <- A.unsafeFreeze marr- let !t = Text arr o u- marr' <- A.new chunkSize- ts <- inlineInterleaveST (loop marr' 0 0 chunkSize s)- return $ t : ts- | otherwise = do- n <- unsafeWrite marr (o+u) (safe c)- loop marr o (u+n) (l-n) cs- in loop p0 o0 u0 l0 str- where- chunkSize = smallChunkSize-{-# INLINE fromString #-}---- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying------ * @'toLazyText' ('fromLazyText' t) = t@----fromLazyText :: L.Text -> Builder-fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++)-{-# INLINE fromLazyText #-}------------------------------------------------------------------------------ Our internal buffer type-data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s)- {-# UNPACK #-} !Int -- offset- {-# UNPACK #-} !Int -- used units- {-# UNPACK #-} !Int -- length left------------------------------------------------------------------------------ | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default--- buffer size. The construction work takes place if and when the--- relevant part of the lazy @Text@ is demanded.-toLazyText :: Builder -> L.Text-toLazyText = toLazyTextWith smallChunkSize---- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given--- size for the initial buffer. The construction work takes place if--- and when the relevant part of the lazy @Text@ is demanded.------ If the initial buffer is too small to hold all data, subsequent--- buffers will be the default buffer size.-toLazyTextWith :: Int -> Builder -> L.Text-toLazyTextWith chunkSize m = L.fromChunks (runST $- newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return [])))---- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any,--- yielding a new chunk in the result lazy @Text@.-flush :: Builder-flush = Builder $ \ k buf@(Buffer p o u l) ->- if u == 0- then k buf- else do arr <- A.unsafeFreeze p- let !b = Buffer p (o+u) 0 l- !t = Text arr o u- ts <- inlineInterleaveST (k b)- return $! t : ts-{-# INLINE [1] flush #-}--- defer inlining so that flush/flush rule may fire.------------------------------------------------------------------------------ | Sequence an ST operation on the buffer-withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder-withBuffer f = Builder $ \k buf -> f buf >>= k-{-# INLINE withBuffer #-}---- | Get the size of the buffer-withSize :: (Int -> Builder) -> Builder-withSize f = Builder $ \ k buf@(Buffer _ _ _ l) ->- runBuilder (f l) k buf-{-# INLINE withSize #-}---- | Map the resulting list of texts.-mapBuilder :: ([S.Text] -> [S.Text]) -> Builder-mapBuilder f = Builder (fmap f .)------------------------------------------------------------------------------ | Ensure that there are at least @n@ many elements available.-ensureFree :: Int -> Builder-ensureFree !n = withSize $ \ l ->- if n <= l- then empty- else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize)))-{-# INLINE [0] ensureFree #-}--writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder-writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f)-{-# INLINE [0] writeAtMost #-}---- | Ensure that @n@ many elements are available, and then use @f@ to--- write some elements into the memory.-writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder-writeN n f = writeAtMost n (\ p o -> f p o >> return n)-{-# INLINE writeN #-}--writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s)-writeBuffer f (Buffer p o u l) = do- n <- f p (o+u)- return $! Buffer p o (u+n) (l-n)-{-# INLINE writeBuffer #-}--newBuffer :: Int -> ST s (Buffer s)-newBuffer size = do- arr <- A.new size- return $! Buffer arr 0 0 size-{-# INLINE newBuffer #-}----------------------------------------------------------------------------- Some nice rules for Builder---- This function makes GHC understand that 'writeN' and 'ensureFree'--- are *not* recursive in the precense of the rewrite rules below.--- This is not needed with GHC 7+.-append' :: Builder -> Builder -> Builder-append' (Builder f) (Builder g) = Builder (f . g)-{-# INLINE append' #-}--{-# RULES--"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)- (g::forall s. A.MArray s -> Int -> ST s Int) ws.- append (writeAtMost a f) (append (writeAtMost b g) ws) =- append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->- g marr (o+n) >>= \ m ->- let s = n+m in s `seq` return s)) ws--"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)- (g::forall s. A.MArray s -> Int -> ST s Int).- append (writeAtMost a f) (writeAtMost b g) =- writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->- g marr (o+n) >>= \ m ->- let s = n+m in s `seq` return s)--"ensureFree/ensureFree" forall a b .- append (ensureFree a) (ensureFree b) = ensureFree (max a b)--"flush/flush"- append flush flush = flush-- #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} +{-# OPTIONS_HADDOCK not-home #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Data.Text.Internal.Builder +-- Copyright : (c) 2013 Bryan O'Sullivan +-- (c) 2010 Johan Tibell +-- License : BSD-style (see LICENSE) +-- +-- Maintainer : Johan Tibell <johan.tibell@gmail.com> +-- Stability : experimental +-- Portability : portable to Hugs and GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. +-- +-- To get maximum performance when building lazy @Text@ values using a +-- builder, associate @mappend@ calls to the right. For example, +-- prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. +-- +----------------------------------------------------------------------------- + +module Data.Text.Internal.Builder + ( -- * Public API + -- ** The Builder type + Builder + , LazyTextBuilder + , toLazyText + , toLazyTextWith + + -- ** Constructing Builders + , singleton + , fromText + , fromLazyText + , fromString + + -- ** Flushing the buffer state + , flush + + -- * Internal functions + , append' + , ensureFree + , writeN + ) where + +import Control.Monad.ST (ST, runST) +import Data.Monoid (Monoid(..)) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Lazy (smallChunkSize) +import Data.Text.Unsafe (inlineInterleaveST) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Prelude hiding (map, putChar) + +import qualified Data.String as String +import qualified Data.Text as S +import qualified Data.Text.Array as A +import qualified Data.Text.Lazy as L + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +------------------------------------------------------------------------ + +-- | A @Builder@ is an efficient way to build lazy @Text@ values. +-- There are several functions for constructing builders, but only one +-- to inspect them: to extract any data, you have to turn them into +-- lazy @Text@ values using @toLazyText@. +-- +-- Internally, a builder constructs a lazy @Text@ by filling arrays +-- piece by piece. As each buffer is filled, it is \'popped\' off, to +-- become a new chunk of the resulting lazy @Text@. All this is +-- hidden from the user of the @Builder@. +newtype Builder = Builder { + -- Invariant (from Data.Text.Lazy): + -- The lists include no null Texts. + runBuilder :: forall s. (Buffer s -> ST s [S.Text]) + -> Buffer s + -> ST s [S.Text] + } + +-- | @since 2.1.2 +type LazyTextBuilder = Builder + +instance Semigroup Builder where + (<>) = append + {-# INLINE (<>) #-} + +instance Monoid Builder where + mempty = empty + {-# INLINE mempty #-} + mappend = (<>) + {-# INLINE mappend #-} + mconcat = foldr mappend Data.Monoid.mempty + {-# INLINE mconcat #-} + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Builder +-- "\65533" +instance String.IsString Builder where + fromString = fromString + {-# INLINE fromString #-} + +instance Show Builder where + show = show . toLazyText + +instance Eq Builder where + a == b = toLazyText a == toLazyText b + +instance Ord Builder where + a <= b = toLazyText a <= toLazyText b + +------------------------------------------------------------------------ + +-- | /O(1)./ The empty @Builder@, satisfying +-- +-- * @'toLazyText' 'empty' = 'L.empty'@ +-- +empty :: Builder +empty = Builder (\ k buf -> k buf) +{-# INLINE empty #-} + +-- | /O(1)./ A @Builder@ taking a single character, satisfying +-- +-- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ +-- +singleton :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> Builder +singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) +{-# INLINE singleton #-} + +------------------------------------------------------------------------ + +-- | /O(1)./ The concatenation of two builders, an associative +-- operation with identity 'empty', satisfying +-- +-- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ +-- +append :: Builder -> Builder -> Builder +append (Builder f) (Builder g) = Builder (f . g) +{-# INLINE [0] append #-} + +-- TODO: Experiment to find the right threshold. +copyLimit :: Int +copyLimit = 128 + +-- This function attempts to merge small @Text@ values instead of +-- treating each value as its own chunk. We may not always want this. + +-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying +-- +-- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ +-- +fromText :: S.Text -> Builder +fromText t@(Text arr off l) + | S.null t = empty + | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off + | otherwise = flush `append` mapBuilder (t :) +{-# INLINE [1] fromText #-} + +{-# RULES +"fromText/pack" forall s . + fromText (S.pack s) = fromString s + #-} + +-- | /O(1)./ A Builder taking a @String@, satisfying +-- +-- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ +-- +-- Performs replacement on invalid scalar values: +-- +-- >>> fromString "\55555" +-- "\65533" +-- +-- @since 1.2.0.0 +fromString :: String -> Builder +fromString str = Builder $ \k (Buffer p0 o0 u0) -> do + len <- A.getSizeofMArray p0 + -- `end` is 3 bytes before the actual end of `marr` + -- to make sure there's room for a 4-byte UTF-8 code point. + let loop !marr !o !u !_ [] = k (Buffer marr o u) + loop marr o u end s@(c:cs) + | u >= end = do + A.shrinkM marr (o + u) + arr <- A.unsafeFreeze marr + let !t = Text arr o u + marr' <- A.new chunkSize + ts <- inlineInterleaveST (loop marr' 0 0 (chunkSize - 3) s) + return $ t : ts + | otherwise = do + n <- unsafeWrite marr (o+u) (safe c) + loop marr o (u+n) end cs + loop p0 o0 u0 (len - o0 - 3) str + where + chunkSize = smallChunkSize +{-# INLINEABLE fromString #-} + +-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying +-- +-- * @'toLazyText' ('fromLazyText' t) = t@ +-- +fromLazyText :: L.Text -> Builder +fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) +{-# INLINE fromLazyText #-} + +------------------------------------------------------------------------ + +-- Our internal buffer type +data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) + {-# UNPACK #-} !Int -- offset + {-# UNPACK #-} !Int -- used units + +------------------------------------------------------------------------ + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default +-- buffer size. The construction work takes place if and when the +-- relevant part of the lazy @Text@ is demanded. +toLazyText :: Builder -> L.Text +toLazyText = toLazyTextWith smallChunkSize + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given +-- size for the initial buffer. The construction work takes place if +-- and when the relevant part of the lazy @Text@ is demanded. +-- +-- If the initial buffer is too small to hold all data, subsequent +-- buffers will be the default buffer size. +toLazyTextWith :: Int -> Builder -> L.Text +toLazyTextWith chunkSize m = L.fromChunks (runST $ + newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) + +-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, +-- yielding a new chunk in the result lazy @Text@. +flush :: Builder +flush = Builder $ \ k buf@(Buffer p o u) -> + if u == 0 + then k buf + else do arr <- A.unsafeFreeze p + let !b = Buffer p (o+u) 0 + !t = Text arr o u + ts <- inlineInterleaveST (k b) + return $! t : ts +{-# INLINE [1] flush #-} +-- defer inlining so that flush/flush rule may fire. + +------------------------------------------------------------------------ + +-- | Sequence an ST operation on the buffer +withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder +withBuffer f = Builder $ \k buf -> f buf >>= k +{-# INLINE withBuffer #-} + +-- | Get the size of the buffer +withSize :: (Int -> Builder) -> Builder +withSize f = Builder $ \ k buf@(Buffer arr offset used) -> do + len <- A.getSizeofMArray arr + runBuilder (f (len - offset - used)) k buf +{-# INLINE withSize #-} + +-- | Map the resulting list of texts. +mapBuilder :: ([S.Text] -> [S.Text]) -> Builder +mapBuilder f = Builder (fmap f .) + +------------------------------------------------------------------------ + +-- | Ensure that there are at least @n@ many elements available. +ensureFree :: Int -> Builder +ensureFree !n = withSize $ \ l -> + if n <= l + then empty + else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) +{-# INLINE [0] ensureFree #-} + +writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder +writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) +{-# INLINE [0] writeAtMost #-} + +-- | Ensure that @n@ many elements are available, and then use @f@ to +-- write some elements into the memory. +writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder +writeN n f = writeAtMost n (\ p o -> f p o >> return n) +{-# INLINE writeN #-} + +writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) +writeBuffer f (Buffer p o u) = do + n <- f p (o+u) + return $! Buffer p o (u+n) +{-# INLINE writeBuffer #-} + +newBuffer :: Int -> ST s (Buffer s) +newBuffer size = do + arr <- A.new size + return $! Buffer arr 0 0 +{-# INLINE newBuffer #-} + +------------------------------------------------------------------------ +-- Some nice rules for Builder + +-- This function makes GHC understand that 'writeN' and 'ensureFree' +-- are *not* recursive in the presence of the rewrite rules below. +-- This is not needed with GHC 7+. +append' :: Builder -> Builder -> Builder +append' (Builder f) (Builder g) = Builder (f . g) +{-# INLINE append' #-} + +{-# RULES + +"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int) ws. + append (writeAtMost a f) (append (writeAtMost b g) ws) = + append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s)) ws + +"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int). + append (writeAtMost a f) (writeAtMost b g) = + writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s) + +"ensureFree/ensureFree" forall a b . + append (ensureFree a) (ensureFree b) = ensureFree (max a b) + +"flush/flush" + append flush flush = flush + + #-}
src/Data/Text/Internal/Builder/Int/Digits.hs view
@@ -1,5 +1,3 @@-{-# LANGUAGE OverloadedStrings #-}- -- Module: Data.Text.Internal.Builder.Int.Digits -- Copyright: (c) 2013 Bryan O'Sullivan -- License: BSD-style@@ -16,11 +14,12 @@ module Data.Text.Internal.Builder.Int.Digits (digits) where -import Data.ByteString.Char8 (ByteString)+import Data.ByteString.Char8 (ByteString, pack) digits :: ByteString-digits = "0001020304050607080910111213141516171819\- \2021222324252627282930313233343536373839\- \4041424344454647484950515253545556575859\- \6061626364656667686970717273747576777879\- \8081828384858687888990919293949596979899"+digits = pack+ "0001020304050607080910111213141516171819\+ \2021222324252627282930313233343536373839\+ \4041424344454647484950515253545556575859\+ \6061626364656667686970717273747576777879\+ \8081828384858687888990919293949596979899"
src/Data/Text/Internal/ByteStringCompat.hs view
@@ -8,14 +8,8 @@ import Foreign.ForeignPtr (ForeignPtr) #if !MIN_VERSION_bytestring(0,11,0)-#if MIN_VERSION_base(4,10,0) import GHC.ForeignPtr (plusForeignPtr)-#else-import GHC.ForeignPtr (ForeignPtr(ForeignPtr))-import GHC.Types (Int (..))-import GHC.Prim (plusAddr#) #endif-#endif mkBS :: ForeignPtr Word8 -> Int -> ByteString #if MIN_VERSION_bytestring(0,11,0)@@ -32,25 +26,3 @@ withBS (PS !sfp !soff !slen) kont = kont (plusForeignPtr sfp soff) slen #endif {-# INLINE withBS #-}--#if !MIN_VERSION_bytestring(0,11,0)-#if !MIN_VERSION_base(4,10,0)--- |Advances the given address by the given offset in bytes.------ The new 'ForeignPtr' shares the finalizer of the original,--- equivalent from a finalization standpoint to just creating another--- reference to the original. That is, the finalizer will not be--- called before the new 'ForeignPtr' is unreachable, nor will it be--- called an additional time due to this call, and the finalizer will--- be called with the same address that it would have had this call--- not happened, *not* the new address.-plusForeignPtr :: ForeignPtr a -> Int -> ForeignPtr b-plusForeignPtr (ForeignPtr addr guts) (I# offset) = ForeignPtr (plusAddr# addr offset) guts-{-# INLINE [0] plusForeignPtr #-}-{-# RULES-"ByteString plusForeignPtr/0" forall fp .- plusForeignPtr fp 0 = fp- #-}-#endif-#endif-
+ src/Data/Text/Internal/Encoding.hs view
@@ -0,0 +1,535 @@+{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash,+ UnliftedFFITypes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE Trustworthy #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE ViewPatterns #-}++-- |+-- Module : Data.Text.Internal.Builder+-- License : BSD-style (see LICENSE)+-- Stability : experimental+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- Internals of "Data.Text.Encoding".+--+-- @since 2.0.2+module Data.Text.Internal.Encoding+ ( validateUtf8Chunk+ , validateUtf8More+ , decodeUtf8Chunk+ , decodeUtf8More+ , decodeUtf8With1+ , decodeUtf8With2+ , Utf8State+ , startUtf8State+ , StrictTextBuilder()+ , StrictBuilder+ , strictBuilderToText+ , textToStrictBuilder++ -- * Internal+ , skipIncomplete+ , getCompleteLen+ , getPartialUtf8+ ) where++#if defined(ASSERTS)+import Control.Exception (assert)+#endif+import Data.Bits ((.&.), shiftL, shiftR)+import Data.ByteString (ByteString)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup (Semigroup(..))+#endif+import Data.Word (Word32, Word8)+import Foreign.Storable (pokeElemOff)+import Data.Text.Encoding.Error (OnDecodeError)+import Data.Text.Internal (Text(..))+import Data.Text.Internal.Encoding.Utf8+ (DecoderState, utf8AcceptState, utf8RejectState, updateDecoderState)+import Data.Text.Internal.StrictBuilder (StrictBuilder, StrictTextBuilder)+import qualified Data.ByteString as B+import qualified Data.ByteString.Internal as BI+import qualified Data.ByteString.Short.Internal as SBS+import qualified Data.Text.Array as A+import qualified Data.Text.Internal.StrictBuilder as SB+#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif++#ifdef SIMDUTF+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Foreign.C.Types (CSize(..))+import Foreign.C.Types (CInt(..))+import Foreign.Ptr (Ptr)+#endif++-- | Use 'StrictBuilder' to build 'Text'.+--+-- @since 2.0.2+strictBuilderToText :: StrictTextBuilder -> Text+strictBuilderToText = SB.toText++-- | Copy 'Text' in a 'StrictBuilder'+--+-- @since 2.0.2+textToStrictBuilder :: Text -> StrictTextBuilder+textToStrictBuilder = SB.fromText++-- | State of decoding a 'ByteString' in UTF-8.+-- Enables incremental decoding ('validateUtf8Chunk', 'validateUtf8More',+-- 'decodeUtf8Chunk', 'decodeUtf8More').+--+-- @since 2.0.2++-- Internal invariant:+-- the first component is the initial state if and only if+-- the second component is empty.+--+-- @+-- 'utf9CodePointState' s = 'utf8StartState'+-- <=>+-- 'partialUtf8CodePoint' s = 'PartialUtf8CodePoint' 0+-- @+data Utf8State = Utf8State+ { -- | State of the UTF-8 state machine.+ utf8CodePointState :: {-# UNPACK #-} !DecoderState+ -- | Bytes of the currently incomplete code point (if any).+ , partialUtf8CodePoint :: {-# UNPACK #-} !PartialUtf8CodePoint+ }+ deriving (Eq, Show)++-- | Initial 'Utf8State'.+--+-- @since 2.0.2+startUtf8State :: Utf8State+startUtf8State = Utf8State utf8AcceptState partUtf8Empty++-- | Prefix of a UTF-8 code point encoded in 4 bytes,+-- possibly empty.+--+-- - The most significant byte contains the number of bytes,+-- between 0 and 3.+-- - The remaining bytes hold the incomplete code point.+-- - Unused bytes must be 0.+--+-- All of operations available on it are the functions below.+-- The constructor should never be used outside of those.+--+-- @since 2.0.2+newtype PartialUtf8CodePoint = PartialUtf8CodePoint Word32+ deriving (Eq, Show)++-- | Empty prefix.+partUtf8Empty :: PartialUtf8CodePoint+partUtf8Empty = PartialUtf8CodePoint 0++-- | Length of the partial code point, stored in the most significant byte.+partUtf8Len :: PartialUtf8CodePoint -> Int+partUtf8Len (PartialUtf8CodePoint w) = fromIntegral $ w `shiftR` 24++-- | Length of the code point once completed (it is known in the first byte).+-- 0 if empty.+partUtf8CompleteLen :: PartialUtf8CodePoint -> Int+partUtf8CompleteLen c@(PartialUtf8CodePoint w)+ | partUtf8Len c == 0 = 0+ | 0xf0 <= firstByte = 4+ | 0xe0 <= firstByte = 3+ | 0xc2 <= firstByte = 2+ | otherwise = 0+ where+ firstByte = (w `shiftR` 16) .&. 255++-- | Get the @n@-th byte, assuming it is within bounds: @0 <= n < partUtf8Len c@.+--+-- Unsafe: no bounds checking.+partUtf8UnsafeIndex ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ PartialUtf8CodePoint -> Int -> Word8+partUtf8UnsafeIndex _c@(PartialUtf8CodePoint w) n =+#if defined(ASSERTS)+ assert (0 <= n && n < partUtf8Len _c) $+#endif+ fromIntegral $ w `shiftR` (16 - 8 * n)++-- | Append some bytes.+--+-- Unsafe: no bounds checking.+partUtf8UnsafeAppend ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ PartialUtf8CodePoint -> ByteString -> PartialUtf8CodePoint+partUtf8UnsafeAppend c@(PartialUtf8CodePoint word) bs =+#if defined(ASSERTS)+ assert (lenc + lenbs <= 3) $+#endif+ PartialUtf8CodePoint $+ tryPush 0 $ tryPush 1 $ tryPush 2 $ word + (fromIntegral lenbs `shiftL` 24)+ where+ lenc = partUtf8Len c+ lenbs = B.length bs+ tryPush i w =+ if i < lenbs+ then w + (fromIntegral (B.index bs i) `shiftL` fromIntegral (16 - 8 * (lenc + i)))+ else w++-- | Fold a 'PartialUtf8CodePoint'. This avoids recursion so it can unfold to straightline code.+{-# INLINE partUtf8Foldr #-}+partUtf8Foldr :: (Word8 -> a -> a) -> a -> PartialUtf8CodePoint -> a+partUtf8Foldr f x0 c = case partUtf8Len c of+ 0 -> x0+ 1 -> build 0 x0+ 2 -> build 0 (build 1 x0)+ _ -> build 0 (build 1 (build 2 x0))+ where+ build i x = f (partUtf8UnsafeIndex c i) x++-- | Convert 'PartialUtf8CodePoint' to 'ByteString'.+partUtf8ToByteString :: PartialUtf8CodePoint -> B.ByteString+partUtf8ToByteString c = BI.unsafeCreate (partUtf8Len c) $ \ptr ->+ partUtf8Foldr (\w k i -> pokeElemOff ptr i w >> k (i+1)) (\_ -> pure ()) c 0++-- | Exported for testing.+getCompleteLen :: Utf8State -> Int+getCompleteLen = partUtf8CompleteLen . partialUtf8CodePoint++-- | Exported for testing.+getPartialUtf8 :: Utf8State -> B.ByteString+getPartialUtf8 = partUtf8ToByteString . partialUtf8CodePoint++#ifdef SIMDUTF+foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8+ :: Ptr Word8 -> CSize -> IO CInt+#endif++-- | Validate a 'ByteString' as UTF-8-encoded text. To be continued using 'validateUtf8More'.+--+-- See also 'validateUtf8More' for details on the result of this function.+--+-- @+-- 'validateUtf8Chunk' = 'validateUtf8More' 'startUtf8State'+-- @+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- 'validateUtf8Chunk' chunk = (n, ms)+-- @+--+-- - The prefix is valid UTF-8. In particular, it should be accepted+-- by this validation:+--+-- @+-- 'validateUtf8Chunk' ('Data.ByteString.take' n chunk) = (n, Just 'startUtf8State')+-- @+validateUtf8Chunk :: ByteString -> (Int, Maybe Utf8State)+validateUtf8Chunk bs = validateUtf8ChunkFrom 0 bs (,)++-- Assume bytes up to offset @ofs@ have been validated already.+--+-- Using CPS lets us inline the continuation and avoid allocating a @Maybe@+-- in the @decode...@ functions.+{-# INLINE validateUtf8ChunkFrom #-}+validateUtf8ChunkFrom :: forall r. Int -> ByteString -> (Int -> Maybe Utf8State -> r) -> r+validateUtf8ChunkFrom ofs bs k+ -- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0.+ -- MIN_VERSION_bytestring does not allow us to differentiate+ -- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+.+#if defined(SIMDUTF) || MIN_VERSION_bytestring(0,12,1)+ | guessUtf8Boundary > 0 &&+ -- the rest of the bytestring is valid utf-8 up to the boundary+ (+#ifdef SIMDUTF+ withBS (B.drop ofs bs) $ \ fp _ -> unsafeDupablePerformIO $+ unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$>+ c_is_valid_utf8 ptr (fromIntegral guessUtf8Boundary)+#else+ B.isValidUtf8 $ B.take guessUtf8Boundary (B.drop ofs bs)+#endif+ ) = slowValidateUtf8ChunkFrom (ofs + guessUtf8Boundary)+ -- No+ | otherwise = slowValidateUtf8ChunkFrom ofs+ where+ len = B.length bs - ofs+ isBoundary n p = len >= n && p (B.index bs (ofs + len - n))+ guessUtf8Boundary+ | isBoundary 1 (<= 0x80) = len -- last char is ASCII (common short-circuit)+ | isBoundary 1 (0xc2 <=) = len - 1 -- last char starts a two-(or more-)byte code point+ | isBoundary 2 (0xe0 <=) = len - 2 -- pre-last char starts a three-or-four-byte code point+ | isBoundary 3 (0xf0 <=) = len - 3 -- third to last char starts a four-byte code point+ | otherwise = len+#else+ = slowValidateUtf8ChunkFrom ofs+ where+#endif+ -- A pure Haskell implementation of validateUtf8More.+ -- Ideally the primitives 'B.isValidUtf8' or 'c_is_valid_utf8' should give us+ -- indices to let us avoid this function.+ slowValidateUtf8ChunkFrom :: Int -> r+ slowValidateUtf8ChunkFrom ofs1 = slowLoop ofs1 ofs1 utf8AcceptState++ slowLoop !utf8End i s+ | i < B.length bs =+ case updateDecoderState (B.index bs i) s of+ s' | s' == utf8RejectState -> k utf8End Nothing+ | s' == utf8AcceptState -> slowLoop (i + 1) (i + 1) s'+ | otherwise -> slowLoop utf8End (i + 1) s'+ | otherwise = k utf8End (Just (Utf8State s (partUtf8UnsafeAppend partUtf8Empty (B.drop utf8End bs))))++-- | Validate another 'ByteString' chunk in an ongoing stream of UTF-8-encoded text.+--+-- Returns a pair:+--+-- 1. The first component @n@ is the end position, relative to the current+-- chunk, of the longest prefix of the accumulated bytestring which is valid UTF-8.+-- @n@ may be negative: that happens when an incomplete code point started in+-- a previous chunk and is not completed by the current chunk (either+-- that code point is still incomplete, or it is broken by an invalid byte).+--+-- 2. The second component @ms@ indicates the following:+--+-- - if @ms = Nothing@, the remainder of the chunk contains an invalid byte,+-- within four bytes from position @n@;+-- - if @ms = Just s'@, you can carry on validating another chunk+-- by calling 'validateUtf8More' with the new state @s'@.+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- 'validateUtf8More' s chunk = (n, ms)+-- @+--+-- - If the chunk is invalid, it cannot be extended to be valid.+--+-- @+-- ms = Nothing+-- ==> 'validateUtf8More' s (chunk '<>' more) = (n, Nothing)+-- @+--+-- - Validating two chunks sequentially is the same as validating them+-- together at once:+--+-- @+-- ms = Just s'+-- ==> 'validateUtf8More' s (chunk '<>' more) = 'Data.Bifunctor.first' ('Data.ByteString.length' chunk '+') ('validateUtf8More' s' more)+-- @+validateUtf8More :: Utf8State -> ByteString -> (Int, Maybe Utf8State)+validateUtf8More st bs = validateUtf8MoreCont st bs (,)++-- CPS: inlining the continuation lets us make more tail calls and avoid+-- allocating a @Maybe@ in @decodeWith1/2@.+{-# INLINE validateUtf8MoreCont #-}+validateUtf8MoreCont :: Utf8State -> ByteString -> (Int -> Maybe Utf8State -> r) -> r+validateUtf8MoreCont st@(Utf8State s0 part) bs k+ | len > 0 = loop 0 s0+ | otherwise = k (- partUtf8Len part) (Just st)+ where+ len = B.length bs+ -- Complete an incomplete code point (if there is one)+ -- and then jump to validateUtf8ChunkFrom+ loop !i s+ | s == utf8AcceptState = validateUtf8ChunkFrom i bs k+ | i < len =+ case updateDecoderState (B.index bs i) s of+ s' | s' == utf8RejectState -> k (- partUtf8Len part) Nothing+ | otherwise -> loop (i + 1) s'+ | otherwise = k (- partUtf8Len part) (Just (Utf8State s (partUtf8UnsafeAppend part bs)))++-- Eta-expanded to inline partUtf8Foldr+partUtf8ToStrictBuilder :: PartialUtf8CodePoint -> StrictTextBuilder+partUtf8ToStrictBuilder c =+ partUtf8Foldr ((<>) . SB.unsafeFromWord8) mempty c++utf8StateToStrictBuilder :: Utf8State -> StrictTextBuilder+utf8StateToStrictBuilder = partUtf8ToStrictBuilder . partialUtf8CodePoint++-- | Decode another chunk in an ongoing UTF-8 stream.+--+-- Returns a triple:+--+-- 1. A 'StrictBuilder' for the decoded chunk of text. You can accumulate+-- chunks with @('<>')@ or output them with 'SB.toText'.+-- 2. The undecoded remainder of the given chunk, for diagnosing errors+-- and resuming (presumably after skipping some bytes).+-- 3. 'Just' the new state, or 'Nothing' if an invalid byte was encountered+-- (it will be within the first 4 bytes of the undecoded remainder).+--+-- @since 2.0.2+--+-- === Properties+--+-- Given:+--+-- @+-- (pre, suf, ms) = 'decodeUtf8More' s chunk+-- @+--+-- 1. If the output @pre@ is nonempty (alternatively, if @length chunk > length suf@)+--+-- @+-- s2b pre \`'Data.ByteString.append'\` suf = p2b s \`'Data.ByteString.append'\` chunk+-- @+--+-- where+--+-- @+-- s2b = 'Data.Text.Encoding.encodeUtf8' . 'Data.Text.Encoding.toText'+-- p2b = 'Data.Text.Internal.Encoding.partUtf8ToByteString'+-- @+--+-- 2. If the output @pre@ is empty (alternatively, if @length chunk = length suf@)+--+-- @suf = chunk@+--+-- 3. Decoding chunks separately is equivalent to decoding their concatenation.+--+-- Given:+--+-- @+-- (pre1, suf1, Just s1) = 'decodeUtf8More' s chunk1+-- (pre2, suf2, ms2) = 'decodeUtf8More' s1 chunk2+-- (pre3, suf3, ms3) = 'decodeUtf8More' s (chunk1 \`B.append\` chunk2)+-- @+--+-- we have:+--+-- @+-- s2b (pre1 '<>' pre2) = s2b pre3+-- ms2 = ms3+-- @+decodeUtf8More :: Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State)+decodeUtf8More s bs =+ validateUtf8MoreCont s bs $ \len ms ->+ let builder | len <= 0 = mempty+ | otherwise = utf8StateToStrictBuilder s+ <> SB.unsafeFromByteString (B.take len bs)+ in (builder, B.drop len bs, ms)++-- | Decode a chunk of UTF-8 text. To be continued with 'decodeUtf8More'.+--+-- See 'decodeUtf8More' for details on the result.+--+-- @since 2.0.2+--+-- === Properties+--+-- @+-- 'decodeUtf8Chunk' = 'decodeUtf8More' 'startUtf8State'+-- @+--+-- Given:+--+-- @+-- 'decodeUtf8Chunk' chunk = (builder, rest, ms)+-- @+--+-- @builder@ is a prefix and @rest@ is a suffix of @chunk@.+--+-- @+-- 'Data.Text.Encoding.encodeUtf8' ('Data.Text.Encoding.strictBuilderToText' builder) '<>' rest = chunk+-- @+decodeUtf8Chunk :: ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State)+decodeUtf8Chunk = decodeUtf8More startUtf8State++-- | Call the error handler on each byte of the partial code point stored in+-- 'Utf8State' and append the results.+--+-- Exported for use in lazy 'Data.Text.Lazy.Encoding.decodeUtf8With'.+--+-- @since 2.0.2+{-# INLINE skipIncomplete #-}+skipIncomplete :: OnDecodeError -> String -> Utf8State -> StrictTextBuilder+skipIncomplete onErr msg s =+ partUtf8Foldr+ ((<>) . handleUtf8Error onErr msg)+ mempty (partialUtf8CodePoint s)++{-# INLINE handleUtf8Error #-}+handleUtf8Error :: OnDecodeError -> String -> Word8 -> StrictTextBuilder+handleUtf8Error onErr msg w = case onErr msg (Just w) of+ Just c -> SB.fromChar c+ Nothing -> mempty++-- | Helper for 'Data.Text.Encoding.decodeUtf8With'.+--+-- @since 2.0.2++-- This could be shorter by calling 'decodeUtf8With2' directly, but we make the+-- first call validateUtf8Chunk directly to return even faster in successful+-- cases.+decodeUtf8With1 ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ OnDecodeError -> String -> ByteString -> Text+decodeUtf8With1 onErr msg bs = validateUtf8ChunkFrom 0 bs $ \len ms -> case ms of+ Just s+ | len == B.length bs ->+ let !(SBS.SBS arr) = SBS.toShort bs in+ Text (A.ByteArray arr) 0 len+ | otherwise -> SB.toText $+ SB.unsafeFromByteString (B.take len bs) <> skipIncomplete onErr msg s+ Nothing ->+ let (builder, _, s) = decodeUtf8With2 onErr msg startUtf8State (B.drop (len + 1) bs) in+ SB.toText $+ SB.unsafeFromByteString (B.take len bs) <>+ handleUtf8Error onErr msg (B.index bs len) <>+ builder <>+ skipIncomplete onErr msg s++-- | Helper for 'Data.Text.Encoding.decodeUtf8With',+-- 'Data.Text.Encoding.streamDecodeUtf8With', and lazy+-- 'Data.Text.Lazy.Encoding.decodeUtf8With',+-- which use an 'OnDecodeError' to process bad bytes.+--+-- See 'decodeUtf8Chunk' for a more flexible alternative.+--+-- @since 2.0.2+decodeUtf8With2 ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ OnDecodeError -> String -> Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Utf8State)+decodeUtf8With2 onErr msg s0 bs = loop s0 0 mempty+ where+ loop s i !builder =+ let nonEmptyPrefix len = builder+ <> utf8StateToStrictBuilder s+ <> SB.unsafeFromByteString (B.take len (B.drop i bs))+ in validateUtf8MoreCont s (B.drop i bs) $ \len ms -> case ms of+ Nothing ->+ if len < 0+ then+ -- If the first byte cannot complete the partial code point in s,+ -- retry from startUtf8State.+ let builder' = builder <> skipIncomplete onErr msg s+ -- Note: loop is strict on builder, so if onErr raises an error it will+ -- be forced here, short-circuiting the loop as desired.+ in loop startUtf8State i builder'+ else+ let builder' = nonEmptyPrefix len+ <> handleUtf8Error onErr msg (B.index bs (i + len))+ in loop startUtf8State (i + len + 1) builder'+ Just s' ->+ let builder' = if len <= 0 then builder else nonEmptyPrefix len+ undecoded = if B.length bs >= partUtf8Len (partialUtf8CodePoint s')+ then B.drop (i + len) bs -- Reuse bs if possible+ else partUtf8ToByteString (partialUtf8CodePoint s')+ in (builder', undecoded, s')
src/Data/Text/Internal/Encoding/Fusion.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} -- | -- Module : Data.Text.Internal.Encoding.Fusion@@ -38,7 +38,7 @@ import Control.Exception (assert) #endif import Data.Bits (shiftL, shiftR)-import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)+import Data.ByteString.Internal (ByteString(..), mallocByteString) import Data.Text.Internal.Fusion (Step(..), Stream(..)) import Data.Text.Internal.Fusion.Size import Data.Text.Encoding.Error@@ -47,6 +47,7 @@ import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) import Data.Word (Word8, Word16, Word32) import Foreign.ForeignPtr (ForeignPtr)+import Foreign.Marshal.Utils (copyBytes) import Foreign.Storable (pokeByteOff) import qualified Data.ByteString as B import qualified Data.ByteString.Unsafe as B@@ -197,7 +198,7 @@ dest <- mallocByteString destLen unsafeWithForeignPtr src $ \src' -> unsafeWithForeignPtr dest $ \dest' ->- memcpy dest' src' srcLen+ copyBytes dest' src' srcLen return dest decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8
src/Data/Text/Internal/Encoding/Utf8.hs view
@@ -1,292 +1,318 @@-{-# LANGUAGE CPP, MagicHash, BangPatterns #-}---- |--- Module : Data.Text.Internal.Encoding.Utf8--- Copyright : (c) 2008, 2009 Tom Harper,--- (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Basic UTF-8 validation and character manipulation.-module Data.Text.Internal.Encoding.Utf8- ( utf8Length- , utf8LengthByLeader- -- Decomposition- , ord2- , ord3- , ord4- -- Construction- , chr2- , chr3- , chr4- -- * Validation- , validate1- , validate2- , validate3- , validate4- -- * Naive decoding- , DecoderResult(..)- , DecoderState(..)- , CodePoint(..)- , utf8DecodeStart- , utf8DecodeContinue- ) where--#if defined(ASSERTS)-import Control.Exception (assert)-import GHC.Stack (HasCallStack)-#endif-import Data.Bits (Bits(..), FiniteBits(..))-import Data.Char (ord, chr)-import GHC.Exts-import GHC.Word (Word8(..))--#if !MIN_VERSION_base(4,16,0)--- harmless to import, except for warnings that it is unused.-import Data.Text.Internal.PrimCompat (word8ToWord#)-#endif--default(Int)--between :: Word8 -- ^ byte to check- -> Word8 -- ^ lower bound- -> Word8 -- ^ upper bound- -> Bool-between x y z = x >= y && x <= z-{-# INLINE between #-}---- This is a branchless version of--- utf8Length c--- | ord c < 0x80 = 1--- | ord c < 0x800 = 2--- | ord c < 0x10000 = 3--- | otherwise = 4--- Implementation suggested by Alex Mason.---- | @since 2.0-utf8Length :: Char -> Int-utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#)))-{-# INLINE utf8Length #-}---- This is a branchless version of--- utf8LengthByLeader w--- | w < 0x80 = 1--- | w < 0xE0 = 2--- | w < 0xF0 = 3--- | otherwise = 4------ c `xor` I# (c# <=# 0#) is a branchless equivalent of c `max` 1.--- It is crucial to write c# <=# 0# and not c# ==# 0#, otherwise--- GHC is tempted to "optimize" by introduction of branches.---- | @since 2.0-utf8LengthByLeader :: Word8 -> Int-utf8LengthByLeader w = c `xor` I# (c# <=# 0#)- where- !c@(I# c#) = countLeadingZeros (complement w)-{-# INLINE utf8LengthByLeader #-}--ord2 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8)-ord2 c =-#if defined(ASSERTS)- assert (n >= 0x80 && n <= 0x07ff)-#endif- (x1,x2)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 6) + 0xC0- x2 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord2 #-}--ord3 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8,Word8)-ord3 c =-#if defined(ASSERTS)- assert (n >= 0x0800 && n <= 0xffff)-#endif- (x1,x2,x3)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 12) + 0xE0- x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80- x3 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord3 #-}--ord4 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8,Word8,Word8)-ord4 c =-#if defined(ASSERTS)- assert (n >= 0x10000)-#endif- (x1,x2,x3,x4)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 18) + 0xF0- x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80- x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80- x4 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord4 #-}--chr2 :: Word8 -> Word8 -> Char-chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6#- !z2# = y2# -# 0x80#-{-# INLINE chr2 #-}--chr3 :: Word8 -> Word8 -> Word8 -> Char-chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !y3# = word2Int# (word8ToWord# x3#)- !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12#- !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6#- !z3# = y3# -# 0x80#-{-# INLINE chr3 #-}--chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char-chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) =- C# (chr# (z1# +# z2# +# z3# +# z4#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !y3# = word2Int# (word8ToWord# x3#)- !y4# = word2Int# (word8ToWord# x4#)- !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18#- !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12#- !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6#- !z4# = y4# -# 0x80#-{-# INLINE chr4 #-}--validate1 :: Word8 -> Bool-validate1 x1 = x1 <= 0x7F-{-# INLINE validate1 #-}--validate2 :: Word8 -> Word8 -> Bool-validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF-{-# INLINE validate2 #-}--validate3 :: Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate3 #-}-validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4- where- validate3_1 = (x1 == 0xE0) &&- between x2 0xA0 0xBF &&- between x3 0x80 0xBF- validate3_2 = between x1 0xE1 0xEC &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF- validate3_3 = x1 == 0xED &&- between x2 0x80 0x9F &&- between x3 0x80 0xBF- validate3_4 = between x1 0xEE 0xEF &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF--validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate4 #-}-validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3- where- validate4_1 = x1 == 0xF0 &&- between x2 0x90 0xBF &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF- validate4_2 = between x1 0xF1 0xF3 &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF- validate4_3 = x1 == 0xF4 &&- between x2 0x80 0x8F &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF--intToWord8 :: Int -> Word8-intToWord8 = fromIntegral--word8ToInt :: Word8 -> Int-word8ToInt = fromIntegral------------------------------------------------------------------------------------ Naive UTF8 decoder.--- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine.--newtype ByteClass = ByteClass Word8--byteToClass :: Word8 -> ByteClass-byteToClass n = ByteClass (W8# el#)- where- !(I# n#) = word8ToInt n- el# = indexWord8OffAddr# table# n#-- table# :: Addr#- table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"#--newtype DecoderState = DecoderState Word8- deriving (Eq)--utf8AcceptState :: DecoderState-utf8AcceptState = DecoderState 0--utf8RejectState :: DecoderState-utf8RejectState = DecoderState 12--updateState :: ByteClass -> DecoderState -> DecoderState-updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#)- where- !(I# n#) = word8ToInt (c + s)- el# = indexWord8OffAddr# table# n#-- table# :: Addr#- table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"#--newtype CodePoint = CodePoint Int---- | @since 2.0-data DecoderResult- = Accept !Char- | Incomplete !DecoderState !CodePoint- | Reject---- | @since 2.0-utf8DecodeStart :: Word8 -> DecoderResult-utf8DecodeStart !w- | st == utf8AcceptState = Accept (chr (word8ToInt w))- | st == utf8RejectState = Reject- | otherwise = Incomplete st (CodePoint cp)- where- cl@(ByteClass cl') = byteToClass w- st = updateState cl utf8AcceptState- cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w---- | @since 2.0-utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult-utf8DecodeContinue !w !st (CodePoint !cp)- | st' == utf8AcceptState = Accept (chr cp')- | st' == utf8RejectState = Reject- | otherwise = Incomplete st' (CodePoint cp')- where- cl = byteToClass w- st' = updateState cl st- cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f)+{-# LANGUAGE CPP, MagicHash, BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Utf8 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-8 validation and character manipulation. +module Data.Text.Internal.Encoding.Utf8 + ( utf8Length + , utf8LengthByLeader + -- Decomposition + , ord2 + , ord3 + , ord4 + -- Construction + , chr2 + , chr3 + , chr4 + -- * Validation + , validate1 + , validate2 + , validate3 + , validate4 + -- * Naive decoding + , DecoderState(..) + , utf8AcceptState + , utf8RejectState + , updateDecoderState + , DecoderResult(..) + , CodePoint(..) + , utf8DecodeStart + , utf8DecodeContinue + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Bits (Bits(..)) +import Data.Char (ord, chr) +import GHC.Exts +import GHC.Word (Word8(..)) + +#if !MIN_VERSION_base(4,16,0) +-- harmless to import, except for warnings that it is unused. +import Data.Text.Internal.PrimCompat (word8ToWord#) +#endif + +default(Int) + +between :: Word8 -- ^ byte to check + -> Word8 -- ^ lower bound + -> Word8 -- ^ upper bound + -> Bool +between x y z = x >= y && x <= z +{-# INLINE between #-} + +-- This is a branchless version of +-- utf8Length c +-- | ord c < 0x80 = 1 +-- | ord c < 0x800 = 2 +-- | ord c < 0x10000 = 3 +-- | otherwise = 4 +-- Implementation suggested by Alex Mason. + +-- | Measure byte length of UTF-8 encoding for a given character. +-- +-- @since 2.0 +utf8Length :: Char -> Int +utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) +{-# INLINE utf8Length #-} + +-- | Measure byte length of UTF-8 encoding for characters, +-- starting with a given byte. +-- +-- @since 2.0 +utf8LengthByLeader :: Word8 -> Int +utf8LengthByLeader w + | w < 0x80 = 1 + | w < 0xE0 = 2 + | w < 0xF0 = 3 + | otherwise = 4 +{-# INLINE utf8LengthByLeader #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 2 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8) +ord2 c = +#if defined(ASSERTS) + assert (n >= 0x80 && n <= 0x07ff) +#endif + (x1,x2) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 + x2 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord2 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 3 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord3 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8) +ord3 c = +#if defined(ASSERTS) + assert (n >= 0x0800 && n <= 0xffff) +#endif + (x1,x2,x3) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 + x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x3 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord3 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 4 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord4 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8,Word8) +ord4 c = +#if defined(ASSERTS) + assert (n >= 0x10000) +#endif + (x1,x2,x3,x4) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 18) + 0xF0 + x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 + x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x4 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord4 #-} + +-- | @since 1.1.0.0 +chr2 :: Word8 -> Word8 -> Char +chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# + !z2# = y2# -# 0x80# +{-# INLINE chr2 #-} + +-- | @since 1.1.0.0 +chr3 :: Word8 -> Word8 -> Word8 -> Char +chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# + !z3# = y3# -# 0x80# +{-# INLINE chr3 #-} + +-- | @since 1.1.0.0 +chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char +chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = + C# (chr# (z1# +# z2# +# z3# +# z4#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !y4# = word2Int# (word8ToWord# x4#) + !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# + !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# + !z4# = y4# -# 0x80# +{-# INLINE chr4 #-} + +-- | @since 1.1.0.0 +validate1 :: Word8 -> Bool +validate1 x1 = x1 <= 0x7F +{-# INLINE validate1 #-} + +-- | @since 1.1.0.0 +validate2 :: Word8 -> Word8 -> Bool +validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF +{-# INLINE validate2 #-} + +-- | @since 1.1.0.0 +validate3 :: Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate3 #-} +validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 + where + validate3_1 = (x1 == 0xE0) && + between x2 0xA0 0xBF && + between x3 0x80 0xBF + validate3_2 = between x1 0xE1 0xEC && + between x2 0x80 0xBF && + between x3 0x80 0xBF + validate3_3 = x1 == 0xED && + between x2 0x80 0x9F && + between x3 0x80 0xBF + validate3_4 = between x1 0xEE 0xEF && + between x2 0x80 0xBF && + between x3 0x80 0xBF + +-- | @since 1.1.0.0 +validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate4 #-} +validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 + where + validate4_1 = x1 == 0xF0 && + between x2 0x90 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_2 = between x1 0xF1 0xF3 && + between x2 0x80 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_3 = x1 == 0xF4 && + between x2 0x80 0x8F && + between x3 0x80 0xBF && + between x4 0x80 0xBF + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +------------------------------------------------------------------------------- +-- Naive UTF8 decoder. +-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. + +newtype ByteClass = ByteClass Word8 + +byteToClass :: Word8 -> ByteClass +byteToClass n = ByteClass (W8# el#) + where + !(I# n#) = word8ToInt n + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# + +-- | @since 2.0 +newtype DecoderState = DecoderState Word8 + deriving (Eq, Show) + +-- | @since 2.0.2 +utf8AcceptState :: DecoderState +utf8AcceptState = DecoderState 0 + +-- | @since 2.0.2 +utf8RejectState :: DecoderState +utf8RejectState = DecoderState 12 + +updateState :: ByteClass -> DecoderState -> DecoderState +updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) + where + !(I# n#) = word8ToInt (c + s) + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# + +-- | @since 2.0.2 +updateDecoderState :: Word8 -> DecoderState -> DecoderState +updateDecoderState b s = updateState (byteToClass b) s + +-- | @since 2.0 +newtype CodePoint = CodePoint Int + +-- | @since 2.0 +data DecoderResult + = Accept !Char + | Incomplete !DecoderState !CodePoint + | Reject + +-- | @since 2.0 +utf8DecodeStart :: Word8 -> DecoderResult +utf8DecodeStart !w + | st == utf8AcceptState = Accept (chr (word8ToInt w)) + | st == utf8RejectState = Reject + | otherwise = Incomplete st (CodePoint cp) + where + cl@(ByteClass cl') = byteToClass w + st = updateState cl utf8AcceptState + cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w + +-- | @since 2.0 +utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult +utf8DecodeContinue !w !st (CodePoint !cp) + | st' == utf8AcceptState = Accept (chr cp') + | st' == utf8RejectState = Reject + | otherwise = Incomplete st' (CodePoint cp') + where + cl = byteToClass w + st' = updateState cl st + cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f)
src/Data/Text/Internal/Fusion.hs view
@@ -25,6 +25,7 @@ -- * Creation and elimination , stream+ , streamLn , unstream , reverseStream @@ -49,8 +50,8 @@ , countChar ) where -import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int,- Num(..), Ord(..), ($),+import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int,+ Num(..), Ord(..), ($), (&&), otherwise) import Data.Bits (shiftL, shiftR) import Data.Text.Internal (Text(..))@@ -78,12 +79,33 @@ HasCallStack => #endif Text -> Stream Char-stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) len)+stream = stream' False+{-# INLINE [0] stream #-}++-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@+--+-- @since 2.1.2+streamLn ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ Text -> Stream Char+streamLn = stream' True++-- | Shared implementation of 'stream' and 'streamLn'.+stream' ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ Bool -> Text -> Stream Char+stream' addNl (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) maxLen) where+ maxLen = if addNl then len + 1 else len !end = off+len next !i- | i >= end = Done- | otherwise = Yield chr (i + l)+ | i < end = Yield chr (i + l)+ | addNl && i == end = Yield '\n' (i + 1)+ | otherwise = Done where n0 = A.unsafeIndex arr i n1 = A.unsafeIndex arr (i + 1)@@ -96,7 +118,7 @@ 2 -> U8.chr2 n0 n1 3 -> U8.chr3 n0 n1 n2 _ -> U8.chr4 n0 n1 n2 n3-{-# INLINE [0] stream #-}+{-# INLINE [0] stream' #-} -- | /O(n)/ Converts 'Text' into a 'Stream' 'Char', but iterates -- backwards through the text.
src/Data/Text/Internal/Fusion/CaseMapping.hs view
@@ -1,6420 +1,7984 @@--- AUTOMATICALLY GENERATED - DO NOT EDIT--- Generated by scripts/CaseMapping.hs--- CaseFolding-14.0.0.txt--- Date: 2021-03-08, 19:35:41 GMT--- SpecialCasing-14.0.0.txt--- Date: 2021-03-08, 19:35:55 GMT--{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}-{-# OPTIONS_GHC -Wno-partial-type-signatures #-}-module Data.Text.Internal.Fusion.CaseMapping where-import GHC.Int-import GHC.Exts-unI64 :: Int64 -> _ {- unboxed Int64 -}-unI64 (I64# n) = n--upperMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE upperMapping #-}-upperMapping = \case- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 174063699- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 146800710- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 153092166- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 159383622- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 321057542111302- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 334251681644614- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 176160851- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 176160851- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2856322357- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2831156548- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2795504964- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2808087876- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2831156558- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2812282180- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 163578556- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429849- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429861- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778634- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373256- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390036- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584343- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584345- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200769- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459557- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987429- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498533- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720293- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025681- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025687- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918745- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429849- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025689- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651609- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918757- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429861- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459553- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025701- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651621- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025705- -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI- '\x1f80'# -> unI64 1931484936- -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI- '\x1f81'# -> unI64 1931484937- -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f82'# -> unI64 1931484938- -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f83'# -> unI64 1931484939- -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f84'# -> unI64 1931484940- -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f85'# -> unI64 1931484941- -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f86'# -> unI64 1931484942- -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f87'# -> unI64 1931484943- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI- '\x1f88'# -> unI64 1931484936- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI- '\x1f89'# -> unI64 1931484937- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f8a'# -> unI64 1931484938- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f8b'# -> unI64 1931484939- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f8c'# -> unI64 1931484940- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f8d'# -> unI64 1931484941- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8e'# -> unI64 1931484942- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8f'# -> unI64 1931484943- -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI- '\x1f90'# -> unI64 1931484968- -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI- '\x1f91'# -> unI64 1931484969- -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f92'# -> unI64 1931484970- -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f93'# -> unI64 1931484971- -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f94'# -> unI64 1931484972- -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f95'# -> unI64 1931484973- -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f96'# -> unI64 1931484974- -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f97'# -> unI64 1931484975- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI- '\x1f98'# -> unI64 1931484968- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI- '\x1f99'# -> unI64 1931484969- -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f9a'# -> unI64 1931484970- -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f9b'# -> unI64 1931484971- -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f9c'# -> unI64 1931484972- -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f9d'# -> unI64 1931484973- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9e'# -> unI64 1931484974- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9f'# -> unI64 1931484975- -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI- '\x1fa0'# -> unI64 1931485032- -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI- '\x1fa1'# -> unI64 1931485033- -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1fa2'# -> unI64 1931485034- -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1fa3'# -> unI64 1931485035- -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1fa4'# -> unI64 1931485036- -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1fa5'# -> unI64 1931485037- -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa6'# -> unI64 1931485038- -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa7'# -> unI64 1931485039- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI- '\x1fa8'# -> unI64 1931485032- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI- '\x1fa9'# -> unI64 1931485033- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1faa'# -> unI64 1931485034- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1fab'# -> unI64 1931485035- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1fac'# -> unI64 1931485036- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1fad'# -> unI64 1931485037- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1fae'# -> unI64 1931485038- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1faf'# -> unI64 1931485039- -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI- '\x1fb3'# -> unI64 1931477905- -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI- '\x1fbc'# -> unI64 1931477905- -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI- '\x1fc3'# -> unI64 1931477911- -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI- '\x1fcc'# -> unI64 1931477911- -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI- '\x1ff3'# -> unI64 1931477929- -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI- '\x1ffc'# -> unI64 1931477929- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1931485114- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1931477894- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1931485130- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1931477897- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1931485178- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1931477903- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 4050602585752465- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 4050602585752471- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 4050602585752489- '\x0061'# -> unI64 65- '\x0062'# -> unI64 66- '\x0063'# -> unI64 67- '\x0064'# -> unI64 68- '\x0065'# -> unI64 69- '\x0066'# -> unI64 70- '\x0067'# -> unI64 71- '\x0068'# -> unI64 72- '\x0069'# -> unI64 73- '\x006a'# -> unI64 74- '\x006b'# -> unI64 75- '\x006c'# -> unI64 76- '\x006d'# -> unI64 77- '\x006e'# -> unI64 78- '\x006f'# -> unI64 79- '\x0070'# -> unI64 80- '\x0071'# -> unI64 81- '\x0072'# -> unI64 82- '\x0073'# -> unI64 83- '\x0074'# -> unI64 84- '\x0075'# -> unI64 85- '\x0076'# -> unI64 86- '\x0077'# -> unI64 87- '\x0078'# -> unI64 88- '\x0079'# -> unI64 89- '\x007a'# -> unI64 90- '\x00b5'# -> unI64 924- '\x00e0'# -> unI64 192- '\x00e1'# -> unI64 193- '\x00e2'# -> unI64 194- '\x00e3'# -> unI64 195- '\x00e4'# -> unI64 196- '\x00e5'# -> unI64 197- '\x00e6'# -> unI64 198- '\x00e7'# -> unI64 199- '\x00e8'# -> unI64 200- '\x00e9'# -> unI64 201- '\x00ea'# -> unI64 202- '\x00eb'# -> unI64 203- '\x00ec'# -> unI64 204- '\x00ed'# -> unI64 205- '\x00ee'# -> unI64 206- '\x00ef'# -> unI64 207- '\x00f0'# -> unI64 208- '\x00f1'# -> unI64 209- '\x00f2'# -> unI64 210- '\x00f3'# -> unI64 211- '\x00f4'# -> unI64 212- '\x00f5'# -> unI64 213- '\x00f6'# -> unI64 214- '\x00f8'# -> unI64 216- '\x00f9'# -> unI64 217- '\x00fa'# -> unI64 218- '\x00fb'# -> unI64 219- '\x00fc'# -> unI64 220- '\x00fd'# -> unI64 221- '\x00fe'# -> unI64 222- '\x00ff'# -> unI64 376- '\x0101'# -> unI64 256- '\x0103'# -> unI64 258- '\x0105'# -> unI64 260- '\x0107'# -> unI64 262- '\x0109'# -> unI64 264- '\x010b'# -> unI64 266- '\x010d'# -> unI64 268- '\x010f'# -> unI64 270- '\x0111'# -> unI64 272- '\x0113'# -> unI64 274- '\x0115'# -> unI64 276- '\x0117'# -> unI64 278- '\x0119'# -> unI64 280- '\x011b'# -> unI64 282- '\x011d'# -> unI64 284- '\x011f'# -> unI64 286- '\x0121'# -> unI64 288- '\x0123'# -> unI64 290- '\x0125'# -> unI64 292- '\x0127'# -> unI64 294- '\x0129'# -> unI64 296- '\x012b'# -> unI64 298- '\x012d'# -> unI64 300- '\x012f'# -> unI64 302- '\x0131'# -> unI64 73- '\x0133'# -> unI64 306- '\x0135'# -> unI64 308- '\x0137'# -> unI64 310- '\x013a'# -> unI64 313- '\x013c'# -> unI64 315- '\x013e'# -> unI64 317- '\x0140'# -> unI64 319- '\x0142'# -> unI64 321- '\x0144'# -> unI64 323- '\x0146'# -> unI64 325- '\x0148'# -> unI64 327- '\x014b'# -> unI64 330- '\x014d'# -> unI64 332- '\x014f'# -> unI64 334- '\x0151'# -> unI64 336- '\x0153'# -> unI64 338- '\x0155'# -> unI64 340- '\x0157'# -> unI64 342- '\x0159'# -> unI64 344- '\x015b'# -> unI64 346- '\x015d'# -> unI64 348- '\x015f'# -> unI64 350- '\x0161'# -> unI64 352- '\x0163'# -> unI64 354- '\x0165'# -> unI64 356- '\x0167'# -> unI64 358- '\x0169'# -> unI64 360- '\x016b'# -> unI64 362- '\x016d'# -> unI64 364- '\x016f'# -> unI64 366- '\x0171'# -> unI64 368- '\x0173'# -> unI64 370- '\x0175'# -> unI64 372- '\x0177'# -> unI64 374- '\x017a'# -> unI64 377- '\x017c'# -> unI64 379- '\x017e'# -> unI64 381- '\x017f'# -> unI64 83- '\x0180'# -> unI64 579- '\x0183'# -> unI64 386- '\x0185'# -> unI64 388- '\x0188'# -> unI64 391- '\x018c'# -> unI64 395- '\x0192'# -> unI64 401- '\x0195'# -> unI64 502- '\x0199'# -> unI64 408- '\x019a'# -> unI64 573- '\x019e'# -> unI64 544- '\x01a1'# -> unI64 416- '\x01a3'# -> unI64 418- '\x01a5'# -> unI64 420- '\x01a8'# -> unI64 423- '\x01ad'# -> unI64 428- '\x01b0'# -> unI64 431- '\x01b4'# -> unI64 435- '\x01b6'# -> unI64 437- '\x01b9'# -> unI64 440- '\x01bd'# -> unI64 444- '\x01bf'# -> unI64 503- '\x01c5'# -> unI64 452- '\x01c6'# -> unI64 452- '\x01c8'# -> unI64 455- '\x01c9'# -> unI64 455- '\x01cb'# -> unI64 458- '\x01cc'# -> unI64 458- '\x01ce'# -> unI64 461- '\x01d0'# -> unI64 463- '\x01d2'# -> unI64 465- '\x01d4'# -> unI64 467- '\x01d6'# -> unI64 469- '\x01d8'# -> unI64 471- '\x01da'# -> unI64 473- '\x01dc'# -> unI64 475- '\x01dd'# -> unI64 398- '\x01df'# -> unI64 478- '\x01e1'# -> unI64 480- '\x01e3'# -> unI64 482- '\x01e5'# -> unI64 484- '\x01e7'# -> unI64 486- '\x01e9'# -> unI64 488- '\x01eb'# -> unI64 490- '\x01ed'# -> unI64 492- '\x01ef'# -> unI64 494- '\x01f2'# -> unI64 497- '\x01f3'# -> unI64 497- '\x01f5'# -> unI64 500- '\x01f9'# -> unI64 504- '\x01fb'# -> unI64 506- '\x01fd'# -> unI64 508- '\x01ff'# -> unI64 510- '\x0201'# -> unI64 512- '\x0203'# -> unI64 514- '\x0205'# -> unI64 516- '\x0207'# -> unI64 518- '\x0209'# -> unI64 520- '\x020b'# -> unI64 522- '\x020d'# -> unI64 524- '\x020f'# -> unI64 526- '\x0211'# -> unI64 528- '\x0213'# -> unI64 530- '\x0215'# -> unI64 532- '\x0217'# -> unI64 534- '\x0219'# -> unI64 536- '\x021b'# -> unI64 538- '\x021d'# -> unI64 540- '\x021f'# -> unI64 542- '\x0223'# -> unI64 546- '\x0225'# -> unI64 548- '\x0227'# -> unI64 550- '\x0229'# -> unI64 552- '\x022b'# -> unI64 554- '\x022d'# -> unI64 556- '\x022f'# -> unI64 558- '\x0231'# -> unI64 560- '\x0233'# -> unI64 562- '\x023c'# -> unI64 571- '\x023f'# -> unI64 11390- '\x0240'# -> unI64 11391- '\x0242'# -> unI64 577- '\x0247'# -> unI64 582- '\x0249'# -> unI64 584- '\x024b'# -> unI64 586- '\x024d'# -> unI64 588- '\x024f'# -> unI64 590- '\x0250'# -> unI64 11375- '\x0251'# -> unI64 11373- '\x0252'# -> unI64 11376- '\x0253'# -> unI64 385- '\x0254'# -> unI64 390- '\x0256'# -> unI64 393- '\x0257'# -> unI64 394- '\x0259'# -> unI64 399- '\x025b'# -> unI64 400- '\x025c'# -> unI64 42923- '\x0260'# -> unI64 403- '\x0261'# -> unI64 42924- '\x0263'# -> unI64 404- '\x0265'# -> unI64 42893- '\x0266'# -> unI64 42922- '\x0268'# -> unI64 407- '\x0269'# -> unI64 406- '\x026a'# -> unI64 42926- '\x026b'# -> unI64 11362- '\x026c'# -> unI64 42925- '\x026f'# -> unI64 412- '\x0271'# -> unI64 11374- '\x0272'# -> unI64 413- '\x0275'# -> unI64 415- '\x027d'# -> unI64 11364- '\x0280'# -> unI64 422- '\x0282'# -> unI64 42949- '\x0283'# -> unI64 425- '\x0287'# -> unI64 42929- '\x0288'# -> unI64 430- '\x0289'# -> unI64 580- '\x028a'# -> unI64 433- '\x028b'# -> unI64 434- '\x028c'# -> unI64 581- '\x0292'# -> unI64 439- '\x029d'# -> unI64 42930- '\x029e'# -> unI64 42928- '\x0345'# -> unI64 921- '\x0371'# -> unI64 880- '\x0373'# -> unI64 882- '\x0377'# -> unI64 886- '\x037b'# -> unI64 1021- '\x037c'# -> unI64 1022- '\x037d'# -> unI64 1023- '\x03ac'# -> unI64 902- '\x03ad'# -> unI64 904- '\x03ae'# -> unI64 905- '\x03af'# -> unI64 906- '\x03b1'# -> unI64 913- '\x03b2'# -> unI64 914- '\x03b3'# -> unI64 915- '\x03b4'# -> unI64 916- '\x03b5'# -> unI64 917- '\x03b6'# -> unI64 918- '\x03b7'# -> unI64 919- '\x03b8'# -> unI64 920- '\x03b9'# -> unI64 921- '\x03ba'# -> unI64 922- '\x03bb'# -> unI64 923- '\x03bc'# -> unI64 924- '\x03bd'# -> unI64 925- '\x03be'# -> unI64 926- '\x03bf'# -> unI64 927- '\x03c0'# -> unI64 928- '\x03c1'# -> unI64 929- '\x03c2'# -> unI64 931- '\x03c3'# -> unI64 931- '\x03c4'# -> unI64 932- '\x03c5'# -> unI64 933- '\x03c6'# -> unI64 934- '\x03c7'# -> unI64 935- '\x03c8'# -> unI64 936- '\x03c9'# -> unI64 937- '\x03ca'# -> unI64 938- '\x03cb'# -> unI64 939- '\x03cc'# -> unI64 908- '\x03cd'# -> unI64 910- '\x03ce'# -> unI64 911- '\x03d0'# -> unI64 914- '\x03d1'# -> unI64 920- '\x03d5'# -> unI64 934- '\x03d6'# -> unI64 928- '\x03d7'# -> unI64 975- '\x03d9'# -> unI64 984- '\x03db'# -> unI64 986- '\x03dd'# -> unI64 988- '\x03df'# -> unI64 990- '\x03e1'# -> unI64 992- '\x03e3'# -> unI64 994- '\x03e5'# -> unI64 996- '\x03e7'# -> unI64 998- '\x03e9'# -> unI64 1000- '\x03eb'# -> unI64 1002- '\x03ed'# -> unI64 1004- '\x03ef'# -> unI64 1006- '\x03f0'# -> unI64 922- '\x03f1'# -> unI64 929- '\x03f2'# -> unI64 1017- '\x03f3'# -> unI64 895- '\x03f5'# -> unI64 917- '\x03f8'# -> unI64 1015- '\x03fb'# -> unI64 1018- '\x0430'# -> unI64 1040- '\x0431'# -> unI64 1041- '\x0432'# -> unI64 1042- '\x0433'# -> unI64 1043- '\x0434'# -> unI64 1044- '\x0435'# -> unI64 1045- '\x0436'# -> unI64 1046- '\x0437'# -> unI64 1047- '\x0438'# -> unI64 1048- '\x0439'# -> unI64 1049- '\x043a'# -> unI64 1050- '\x043b'# -> unI64 1051- '\x043c'# -> unI64 1052- '\x043d'# -> unI64 1053- '\x043e'# -> unI64 1054- '\x043f'# -> unI64 1055- '\x0440'# -> unI64 1056- '\x0441'# -> unI64 1057- '\x0442'# -> unI64 1058- '\x0443'# -> unI64 1059- '\x0444'# -> unI64 1060- '\x0445'# -> unI64 1061- '\x0446'# -> unI64 1062- '\x0447'# -> unI64 1063- '\x0448'# -> unI64 1064- '\x0449'# -> unI64 1065- '\x044a'# -> unI64 1066- '\x044b'# -> unI64 1067- '\x044c'# -> unI64 1068- '\x044d'# -> unI64 1069- '\x044e'# -> unI64 1070- '\x044f'# -> unI64 1071- '\x0450'# -> unI64 1024- '\x0451'# -> unI64 1025- '\x0452'# -> unI64 1026- '\x0453'# -> unI64 1027- '\x0454'# -> unI64 1028- '\x0455'# -> unI64 1029- '\x0456'# -> unI64 1030- '\x0457'# -> unI64 1031- '\x0458'# -> unI64 1032- '\x0459'# -> unI64 1033- '\x045a'# -> unI64 1034- '\x045b'# -> unI64 1035- '\x045c'# -> unI64 1036- '\x045d'# -> unI64 1037- '\x045e'# -> unI64 1038- '\x045f'# -> unI64 1039- '\x0461'# -> unI64 1120- '\x0463'# -> unI64 1122- '\x0465'# -> unI64 1124- '\x0467'# -> unI64 1126- '\x0469'# -> unI64 1128- '\x046b'# -> unI64 1130- '\x046d'# -> unI64 1132- '\x046f'# -> unI64 1134- '\x0471'# -> unI64 1136- '\x0473'# -> unI64 1138- '\x0475'# -> unI64 1140- '\x0477'# -> unI64 1142- '\x0479'# -> unI64 1144- '\x047b'# -> unI64 1146- '\x047d'# -> unI64 1148- '\x047f'# -> unI64 1150- '\x0481'# -> unI64 1152- '\x048b'# -> unI64 1162- '\x048d'# -> unI64 1164- '\x048f'# -> unI64 1166- '\x0491'# -> unI64 1168- '\x0493'# -> unI64 1170- '\x0495'# -> unI64 1172- '\x0497'# -> unI64 1174- '\x0499'# -> unI64 1176- '\x049b'# -> unI64 1178- '\x049d'# -> unI64 1180- '\x049f'# -> unI64 1182- '\x04a1'# -> unI64 1184- '\x04a3'# -> unI64 1186- '\x04a5'# -> unI64 1188- '\x04a7'# -> unI64 1190- '\x04a9'# -> unI64 1192- '\x04ab'# -> unI64 1194- '\x04ad'# -> unI64 1196- '\x04af'# -> unI64 1198- '\x04b1'# -> unI64 1200- '\x04b3'# -> unI64 1202- '\x04b5'# -> unI64 1204- '\x04b7'# -> unI64 1206- '\x04b9'# -> unI64 1208- '\x04bb'# -> unI64 1210- '\x04bd'# -> unI64 1212- '\x04bf'# -> unI64 1214- '\x04c2'# -> unI64 1217- '\x04c4'# -> unI64 1219- '\x04c6'# -> unI64 1221- '\x04c8'# -> unI64 1223- '\x04ca'# -> unI64 1225- '\x04cc'# -> unI64 1227- '\x04ce'# -> unI64 1229- '\x04cf'# -> unI64 1216- '\x04d1'# -> unI64 1232- '\x04d3'# -> unI64 1234- '\x04d5'# -> unI64 1236- '\x04d7'# -> unI64 1238- '\x04d9'# -> unI64 1240- '\x04db'# -> unI64 1242- '\x04dd'# -> unI64 1244- '\x04df'# -> unI64 1246- '\x04e1'# -> unI64 1248- '\x04e3'# -> unI64 1250- '\x04e5'# -> unI64 1252- '\x04e7'# -> unI64 1254- '\x04e9'# -> unI64 1256- '\x04eb'# -> unI64 1258- '\x04ed'# -> unI64 1260- '\x04ef'# -> unI64 1262- '\x04f1'# -> unI64 1264- '\x04f3'# -> unI64 1266- '\x04f5'# -> unI64 1268- '\x04f7'# -> unI64 1270- '\x04f9'# -> unI64 1272- '\x04fb'# -> unI64 1274- '\x04fd'# -> unI64 1276- '\x04ff'# -> unI64 1278- '\x0501'# -> unI64 1280- '\x0503'# -> unI64 1282- '\x0505'# -> unI64 1284- '\x0507'# -> unI64 1286- '\x0509'# -> unI64 1288- '\x050b'# -> unI64 1290- '\x050d'# -> unI64 1292- '\x050f'# -> unI64 1294- '\x0511'# -> unI64 1296- '\x0513'# -> unI64 1298- '\x0515'# -> unI64 1300- '\x0517'# -> unI64 1302- '\x0519'# -> unI64 1304- '\x051b'# -> unI64 1306- '\x051d'# -> unI64 1308- '\x051f'# -> unI64 1310- '\x0521'# -> unI64 1312- '\x0523'# -> unI64 1314- '\x0525'# -> unI64 1316- '\x0527'# -> unI64 1318- '\x0529'# -> unI64 1320- '\x052b'# -> unI64 1322- '\x052d'# -> unI64 1324- '\x052f'# -> unI64 1326- '\x0561'# -> unI64 1329- '\x0562'# -> unI64 1330- '\x0563'# -> unI64 1331- '\x0564'# -> unI64 1332- '\x0565'# -> unI64 1333- '\x0566'# -> unI64 1334- '\x0567'# -> unI64 1335- '\x0568'# -> unI64 1336- '\x0569'# -> unI64 1337- '\x056a'# -> unI64 1338- '\x056b'# -> unI64 1339- '\x056c'# -> unI64 1340- '\x056d'# -> unI64 1341- '\x056e'# -> unI64 1342- '\x056f'# -> unI64 1343- '\x0570'# -> unI64 1344- '\x0571'# -> unI64 1345- '\x0572'# -> unI64 1346- '\x0573'# -> unI64 1347- '\x0574'# -> unI64 1348- '\x0575'# -> unI64 1349- '\x0576'# -> unI64 1350- '\x0577'# -> unI64 1351- '\x0578'# -> unI64 1352- '\x0579'# -> unI64 1353- '\x057a'# -> unI64 1354- '\x057b'# -> unI64 1355- '\x057c'# -> unI64 1356- '\x057d'# -> unI64 1357- '\x057e'# -> unI64 1358- '\x057f'# -> unI64 1359- '\x0580'# -> unI64 1360- '\x0581'# -> unI64 1361- '\x0582'# -> unI64 1362- '\x0583'# -> unI64 1363- '\x0584'# -> unI64 1364- '\x0585'# -> unI64 1365- '\x0586'# -> unI64 1366- '\x10d0'# -> unI64 7312- '\x10d1'# -> unI64 7313- '\x10d2'# -> unI64 7314- '\x10d3'# -> unI64 7315- '\x10d4'# -> unI64 7316- '\x10d5'# -> unI64 7317- '\x10d6'# -> unI64 7318- '\x10d7'# -> unI64 7319- '\x10d8'# -> unI64 7320- '\x10d9'# -> unI64 7321- '\x10da'# -> unI64 7322- '\x10db'# -> unI64 7323- '\x10dc'# -> unI64 7324- '\x10dd'# -> unI64 7325- '\x10de'# -> unI64 7326- '\x10df'# -> unI64 7327- '\x10e0'# -> unI64 7328- '\x10e1'# -> unI64 7329- '\x10e2'# -> unI64 7330- '\x10e3'# -> unI64 7331- '\x10e4'# -> unI64 7332- '\x10e5'# -> unI64 7333- '\x10e6'# -> unI64 7334- '\x10e7'# -> unI64 7335- '\x10e8'# -> unI64 7336- '\x10e9'# -> unI64 7337- '\x10ea'# -> unI64 7338- '\x10eb'# -> unI64 7339- '\x10ec'# -> unI64 7340- '\x10ed'# -> unI64 7341- '\x10ee'# -> unI64 7342- '\x10ef'# -> unI64 7343- '\x10f0'# -> unI64 7344- '\x10f1'# -> unI64 7345- '\x10f2'# -> unI64 7346- '\x10f3'# -> unI64 7347- '\x10f4'# -> unI64 7348- '\x10f5'# -> unI64 7349- '\x10f6'# -> unI64 7350- '\x10f7'# -> unI64 7351- '\x10f8'# -> unI64 7352- '\x10f9'# -> unI64 7353- '\x10fa'# -> unI64 7354- '\x10fd'# -> unI64 7357- '\x10fe'# -> unI64 7358- '\x10ff'# -> unI64 7359- '\x13f8'# -> unI64 5104- '\x13f9'# -> unI64 5105- '\x13fa'# -> unI64 5106- '\x13fb'# -> unI64 5107- '\x13fc'# -> unI64 5108- '\x13fd'# -> unI64 5109- '\x1c80'# -> unI64 1042- '\x1c81'# -> unI64 1044- '\x1c82'# -> unI64 1054- '\x1c83'# -> unI64 1057- '\x1c84'# -> unI64 1058- '\x1c85'# -> unI64 1058- '\x1c86'# -> unI64 1066- '\x1c87'# -> unI64 1122- '\x1c88'# -> unI64 42570- '\x1d79'# -> unI64 42877- '\x1d7d'# -> unI64 11363- '\x1d8e'# -> unI64 42950- '\x1e01'# -> unI64 7680- '\x1e03'# -> unI64 7682- '\x1e05'# -> unI64 7684- '\x1e07'# -> unI64 7686- '\x1e09'# -> unI64 7688- '\x1e0b'# -> unI64 7690- '\x1e0d'# -> unI64 7692- '\x1e0f'# -> unI64 7694- '\x1e11'# -> unI64 7696- '\x1e13'# -> unI64 7698- '\x1e15'# -> unI64 7700- '\x1e17'# -> unI64 7702- '\x1e19'# -> unI64 7704- '\x1e1b'# -> unI64 7706- '\x1e1d'# -> unI64 7708- '\x1e1f'# -> unI64 7710- '\x1e21'# -> unI64 7712- '\x1e23'# -> unI64 7714- '\x1e25'# -> unI64 7716- '\x1e27'# -> unI64 7718- '\x1e29'# -> unI64 7720- '\x1e2b'# -> unI64 7722- '\x1e2d'# -> unI64 7724- '\x1e2f'# -> unI64 7726- '\x1e31'# -> unI64 7728- '\x1e33'# -> unI64 7730- '\x1e35'# -> unI64 7732- '\x1e37'# -> unI64 7734- '\x1e39'# -> unI64 7736- '\x1e3b'# -> unI64 7738- '\x1e3d'# -> unI64 7740- '\x1e3f'# -> unI64 7742- '\x1e41'# -> unI64 7744- '\x1e43'# -> unI64 7746- '\x1e45'# -> unI64 7748- '\x1e47'# -> unI64 7750- '\x1e49'# -> unI64 7752- '\x1e4b'# -> unI64 7754- '\x1e4d'# -> unI64 7756- '\x1e4f'# -> unI64 7758- '\x1e51'# -> unI64 7760- '\x1e53'# -> unI64 7762- '\x1e55'# -> unI64 7764- '\x1e57'# -> unI64 7766- '\x1e59'# -> unI64 7768- '\x1e5b'# -> unI64 7770- '\x1e5d'# -> unI64 7772- '\x1e5f'# -> unI64 7774- '\x1e61'# -> unI64 7776- '\x1e63'# -> unI64 7778- '\x1e65'# -> unI64 7780- '\x1e67'# -> unI64 7782- '\x1e69'# -> unI64 7784- '\x1e6b'# -> unI64 7786- '\x1e6d'# -> unI64 7788- '\x1e6f'# -> unI64 7790- '\x1e71'# -> unI64 7792- '\x1e73'# -> unI64 7794- '\x1e75'# -> unI64 7796- '\x1e77'# -> unI64 7798- '\x1e79'# -> unI64 7800- '\x1e7b'# -> unI64 7802- '\x1e7d'# -> unI64 7804- '\x1e7f'# -> unI64 7806- '\x1e81'# -> unI64 7808- '\x1e83'# -> unI64 7810- '\x1e85'# -> unI64 7812- '\x1e87'# -> unI64 7814- '\x1e89'# -> unI64 7816- '\x1e8b'# -> unI64 7818- '\x1e8d'# -> unI64 7820- '\x1e8f'# -> unI64 7822- '\x1e91'# -> unI64 7824- '\x1e93'# -> unI64 7826- '\x1e95'# -> unI64 7828- '\x1e9b'# -> unI64 7776- '\x1ea1'# -> unI64 7840- '\x1ea3'# -> unI64 7842- '\x1ea5'# -> unI64 7844- '\x1ea7'# -> unI64 7846- '\x1ea9'# -> unI64 7848- '\x1eab'# -> unI64 7850- '\x1ead'# -> unI64 7852- '\x1eaf'# -> unI64 7854- '\x1eb1'# -> unI64 7856- '\x1eb3'# -> unI64 7858- '\x1eb5'# -> unI64 7860- '\x1eb7'# -> unI64 7862- '\x1eb9'# -> unI64 7864- '\x1ebb'# -> unI64 7866- '\x1ebd'# -> unI64 7868- '\x1ebf'# -> unI64 7870- '\x1ec1'# -> unI64 7872- '\x1ec3'# -> unI64 7874- '\x1ec5'# -> unI64 7876- '\x1ec7'# -> unI64 7878- '\x1ec9'# -> unI64 7880- '\x1ecb'# -> unI64 7882- '\x1ecd'# -> unI64 7884- '\x1ecf'# -> unI64 7886- '\x1ed1'# -> unI64 7888- '\x1ed3'# -> unI64 7890- '\x1ed5'# -> unI64 7892- '\x1ed7'# -> unI64 7894- '\x1ed9'# -> unI64 7896- '\x1edb'# -> unI64 7898- '\x1edd'# -> unI64 7900- '\x1edf'# -> unI64 7902- '\x1ee1'# -> unI64 7904- '\x1ee3'# -> unI64 7906- '\x1ee5'# -> unI64 7908- '\x1ee7'# -> unI64 7910- '\x1ee9'# -> unI64 7912- '\x1eeb'# -> unI64 7914- '\x1eed'# -> unI64 7916- '\x1eef'# -> unI64 7918- '\x1ef1'# -> unI64 7920- '\x1ef3'# -> unI64 7922- '\x1ef5'# -> unI64 7924- '\x1ef7'# -> unI64 7926- '\x1ef9'# -> unI64 7928- '\x1efb'# -> unI64 7930- '\x1efd'# -> unI64 7932- '\x1eff'# -> unI64 7934- '\x1f00'# -> unI64 7944- '\x1f01'# -> unI64 7945- '\x1f02'# -> unI64 7946- '\x1f03'# -> unI64 7947- '\x1f04'# -> unI64 7948- '\x1f05'# -> unI64 7949- '\x1f06'# -> unI64 7950- '\x1f07'# -> unI64 7951- '\x1f10'# -> unI64 7960- '\x1f11'# -> unI64 7961- '\x1f12'# -> unI64 7962- '\x1f13'# -> unI64 7963- '\x1f14'# -> unI64 7964- '\x1f15'# -> unI64 7965- '\x1f20'# -> unI64 7976- '\x1f21'# -> unI64 7977- '\x1f22'# -> unI64 7978- '\x1f23'# -> unI64 7979- '\x1f24'# -> unI64 7980- '\x1f25'# -> unI64 7981- '\x1f26'# -> unI64 7982- '\x1f27'# -> unI64 7983- '\x1f30'# -> unI64 7992- '\x1f31'# -> unI64 7993- '\x1f32'# -> unI64 7994- '\x1f33'# -> unI64 7995- '\x1f34'# -> unI64 7996- '\x1f35'# -> unI64 7997- '\x1f36'# -> unI64 7998- '\x1f37'# -> unI64 7999- '\x1f40'# -> unI64 8008- '\x1f41'# -> unI64 8009- '\x1f42'# -> unI64 8010- '\x1f43'# -> unI64 8011- '\x1f44'# -> unI64 8012- '\x1f45'# -> unI64 8013- '\x1f51'# -> unI64 8025- '\x1f53'# -> unI64 8027- '\x1f55'# -> unI64 8029- '\x1f57'# -> unI64 8031- '\x1f60'# -> unI64 8040- '\x1f61'# -> unI64 8041- '\x1f62'# -> unI64 8042- '\x1f63'# -> unI64 8043- '\x1f64'# -> unI64 8044- '\x1f65'# -> unI64 8045- '\x1f66'# -> unI64 8046- '\x1f67'# -> unI64 8047- '\x1f70'# -> unI64 8122- '\x1f71'# -> unI64 8123- '\x1f72'# -> unI64 8136- '\x1f73'# -> unI64 8137- '\x1f74'# -> unI64 8138- '\x1f75'# -> unI64 8139- '\x1f76'# -> unI64 8154- '\x1f77'# -> unI64 8155- '\x1f78'# -> unI64 8184- '\x1f79'# -> unI64 8185- '\x1f7a'# -> unI64 8170- '\x1f7b'# -> unI64 8171- '\x1f7c'# -> unI64 8186- '\x1f7d'# -> unI64 8187- '\x1fb0'# -> unI64 8120- '\x1fb1'# -> unI64 8121- '\x1fbe'# -> unI64 921- '\x1fd0'# -> unI64 8152- '\x1fd1'# -> unI64 8153- '\x1fe0'# -> unI64 8168- '\x1fe1'# -> unI64 8169- '\x1fe5'# -> unI64 8172- '\x214e'# -> unI64 8498- '\x2170'# -> unI64 8544- '\x2171'# -> unI64 8545- '\x2172'# -> unI64 8546- '\x2173'# -> unI64 8547- '\x2174'# -> unI64 8548- '\x2175'# -> unI64 8549- '\x2176'# -> unI64 8550- '\x2177'# -> unI64 8551- '\x2178'# -> unI64 8552- '\x2179'# -> unI64 8553- '\x217a'# -> unI64 8554- '\x217b'# -> unI64 8555- '\x217c'# -> unI64 8556- '\x217d'# -> unI64 8557- '\x217e'# -> unI64 8558- '\x217f'# -> unI64 8559- '\x2184'# -> unI64 8579- '\x24d0'# -> unI64 9398- '\x24d1'# -> unI64 9399- '\x24d2'# -> unI64 9400- '\x24d3'# -> unI64 9401- '\x24d4'# -> unI64 9402- '\x24d5'# -> unI64 9403- '\x24d6'# -> unI64 9404- '\x24d7'# -> unI64 9405- '\x24d8'# -> unI64 9406- '\x24d9'# -> unI64 9407- '\x24da'# -> unI64 9408- '\x24db'# -> unI64 9409- '\x24dc'# -> unI64 9410- '\x24dd'# -> unI64 9411- '\x24de'# -> unI64 9412- '\x24df'# -> unI64 9413- '\x24e0'# -> unI64 9414- '\x24e1'# -> unI64 9415- '\x24e2'# -> unI64 9416- '\x24e3'# -> unI64 9417- '\x24e4'# -> unI64 9418- '\x24e5'# -> unI64 9419- '\x24e6'# -> unI64 9420- '\x24e7'# -> unI64 9421- '\x24e8'# -> unI64 9422- '\x24e9'# -> unI64 9423- '\x2c30'# -> unI64 11264- '\x2c31'# -> unI64 11265- '\x2c32'# -> unI64 11266- '\x2c33'# -> unI64 11267- '\x2c34'# -> unI64 11268- '\x2c35'# -> unI64 11269- '\x2c36'# -> unI64 11270- '\x2c37'# -> unI64 11271- '\x2c38'# -> unI64 11272- '\x2c39'# -> unI64 11273- '\x2c3a'# -> unI64 11274- '\x2c3b'# -> unI64 11275- '\x2c3c'# -> unI64 11276- '\x2c3d'# -> unI64 11277- '\x2c3e'# -> unI64 11278- '\x2c3f'# -> unI64 11279- '\x2c40'# -> unI64 11280- '\x2c41'# -> unI64 11281- '\x2c42'# -> unI64 11282- '\x2c43'# -> unI64 11283- '\x2c44'# -> unI64 11284- '\x2c45'# -> unI64 11285- '\x2c46'# -> unI64 11286- '\x2c47'# -> unI64 11287- '\x2c48'# -> unI64 11288- '\x2c49'# -> unI64 11289- '\x2c4a'# -> unI64 11290- '\x2c4b'# -> unI64 11291- '\x2c4c'# -> unI64 11292- '\x2c4d'# -> unI64 11293- '\x2c4e'# -> unI64 11294- '\x2c4f'# -> unI64 11295- '\x2c50'# -> unI64 11296- '\x2c51'# -> unI64 11297- '\x2c52'# -> unI64 11298- '\x2c53'# -> unI64 11299- '\x2c54'# -> unI64 11300- '\x2c55'# -> unI64 11301- '\x2c56'# -> unI64 11302- '\x2c57'# -> unI64 11303- '\x2c58'# -> unI64 11304- '\x2c59'# -> unI64 11305- '\x2c5a'# -> unI64 11306- '\x2c5b'# -> unI64 11307- '\x2c5c'# -> unI64 11308- '\x2c5d'# -> unI64 11309- '\x2c5e'# -> unI64 11310- '\x2c5f'# -> unI64 11311- '\x2c61'# -> unI64 11360- '\x2c65'# -> unI64 570- '\x2c66'# -> unI64 574- '\x2c68'# -> unI64 11367- '\x2c6a'# -> unI64 11369- '\x2c6c'# -> unI64 11371- '\x2c73'# -> unI64 11378- '\x2c76'# -> unI64 11381- '\x2c81'# -> unI64 11392- '\x2c83'# -> unI64 11394- '\x2c85'# -> unI64 11396- '\x2c87'# -> unI64 11398- '\x2c89'# -> unI64 11400- '\x2c8b'# -> unI64 11402- '\x2c8d'# -> unI64 11404- '\x2c8f'# -> unI64 11406- '\x2c91'# -> unI64 11408- '\x2c93'# -> unI64 11410- '\x2c95'# -> unI64 11412- '\x2c97'# -> unI64 11414- '\x2c99'# -> unI64 11416- '\x2c9b'# -> unI64 11418- '\x2c9d'# -> unI64 11420- '\x2c9f'# -> unI64 11422- '\x2ca1'# -> unI64 11424- '\x2ca3'# -> unI64 11426- '\x2ca5'# -> unI64 11428- '\x2ca7'# -> unI64 11430- '\x2ca9'# -> unI64 11432- '\x2cab'# -> unI64 11434- '\x2cad'# -> unI64 11436- '\x2caf'# -> unI64 11438- '\x2cb1'# -> unI64 11440- '\x2cb3'# -> unI64 11442- '\x2cb5'# -> unI64 11444- '\x2cb7'# -> unI64 11446- '\x2cb9'# -> unI64 11448- '\x2cbb'# -> unI64 11450- '\x2cbd'# -> unI64 11452- '\x2cbf'# -> unI64 11454- '\x2cc1'# -> unI64 11456- '\x2cc3'# -> unI64 11458- '\x2cc5'# -> unI64 11460- '\x2cc7'# -> unI64 11462- '\x2cc9'# -> unI64 11464- '\x2ccb'# -> unI64 11466- '\x2ccd'# -> unI64 11468- '\x2ccf'# -> unI64 11470- '\x2cd1'# -> unI64 11472- '\x2cd3'# -> unI64 11474- '\x2cd5'# -> unI64 11476- '\x2cd7'# -> unI64 11478- '\x2cd9'# -> unI64 11480- '\x2cdb'# -> unI64 11482- '\x2cdd'# -> unI64 11484- '\x2cdf'# -> unI64 11486- '\x2ce1'# -> unI64 11488- '\x2ce3'# -> unI64 11490- '\x2cec'# -> unI64 11499- '\x2cee'# -> unI64 11501- '\x2cf3'# -> unI64 11506- '\x2d00'# -> unI64 4256- '\x2d01'# -> unI64 4257- '\x2d02'# -> unI64 4258- '\x2d03'# -> unI64 4259- '\x2d04'# -> unI64 4260- '\x2d05'# -> unI64 4261- '\x2d06'# -> unI64 4262- '\x2d07'# -> unI64 4263- '\x2d08'# -> unI64 4264- '\x2d09'# -> unI64 4265- '\x2d0a'# -> unI64 4266- '\x2d0b'# -> unI64 4267- '\x2d0c'# -> unI64 4268- '\x2d0d'# -> unI64 4269- '\x2d0e'# -> unI64 4270- '\x2d0f'# -> unI64 4271- '\x2d10'# -> unI64 4272- '\x2d11'# -> unI64 4273- '\x2d12'# -> unI64 4274- '\x2d13'# -> unI64 4275- '\x2d14'# -> unI64 4276- '\x2d15'# -> unI64 4277- '\x2d16'# -> unI64 4278- '\x2d17'# -> unI64 4279- '\x2d18'# -> unI64 4280- '\x2d19'# -> unI64 4281- '\x2d1a'# -> unI64 4282- '\x2d1b'# -> unI64 4283- '\x2d1c'# -> unI64 4284- '\x2d1d'# -> unI64 4285- '\x2d1e'# -> unI64 4286- '\x2d1f'# -> unI64 4287- '\x2d20'# -> unI64 4288- '\x2d21'# -> unI64 4289- '\x2d22'# -> unI64 4290- '\x2d23'# -> unI64 4291- '\x2d24'# -> unI64 4292- '\x2d25'# -> unI64 4293- '\x2d27'# -> unI64 4295- '\x2d2d'# -> unI64 4301- '\xa641'# -> unI64 42560- '\xa643'# -> unI64 42562- '\xa645'# -> unI64 42564- '\xa647'# -> unI64 42566- '\xa649'# -> unI64 42568- '\xa64b'# -> unI64 42570- '\xa64d'# -> unI64 42572- '\xa64f'# -> unI64 42574- '\xa651'# -> unI64 42576- '\xa653'# -> unI64 42578- '\xa655'# -> unI64 42580- '\xa657'# -> unI64 42582- '\xa659'# -> unI64 42584- '\xa65b'# -> unI64 42586- '\xa65d'# -> unI64 42588- '\xa65f'# -> unI64 42590- '\xa661'# -> unI64 42592- '\xa663'# -> unI64 42594- '\xa665'# -> unI64 42596- '\xa667'# -> unI64 42598- '\xa669'# -> unI64 42600- '\xa66b'# -> unI64 42602- '\xa66d'# -> unI64 42604- '\xa681'# -> unI64 42624- '\xa683'# -> unI64 42626- '\xa685'# -> unI64 42628- '\xa687'# -> unI64 42630- '\xa689'# -> unI64 42632- '\xa68b'# -> unI64 42634- '\xa68d'# -> unI64 42636- '\xa68f'# -> unI64 42638- '\xa691'# -> unI64 42640- '\xa693'# -> unI64 42642- '\xa695'# -> unI64 42644- '\xa697'# -> unI64 42646- '\xa699'# -> unI64 42648- '\xa69b'# -> unI64 42650- '\xa723'# -> unI64 42786- '\xa725'# -> unI64 42788- '\xa727'# -> unI64 42790- '\xa729'# -> unI64 42792- '\xa72b'# -> unI64 42794- '\xa72d'# -> unI64 42796- '\xa72f'# -> unI64 42798- '\xa733'# -> unI64 42802- '\xa735'# -> unI64 42804- '\xa737'# -> unI64 42806- '\xa739'# -> unI64 42808- '\xa73b'# -> unI64 42810- '\xa73d'# -> unI64 42812- '\xa73f'# -> unI64 42814- '\xa741'# -> unI64 42816- '\xa743'# -> unI64 42818- '\xa745'# -> unI64 42820- '\xa747'# -> unI64 42822- '\xa749'# -> unI64 42824- '\xa74b'# -> unI64 42826- '\xa74d'# -> unI64 42828- '\xa74f'# -> unI64 42830- '\xa751'# -> unI64 42832- '\xa753'# -> unI64 42834- '\xa755'# -> unI64 42836- '\xa757'# -> unI64 42838- '\xa759'# -> unI64 42840- '\xa75b'# -> unI64 42842- '\xa75d'# -> unI64 42844- '\xa75f'# -> unI64 42846- '\xa761'# -> unI64 42848- '\xa763'# -> unI64 42850- '\xa765'# -> unI64 42852- '\xa767'# -> unI64 42854- '\xa769'# -> unI64 42856- '\xa76b'# -> unI64 42858- '\xa76d'# -> unI64 42860- '\xa76f'# -> unI64 42862- '\xa77a'# -> unI64 42873- '\xa77c'# -> unI64 42875- '\xa77f'# -> unI64 42878- '\xa781'# -> unI64 42880- '\xa783'# -> unI64 42882- '\xa785'# -> unI64 42884- '\xa787'# -> unI64 42886- '\xa78c'# -> unI64 42891- '\xa791'# -> unI64 42896- '\xa793'# -> unI64 42898- '\xa794'# -> unI64 42948- '\xa797'# -> unI64 42902- '\xa799'# -> unI64 42904- '\xa79b'# -> unI64 42906- '\xa79d'# -> unI64 42908- '\xa79f'# -> unI64 42910- '\xa7a1'# -> unI64 42912- '\xa7a3'# -> unI64 42914- '\xa7a5'# -> unI64 42916- '\xa7a7'# -> unI64 42918- '\xa7a9'# -> unI64 42920- '\xa7b5'# -> unI64 42932- '\xa7b7'# -> unI64 42934- '\xa7b9'# -> unI64 42936- '\xa7bb'# -> unI64 42938- '\xa7bd'# -> unI64 42940- '\xa7bf'# -> unI64 42942- '\xa7c1'# -> unI64 42944- '\xa7c3'# -> unI64 42946- '\xa7c8'# -> unI64 42951- '\xa7ca'# -> unI64 42953- '\xa7d1'# -> unI64 42960- '\xa7d7'# -> unI64 42966- '\xa7d9'# -> unI64 42968- '\xa7f6'# -> unI64 42997- '\xab53'# -> unI64 42931- '\xab70'# -> unI64 5024- '\xab71'# -> unI64 5025- '\xab72'# -> unI64 5026- '\xab73'# -> unI64 5027- '\xab74'# -> unI64 5028- '\xab75'# -> unI64 5029- '\xab76'# -> unI64 5030- '\xab77'# -> unI64 5031- '\xab78'# -> unI64 5032- '\xab79'# -> unI64 5033- '\xab7a'# -> unI64 5034- '\xab7b'# -> unI64 5035- '\xab7c'# -> unI64 5036- '\xab7d'# -> unI64 5037- '\xab7e'# -> unI64 5038- '\xab7f'# -> unI64 5039- '\xab80'# -> unI64 5040- '\xab81'# -> unI64 5041- '\xab82'# -> unI64 5042- '\xab83'# -> unI64 5043- '\xab84'# -> unI64 5044- '\xab85'# -> unI64 5045- '\xab86'# -> unI64 5046- '\xab87'# -> unI64 5047- '\xab88'# -> unI64 5048- '\xab89'# -> unI64 5049- '\xab8a'# -> unI64 5050- '\xab8b'# -> unI64 5051- '\xab8c'# -> unI64 5052- '\xab8d'# -> unI64 5053- '\xab8e'# -> unI64 5054- '\xab8f'# -> unI64 5055- '\xab90'# -> unI64 5056- '\xab91'# -> unI64 5057- '\xab92'# -> unI64 5058- '\xab93'# -> unI64 5059- '\xab94'# -> unI64 5060- '\xab95'# -> unI64 5061- '\xab96'# -> unI64 5062- '\xab97'# -> unI64 5063- '\xab98'# -> unI64 5064- '\xab99'# -> unI64 5065- '\xab9a'# -> unI64 5066- '\xab9b'# -> unI64 5067- '\xab9c'# -> unI64 5068- '\xab9d'# -> unI64 5069- '\xab9e'# -> unI64 5070- '\xab9f'# -> unI64 5071- '\xaba0'# -> unI64 5072- '\xaba1'# -> unI64 5073- '\xaba2'# -> unI64 5074- '\xaba3'# -> unI64 5075- '\xaba4'# -> unI64 5076- '\xaba5'# -> unI64 5077- '\xaba6'# -> unI64 5078- '\xaba7'# -> unI64 5079- '\xaba8'# -> unI64 5080- '\xaba9'# -> unI64 5081- '\xabaa'# -> unI64 5082- '\xabab'# -> unI64 5083- '\xabac'# -> unI64 5084- '\xabad'# -> unI64 5085- '\xabae'# -> unI64 5086- '\xabaf'# -> unI64 5087- '\xabb0'# -> unI64 5088- '\xabb1'# -> unI64 5089- '\xabb2'# -> unI64 5090- '\xabb3'# -> unI64 5091- '\xabb4'# -> unI64 5092- '\xabb5'# -> unI64 5093- '\xabb6'# -> unI64 5094- '\xabb7'# -> unI64 5095- '\xabb8'# -> unI64 5096- '\xabb9'# -> unI64 5097- '\xabba'# -> unI64 5098- '\xabbb'# -> unI64 5099- '\xabbc'# -> unI64 5100- '\xabbd'# -> unI64 5101- '\xabbe'# -> unI64 5102- '\xabbf'# -> unI64 5103- '\xff41'# -> unI64 65313- '\xff42'# -> unI64 65314- '\xff43'# -> unI64 65315- '\xff44'# -> unI64 65316- '\xff45'# -> unI64 65317- '\xff46'# -> unI64 65318- '\xff47'# -> unI64 65319- '\xff48'# -> unI64 65320- '\xff49'# -> unI64 65321- '\xff4a'# -> unI64 65322- '\xff4b'# -> unI64 65323- '\xff4c'# -> unI64 65324- '\xff4d'# -> unI64 65325- '\xff4e'# -> unI64 65326- '\xff4f'# -> unI64 65327- '\xff50'# -> unI64 65328- '\xff51'# -> unI64 65329- '\xff52'# -> unI64 65330- '\xff53'# -> unI64 65331- '\xff54'# -> unI64 65332- '\xff55'# -> unI64 65333- '\xff56'# -> unI64 65334- '\xff57'# -> unI64 65335- '\xff58'# -> unI64 65336- '\xff59'# -> unI64 65337- '\xff5a'# -> unI64 65338- '\x10428'# -> unI64 66560- '\x10429'# -> unI64 66561- '\x1042a'# -> unI64 66562- '\x1042b'# -> unI64 66563- '\x1042c'# -> unI64 66564- '\x1042d'# -> unI64 66565- '\x1042e'# -> unI64 66566- '\x1042f'# -> unI64 66567- '\x10430'# -> unI64 66568- '\x10431'# -> unI64 66569- '\x10432'# -> unI64 66570- '\x10433'# -> unI64 66571- '\x10434'# -> unI64 66572- '\x10435'# -> unI64 66573- '\x10436'# -> unI64 66574- '\x10437'# -> unI64 66575- '\x10438'# -> unI64 66576- '\x10439'# -> unI64 66577- '\x1043a'# -> unI64 66578- '\x1043b'# -> unI64 66579- '\x1043c'# -> unI64 66580- '\x1043d'# -> unI64 66581- '\x1043e'# -> unI64 66582- '\x1043f'# -> unI64 66583- '\x10440'# -> unI64 66584- '\x10441'# -> unI64 66585- '\x10442'# -> unI64 66586- '\x10443'# -> unI64 66587- '\x10444'# -> unI64 66588- '\x10445'# -> unI64 66589- '\x10446'# -> unI64 66590- '\x10447'# -> unI64 66591- '\x10448'# -> unI64 66592- '\x10449'# -> unI64 66593- '\x1044a'# -> unI64 66594- '\x1044b'# -> unI64 66595- '\x1044c'# -> unI64 66596- '\x1044d'# -> unI64 66597- '\x1044e'# -> unI64 66598- '\x1044f'# -> unI64 66599- '\x104d8'# -> unI64 66736- '\x104d9'# -> unI64 66737- '\x104da'# -> unI64 66738- '\x104db'# -> unI64 66739- '\x104dc'# -> unI64 66740- '\x104dd'# -> unI64 66741- '\x104de'# -> unI64 66742- '\x104df'# -> unI64 66743- '\x104e0'# -> unI64 66744- '\x104e1'# -> unI64 66745- '\x104e2'# -> unI64 66746- '\x104e3'# -> unI64 66747- '\x104e4'# -> unI64 66748- '\x104e5'# -> unI64 66749- '\x104e6'# -> unI64 66750- '\x104e7'# -> unI64 66751- '\x104e8'# -> unI64 66752- '\x104e9'# -> unI64 66753- '\x104ea'# -> unI64 66754- '\x104eb'# -> unI64 66755- '\x104ec'# -> unI64 66756- '\x104ed'# -> unI64 66757- '\x104ee'# -> unI64 66758- '\x104ef'# -> unI64 66759- '\x104f0'# -> unI64 66760- '\x104f1'# -> unI64 66761- '\x104f2'# -> unI64 66762- '\x104f3'# -> unI64 66763- '\x104f4'# -> unI64 66764- '\x104f5'# -> unI64 66765- '\x104f6'# -> unI64 66766- '\x104f7'# -> unI64 66767- '\x104f8'# -> unI64 66768- '\x104f9'# -> unI64 66769- '\x104fa'# -> unI64 66770- '\x104fb'# -> unI64 66771- '\x10597'# -> unI64 66928- '\x10598'# -> unI64 66929- '\x10599'# -> unI64 66930- '\x1059a'# -> unI64 66931- '\x1059b'# -> unI64 66932- '\x1059c'# -> unI64 66933- '\x1059d'# -> unI64 66934- '\x1059e'# -> unI64 66935- '\x1059f'# -> unI64 66936- '\x105a0'# -> unI64 66937- '\x105a1'# -> unI64 66938- '\x105a3'# -> unI64 66940- '\x105a4'# -> unI64 66941- '\x105a5'# -> unI64 66942- '\x105a6'# -> unI64 66943- '\x105a7'# -> unI64 66944- '\x105a8'# -> unI64 66945- '\x105a9'# -> unI64 66946- '\x105aa'# -> unI64 66947- '\x105ab'# -> unI64 66948- '\x105ac'# -> unI64 66949- '\x105ad'# -> unI64 66950- '\x105ae'# -> unI64 66951- '\x105af'# -> unI64 66952- '\x105b0'# -> unI64 66953- '\x105b1'# -> unI64 66954- '\x105b3'# -> unI64 66956- '\x105b4'# -> unI64 66957- '\x105b5'# -> unI64 66958- '\x105b6'# -> unI64 66959- '\x105b7'# -> unI64 66960- '\x105b8'# -> unI64 66961- '\x105b9'# -> unI64 66962- '\x105bb'# -> unI64 66964- '\x105bc'# -> unI64 66965- '\x10cc0'# -> unI64 68736- '\x10cc1'# -> unI64 68737- '\x10cc2'# -> unI64 68738- '\x10cc3'# -> unI64 68739- '\x10cc4'# -> unI64 68740- '\x10cc5'# -> unI64 68741- '\x10cc6'# -> unI64 68742- '\x10cc7'# -> unI64 68743- '\x10cc8'# -> unI64 68744- '\x10cc9'# -> unI64 68745- '\x10cca'# -> unI64 68746- '\x10ccb'# -> unI64 68747- '\x10ccc'# -> unI64 68748- '\x10ccd'# -> unI64 68749- '\x10cce'# -> unI64 68750- '\x10ccf'# -> unI64 68751- '\x10cd0'# -> unI64 68752- '\x10cd1'# -> unI64 68753- '\x10cd2'# -> unI64 68754- '\x10cd3'# -> unI64 68755- '\x10cd4'# -> unI64 68756- '\x10cd5'# -> unI64 68757- '\x10cd6'# -> unI64 68758- '\x10cd7'# -> unI64 68759- '\x10cd8'# -> unI64 68760- '\x10cd9'# -> unI64 68761- '\x10cda'# -> unI64 68762- '\x10cdb'# -> unI64 68763- '\x10cdc'# -> unI64 68764- '\x10cdd'# -> unI64 68765- '\x10cde'# -> unI64 68766- '\x10cdf'# -> unI64 68767- '\x10ce0'# -> unI64 68768- '\x10ce1'# -> unI64 68769- '\x10ce2'# -> unI64 68770- '\x10ce3'# -> unI64 68771- '\x10ce4'# -> unI64 68772- '\x10ce5'# -> unI64 68773- '\x10ce6'# -> unI64 68774- '\x10ce7'# -> unI64 68775- '\x10ce8'# -> unI64 68776- '\x10ce9'# -> unI64 68777- '\x10cea'# -> unI64 68778- '\x10ceb'# -> unI64 68779- '\x10cec'# -> unI64 68780- '\x10ced'# -> unI64 68781- '\x10cee'# -> unI64 68782- '\x10cef'# -> unI64 68783- '\x10cf0'# -> unI64 68784- '\x10cf1'# -> unI64 68785- '\x10cf2'# -> unI64 68786- '\x118c0'# -> unI64 71840- '\x118c1'# -> unI64 71841- '\x118c2'# -> unI64 71842- '\x118c3'# -> unI64 71843- '\x118c4'# -> unI64 71844- '\x118c5'# -> unI64 71845- '\x118c6'# -> unI64 71846- '\x118c7'# -> unI64 71847- '\x118c8'# -> unI64 71848- '\x118c9'# -> unI64 71849- '\x118ca'# -> unI64 71850- '\x118cb'# -> unI64 71851- '\x118cc'# -> unI64 71852- '\x118cd'# -> unI64 71853- '\x118ce'# -> unI64 71854- '\x118cf'# -> unI64 71855- '\x118d0'# -> unI64 71856- '\x118d1'# -> unI64 71857- '\x118d2'# -> unI64 71858- '\x118d3'# -> unI64 71859- '\x118d4'# -> unI64 71860- '\x118d5'# -> unI64 71861- '\x118d6'# -> unI64 71862- '\x118d7'# -> unI64 71863- '\x118d8'# -> unI64 71864- '\x118d9'# -> unI64 71865- '\x118da'# -> unI64 71866- '\x118db'# -> unI64 71867- '\x118dc'# -> unI64 71868- '\x118dd'# -> unI64 71869- '\x118de'# -> unI64 71870- '\x118df'# -> unI64 71871- '\x16e60'# -> unI64 93760- '\x16e61'# -> unI64 93761- '\x16e62'# -> unI64 93762- '\x16e63'# -> unI64 93763- '\x16e64'# -> unI64 93764- '\x16e65'# -> unI64 93765- '\x16e66'# -> unI64 93766- '\x16e67'# -> unI64 93767- '\x16e68'# -> unI64 93768- '\x16e69'# -> unI64 93769- '\x16e6a'# -> unI64 93770- '\x16e6b'# -> unI64 93771- '\x16e6c'# -> unI64 93772- '\x16e6d'# -> unI64 93773- '\x16e6e'# -> unI64 93774- '\x16e6f'# -> unI64 93775- '\x16e70'# -> unI64 93776- '\x16e71'# -> unI64 93777- '\x16e72'# -> unI64 93778- '\x16e73'# -> unI64 93779- '\x16e74'# -> unI64 93780- '\x16e75'# -> unI64 93781- '\x16e76'# -> unI64 93782- '\x16e77'# -> unI64 93783- '\x16e78'# -> unI64 93784- '\x16e79'# -> unI64 93785- '\x16e7a'# -> unI64 93786- '\x16e7b'# -> unI64 93787- '\x16e7c'# -> unI64 93788- '\x16e7d'# -> unI64 93789- '\x16e7e'# -> unI64 93790- '\x16e7f'# -> unI64 93791- '\x1e922'# -> unI64 125184- '\x1e923'# -> unI64 125185- '\x1e924'# -> unI64 125186- '\x1e925'# -> unI64 125187- '\x1e926'# -> unI64 125188- '\x1e927'# -> unI64 125189- '\x1e928'# -> unI64 125190- '\x1e929'# -> unI64 125191- '\x1e92a'# -> unI64 125192- '\x1e92b'# -> unI64 125193- '\x1e92c'# -> unI64 125194- '\x1e92d'# -> unI64 125195- '\x1e92e'# -> unI64 125196- '\x1e92f'# -> unI64 125197- '\x1e930'# -> unI64 125198- '\x1e931'# -> unI64 125199- '\x1e932'# -> unI64 125200- '\x1e933'# -> unI64 125201- '\x1e934'# -> unI64 125202- '\x1e935'# -> unI64 125203- '\x1e936'# -> unI64 125204- '\x1e937'# -> unI64 125205- '\x1e938'# -> unI64 125206- '\x1e939'# -> unI64 125207- '\x1e93a'# -> unI64 125208- '\x1e93b'# -> unI64 125209- '\x1e93c'# -> unI64 125210- '\x1e93d'# -> unI64 125211- '\x1e93e'# -> unI64 125212- '\x1e93f'# -> unI64 125213- '\x1e940'# -> unI64 125214- '\x1e941'# -> unI64 125215- '\x1e942'# -> unI64 125216- '\x1e943'# -> unI64 125217- _ -> unI64 0-lowerMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE lowerMapping #-}-lowerMapping = \case- -- LATIN CAPITAL LETTER I WITH DOT ABOVE- '\x0130'# -> unI64 1625292905- '\x0041'# -> unI64 97- '\x0042'# -> unI64 98- '\x0043'# -> unI64 99- '\x0044'# -> unI64 100- '\x0045'# -> unI64 101- '\x0046'# -> unI64 102- '\x0047'# -> unI64 103- '\x0048'# -> unI64 104- '\x0049'# -> unI64 105- '\x004a'# -> unI64 106- '\x004b'# -> unI64 107- '\x004c'# -> unI64 108- '\x004d'# -> unI64 109- '\x004e'# -> unI64 110- '\x004f'# -> unI64 111- '\x0050'# -> unI64 112- '\x0051'# -> unI64 113- '\x0052'# -> unI64 114- '\x0053'# -> unI64 115- '\x0054'# -> unI64 116- '\x0055'# -> unI64 117- '\x0056'# -> unI64 118- '\x0057'# -> unI64 119- '\x0058'# -> unI64 120- '\x0059'# -> unI64 121- '\x005a'# -> unI64 122- '\x00c0'# -> unI64 224- '\x00c1'# -> unI64 225- '\x00c2'# -> unI64 226- '\x00c3'# -> unI64 227- '\x00c4'# -> unI64 228- '\x00c5'# -> unI64 229- '\x00c6'# -> unI64 230- '\x00c7'# -> unI64 231- '\x00c8'# -> unI64 232- '\x00c9'# -> unI64 233- '\x00ca'# -> unI64 234- '\x00cb'# -> unI64 235- '\x00cc'# -> unI64 236- '\x00cd'# -> unI64 237- '\x00ce'# -> unI64 238- '\x00cf'# -> unI64 239- '\x00d0'# -> unI64 240- '\x00d1'# -> unI64 241- '\x00d2'# -> unI64 242- '\x00d3'# -> unI64 243- '\x00d4'# -> unI64 244- '\x00d5'# -> unI64 245- '\x00d6'# -> unI64 246- '\x00d8'# -> unI64 248- '\x00d9'# -> unI64 249- '\x00da'# -> unI64 250- '\x00db'# -> unI64 251- '\x00dc'# -> unI64 252- '\x00dd'# -> unI64 253- '\x00de'# -> unI64 254- '\x0100'# -> unI64 257- '\x0102'# -> unI64 259- '\x0104'# -> unI64 261- '\x0106'# -> unI64 263- '\x0108'# -> unI64 265- '\x010a'# -> unI64 267- '\x010c'# -> unI64 269- '\x010e'# -> unI64 271- '\x0110'# -> unI64 273- '\x0112'# -> unI64 275- '\x0114'# -> unI64 277- '\x0116'# -> unI64 279- '\x0118'# -> unI64 281- '\x011a'# -> unI64 283- '\x011c'# -> unI64 285- '\x011e'# -> unI64 287- '\x0120'# -> unI64 289- '\x0122'# -> unI64 291- '\x0124'# -> unI64 293- '\x0126'# -> unI64 295- '\x0128'# -> unI64 297- '\x012a'# -> unI64 299- '\x012c'# -> unI64 301- '\x012e'# -> unI64 303- '\x0132'# -> unI64 307- '\x0134'# -> unI64 309- '\x0136'# -> unI64 311- '\x0139'# -> unI64 314- '\x013b'# -> unI64 316- '\x013d'# -> unI64 318- '\x013f'# -> unI64 320- '\x0141'# -> unI64 322- '\x0143'# -> unI64 324- '\x0145'# -> unI64 326- '\x0147'# -> unI64 328- '\x014a'# -> unI64 331- '\x014c'# -> unI64 333- '\x014e'# -> unI64 335- '\x0150'# -> unI64 337- '\x0152'# -> unI64 339- '\x0154'# -> unI64 341- '\x0156'# -> unI64 343- '\x0158'# -> unI64 345- '\x015a'# -> unI64 347- '\x015c'# -> unI64 349- '\x015e'# -> unI64 351- '\x0160'# -> unI64 353- '\x0162'# -> unI64 355- '\x0164'# -> unI64 357- '\x0166'# -> unI64 359- '\x0168'# -> unI64 361- '\x016a'# -> unI64 363- '\x016c'# -> unI64 365- '\x016e'# -> unI64 367- '\x0170'# -> unI64 369- '\x0172'# -> unI64 371- '\x0174'# -> unI64 373- '\x0176'# -> unI64 375- '\x0178'# -> unI64 255- '\x0179'# -> unI64 378- '\x017b'# -> unI64 380- '\x017d'# -> unI64 382- '\x0181'# -> unI64 595- '\x0182'# -> unI64 387- '\x0184'# -> unI64 389- '\x0186'# -> unI64 596- '\x0187'# -> unI64 392- '\x0189'# -> unI64 598- '\x018a'# -> unI64 599- '\x018b'# -> unI64 396- '\x018e'# -> unI64 477- '\x018f'# -> unI64 601- '\x0190'# -> unI64 603- '\x0191'# -> unI64 402- '\x0193'# -> unI64 608- '\x0194'# -> unI64 611- '\x0196'# -> unI64 617- '\x0197'# -> unI64 616- '\x0198'# -> unI64 409- '\x019c'# -> unI64 623- '\x019d'# -> unI64 626- '\x019f'# -> unI64 629- '\x01a0'# -> unI64 417- '\x01a2'# -> unI64 419- '\x01a4'# -> unI64 421- '\x01a6'# -> unI64 640- '\x01a7'# -> unI64 424- '\x01a9'# -> unI64 643- '\x01ac'# -> unI64 429- '\x01ae'# -> unI64 648- '\x01af'# -> unI64 432- '\x01b1'# -> unI64 650- '\x01b2'# -> unI64 651- '\x01b3'# -> unI64 436- '\x01b5'# -> unI64 438- '\x01b7'# -> unI64 658- '\x01b8'# -> unI64 441- '\x01bc'# -> unI64 445- '\x01c4'# -> unI64 454- '\x01c5'# -> unI64 454- '\x01c7'# -> unI64 457- '\x01c8'# -> unI64 457- '\x01ca'# -> unI64 460- '\x01cb'# -> unI64 460- '\x01cd'# -> unI64 462- '\x01cf'# -> unI64 464- '\x01d1'# -> unI64 466- '\x01d3'# -> unI64 468- '\x01d5'# -> unI64 470- '\x01d7'# -> unI64 472- '\x01d9'# -> unI64 474- '\x01db'# -> unI64 476- '\x01de'# -> unI64 479- '\x01e0'# -> unI64 481- '\x01e2'# -> unI64 483- '\x01e4'# -> unI64 485- '\x01e6'# -> unI64 487- '\x01e8'# -> unI64 489- '\x01ea'# -> unI64 491- '\x01ec'# -> unI64 493- '\x01ee'# -> unI64 495- '\x01f1'# -> unI64 499- '\x01f2'# -> unI64 499- '\x01f4'# -> unI64 501- '\x01f6'# -> unI64 405- '\x01f7'# -> unI64 447- '\x01f8'# -> unI64 505- '\x01fa'# -> unI64 507- '\x01fc'# -> unI64 509- '\x01fe'# -> unI64 511- '\x0200'# -> unI64 513- '\x0202'# -> unI64 515- '\x0204'# -> unI64 517- '\x0206'# -> unI64 519- '\x0208'# -> unI64 521- '\x020a'# -> unI64 523- '\x020c'# -> unI64 525- '\x020e'# -> unI64 527- '\x0210'# -> unI64 529- '\x0212'# -> unI64 531- '\x0214'# -> unI64 533- '\x0216'# -> unI64 535- '\x0218'# -> unI64 537- '\x021a'# -> unI64 539- '\x021c'# -> unI64 541- '\x021e'# -> unI64 543- '\x0220'# -> unI64 414- '\x0222'# -> unI64 547- '\x0224'# -> unI64 549- '\x0226'# -> unI64 551- '\x0228'# -> unI64 553- '\x022a'# -> unI64 555- '\x022c'# -> unI64 557- '\x022e'# -> unI64 559- '\x0230'# -> unI64 561- '\x0232'# -> unI64 563- '\x023a'# -> unI64 11365- '\x023b'# -> unI64 572- '\x023d'# -> unI64 410- '\x023e'# -> unI64 11366- '\x0241'# -> unI64 578- '\x0243'# -> unI64 384- '\x0244'# -> unI64 649- '\x0245'# -> unI64 652- '\x0246'# -> unI64 583- '\x0248'# -> unI64 585- '\x024a'# -> unI64 587- '\x024c'# -> unI64 589- '\x024e'# -> unI64 591- '\x0370'# -> unI64 881- '\x0372'# -> unI64 883- '\x0376'# -> unI64 887- '\x037f'# -> unI64 1011- '\x0386'# -> unI64 940- '\x0388'# -> unI64 941- '\x0389'# -> unI64 942- '\x038a'# -> unI64 943- '\x038c'# -> unI64 972- '\x038e'# -> unI64 973- '\x038f'# -> unI64 974- '\x0391'# -> unI64 945- '\x0392'# -> unI64 946- '\x0393'# -> unI64 947- '\x0394'# -> unI64 948- '\x0395'# -> unI64 949- '\x0396'# -> unI64 950- '\x0397'# -> unI64 951- '\x0398'# -> unI64 952- '\x0399'# -> unI64 953- '\x039a'# -> unI64 954- '\x039b'# -> unI64 955- '\x039c'# -> unI64 956- '\x039d'# -> unI64 957- '\x039e'# -> unI64 958- '\x039f'# -> unI64 959- '\x03a0'# -> unI64 960- '\x03a1'# -> unI64 961- '\x03a3'# -> unI64 963- '\x03a4'# -> unI64 964- '\x03a5'# -> unI64 965- '\x03a6'# -> unI64 966- '\x03a7'# -> unI64 967- '\x03a8'# -> unI64 968- '\x03a9'# -> unI64 969- '\x03aa'# -> unI64 970- '\x03ab'# -> unI64 971- '\x03cf'# -> unI64 983- '\x03d8'# -> unI64 985- '\x03da'# -> unI64 987- '\x03dc'# -> unI64 989- '\x03de'# -> unI64 991- '\x03e0'# -> unI64 993- '\x03e2'# -> unI64 995- '\x03e4'# -> unI64 997- '\x03e6'# -> unI64 999- '\x03e8'# -> unI64 1001- '\x03ea'# -> unI64 1003- '\x03ec'# -> unI64 1005- '\x03ee'# -> unI64 1007- '\x03f4'# -> unI64 952- '\x03f7'# -> unI64 1016- '\x03f9'# -> unI64 1010- '\x03fa'# -> unI64 1019- '\x03fd'# -> unI64 891- '\x03fe'# -> unI64 892- '\x03ff'# -> unI64 893- '\x0400'# -> unI64 1104- '\x0401'# -> unI64 1105- '\x0402'# -> unI64 1106- '\x0403'# -> unI64 1107- '\x0404'# -> unI64 1108- '\x0405'# -> unI64 1109- '\x0406'# -> unI64 1110- '\x0407'# -> unI64 1111- '\x0408'# -> unI64 1112- '\x0409'# -> unI64 1113- '\x040a'# -> unI64 1114- '\x040b'# -> unI64 1115- '\x040c'# -> unI64 1116- '\x040d'# -> unI64 1117- '\x040e'# -> unI64 1118- '\x040f'# -> unI64 1119- '\x0410'# -> unI64 1072- '\x0411'# -> unI64 1073- '\x0412'# -> unI64 1074- '\x0413'# -> unI64 1075- '\x0414'# -> unI64 1076- '\x0415'# -> unI64 1077- '\x0416'# -> unI64 1078- '\x0417'# -> unI64 1079- '\x0418'# -> unI64 1080- '\x0419'# -> unI64 1081- '\x041a'# -> unI64 1082- '\x041b'# -> unI64 1083- '\x041c'# -> unI64 1084- '\x041d'# -> unI64 1085- '\x041e'# -> unI64 1086- '\x041f'# -> unI64 1087- '\x0420'# -> unI64 1088- '\x0421'# -> unI64 1089- '\x0422'# -> unI64 1090- '\x0423'# -> unI64 1091- '\x0424'# -> unI64 1092- '\x0425'# -> unI64 1093- '\x0426'# -> unI64 1094- '\x0427'# -> unI64 1095- '\x0428'# -> unI64 1096- '\x0429'# -> unI64 1097- '\x042a'# -> unI64 1098- '\x042b'# -> unI64 1099- '\x042c'# -> unI64 1100- '\x042d'# -> unI64 1101- '\x042e'# -> unI64 1102- '\x042f'# -> unI64 1103- '\x0460'# -> unI64 1121- '\x0462'# -> unI64 1123- '\x0464'# -> unI64 1125- '\x0466'# -> unI64 1127- '\x0468'# -> unI64 1129- '\x046a'# -> unI64 1131- '\x046c'# -> unI64 1133- '\x046e'# -> unI64 1135- '\x0470'# -> unI64 1137- '\x0472'# -> unI64 1139- '\x0474'# -> unI64 1141- '\x0476'# -> unI64 1143- '\x0478'# -> unI64 1145- '\x047a'# -> unI64 1147- '\x047c'# -> unI64 1149- '\x047e'# -> unI64 1151- '\x0480'# -> unI64 1153- '\x048a'# -> unI64 1163- '\x048c'# -> unI64 1165- '\x048e'# -> unI64 1167- '\x0490'# -> unI64 1169- '\x0492'# -> unI64 1171- '\x0494'# -> unI64 1173- '\x0496'# -> unI64 1175- '\x0498'# -> unI64 1177- '\x049a'# -> unI64 1179- '\x049c'# -> unI64 1181- '\x049e'# -> unI64 1183- '\x04a0'# -> unI64 1185- '\x04a2'# -> unI64 1187- '\x04a4'# -> unI64 1189- '\x04a6'# -> unI64 1191- '\x04a8'# -> unI64 1193- '\x04aa'# -> unI64 1195- '\x04ac'# -> unI64 1197- '\x04ae'# -> unI64 1199- '\x04b0'# -> unI64 1201- '\x04b2'# -> unI64 1203- '\x04b4'# -> unI64 1205- '\x04b6'# -> unI64 1207- '\x04b8'# -> unI64 1209- '\x04ba'# -> unI64 1211- '\x04bc'# -> unI64 1213- '\x04be'# -> unI64 1215- '\x04c0'# -> unI64 1231- '\x04c1'# -> unI64 1218- '\x04c3'# -> unI64 1220- '\x04c5'# -> unI64 1222- '\x04c7'# -> unI64 1224- '\x04c9'# -> unI64 1226- '\x04cb'# -> unI64 1228- '\x04cd'# -> unI64 1230- '\x04d0'# -> unI64 1233- '\x04d2'# -> unI64 1235- '\x04d4'# -> unI64 1237- '\x04d6'# -> unI64 1239- '\x04d8'# -> unI64 1241- '\x04da'# -> unI64 1243- '\x04dc'# -> unI64 1245- '\x04de'# -> unI64 1247- '\x04e0'# -> unI64 1249- '\x04e2'# -> unI64 1251- '\x04e4'# -> unI64 1253- '\x04e6'# -> unI64 1255- '\x04e8'# -> unI64 1257- '\x04ea'# -> unI64 1259- '\x04ec'# -> unI64 1261- '\x04ee'# -> unI64 1263- '\x04f0'# -> unI64 1265- '\x04f2'# -> unI64 1267- '\x04f4'# -> unI64 1269- '\x04f6'# -> unI64 1271- '\x04f8'# -> unI64 1273- '\x04fa'# -> unI64 1275- '\x04fc'# -> unI64 1277- '\x04fe'# -> unI64 1279- '\x0500'# -> unI64 1281- '\x0502'# -> unI64 1283- '\x0504'# -> unI64 1285- '\x0506'# -> unI64 1287- '\x0508'# -> unI64 1289- '\x050a'# -> unI64 1291- '\x050c'# -> unI64 1293- '\x050e'# -> unI64 1295- '\x0510'# -> unI64 1297- '\x0512'# -> unI64 1299- '\x0514'# -> unI64 1301- '\x0516'# -> unI64 1303- '\x0518'# -> unI64 1305- '\x051a'# -> unI64 1307- '\x051c'# -> unI64 1309- '\x051e'# -> unI64 1311- '\x0520'# -> unI64 1313- '\x0522'# -> unI64 1315- '\x0524'# -> unI64 1317- '\x0526'# -> unI64 1319- '\x0528'# -> unI64 1321- '\x052a'# -> unI64 1323- '\x052c'# -> unI64 1325- '\x052e'# -> unI64 1327- '\x0531'# -> unI64 1377- '\x0532'# -> unI64 1378- '\x0533'# -> unI64 1379- '\x0534'# -> unI64 1380- '\x0535'# -> unI64 1381- '\x0536'# -> unI64 1382- '\x0537'# -> unI64 1383- '\x0538'# -> unI64 1384- '\x0539'# -> unI64 1385- '\x053a'# -> unI64 1386- '\x053b'# -> unI64 1387- '\x053c'# -> unI64 1388- '\x053d'# -> unI64 1389- '\x053e'# -> unI64 1390- '\x053f'# -> unI64 1391- '\x0540'# -> unI64 1392- '\x0541'# -> unI64 1393- '\x0542'# -> unI64 1394- '\x0543'# -> unI64 1395- '\x0544'# -> unI64 1396- '\x0545'# -> unI64 1397- '\x0546'# -> unI64 1398- '\x0547'# -> unI64 1399- '\x0548'# -> unI64 1400- '\x0549'# -> unI64 1401- '\x054a'# -> unI64 1402- '\x054b'# -> unI64 1403- '\x054c'# -> unI64 1404- '\x054d'# -> unI64 1405- '\x054e'# -> unI64 1406- '\x054f'# -> unI64 1407- '\x0550'# -> unI64 1408- '\x0551'# -> unI64 1409- '\x0552'# -> unI64 1410- '\x0553'# -> unI64 1411- '\x0554'# -> unI64 1412- '\x0555'# -> unI64 1413- '\x0556'# -> unI64 1414- '\x10a0'# -> unI64 11520- '\x10a1'# -> unI64 11521- '\x10a2'# -> unI64 11522- '\x10a3'# -> unI64 11523- '\x10a4'# -> unI64 11524- '\x10a5'# -> unI64 11525- '\x10a6'# -> unI64 11526- '\x10a7'# -> unI64 11527- '\x10a8'# -> unI64 11528- '\x10a9'# -> unI64 11529- '\x10aa'# -> unI64 11530- '\x10ab'# -> unI64 11531- '\x10ac'# -> unI64 11532- '\x10ad'# -> unI64 11533- '\x10ae'# -> unI64 11534- '\x10af'# -> unI64 11535- '\x10b0'# -> unI64 11536- '\x10b1'# -> unI64 11537- '\x10b2'# -> unI64 11538- '\x10b3'# -> unI64 11539- '\x10b4'# -> unI64 11540- '\x10b5'# -> unI64 11541- '\x10b6'# -> unI64 11542- '\x10b7'# -> unI64 11543- '\x10b8'# -> unI64 11544- '\x10b9'# -> unI64 11545- '\x10ba'# -> unI64 11546- '\x10bb'# -> unI64 11547- '\x10bc'# -> unI64 11548- '\x10bd'# -> unI64 11549- '\x10be'# -> unI64 11550- '\x10bf'# -> unI64 11551- '\x10c0'# -> unI64 11552- '\x10c1'# -> unI64 11553- '\x10c2'# -> unI64 11554- '\x10c3'# -> unI64 11555- '\x10c4'# -> unI64 11556- '\x10c5'# -> unI64 11557- '\x10c7'# -> unI64 11559- '\x10cd'# -> unI64 11565- '\x13a0'# -> unI64 43888- '\x13a1'# -> unI64 43889- '\x13a2'# -> unI64 43890- '\x13a3'# -> unI64 43891- '\x13a4'# -> unI64 43892- '\x13a5'# -> unI64 43893- '\x13a6'# -> unI64 43894- '\x13a7'# -> unI64 43895- '\x13a8'# -> unI64 43896- '\x13a9'# -> unI64 43897- '\x13aa'# -> unI64 43898- '\x13ab'# -> unI64 43899- '\x13ac'# -> unI64 43900- '\x13ad'# -> unI64 43901- '\x13ae'# -> unI64 43902- '\x13af'# -> unI64 43903- '\x13b0'# -> unI64 43904- '\x13b1'# -> unI64 43905- '\x13b2'# -> unI64 43906- '\x13b3'# -> unI64 43907- '\x13b4'# -> unI64 43908- '\x13b5'# -> unI64 43909- '\x13b6'# -> unI64 43910- '\x13b7'# -> unI64 43911- '\x13b8'# -> unI64 43912- '\x13b9'# -> unI64 43913- '\x13ba'# -> unI64 43914- '\x13bb'# -> unI64 43915- '\x13bc'# -> unI64 43916- '\x13bd'# -> unI64 43917- '\x13be'# -> unI64 43918- '\x13bf'# -> unI64 43919- '\x13c0'# -> unI64 43920- '\x13c1'# -> unI64 43921- '\x13c2'# -> unI64 43922- '\x13c3'# -> unI64 43923- '\x13c4'# -> unI64 43924- '\x13c5'# -> unI64 43925- '\x13c6'# -> unI64 43926- '\x13c7'# -> unI64 43927- '\x13c8'# -> unI64 43928- '\x13c9'# -> unI64 43929- '\x13ca'# -> unI64 43930- '\x13cb'# -> unI64 43931- '\x13cc'# -> unI64 43932- '\x13cd'# -> unI64 43933- '\x13ce'# -> unI64 43934- '\x13cf'# -> unI64 43935- '\x13d0'# -> unI64 43936- '\x13d1'# -> unI64 43937- '\x13d2'# -> unI64 43938- '\x13d3'# -> unI64 43939- '\x13d4'# -> unI64 43940- '\x13d5'# -> unI64 43941- '\x13d6'# -> unI64 43942- '\x13d7'# -> unI64 43943- '\x13d8'# -> unI64 43944- '\x13d9'# -> unI64 43945- '\x13da'# -> unI64 43946- '\x13db'# -> unI64 43947- '\x13dc'# -> unI64 43948- '\x13dd'# -> unI64 43949- '\x13de'# -> unI64 43950- '\x13df'# -> unI64 43951- '\x13e0'# -> unI64 43952- '\x13e1'# -> unI64 43953- '\x13e2'# -> unI64 43954- '\x13e3'# -> unI64 43955- '\x13e4'# -> unI64 43956- '\x13e5'# -> unI64 43957- '\x13e6'# -> unI64 43958- '\x13e7'# -> unI64 43959- '\x13e8'# -> unI64 43960- '\x13e9'# -> unI64 43961- '\x13ea'# -> unI64 43962- '\x13eb'# -> unI64 43963- '\x13ec'# -> unI64 43964- '\x13ed'# -> unI64 43965- '\x13ee'# -> unI64 43966- '\x13ef'# -> unI64 43967- '\x13f0'# -> unI64 5112- '\x13f1'# -> unI64 5113- '\x13f2'# -> unI64 5114- '\x13f3'# -> unI64 5115- '\x13f4'# -> unI64 5116- '\x13f5'# -> unI64 5117- '\x1c90'# -> unI64 4304- '\x1c91'# -> unI64 4305- '\x1c92'# -> unI64 4306- '\x1c93'# -> unI64 4307- '\x1c94'# -> unI64 4308- '\x1c95'# -> unI64 4309- '\x1c96'# -> unI64 4310- '\x1c97'# -> unI64 4311- '\x1c98'# -> unI64 4312- '\x1c99'# -> unI64 4313- '\x1c9a'# -> unI64 4314- '\x1c9b'# -> unI64 4315- '\x1c9c'# -> unI64 4316- '\x1c9d'# -> unI64 4317- '\x1c9e'# -> unI64 4318- '\x1c9f'# -> unI64 4319- '\x1ca0'# -> unI64 4320- '\x1ca1'# -> unI64 4321- '\x1ca2'# -> unI64 4322- '\x1ca3'# -> unI64 4323- '\x1ca4'# -> unI64 4324- '\x1ca5'# -> unI64 4325- '\x1ca6'# -> unI64 4326- '\x1ca7'# -> unI64 4327- '\x1ca8'# -> unI64 4328- '\x1ca9'# -> unI64 4329- '\x1caa'# -> unI64 4330- '\x1cab'# -> unI64 4331- '\x1cac'# -> unI64 4332- '\x1cad'# -> unI64 4333- '\x1cae'# -> unI64 4334- '\x1caf'# -> unI64 4335- '\x1cb0'# -> unI64 4336- '\x1cb1'# -> unI64 4337- '\x1cb2'# -> unI64 4338- '\x1cb3'# -> unI64 4339- '\x1cb4'# -> unI64 4340- '\x1cb5'# -> unI64 4341- '\x1cb6'# -> unI64 4342- '\x1cb7'# -> unI64 4343- '\x1cb8'# -> unI64 4344- '\x1cb9'# -> unI64 4345- '\x1cba'# -> unI64 4346- '\x1cbd'# -> unI64 4349- '\x1cbe'# -> unI64 4350- '\x1cbf'# -> unI64 4351- '\x1e00'# -> unI64 7681- '\x1e02'# -> unI64 7683- '\x1e04'# -> unI64 7685- '\x1e06'# -> unI64 7687- '\x1e08'# -> unI64 7689- '\x1e0a'# -> unI64 7691- '\x1e0c'# -> unI64 7693- '\x1e0e'# -> unI64 7695- '\x1e10'# -> unI64 7697- '\x1e12'# -> unI64 7699- '\x1e14'# -> unI64 7701- '\x1e16'# -> unI64 7703- '\x1e18'# -> unI64 7705- '\x1e1a'# -> unI64 7707- '\x1e1c'# -> unI64 7709- '\x1e1e'# -> unI64 7711- '\x1e20'# -> unI64 7713- '\x1e22'# -> unI64 7715- '\x1e24'# -> unI64 7717- '\x1e26'# -> unI64 7719- '\x1e28'# -> unI64 7721- '\x1e2a'# -> unI64 7723- '\x1e2c'# -> unI64 7725- '\x1e2e'# -> unI64 7727- '\x1e30'# -> unI64 7729- '\x1e32'# -> unI64 7731- '\x1e34'# -> unI64 7733- '\x1e36'# -> unI64 7735- '\x1e38'# -> unI64 7737- '\x1e3a'# -> unI64 7739- '\x1e3c'# -> unI64 7741- '\x1e3e'# -> unI64 7743- '\x1e40'# -> unI64 7745- '\x1e42'# -> unI64 7747- '\x1e44'# -> unI64 7749- '\x1e46'# -> unI64 7751- '\x1e48'# -> unI64 7753- '\x1e4a'# -> unI64 7755- '\x1e4c'# -> unI64 7757- '\x1e4e'# -> unI64 7759- '\x1e50'# -> unI64 7761- '\x1e52'# -> unI64 7763- '\x1e54'# -> unI64 7765- '\x1e56'# -> unI64 7767- '\x1e58'# -> unI64 7769- '\x1e5a'# -> unI64 7771- '\x1e5c'# -> unI64 7773- '\x1e5e'# -> unI64 7775- '\x1e60'# -> unI64 7777- '\x1e62'# -> unI64 7779- '\x1e64'# -> unI64 7781- '\x1e66'# -> unI64 7783- '\x1e68'# -> unI64 7785- '\x1e6a'# -> unI64 7787- '\x1e6c'# -> unI64 7789- '\x1e6e'# -> unI64 7791- '\x1e70'# -> unI64 7793- '\x1e72'# -> unI64 7795- '\x1e74'# -> unI64 7797- '\x1e76'# -> unI64 7799- '\x1e78'# -> unI64 7801- '\x1e7a'# -> unI64 7803- '\x1e7c'# -> unI64 7805- '\x1e7e'# -> unI64 7807- '\x1e80'# -> unI64 7809- '\x1e82'# -> unI64 7811- '\x1e84'# -> unI64 7813- '\x1e86'# -> unI64 7815- '\x1e88'# -> unI64 7817- '\x1e8a'# -> unI64 7819- '\x1e8c'# -> unI64 7821- '\x1e8e'# -> unI64 7823- '\x1e90'# -> unI64 7825- '\x1e92'# -> unI64 7827- '\x1e94'# -> unI64 7829- '\x1e9e'# -> unI64 223- '\x1ea0'# -> unI64 7841- '\x1ea2'# -> unI64 7843- '\x1ea4'# -> unI64 7845- '\x1ea6'# -> unI64 7847- '\x1ea8'# -> unI64 7849- '\x1eaa'# -> unI64 7851- '\x1eac'# -> unI64 7853- '\x1eae'# -> unI64 7855- '\x1eb0'# -> unI64 7857- '\x1eb2'# -> unI64 7859- '\x1eb4'# -> unI64 7861- '\x1eb6'# -> unI64 7863- '\x1eb8'# -> unI64 7865- '\x1eba'# -> unI64 7867- '\x1ebc'# -> unI64 7869- '\x1ebe'# -> unI64 7871- '\x1ec0'# -> unI64 7873- '\x1ec2'# -> unI64 7875- '\x1ec4'# -> unI64 7877- '\x1ec6'# -> unI64 7879- '\x1ec8'# -> unI64 7881- '\x1eca'# -> unI64 7883- '\x1ecc'# -> unI64 7885- '\x1ece'# -> unI64 7887- '\x1ed0'# -> unI64 7889- '\x1ed2'# -> unI64 7891- '\x1ed4'# -> unI64 7893- '\x1ed6'# -> unI64 7895- '\x1ed8'# -> unI64 7897- '\x1eda'# -> unI64 7899- '\x1edc'# -> unI64 7901- '\x1ede'# -> unI64 7903- '\x1ee0'# -> unI64 7905- '\x1ee2'# -> unI64 7907- '\x1ee4'# -> unI64 7909- '\x1ee6'# -> unI64 7911- '\x1ee8'# -> unI64 7913- '\x1eea'# -> unI64 7915- '\x1eec'# -> unI64 7917- '\x1eee'# -> unI64 7919- '\x1ef0'# -> unI64 7921- '\x1ef2'# -> unI64 7923- '\x1ef4'# -> unI64 7925- '\x1ef6'# -> unI64 7927- '\x1ef8'# -> unI64 7929- '\x1efa'# -> unI64 7931- '\x1efc'# -> unI64 7933- '\x1efe'# -> unI64 7935- '\x1f08'# -> unI64 7936- '\x1f09'# -> unI64 7937- '\x1f0a'# -> unI64 7938- '\x1f0b'# -> unI64 7939- '\x1f0c'# -> unI64 7940- '\x1f0d'# -> unI64 7941- '\x1f0e'# -> unI64 7942- '\x1f0f'# -> unI64 7943- '\x1f18'# -> unI64 7952- '\x1f19'# -> unI64 7953- '\x1f1a'# -> unI64 7954- '\x1f1b'# -> unI64 7955- '\x1f1c'# -> unI64 7956- '\x1f1d'# -> unI64 7957- '\x1f28'# -> unI64 7968- '\x1f29'# -> unI64 7969- '\x1f2a'# -> unI64 7970- '\x1f2b'# -> unI64 7971- '\x1f2c'# -> unI64 7972- '\x1f2d'# -> unI64 7973- '\x1f2e'# -> unI64 7974- '\x1f2f'# -> unI64 7975- '\x1f38'# -> unI64 7984- '\x1f39'# -> unI64 7985- '\x1f3a'# -> unI64 7986- '\x1f3b'# -> unI64 7987- '\x1f3c'# -> unI64 7988- '\x1f3d'# -> unI64 7989- '\x1f3e'# -> unI64 7990- '\x1f3f'# -> unI64 7991- '\x1f48'# -> unI64 8000- '\x1f49'# -> unI64 8001- '\x1f4a'# -> unI64 8002- '\x1f4b'# -> unI64 8003- '\x1f4c'# -> unI64 8004- '\x1f4d'# -> unI64 8005- '\x1f59'# -> unI64 8017- '\x1f5b'# -> unI64 8019- '\x1f5d'# -> unI64 8021- '\x1f5f'# -> unI64 8023- '\x1f68'# -> unI64 8032- '\x1f69'# -> unI64 8033- '\x1f6a'# -> unI64 8034- '\x1f6b'# -> unI64 8035- '\x1f6c'# -> unI64 8036- '\x1f6d'# -> unI64 8037- '\x1f6e'# -> unI64 8038- '\x1f6f'# -> unI64 8039- '\x1f88'# -> unI64 8064- '\x1f89'# -> unI64 8065- '\x1f8a'# -> unI64 8066- '\x1f8b'# -> unI64 8067- '\x1f8c'# -> unI64 8068- '\x1f8d'# -> unI64 8069- '\x1f8e'# -> unI64 8070- '\x1f8f'# -> unI64 8071- '\x1f98'# -> unI64 8080- '\x1f99'# -> unI64 8081- '\x1f9a'# -> unI64 8082- '\x1f9b'# -> unI64 8083- '\x1f9c'# -> unI64 8084- '\x1f9d'# -> unI64 8085- '\x1f9e'# -> unI64 8086- '\x1f9f'# -> unI64 8087- '\x1fa8'# -> unI64 8096- '\x1fa9'# -> unI64 8097- '\x1faa'# -> unI64 8098- '\x1fab'# -> unI64 8099- '\x1fac'# -> unI64 8100- '\x1fad'# -> unI64 8101- '\x1fae'# -> unI64 8102- '\x1faf'# -> unI64 8103- '\x1fb8'# -> unI64 8112- '\x1fb9'# -> unI64 8113- '\x1fba'# -> unI64 8048- '\x1fbb'# -> unI64 8049- '\x1fbc'# -> unI64 8115- '\x1fc8'# -> unI64 8050- '\x1fc9'# -> unI64 8051- '\x1fca'# -> unI64 8052- '\x1fcb'# -> unI64 8053- '\x1fcc'# -> unI64 8131- '\x1fd8'# -> unI64 8144- '\x1fd9'# -> unI64 8145- '\x1fda'# -> unI64 8054- '\x1fdb'# -> unI64 8055- '\x1fe8'# -> unI64 8160- '\x1fe9'# -> unI64 8161- '\x1fea'# -> unI64 8058- '\x1feb'# -> unI64 8059- '\x1fec'# -> unI64 8165- '\x1ff8'# -> unI64 8056- '\x1ff9'# -> unI64 8057- '\x1ffa'# -> unI64 8060- '\x1ffb'# -> unI64 8061- '\x1ffc'# -> unI64 8179- '\x2126'# -> unI64 969- '\x212a'# -> unI64 107- '\x212b'# -> unI64 229- '\x2132'# -> unI64 8526- '\x2160'# -> unI64 8560- '\x2161'# -> unI64 8561- '\x2162'# -> unI64 8562- '\x2163'# -> unI64 8563- '\x2164'# -> unI64 8564- '\x2165'# -> unI64 8565- '\x2166'# -> unI64 8566- '\x2167'# -> unI64 8567- '\x2168'# -> unI64 8568- '\x2169'# -> unI64 8569- '\x216a'# -> unI64 8570- '\x216b'# -> unI64 8571- '\x216c'# -> unI64 8572- '\x216d'# -> unI64 8573- '\x216e'# -> unI64 8574- '\x216f'# -> unI64 8575- '\x2183'# -> unI64 8580- '\x24b6'# -> unI64 9424- '\x24b7'# -> unI64 9425- '\x24b8'# -> unI64 9426- '\x24b9'# -> unI64 9427- '\x24ba'# -> unI64 9428- '\x24bb'# -> unI64 9429- '\x24bc'# -> unI64 9430- '\x24bd'# -> unI64 9431- '\x24be'# -> unI64 9432- '\x24bf'# -> unI64 9433- '\x24c0'# -> unI64 9434- '\x24c1'# -> unI64 9435- '\x24c2'# -> unI64 9436- '\x24c3'# -> unI64 9437- '\x24c4'# -> unI64 9438- '\x24c5'# -> unI64 9439- '\x24c6'# -> unI64 9440- '\x24c7'# -> unI64 9441- '\x24c8'# -> unI64 9442- '\x24c9'# -> unI64 9443- '\x24ca'# -> unI64 9444- '\x24cb'# -> unI64 9445- '\x24cc'# -> unI64 9446- '\x24cd'# -> unI64 9447- '\x24ce'# -> unI64 9448- '\x24cf'# -> unI64 9449- '\x2c00'# -> unI64 11312- '\x2c01'# -> unI64 11313- '\x2c02'# -> unI64 11314- '\x2c03'# -> unI64 11315- '\x2c04'# -> unI64 11316- '\x2c05'# -> unI64 11317- '\x2c06'# -> unI64 11318- '\x2c07'# -> unI64 11319- '\x2c08'# -> unI64 11320- '\x2c09'# -> unI64 11321- '\x2c0a'# -> unI64 11322- '\x2c0b'# -> unI64 11323- '\x2c0c'# -> unI64 11324- '\x2c0d'# -> unI64 11325- '\x2c0e'# -> unI64 11326- '\x2c0f'# -> unI64 11327- '\x2c10'# -> unI64 11328- '\x2c11'# -> unI64 11329- '\x2c12'# -> unI64 11330- '\x2c13'# -> unI64 11331- '\x2c14'# -> unI64 11332- '\x2c15'# -> unI64 11333- '\x2c16'# -> unI64 11334- '\x2c17'# -> unI64 11335- '\x2c18'# -> unI64 11336- '\x2c19'# -> unI64 11337- '\x2c1a'# -> unI64 11338- '\x2c1b'# -> unI64 11339- '\x2c1c'# -> unI64 11340- '\x2c1d'# -> unI64 11341- '\x2c1e'# -> unI64 11342- '\x2c1f'# -> unI64 11343- '\x2c20'# -> unI64 11344- '\x2c21'# -> unI64 11345- '\x2c22'# -> unI64 11346- '\x2c23'# -> unI64 11347- '\x2c24'# -> unI64 11348- '\x2c25'# -> unI64 11349- '\x2c26'# -> unI64 11350- '\x2c27'# -> unI64 11351- '\x2c28'# -> unI64 11352- '\x2c29'# -> unI64 11353- '\x2c2a'# -> unI64 11354- '\x2c2b'# -> unI64 11355- '\x2c2c'# -> unI64 11356- '\x2c2d'# -> unI64 11357- '\x2c2e'# -> unI64 11358- '\x2c2f'# -> unI64 11359- '\x2c60'# -> unI64 11361- '\x2c62'# -> unI64 619- '\x2c63'# -> unI64 7549- '\x2c64'# -> unI64 637- '\x2c67'# -> unI64 11368- '\x2c69'# -> unI64 11370- '\x2c6b'# -> unI64 11372- '\x2c6d'# -> unI64 593- '\x2c6e'# -> unI64 625- '\x2c6f'# -> unI64 592- '\x2c70'# -> unI64 594- '\x2c72'# -> unI64 11379- '\x2c75'# -> unI64 11382- '\x2c7e'# -> unI64 575- '\x2c7f'# -> unI64 576- '\x2c80'# -> unI64 11393- '\x2c82'# -> unI64 11395- '\x2c84'# -> unI64 11397- '\x2c86'# -> unI64 11399- '\x2c88'# -> unI64 11401- '\x2c8a'# -> unI64 11403- '\x2c8c'# -> unI64 11405- '\x2c8e'# -> unI64 11407- '\x2c90'# -> unI64 11409- '\x2c92'# -> unI64 11411- '\x2c94'# -> unI64 11413- '\x2c96'# -> unI64 11415- '\x2c98'# -> unI64 11417- '\x2c9a'# -> unI64 11419- '\x2c9c'# -> unI64 11421- '\x2c9e'# -> unI64 11423- '\x2ca0'# -> unI64 11425- '\x2ca2'# -> unI64 11427- '\x2ca4'# -> unI64 11429- '\x2ca6'# -> unI64 11431- '\x2ca8'# -> unI64 11433- '\x2caa'# -> unI64 11435- '\x2cac'# -> unI64 11437- '\x2cae'# -> unI64 11439- '\x2cb0'# -> unI64 11441- '\x2cb2'# -> unI64 11443- '\x2cb4'# -> unI64 11445- '\x2cb6'# -> unI64 11447- '\x2cb8'# -> unI64 11449- '\x2cba'# -> unI64 11451- '\x2cbc'# -> unI64 11453- '\x2cbe'# -> unI64 11455- '\x2cc0'# -> unI64 11457- '\x2cc2'# -> unI64 11459- '\x2cc4'# -> unI64 11461- '\x2cc6'# -> unI64 11463- '\x2cc8'# -> unI64 11465- '\x2cca'# -> unI64 11467- '\x2ccc'# -> unI64 11469- '\x2cce'# -> unI64 11471- '\x2cd0'# -> unI64 11473- '\x2cd2'# -> unI64 11475- '\x2cd4'# -> unI64 11477- '\x2cd6'# -> unI64 11479- '\x2cd8'# -> unI64 11481- '\x2cda'# -> unI64 11483- '\x2cdc'# -> unI64 11485- '\x2cde'# -> unI64 11487- '\x2ce0'# -> unI64 11489- '\x2ce2'# -> unI64 11491- '\x2ceb'# -> unI64 11500- '\x2ced'# -> unI64 11502- '\x2cf2'# -> unI64 11507- '\xa640'# -> unI64 42561- '\xa642'# -> unI64 42563- '\xa644'# -> unI64 42565- '\xa646'# -> unI64 42567- '\xa648'# -> unI64 42569- '\xa64a'# -> unI64 42571- '\xa64c'# -> unI64 42573- '\xa64e'# -> unI64 42575- '\xa650'# -> unI64 42577- '\xa652'# -> unI64 42579- '\xa654'# -> unI64 42581- '\xa656'# -> unI64 42583- '\xa658'# -> unI64 42585- '\xa65a'# -> unI64 42587- '\xa65c'# -> unI64 42589- '\xa65e'# -> unI64 42591- '\xa660'# -> unI64 42593- '\xa662'# -> unI64 42595- '\xa664'# -> unI64 42597- '\xa666'# -> unI64 42599- '\xa668'# -> unI64 42601- '\xa66a'# -> unI64 42603- '\xa66c'# -> unI64 42605- '\xa680'# -> unI64 42625- '\xa682'# -> unI64 42627- '\xa684'# -> unI64 42629- '\xa686'# -> unI64 42631- '\xa688'# -> unI64 42633- '\xa68a'# -> unI64 42635- '\xa68c'# -> unI64 42637- '\xa68e'# -> unI64 42639- '\xa690'# -> unI64 42641- '\xa692'# -> unI64 42643- '\xa694'# -> unI64 42645- '\xa696'# -> unI64 42647- '\xa698'# -> unI64 42649- '\xa69a'# -> unI64 42651- '\xa722'# -> unI64 42787- '\xa724'# -> unI64 42789- '\xa726'# -> unI64 42791- '\xa728'# -> unI64 42793- '\xa72a'# -> unI64 42795- '\xa72c'# -> unI64 42797- '\xa72e'# -> unI64 42799- '\xa732'# -> unI64 42803- '\xa734'# -> unI64 42805- '\xa736'# -> unI64 42807- '\xa738'# -> unI64 42809- '\xa73a'# -> unI64 42811- '\xa73c'# -> unI64 42813- '\xa73e'# -> unI64 42815- '\xa740'# -> unI64 42817- '\xa742'# -> unI64 42819- '\xa744'# -> unI64 42821- '\xa746'# -> unI64 42823- '\xa748'# -> unI64 42825- '\xa74a'# -> unI64 42827- '\xa74c'# -> unI64 42829- '\xa74e'# -> unI64 42831- '\xa750'# -> unI64 42833- '\xa752'# -> unI64 42835- '\xa754'# -> unI64 42837- '\xa756'# -> unI64 42839- '\xa758'# -> unI64 42841- '\xa75a'# -> unI64 42843- '\xa75c'# -> unI64 42845- '\xa75e'# -> unI64 42847- '\xa760'# -> unI64 42849- '\xa762'# -> unI64 42851- '\xa764'# -> unI64 42853- '\xa766'# -> unI64 42855- '\xa768'# -> unI64 42857- '\xa76a'# -> unI64 42859- '\xa76c'# -> unI64 42861- '\xa76e'# -> unI64 42863- '\xa779'# -> unI64 42874- '\xa77b'# -> unI64 42876- '\xa77d'# -> unI64 7545- '\xa77e'# -> unI64 42879- '\xa780'# -> unI64 42881- '\xa782'# -> unI64 42883- '\xa784'# -> unI64 42885- '\xa786'# -> unI64 42887- '\xa78b'# -> unI64 42892- '\xa78d'# -> unI64 613- '\xa790'# -> unI64 42897- '\xa792'# -> unI64 42899- '\xa796'# -> unI64 42903- '\xa798'# -> unI64 42905- '\xa79a'# -> unI64 42907- '\xa79c'# -> unI64 42909- '\xa79e'# -> unI64 42911- '\xa7a0'# -> unI64 42913- '\xa7a2'# -> unI64 42915- '\xa7a4'# -> unI64 42917- '\xa7a6'# -> unI64 42919- '\xa7a8'# -> unI64 42921- '\xa7aa'# -> unI64 614- '\xa7ab'# -> unI64 604- '\xa7ac'# -> unI64 609- '\xa7ad'# -> unI64 620- '\xa7ae'# -> unI64 618- '\xa7b0'# -> unI64 670- '\xa7b1'# -> unI64 647- '\xa7b2'# -> unI64 669- '\xa7b3'# -> unI64 43859- '\xa7b4'# -> unI64 42933- '\xa7b6'# -> unI64 42935- '\xa7b8'# -> unI64 42937- '\xa7ba'# -> unI64 42939- '\xa7bc'# -> unI64 42941- '\xa7be'# -> unI64 42943- '\xa7c0'# -> unI64 42945- '\xa7c2'# -> unI64 42947- '\xa7c4'# -> unI64 42900- '\xa7c5'# -> unI64 642- '\xa7c6'# -> unI64 7566- '\xa7c7'# -> unI64 42952- '\xa7c9'# -> unI64 42954- '\xa7d0'# -> unI64 42961- '\xa7d6'# -> unI64 42967- '\xa7d8'# -> unI64 42969- '\xa7f5'# -> unI64 42998- '\xff21'# -> unI64 65345- '\xff22'# -> unI64 65346- '\xff23'# -> unI64 65347- '\xff24'# -> unI64 65348- '\xff25'# -> unI64 65349- '\xff26'# -> unI64 65350- '\xff27'# -> unI64 65351- '\xff28'# -> unI64 65352- '\xff29'# -> unI64 65353- '\xff2a'# -> unI64 65354- '\xff2b'# -> unI64 65355- '\xff2c'# -> unI64 65356- '\xff2d'# -> unI64 65357- '\xff2e'# -> unI64 65358- '\xff2f'# -> unI64 65359- '\xff30'# -> unI64 65360- '\xff31'# -> unI64 65361- '\xff32'# -> unI64 65362- '\xff33'# -> unI64 65363- '\xff34'# -> unI64 65364- '\xff35'# -> unI64 65365- '\xff36'# -> unI64 65366- '\xff37'# -> unI64 65367- '\xff38'# -> unI64 65368- '\xff39'# -> unI64 65369- '\xff3a'# -> unI64 65370- '\x10400'# -> unI64 66600- '\x10401'# -> unI64 66601- '\x10402'# -> unI64 66602- '\x10403'# -> unI64 66603- '\x10404'# -> unI64 66604- '\x10405'# -> unI64 66605- '\x10406'# -> unI64 66606- '\x10407'# -> unI64 66607- '\x10408'# -> unI64 66608- '\x10409'# -> unI64 66609- '\x1040a'# -> unI64 66610- '\x1040b'# -> unI64 66611- '\x1040c'# -> unI64 66612- '\x1040d'# -> unI64 66613- '\x1040e'# -> unI64 66614- '\x1040f'# -> unI64 66615- '\x10410'# -> unI64 66616- '\x10411'# -> unI64 66617- '\x10412'# -> unI64 66618- '\x10413'# -> unI64 66619- '\x10414'# -> unI64 66620- '\x10415'# -> unI64 66621- '\x10416'# -> unI64 66622- '\x10417'# -> unI64 66623- '\x10418'# -> unI64 66624- '\x10419'# -> unI64 66625- '\x1041a'# -> unI64 66626- '\x1041b'# -> unI64 66627- '\x1041c'# -> unI64 66628- '\x1041d'# -> unI64 66629- '\x1041e'# -> unI64 66630- '\x1041f'# -> unI64 66631- '\x10420'# -> unI64 66632- '\x10421'# -> unI64 66633- '\x10422'# -> unI64 66634- '\x10423'# -> unI64 66635- '\x10424'# -> unI64 66636- '\x10425'# -> unI64 66637- '\x10426'# -> unI64 66638- '\x10427'# -> unI64 66639- '\x104b0'# -> unI64 66776- '\x104b1'# -> unI64 66777- '\x104b2'# -> unI64 66778- '\x104b3'# -> unI64 66779- '\x104b4'# -> unI64 66780- '\x104b5'# -> unI64 66781- '\x104b6'# -> unI64 66782- '\x104b7'# -> unI64 66783- '\x104b8'# -> unI64 66784- '\x104b9'# -> unI64 66785- '\x104ba'# -> unI64 66786- '\x104bb'# -> unI64 66787- '\x104bc'# -> unI64 66788- '\x104bd'# -> unI64 66789- '\x104be'# -> unI64 66790- '\x104bf'# -> unI64 66791- '\x104c0'# -> unI64 66792- '\x104c1'# -> unI64 66793- '\x104c2'# -> unI64 66794- '\x104c3'# -> unI64 66795- '\x104c4'# -> unI64 66796- '\x104c5'# -> unI64 66797- '\x104c6'# -> unI64 66798- '\x104c7'# -> unI64 66799- '\x104c8'# -> unI64 66800- '\x104c9'# -> unI64 66801- '\x104ca'# -> unI64 66802- '\x104cb'# -> unI64 66803- '\x104cc'# -> unI64 66804- '\x104cd'# -> unI64 66805- '\x104ce'# -> unI64 66806- '\x104cf'# -> unI64 66807- '\x104d0'# -> unI64 66808- '\x104d1'# -> unI64 66809- '\x104d2'# -> unI64 66810- '\x104d3'# -> unI64 66811- '\x10570'# -> unI64 66967- '\x10571'# -> unI64 66968- '\x10572'# -> unI64 66969- '\x10573'# -> unI64 66970- '\x10574'# -> unI64 66971- '\x10575'# -> unI64 66972- '\x10576'# -> unI64 66973- '\x10577'# -> unI64 66974- '\x10578'# -> unI64 66975- '\x10579'# -> unI64 66976- '\x1057a'# -> unI64 66977- '\x1057c'# -> unI64 66979- '\x1057d'# -> unI64 66980- '\x1057e'# -> unI64 66981- '\x1057f'# -> unI64 66982- '\x10580'# -> unI64 66983- '\x10581'# -> unI64 66984- '\x10582'# -> unI64 66985- '\x10583'# -> unI64 66986- '\x10584'# -> unI64 66987- '\x10585'# -> unI64 66988- '\x10586'# -> unI64 66989- '\x10587'# -> unI64 66990- '\x10588'# -> unI64 66991- '\x10589'# -> unI64 66992- '\x1058a'# -> unI64 66993- '\x1058c'# -> unI64 66995- '\x1058d'# -> unI64 66996- '\x1058e'# -> unI64 66997- '\x1058f'# -> unI64 66998- '\x10590'# -> unI64 66999- '\x10591'# -> unI64 67000- '\x10592'# -> unI64 67001- '\x10594'# -> unI64 67003- '\x10595'# -> unI64 67004- '\x10c80'# -> unI64 68800- '\x10c81'# -> unI64 68801- '\x10c82'# -> unI64 68802- '\x10c83'# -> unI64 68803- '\x10c84'# -> unI64 68804- '\x10c85'# -> unI64 68805- '\x10c86'# -> unI64 68806- '\x10c87'# -> unI64 68807- '\x10c88'# -> unI64 68808- '\x10c89'# -> unI64 68809- '\x10c8a'# -> unI64 68810- '\x10c8b'# -> unI64 68811- '\x10c8c'# -> unI64 68812- '\x10c8d'# -> unI64 68813- '\x10c8e'# -> unI64 68814- '\x10c8f'# -> unI64 68815- '\x10c90'# -> unI64 68816- '\x10c91'# -> unI64 68817- '\x10c92'# -> unI64 68818- '\x10c93'# -> unI64 68819- '\x10c94'# -> unI64 68820- '\x10c95'# -> unI64 68821- '\x10c96'# -> unI64 68822- '\x10c97'# -> unI64 68823- '\x10c98'# -> unI64 68824- '\x10c99'# -> unI64 68825- '\x10c9a'# -> unI64 68826- '\x10c9b'# -> unI64 68827- '\x10c9c'# -> unI64 68828- '\x10c9d'# -> unI64 68829- '\x10c9e'# -> unI64 68830- '\x10c9f'# -> unI64 68831- '\x10ca0'# -> unI64 68832- '\x10ca1'# -> unI64 68833- '\x10ca2'# -> unI64 68834- '\x10ca3'# -> unI64 68835- '\x10ca4'# -> unI64 68836- '\x10ca5'# -> unI64 68837- '\x10ca6'# -> unI64 68838- '\x10ca7'# -> unI64 68839- '\x10ca8'# -> unI64 68840- '\x10ca9'# -> unI64 68841- '\x10caa'# -> unI64 68842- '\x10cab'# -> unI64 68843- '\x10cac'# -> unI64 68844- '\x10cad'# -> unI64 68845- '\x10cae'# -> unI64 68846- '\x10caf'# -> unI64 68847- '\x10cb0'# -> unI64 68848- '\x10cb1'# -> unI64 68849- '\x10cb2'# -> unI64 68850- '\x118a0'# -> unI64 71872- '\x118a1'# -> unI64 71873- '\x118a2'# -> unI64 71874- '\x118a3'# -> unI64 71875- '\x118a4'# -> unI64 71876- '\x118a5'# -> unI64 71877- '\x118a6'# -> unI64 71878- '\x118a7'# -> unI64 71879- '\x118a8'# -> unI64 71880- '\x118a9'# -> unI64 71881- '\x118aa'# -> unI64 71882- '\x118ab'# -> unI64 71883- '\x118ac'# -> unI64 71884- '\x118ad'# -> unI64 71885- '\x118ae'# -> unI64 71886- '\x118af'# -> unI64 71887- '\x118b0'# -> unI64 71888- '\x118b1'# -> unI64 71889- '\x118b2'# -> unI64 71890- '\x118b3'# -> unI64 71891- '\x118b4'# -> unI64 71892- '\x118b5'# -> unI64 71893- '\x118b6'# -> unI64 71894- '\x118b7'# -> unI64 71895- '\x118b8'# -> unI64 71896- '\x118b9'# -> unI64 71897- '\x118ba'# -> unI64 71898- '\x118bb'# -> unI64 71899- '\x118bc'# -> unI64 71900- '\x118bd'# -> unI64 71901- '\x118be'# -> unI64 71902- '\x118bf'# -> unI64 71903- '\x16e40'# -> unI64 93792- '\x16e41'# -> unI64 93793- '\x16e42'# -> unI64 93794- '\x16e43'# -> unI64 93795- '\x16e44'# -> unI64 93796- '\x16e45'# -> unI64 93797- '\x16e46'# -> unI64 93798- '\x16e47'# -> unI64 93799- '\x16e48'# -> unI64 93800- '\x16e49'# -> unI64 93801- '\x16e4a'# -> unI64 93802- '\x16e4b'# -> unI64 93803- '\x16e4c'# -> unI64 93804- '\x16e4d'# -> unI64 93805- '\x16e4e'# -> unI64 93806- '\x16e4f'# -> unI64 93807- '\x16e50'# -> unI64 93808- '\x16e51'# -> unI64 93809- '\x16e52'# -> unI64 93810- '\x16e53'# -> unI64 93811- '\x16e54'# -> unI64 93812- '\x16e55'# -> unI64 93813- '\x16e56'# -> unI64 93814- '\x16e57'# -> unI64 93815- '\x16e58'# -> unI64 93816- '\x16e59'# -> unI64 93817- '\x16e5a'# -> unI64 93818- '\x16e5b'# -> unI64 93819- '\x16e5c'# -> unI64 93820- '\x16e5d'# -> unI64 93821- '\x16e5e'# -> unI64 93822- '\x16e5f'# -> unI64 93823- '\x1e900'# -> unI64 125218- '\x1e901'# -> unI64 125219- '\x1e902'# -> unI64 125220- '\x1e903'# -> unI64 125221- '\x1e904'# -> unI64 125222- '\x1e905'# -> unI64 125223- '\x1e906'# -> unI64 125224- '\x1e907'# -> unI64 125225- '\x1e908'# -> unI64 125226- '\x1e909'# -> unI64 125227- '\x1e90a'# -> unI64 125228- '\x1e90b'# -> unI64 125229- '\x1e90c'# -> unI64 125230- '\x1e90d'# -> unI64 125231- '\x1e90e'# -> unI64 125232- '\x1e90f'# -> unI64 125233- '\x1e910'# -> unI64 125234- '\x1e911'# -> unI64 125235- '\x1e912'# -> unI64 125236- '\x1e913'# -> unI64 125237- '\x1e914'# -> unI64 125238- '\x1e915'# -> unI64 125239- '\x1e916'# -> unI64 125240- '\x1e917'# -> unI64 125241- '\x1e918'# -> unI64 125242- '\x1e919'# -> unI64 125243- '\x1e91a'# -> unI64 125244- '\x1e91b'# -> unI64 125245- '\x1e91c'# -> unI64 125246- '\x1e91d'# -> unI64 125247- '\x1e91e'# -> unI64 125248- '\x1e91f'# -> unI64 125249- '\x1e920'# -> unI64 125250- '\x1e921'# -> unI64 125251- _ -> unI64 0-titleMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE titleMapping #-}-titleMapping = \case- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 241172563- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 213909574- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 220201030- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 226492486- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 461795097575494- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 474989237108806- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 243269715- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 243269715- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2956985653- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2931819844- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2896168260- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2908751172- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2931819854- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2912945476- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 163578556- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429849- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429861- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778634- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373256- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390036- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584343- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584345- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200769- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459557- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987429- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498533- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720293- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025681- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025687- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918745- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429849- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025689- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651609- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918757- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429861- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459553- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025701- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651621- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025705- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1755324346- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1755317126- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1755324362- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1755317129- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1755324410- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1755317135- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 3681166678819729- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 3681166678819735- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 3681166678819753- '\x0061'# -> unI64 65- '\x0062'# -> unI64 66- '\x0063'# -> unI64 67- '\x0064'# -> unI64 68- '\x0065'# -> unI64 69- '\x0066'# -> unI64 70- '\x0067'# -> unI64 71- '\x0068'# -> unI64 72- '\x0069'# -> unI64 73- '\x006a'# -> unI64 74- '\x006b'# -> unI64 75- '\x006c'# -> unI64 76- '\x006d'# -> unI64 77- '\x006e'# -> unI64 78- '\x006f'# -> unI64 79- '\x0070'# -> unI64 80- '\x0071'# -> unI64 81- '\x0072'# -> unI64 82- '\x0073'# -> unI64 83- '\x0074'# -> unI64 84- '\x0075'# -> unI64 85- '\x0076'# -> unI64 86- '\x0077'# -> unI64 87- '\x0078'# -> unI64 88- '\x0079'# -> unI64 89- '\x007a'# -> unI64 90- '\x00b5'# -> unI64 924- '\x00e0'# -> unI64 192- '\x00e1'# -> unI64 193- '\x00e2'# -> unI64 194- '\x00e3'# -> unI64 195- '\x00e4'# -> unI64 196- '\x00e5'# -> unI64 197- '\x00e6'# -> unI64 198- '\x00e7'# -> unI64 199- '\x00e8'# -> unI64 200- '\x00e9'# -> unI64 201- '\x00ea'# -> unI64 202- '\x00eb'# -> unI64 203- '\x00ec'# -> unI64 204- '\x00ed'# -> unI64 205- '\x00ee'# -> unI64 206- '\x00ef'# -> unI64 207- '\x00f0'# -> unI64 208- '\x00f1'# -> unI64 209- '\x00f2'# -> unI64 210- '\x00f3'# -> unI64 211- '\x00f4'# -> unI64 212- '\x00f5'# -> unI64 213- '\x00f6'# -> unI64 214- '\x00f8'# -> unI64 216- '\x00f9'# -> unI64 217- '\x00fa'# -> unI64 218- '\x00fb'# -> unI64 219- '\x00fc'# -> unI64 220- '\x00fd'# -> unI64 221- '\x00fe'# -> unI64 222- '\x00ff'# -> unI64 376- '\x0101'# -> unI64 256- '\x0103'# -> unI64 258- '\x0105'# -> unI64 260- '\x0107'# -> unI64 262- '\x0109'# -> unI64 264- '\x010b'# -> unI64 266- '\x010d'# -> unI64 268- '\x010f'# -> unI64 270- '\x0111'# -> unI64 272- '\x0113'# -> unI64 274- '\x0115'# -> unI64 276- '\x0117'# -> unI64 278- '\x0119'# -> unI64 280- '\x011b'# -> unI64 282- '\x011d'# -> unI64 284- '\x011f'# -> unI64 286- '\x0121'# -> unI64 288- '\x0123'# -> unI64 290- '\x0125'# -> unI64 292- '\x0127'# -> unI64 294- '\x0129'# -> unI64 296- '\x012b'# -> unI64 298- '\x012d'# -> unI64 300- '\x012f'# -> unI64 302- '\x0131'# -> unI64 73- '\x0133'# -> unI64 306- '\x0135'# -> unI64 308- '\x0137'# -> unI64 310- '\x013a'# -> unI64 313- '\x013c'# -> unI64 315- '\x013e'# -> unI64 317- '\x0140'# -> unI64 319- '\x0142'# -> unI64 321- '\x0144'# -> unI64 323- '\x0146'# -> unI64 325- '\x0148'# -> unI64 327- '\x014b'# -> unI64 330- '\x014d'# -> unI64 332- '\x014f'# -> unI64 334- '\x0151'# -> unI64 336- '\x0153'# -> unI64 338- '\x0155'# -> unI64 340- '\x0157'# -> unI64 342- '\x0159'# -> unI64 344- '\x015b'# -> unI64 346- '\x015d'# -> unI64 348- '\x015f'# -> unI64 350- '\x0161'# -> unI64 352- '\x0163'# -> unI64 354- '\x0165'# -> unI64 356- '\x0167'# -> unI64 358- '\x0169'# -> unI64 360- '\x016b'# -> unI64 362- '\x016d'# -> unI64 364- '\x016f'# -> unI64 366- '\x0171'# -> unI64 368- '\x0173'# -> unI64 370- '\x0175'# -> unI64 372- '\x0177'# -> unI64 374- '\x017a'# -> unI64 377- '\x017c'# -> unI64 379- '\x017e'# -> unI64 381- '\x017f'# -> unI64 83- '\x0180'# -> unI64 579- '\x0183'# -> unI64 386- '\x0185'# -> unI64 388- '\x0188'# -> unI64 391- '\x018c'# -> unI64 395- '\x0192'# -> unI64 401- '\x0195'# -> unI64 502- '\x0199'# -> unI64 408- '\x019a'# -> unI64 573- '\x019e'# -> unI64 544- '\x01a1'# -> unI64 416- '\x01a3'# -> unI64 418- '\x01a5'# -> unI64 420- '\x01a8'# -> unI64 423- '\x01ad'# -> unI64 428- '\x01b0'# -> unI64 431- '\x01b4'# -> unI64 435- '\x01b6'# -> unI64 437- '\x01b9'# -> unI64 440- '\x01bd'# -> unI64 444- '\x01bf'# -> unI64 503- '\x01c4'# -> unI64 453- '\x01c6'# -> unI64 453- '\x01c7'# -> unI64 456- '\x01c9'# -> unI64 456- '\x01ca'# -> unI64 459- '\x01cc'# -> unI64 459- '\x01ce'# -> unI64 461- '\x01d0'# -> unI64 463- '\x01d2'# -> unI64 465- '\x01d4'# -> unI64 467- '\x01d6'# -> unI64 469- '\x01d8'# -> unI64 471- '\x01da'# -> unI64 473- '\x01dc'# -> unI64 475- '\x01dd'# -> unI64 398- '\x01df'# -> unI64 478- '\x01e1'# -> unI64 480- '\x01e3'# -> unI64 482- '\x01e5'# -> unI64 484- '\x01e7'# -> unI64 486- '\x01e9'# -> unI64 488- '\x01eb'# -> unI64 490- '\x01ed'# -> unI64 492- '\x01ef'# -> unI64 494- '\x01f1'# -> unI64 498- '\x01f3'# -> unI64 498- '\x01f5'# -> unI64 500- '\x01f9'# -> unI64 504- '\x01fb'# -> unI64 506- '\x01fd'# -> unI64 508- '\x01ff'# -> unI64 510- '\x0201'# -> unI64 512- '\x0203'# -> unI64 514- '\x0205'# -> unI64 516- '\x0207'# -> unI64 518- '\x0209'# -> unI64 520- '\x020b'# -> unI64 522- '\x020d'# -> unI64 524- '\x020f'# -> unI64 526- '\x0211'# -> unI64 528- '\x0213'# -> unI64 530- '\x0215'# -> unI64 532- '\x0217'# -> unI64 534- '\x0219'# -> unI64 536- '\x021b'# -> unI64 538- '\x021d'# -> unI64 540- '\x021f'# -> unI64 542- '\x0223'# -> unI64 546- '\x0225'# -> unI64 548- '\x0227'# -> unI64 550- '\x0229'# -> unI64 552- '\x022b'# -> unI64 554- '\x022d'# -> unI64 556- '\x022f'# -> unI64 558- '\x0231'# -> unI64 560- '\x0233'# -> unI64 562- '\x023c'# -> unI64 571- '\x023f'# -> unI64 11390- '\x0240'# -> unI64 11391- '\x0242'# -> unI64 577- '\x0247'# -> unI64 582- '\x0249'# -> unI64 584- '\x024b'# -> unI64 586- '\x024d'# -> unI64 588- '\x024f'# -> unI64 590- '\x0250'# -> unI64 11375- '\x0251'# -> unI64 11373- '\x0252'# -> unI64 11376- '\x0253'# -> unI64 385- '\x0254'# -> unI64 390- '\x0256'# -> unI64 393- '\x0257'# -> unI64 394- '\x0259'# -> unI64 399- '\x025b'# -> unI64 400- '\x025c'# -> unI64 42923- '\x0260'# -> unI64 403- '\x0261'# -> unI64 42924- '\x0263'# -> unI64 404- '\x0265'# -> unI64 42893- '\x0266'# -> unI64 42922- '\x0268'# -> unI64 407- '\x0269'# -> unI64 406- '\x026a'# -> unI64 42926- '\x026b'# -> unI64 11362- '\x026c'# -> unI64 42925- '\x026f'# -> unI64 412- '\x0271'# -> unI64 11374- '\x0272'# -> unI64 413- '\x0275'# -> unI64 415- '\x027d'# -> unI64 11364- '\x0280'# -> unI64 422- '\x0282'# -> unI64 42949- '\x0283'# -> unI64 425- '\x0287'# -> unI64 42929- '\x0288'# -> unI64 430- '\x0289'# -> unI64 580- '\x028a'# -> unI64 433- '\x028b'# -> unI64 434- '\x028c'# -> unI64 581- '\x0292'# -> unI64 439- '\x029d'# -> unI64 42930- '\x029e'# -> unI64 42928- '\x0345'# -> unI64 921- '\x0371'# -> unI64 880- '\x0373'# -> unI64 882- '\x0377'# -> unI64 886- '\x037b'# -> unI64 1021- '\x037c'# -> unI64 1022- '\x037d'# -> unI64 1023- '\x03ac'# -> unI64 902- '\x03ad'# -> unI64 904- '\x03ae'# -> unI64 905- '\x03af'# -> unI64 906- '\x03b1'# -> unI64 913- '\x03b2'# -> unI64 914- '\x03b3'# -> unI64 915- '\x03b4'# -> unI64 916- '\x03b5'# -> unI64 917- '\x03b6'# -> unI64 918- '\x03b7'# -> unI64 919- '\x03b8'# -> unI64 920- '\x03b9'# -> unI64 921- '\x03ba'# -> unI64 922- '\x03bb'# -> unI64 923- '\x03bc'# -> unI64 924- '\x03bd'# -> unI64 925- '\x03be'# -> unI64 926- '\x03bf'# -> unI64 927- '\x03c0'# -> unI64 928- '\x03c1'# -> unI64 929- '\x03c2'# -> unI64 931- '\x03c3'# -> unI64 931- '\x03c4'# -> unI64 932- '\x03c5'# -> unI64 933- '\x03c6'# -> unI64 934- '\x03c7'# -> unI64 935- '\x03c8'# -> unI64 936- '\x03c9'# -> unI64 937- '\x03ca'# -> unI64 938- '\x03cb'# -> unI64 939- '\x03cc'# -> unI64 908- '\x03cd'# -> unI64 910- '\x03ce'# -> unI64 911- '\x03d0'# -> unI64 914- '\x03d1'# -> unI64 920- '\x03d5'# -> unI64 934- '\x03d6'# -> unI64 928- '\x03d7'# -> unI64 975- '\x03d9'# -> unI64 984- '\x03db'# -> unI64 986- '\x03dd'# -> unI64 988- '\x03df'# -> unI64 990- '\x03e1'# -> unI64 992- '\x03e3'# -> unI64 994- '\x03e5'# -> unI64 996- '\x03e7'# -> unI64 998- '\x03e9'# -> unI64 1000- '\x03eb'# -> unI64 1002- '\x03ed'# -> unI64 1004- '\x03ef'# -> unI64 1006- '\x03f0'# -> unI64 922- '\x03f1'# -> unI64 929- '\x03f2'# -> unI64 1017- '\x03f3'# -> unI64 895- '\x03f5'# -> unI64 917- '\x03f8'# -> unI64 1015- '\x03fb'# -> unI64 1018- '\x0430'# -> unI64 1040- '\x0431'# -> unI64 1041- '\x0432'# -> unI64 1042- '\x0433'# -> unI64 1043- '\x0434'# -> unI64 1044- '\x0435'# -> unI64 1045- '\x0436'# -> unI64 1046- '\x0437'# -> unI64 1047- '\x0438'# -> unI64 1048- '\x0439'# -> unI64 1049- '\x043a'# -> unI64 1050- '\x043b'# -> unI64 1051- '\x043c'# -> unI64 1052- '\x043d'# -> unI64 1053- '\x043e'# -> unI64 1054- '\x043f'# -> unI64 1055- '\x0440'# -> unI64 1056- '\x0441'# -> unI64 1057- '\x0442'# -> unI64 1058- '\x0443'# -> unI64 1059- '\x0444'# -> unI64 1060- '\x0445'# -> unI64 1061- '\x0446'# -> unI64 1062- '\x0447'# -> unI64 1063- '\x0448'# -> unI64 1064- '\x0449'# -> unI64 1065- '\x044a'# -> unI64 1066- '\x044b'# -> unI64 1067- '\x044c'# -> unI64 1068- '\x044d'# -> unI64 1069- '\x044e'# -> unI64 1070- '\x044f'# -> unI64 1071- '\x0450'# -> unI64 1024- '\x0451'# -> unI64 1025- '\x0452'# -> unI64 1026- '\x0453'# -> unI64 1027- '\x0454'# -> unI64 1028- '\x0455'# -> unI64 1029- '\x0456'# -> unI64 1030- '\x0457'# -> unI64 1031- '\x0458'# -> unI64 1032- '\x0459'# -> unI64 1033- '\x045a'# -> unI64 1034- '\x045b'# -> unI64 1035- '\x045c'# -> unI64 1036- '\x045d'# -> unI64 1037- '\x045e'# -> unI64 1038- '\x045f'# -> unI64 1039- '\x0461'# -> unI64 1120- '\x0463'# -> unI64 1122- '\x0465'# -> unI64 1124- '\x0467'# -> unI64 1126- '\x0469'# -> unI64 1128- '\x046b'# -> unI64 1130- '\x046d'# -> unI64 1132- '\x046f'# -> unI64 1134- '\x0471'# -> unI64 1136- '\x0473'# -> unI64 1138- '\x0475'# -> unI64 1140- '\x0477'# -> unI64 1142- '\x0479'# -> unI64 1144- '\x047b'# -> unI64 1146- '\x047d'# -> unI64 1148- '\x047f'# -> unI64 1150- '\x0481'# -> unI64 1152- '\x048b'# -> unI64 1162- '\x048d'# -> unI64 1164- '\x048f'# -> unI64 1166- '\x0491'# -> unI64 1168- '\x0493'# -> unI64 1170- '\x0495'# -> unI64 1172- '\x0497'# -> unI64 1174- '\x0499'# -> unI64 1176- '\x049b'# -> unI64 1178- '\x049d'# -> unI64 1180- '\x049f'# -> unI64 1182- '\x04a1'# -> unI64 1184- '\x04a3'# -> unI64 1186- '\x04a5'# -> unI64 1188- '\x04a7'# -> unI64 1190- '\x04a9'# -> unI64 1192- '\x04ab'# -> unI64 1194- '\x04ad'# -> unI64 1196- '\x04af'# -> unI64 1198- '\x04b1'# -> unI64 1200- '\x04b3'# -> unI64 1202- '\x04b5'# -> unI64 1204- '\x04b7'# -> unI64 1206- '\x04b9'# -> unI64 1208- '\x04bb'# -> unI64 1210- '\x04bd'# -> unI64 1212- '\x04bf'# -> unI64 1214- '\x04c2'# -> unI64 1217- '\x04c4'# -> unI64 1219- '\x04c6'# -> unI64 1221- '\x04c8'# -> unI64 1223- '\x04ca'# -> unI64 1225- '\x04cc'# -> unI64 1227- '\x04ce'# -> unI64 1229- '\x04cf'# -> unI64 1216- '\x04d1'# -> unI64 1232- '\x04d3'# -> unI64 1234- '\x04d5'# -> unI64 1236- '\x04d7'# -> unI64 1238- '\x04d9'# -> unI64 1240- '\x04db'# -> unI64 1242- '\x04dd'# -> unI64 1244- '\x04df'# -> unI64 1246- '\x04e1'# -> unI64 1248- '\x04e3'# -> unI64 1250- '\x04e5'# -> unI64 1252- '\x04e7'# -> unI64 1254- '\x04e9'# -> unI64 1256- '\x04eb'# -> unI64 1258- '\x04ed'# -> unI64 1260- '\x04ef'# -> unI64 1262- '\x04f1'# -> unI64 1264- '\x04f3'# -> unI64 1266- '\x04f5'# -> unI64 1268- '\x04f7'# -> unI64 1270- '\x04f9'# -> unI64 1272- '\x04fb'# -> unI64 1274- '\x04fd'# -> unI64 1276- '\x04ff'# -> unI64 1278- '\x0501'# -> unI64 1280- '\x0503'# -> unI64 1282- '\x0505'# -> unI64 1284- '\x0507'# -> unI64 1286- '\x0509'# -> unI64 1288- '\x050b'# -> unI64 1290- '\x050d'# -> unI64 1292- '\x050f'# -> unI64 1294- '\x0511'# -> unI64 1296- '\x0513'# -> unI64 1298- '\x0515'# -> unI64 1300- '\x0517'# -> unI64 1302- '\x0519'# -> unI64 1304- '\x051b'# -> unI64 1306- '\x051d'# -> unI64 1308- '\x051f'# -> unI64 1310- '\x0521'# -> unI64 1312- '\x0523'# -> unI64 1314- '\x0525'# -> unI64 1316- '\x0527'# -> unI64 1318- '\x0529'# -> unI64 1320- '\x052b'# -> unI64 1322- '\x052d'# -> unI64 1324- '\x052f'# -> unI64 1326- '\x0561'# -> unI64 1329- '\x0562'# -> unI64 1330- '\x0563'# -> unI64 1331- '\x0564'# -> unI64 1332- '\x0565'# -> unI64 1333- '\x0566'# -> unI64 1334- '\x0567'# -> unI64 1335- '\x0568'# -> unI64 1336- '\x0569'# -> unI64 1337- '\x056a'# -> unI64 1338- '\x056b'# -> unI64 1339- '\x056c'# -> unI64 1340- '\x056d'# -> unI64 1341- '\x056e'# -> unI64 1342- '\x056f'# -> unI64 1343- '\x0570'# -> unI64 1344- '\x0571'# -> unI64 1345- '\x0572'# -> unI64 1346- '\x0573'# -> unI64 1347- '\x0574'# -> unI64 1348- '\x0575'# -> unI64 1349- '\x0576'# -> unI64 1350- '\x0577'# -> unI64 1351- '\x0578'# -> unI64 1352- '\x0579'# -> unI64 1353- '\x057a'# -> unI64 1354- '\x057b'# -> unI64 1355- '\x057c'# -> unI64 1356- '\x057d'# -> unI64 1357- '\x057e'# -> unI64 1358- '\x057f'# -> unI64 1359- '\x0580'# -> unI64 1360- '\x0581'# -> unI64 1361- '\x0582'# -> unI64 1362- '\x0583'# -> unI64 1363- '\x0584'# -> unI64 1364- '\x0585'# -> unI64 1365- '\x0586'# -> unI64 1366- '\x13f8'# -> unI64 5104- '\x13f9'# -> unI64 5105- '\x13fa'# -> unI64 5106- '\x13fb'# -> unI64 5107- '\x13fc'# -> unI64 5108- '\x13fd'# -> unI64 5109- '\x1c80'# -> unI64 1042- '\x1c81'# -> unI64 1044- '\x1c82'# -> unI64 1054- '\x1c83'# -> unI64 1057- '\x1c84'# -> unI64 1058- '\x1c85'# -> unI64 1058- '\x1c86'# -> unI64 1066- '\x1c87'# -> unI64 1122- '\x1c88'# -> unI64 42570- '\x1d79'# -> unI64 42877- '\x1d7d'# -> unI64 11363- '\x1d8e'# -> unI64 42950- '\x1e01'# -> unI64 7680- '\x1e03'# -> unI64 7682- '\x1e05'# -> unI64 7684- '\x1e07'# -> unI64 7686- '\x1e09'# -> unI64 7688- '\x1e0b'# -> unI64 7690- '\x1e0d'# -> unI64 7692- '\x1e0f'# -> unI64 7694- '\x1e11'# -> unI64 7696- '\x1e13'# -> unI64 7698- '\x1e15'# -> unI64 7700- '\x1e17'# -> unI64 7702- '\x1e19'# -> unI64 7704- '\x1e1b'# -> unI64 7706- '\x1e1d'# -> unI64 7708- '\x1e1f'# -> unI64 7710- '\x1e21'# -> unI64 7712- '\x1e23'# -> unI64 7714- '\x1e25'# -> unI64 7716- '\x1e27'# -> unI64 7718- '\x1e29'# -> unI64 7720- '\x1e2b'# -> unI64 7722- '\x1e2d'# -> unI64 7724- '\x1e2f'# -> unI64 7726- '\x1e31'# -> unI64 7728- '\x1e33'# -> unI64 7730- '\x1e35'# -> unI64 7732- '\x1e37'# -> unI64 7734- '\x1e39'# -> unI64 7736- '\x1e3b'# -> unI64 7738- '\x1e3d'# -> unI64 7740- '\x1e3f'# -> unI64 7742- '\x1e41'# -> unI64 7744- '\x1e43'# -> unI64 7746- '\x1e45'# -> unI64 7748- '\x1e47'# -> unI64 7750- '\x1e49'# -> unI64 7752- '\x1e4b'# -> unI64 7754- '\x1e4d'# -> unI64 7756- '\x1e4f'# -> unI64 7758- '\x1e51'# -> unI64 7760- '\x1e53'# -> unI64 7762- '\x1e55'# -> unI64 7764- '\x1e57'# -> unI64 7766- '\x1e59'# -> unI64 7768- '\x1e5b'# -> unI64 7770- '\x1e5d'# -> unI64 7772- '\x1e5f'# -> unI64 7774- '\x1e61'# -> unI64 7776- '\x1e63'# -> unI64 7778- '\x1e65'# -> unI64 7780- '\x1e67'# -> unI64 7782- '\x1e69'# -> unI64 7784- '\x1e6b'# -> unI64 7786- '\x1e6d'# -> unI64 7788- '\x1e6f'# -> unI64 7790- '\x1e71'# -> unI64 7792- '\x1e73'# -> unI64 7794- '\x1e75'# -> unI64 7796- '\x1e77'# -> unI64 7798- '\x1e79'# -> unI64 7800- '\x1e7b'# -> unI64 7802- '\x1e7d'# -> unI64 7804- '\x1e7f'# -> unI64 7806- '\x1e81'# -> unI64 7808- '\x1e83'# -> unI64 7810- '\x1e85'# -> unI64 7812- '\x1e87'# -> unI64 7814- '\x1e89'# -> unI64 7816- '\x1e8b'# -> unI64 7818- '\x1e8d'# -> unI64 7820- '\x1e8f'# -> unI64 7822- '\x1e91'# -> unI64 7824- '\x1e93'# -> unI64 7826- '\x1e95'# -> unI64 7828- '\x1e9b'# -> unI64 7776- '\x1ea1'# -> unI64 7840- '\x1ea3'# -> unI64 7842- '\x1ea5'# -> unI64 7844- '\x1ea7'# -> unI64 7846- '\x1ea9'# -> unI64 7848- '\x1eab'# -> unI64 7850- '\x1ead'# -> unI64 7852- '\x1eaf'# -> unI64 7854- '\x1eb1'# -> unI64 7856- '\x1eb3'# -> unI64 7858- '\x1eb5'# -> unI64 7860- '\x1eb7'# -> unI64 7862- '\x1eb9'# -> unI64 7864- '\x1ebb'# -> unI64 7866- '\x1ebd'# -> unI64 7868- '\x1ebf'# -> unI64 7870- '\x1ec1'# -> unI64 7872- '\x1ec3'# -> unI64 7874- '\x1ec5'# -> unI64 7876- '\x1ec7'# -> unI64 7878- '\x1ec9'# -> unI64 7880- '\x1ecb'# -> unI64 7882- '\x1ecd'# -> unI64 7884- '\x1ecf'# -> unI64 7886- '\x1ed1'# -> unI64 7888- '\x1ed3'# -> unI64 7890- '\x1ed5'# -> unI64 7892- '\x1ed7'# -> unI64 7894- '\x1ed9'# -> unI64 7896- '\x1edb'# -> unI64 7898- '\x1edd'# -> unI64 7900- '\x1edf'# -> unI64 7902- '\x1ee1'# -> unI64 7904- '\x1ee3'# -> unI64 7906- '\x1ee5'# -> unI64 7908- '\x1ee7'# -> unI64 7910- '\x1ee9'# -> unI64 7912- '\x1eeb'# -> unI64 7914- '\x1eed'# -> unI64 7916- '\x1eef'# -> unI64 7918- '\x1ef1'# -> unI64 7920- '\x1ef3'# -> unI64 7922- '\x1ef5'# -> unI64 7924- '\x1ef7'# -> unI64 7926- '\x1ef9'# -> unI64 7928- '\x1efb'# -> unI64 7930- '\x1efd'# -> unI64 7932- '\x1eff'# -> unI64 7934- '\x1f00'# -> unI64 7944- '\x1f01'# -> unI64 7945- '\x1f02'# -> unI64 7946- '\x1f03'# -> unI64 7947- '\x1f04'# -> unI64 7948- '\x1f05'# -> unI64 7949- '\x1f06'# -> unI64 7950- '\x1f07'# -> unI64 7951- '\x1f10'# -> unI64 7960- '\x1f11'# -> unI64 7961- '\x1f12'# -> unI64 7962- '\x1f13'# -> unI64 7963- '\x1f14'# -> unI64 7964- '\x1f15'# -> unI64 7965- '\x1f20'# -> unI64 7976- '\x1f21'# -> unI64 7977- '\x1f22'# -> unI64 7978- '\x1f23'# -> unI64 7979- '\x1f24'# -> unI64 7980- '\x1f25'# -> unI64 7981- '\x1f26'# -> unI64 7982- '\x1f27'# -> unI64 7983- '\x1f30'# -> unI64 7992- '\x1f31'# -> unI64 7993- '\x1f32'# -> unI64 7994- '\x1f33'# -> unI64 7995- '\x1f34'# -> unI64 7996- '\x1f35'# -> unI64 7997- '\x1f36'# -> unI64 7998- '\x1f37'# -> unI64 7999- '\x1f40'# -> unI64 8008- '\x1f41'# -> unI64 8009- '\x1f42'# -> unI64 8010- '\x1f43'# -> unI64 8011- '\x1f44'# -> unI64 8012- '\x1f45'# -> unI64 8013- '\x1f51'# -> unI64 8025- '\x1f53'# -> unI64 8027- '\x1f55'# -> unI64 8029- '\x1f57'# -> unI64 8031- '\x1f60'# -> unI64 8040- '\x1f61'# -> unI64 8041- '\x1f62'# -> unI64 8042- '\x1f63'# -> unI64 8043- '\x1f64'# -> unI64 8044- '\x1f65'# -> unI64 8045- '\x1f66'# -> unI64 8046- '\x1f67'# -> unI64 8047- '\x1f70'# -> unI64 8122- '\x1f71'# -> unI64 8123- '\x1f72'# -> unI64 8136- '\x1f73'# -> unI64 8137- '\x1f74'# -> unI64 8138- '\x1f75'# -> unI64 8139- '\x1f76'# -> unI64 8154- '\x1f77'# -> unI64 8155- '\x1f78'# -> unI64 8184- '\x1f79'# -> unI64 8185- '\x1f7a'# -> unI64 8170- '\x1f7b'# -> unI64 8171- '\x1f7c'# -> unI64 8186- '\x1f7d'# -> unI64 8187- '\x1f80'# -> unI64 8072- '\x1f81'# -> unI64 8073- '\x1f82'# -> unI64 8074- '\x1f83'# -> unI64 8075- '\x1f84'# -> unI64 8076- '\x1f85'# -> unI64 8077- '\x1f86'# -> unI64 8078- '\x1f87'# -> unI64 8079- '\x1f90'# -> unI64 8088- '\x1f91'# -> unI64 8089- '\x1f92'# -> unI64 8090- '\x1f93'# -> unI64 8091- '\x1f94'# -> unI64 8092- '\x1f95'# -> unI64 8093- '\x1f96'# -> unI64 8094- '\x1f97'# -> unI64 8095- '\x1fa0'# -> unI64 8104- '\x1fa1'# -> unI64 8105- '\x1fa2'# -> unI64 8106- '\x1fa3'# -> unI64 8107- '\x1fa4'# -> unI64 8108- '\x1fa5'# -> unI64 8109- '\x1fa6'# -> unI64 8110- '\x1fa7'# -> unI64 8111- '\x1fb0'# -> unI64 8120- '\x1fb1'# -> unI64 8121- '\x1fb3'# -> unI64 8124- '\x1fbe'# -> unI64 921- '\x1fc3'# -> unI64 8140- '\x1fd0'# -> unI64 8152- '\x1fd1'# -> unI64 8153- '\x1fe0'# -> unI64 8168- '\x1fe1'# -> unI64 8169- '\x1fe5'# -> unI64 8172- '\x1ff3'# -> unI64 8188- '\x214e'# -> unI64 8498- '\x2170'# -> unI64 8544- '\x2171'# -> unI64 8545- '\x2172'# -> unI64 8546- '\x2173'# -> unI64 8547- '\x2174'# -> unI64 8548- '\x2175'# -> unI64 8549- '\x2176'# -> unI64 8550- '\x2177'# -> unI64 8551- '\x2178'# -> unI64 8552- '\x2179'# -> unI64 8553- '\x217a'# -> unI64 8554- '\x217b'# -> unI64 8555- '\x217c'# -> unI64 8556- '\x217d'# -> unI64 8557- '\x217e'# -> unI64 8558- '\x217f'# -> unI64 8559- '\x2184'# -> unI64 8579- '\x24d0'# -> unI64 9398- '\x24d1'# -> unI64 9399- '\x24d2'# -> unI64 9400- '\x24d3'# -> unI64 9401- '\x24d4'# -> unI64 9402- '\x24d5'# -> unI64 9403- '\x24d6'# -> unI64 9404- '\x24d7'# -> unI64 9405- '\x24d8'# -> unI64 9406- '\x24d9'# -> unI64 9407- '\x24da'# -> unI64 9408- '\x24db'# -> unI64 9409- '\x24dc'# -> unI64 9410- '\x24dd'# -> unI64 9411- '\x24de'# -> unI64 9412- '\x24df'# -> unI64 9413- '\x24e0'# -> unI64 9414- '\x24e1'# -> unI64 9415- '\x24e2'# -> unI64 9416- '\x24e3'# -> unI64 9417- '\x24e4'# -> unI64 9418- '\x24e5'# -> unI64 9419- '\x24e6'# -> unI64 9420- '\x24e7'# -> unI64 9421- '\x24e8'# -> unI64 9422- '\x24e9'# -> unI64 9423- '\x2c30'# -> unI64 11264- '\x2c31'# -> unI64 11265- '\x2c32'# -> unI64 11266- '\x2c33'# -> unI64 11267- '\x2c34'# -> unI64 11268- '\x2c35'# -> unI64 11269- '\x2c36'# -> unI64 11270- '\x2c37'# -> unI64 11271- '\x2c38'# -> unI64 11272- '\x2c39'# -> unI64 11273- '\x2c3a'# -> unI64 11274- '\x2c3b'# -> unI64 11275- '\x2c3c'# -> unI64 11276- '\x2c3d'# -> unI64 11277- '\x2c3e'# -> unI64 11278- '\x2c3f'# -> unI64 11279- '\x2c40'# -> unI64 11280- '\x2c41'# -> unI64 11281- '\x2c42'# -> unI64 11282- '\x2c43'# -> unI64 11283- '\x2c44'# -> unI64 11284- '\x2c45'# -> unI64 11285- '\x2c46'# -> unI64 11286- '\x2c47'# -> unI64 11287- '\x2c48'# -> unI64 11288- '\x2c49'# -> unI64 11289- '\x2c4a'# -> unI64 11290- '\x2c4b'# -> unI64 11291- '\x2c4c'# -> unI64 11292- '\x2c4d'# -> unI64 11293- '\x2c4e'# -> unI64 11294- '\x2c4f'# -> unI64 11295- '\x2c50'# -> unI64 11296- '\x2c51'# -> unI64 11297- '\x2c52'# -> unI64 11298- '\x2c53'# -> unI64 11299- '\x2c54'# -> unI64 11300- '\x2c55'# -> unI64 11301- '\x2c56'# -> unI64 11302- '\x2c57'# -> unI64 11303- '\x2c58'# -> unI64 11304- '\x2c59'# -> unI64 11305- '\x2c5a'# -> unI64 11306- '\x2c5b'# -> unI64 11307- '\x2c5c'# -> unI64 11308- '\x2c5d'# -> unI64 11309- '\x2c5e'# -> unI64 11310- '\x2c5f'# -> unI64 11311- '\x2c61'# -> unI64 11360- '\x2c65'# -> unI64 570- '\x2c66'# -> unI64 574- '\x2c68'# -> unI64 11367- '\x2c6a'# -> unI64 11369- '\x2c6c'# -> unI64 11371- '\x2c73'# -> unI64 11378- '\x2c76'# -> unI64 11381- '\x2c81'# -> unI64 11392- '\x2c83'# -> unI64 11394- '\x2c85'# -> unI64 11396- '\x2c87'# -> unI64 11398- '\x2c89'# -> unI64 11400- '\x2c8b'# -> unI64 11402- '\x2c8d'# -> unI64 11404- '\x2c8f'# -> unI64 11406- '\x2c91'# -> unI64 11408- '\x2c93'# -> unI64 11410- '\x2c95'# -> unI64 11412- '\x2c97'# -> unI64 11414- '\x2c99'# -> unI64 11416- '\x2c9b'# -> unI64 11418- '\x2c9d'# -> unI64 11420- '\x2c9f'# -> unI64 11422- '\x2ca1'# -> unI64 11424- '\x2ca3'# -> unI64 11426- '\x2ca5'# -> unI64 11428- '\x2ca7'# -> unI64 11430- '\x2ca9'# -> unI64 11432- '\x2cab'# -> unI64 11434- '\x2cad'# -> unI64 11436- '\x2caf'# -> unI64 11438- '\x2cb1'# -> unI64 11440- '\x2cb3'# -> unI64 11442- '\x2cb5'# -> unI64 11444- '\x2cb7'# -> unI64 11446- '\x2cb9'# -> unI64 11448- '\x2cbb'# -> unI64 11450- '\x2cbd'# -> unI64 11452- '\x2cbf'# -> unI64 11454- '\x2cc1'# -> unI64 11456- '\x2cc3'# -> unI64 11458- '\x2cc5'# -> unI64 11460- '\x2cc7'# -> unI64 11462- '\x2cc9'# -> unI64 11464- '\x2ccb'# -> unI64 11466- '\x2ccd'# -> unI64 11468- '\x2ccf'# -> unI64 11470- '\x2cd1'# -> unI64 11472- '\x2cd3'# -> unI64 11474- '\x2cd5'# -> unI64 11476- '\x2cd7'# -> unI64 11478- '\x2cd9'# -> unI64 11480- '\x2cdb'# -> unI64 11482- '\x2cdd'# -> unI64 11484- '\x2cdf'# -> unI64 11486- '\x2ce1'# -> unI64 11488- '\x2ce3'# -> unI64 11490- '\x2cec'# -> unI64 11499- '\x2cee'# -> unI64 11501- '\x2cf3'# -> unI64 11506- '\x2d00'# -> unI64 4256- '\x2d01'# -> unI64 4257- '\x2d02'# -> unI64 4258- '\x2d03'# -> unI64 4259- '\x2d04'# -> unI64 4260- '\x2d05'# -> unI64 4261- '\x2d06'# -> unI64 4262- '\x2d07'# -> unI64 4263- '\x2d08'# -> unI64 4264- '\x2d09'# -> unI64 4265- '\x2d0a'# -> unI64 4266- '\x2d0b'# -> unI64 4267- '\x2d0c'# -> unI64 4268- '\x2d0d'# -> unI64 4269- '\x2d0e'# -> unI64 4270- '\x2d0f'# -> unI64 4271- '\x2d10'# -> unI64 4272- '\x2d11'# -> unI64 4273- '\x2d12'# -> unI64 4274- '\x2d13'# -> unI64 4275- '\x2d14'# -> unI64 4276- '\x2d15'# -> unI64 4277- '\x2d16'# -> unI64 4278- '\x2d17'# -> unI64 4279- '\x2d18'# -> unI64 4280- '\x2d19'# -> unI64 4281- '\x2d1a'# -> unI64 4282- '\x2d1b'# -> unI64 4283- '\x2d1c'# -> unI64 4284- '\x2d1d'# -> unI64 4285- '\x2d1e'# -> unI64 4286- '\x2d1f'# -> unI64 4287- '\x2d20'# -> unI64 4288- '\x2d21'# -> unI64 4289- '\x2d22'# -> unI64 4290- '\x2d23'# -> unI64 4291- '\x2d24'# -> unI64 4292- '\x2d25'# -> unI64 4293- '\x2d27'# -> unI64 4295- '\x2d2d'# -> unI64 4301- '\xa641'# -> unI64 42560- '\xa643'# -> unI64 42562- '\xa645'# -> unI64 42564- '\xa647'# -> unI64 42566- '\xa649'# -> unI64 42568- '\xa64b'# -> unI64 42570- '\xa64d'# -> unI64 42572- '\xa64f'# -> unI64 42574- '\xa651'# -> unI64 42576- '\xa653'# -> unI64 42578- '\xa655'# -> unI64 42580- '\xa657'# -> unI64 42582- '\xa659'# -> unI64 42584- '\xa65b'# -> unI64 42586- '\xa65d'# -> unI64 42588- '\xa65f'# -> unI64 42590- '\xa661'# -> unI64 42592- '\xa663'# -> unI64 42594- '\xa665'# -> unI64 42596- '\xa667'# -> unI64 42598- '\xa669'# -> unI64 42600- '\xa66b'# -> unI64 42602- '\xa66d'# -> unI64 42604- '\xa681'# -> unI64 42624- '\xa683'# -> unI64 42626- '\xa685'# -> unI64 42628- '\xa687'# -> unI64 42630- '\xa689'# -> unI64 42632- '\xa68b'# -> unI64 42634- '\xa68d'# -> unI64 42636- '\xa68f'# -> unI64 42638- '\xa691'# -> unI64 42640- '\xa693'# -> unI64 42642- '\xa695'# -> unI64 42644- '\xa697'# -> unI64 42646- '\xa699'# -> unI64 42648- '\xa69b'# -> unI64 42650- '\xa723'# -> unI64 42786- '\xa725'# -> unI64 42788- '\xa727'# -> unI64 42790- '\xa729'# -> unI64 42792- '\xa72b'# -> unI64 42794- '\xa72d'# -> unI64 42796- '\xa72f'# -> unI64 42798- '\xa733'# -> unI64 42802- '\xa735'# -> unI64 42804- '\xa737'# -> unI64 42806- '\xa739'# -> unI64 42808- '\xa73b'# -> unI64 42810- '\xa73d'# -> unI64 42812- '\xa73f'# -> unI64 42814- '\xa741'# -> unI64 42816- '\xa743'# -> unI64 42818- '\xa745'# -> unI64 42820- '\xa747'# -> unI64 42822- '\xa749'# -> unI64 42824- '\xa74b'# -> unI64 42826- '\xa74d'# -> unI64 42828- '\xa74f'# -> unI64 42830- '\xa751'# -> unI64 42832- '\xa753'# -> unI64 42834- '\xa755'# -> unI64 42836- '\xa757'# -> unI64 42838- '\xa759'# -> unI64 42840- '\xa75b'# -> unI64 42842- '\xa75d'# -> unI64 42844- '\xa75f'# -> unI64 42846- '\xa761'# -> unI64 42848- '\xa763'# -> unI64 42850- '\xa765'# -> unI64 42852- '\xa767'# -> unI64 42854- '\xa769'# -> unI64 42856- '\xa76b'# -> unI64 42858- '\xa76d'# -> unI64 42860- '\xa76f'# -> unI64 42862- '\xa77a'# -> unI64 42873- '\xa77c'# -> unI64 42875- '\xa77f'# -> unI64 42878- '\xa781'# -> unI64 42880- '\xa783'# -> unI64 42882- '\xa785'# -> unI64 42884- '\xa787'# -> unI64 42886- '\xa78c'# -> unI64 42891- '\xa791'# -> unI64 42896- '\xa793'# -> unI64 42898- '\xa794'# -> unI64 42948- '\xa797'# -> unI64 42902- '\xa799'# -> unI64 42904- '\xa79b'# -> unI64 42906- '\xa79d'# -> unI64 42908- '\xa79f'# -> unI64 42910- '\xa7a1'# -> unI64 42912- '\xa7a3'# -> unI64 42914- '\xa7a5'# -> unI64 42916- '\xa7a7'# -> unI64 42918- '\xa7a9'# -> unI64 42920- '\xa7b5'# -> unI64 42932- '\xa7b7'# -> unI64 42934- '\xa7b9'# -> unI64 42936- '\xa7bb'# -> unI64 42938- '\xa7bd'# -> unI64 42940- '\xa7bf'# -> unI64 42942- '\xa7c1'# -> unI64 42944- '\xa7c3'# -> unI64 42946- '\xa7c8'# -> unI64 42951- '\xa7ca'# -> unI64 42953- '\xa7d1'# -> unI64 42960- '\xa7d7'# -> unI64 42966- '\xa7d9'# -> unI64 42968- '\xa7f6'# -> unI64 42997- '\xab53'# -> unI64 42931- '\xab70'# -> unI64 5024- '\xab71'# -> unI64 5025- '\xab72'# -> unI64 5026- '\xab73'# -> unI64 5027- '\xab74'# -> unI64 5028- '\xab75'# -> unI64 5029- '\xab76'# -> unI64 5030- '\xab77'# -> unI64 5031- '\xab78'# -> unI64 5032- '\xab79'# -> unI64 5033- '\xab7a'# -> unI64 5034- '\xab7b'# -> unI64 5035- '\xab7c'# -> unI64 5036- '\xab7d'# -> unI64 5037- '\xab7e'# -> unI64 5038- '\xab7f'# -> unI64 5039- '\xab80'# -> unI64 5040- '\xab81'# -> unI64 5041- '\xab82'# -> unI64 5042- '\xab83'# -> unI64 5043- '\xab84'# -> unI64 5044- '\xab85'# -> unI64 5045- '\xab86'# -> unI64 5046- '\xab87'# -> unI64 5047- '\xab88'# -> unI64 5048- '\xab89'# -> unI64 5049- '\xab8a'# -> unI64 5050- '\xab8b'# -> unI64 5051- '\xab8c'# -> unI64 5052- '\xab8d'# -> unI64 5053- '\xab8e'# -> unI64 5054- '\xab8f'# -> unI64 5055- '\xab90'# -> unI64 5056- '\xab91'# -> unI64 5057- '\xab92'# -> unI64 5058- '\xab93'# -> unI64 5059- '\xab94'# -> unI64 5060- '\xab95'# -> unI64 5061- '\xab96'# -> unI64 5062- '\xab97'# -> unI64 5063- '\xab98'# -> unI64 5064- '\xab99'# -> unI64 5065- '\xab9a'# -> unI64 5066- '\xab9b'# -> unI64 5067- '\xab9c'# -> unI64 5068- '\xab9d'# -> unI64 5069- '\xab9e'# -> unI64 5070- '\xab9f'# -> unI64 5071- '\xaba0'# -> unI64 5072- '\xaba1'# -> unI64 5073- '\xaba2'# -> unI64 5074- '\xaba3'# -> unI64 5075- '\xaba4'# -> unI64 5076- '\xaba5'# -> unI64 5077- '\xaba6'# -> unI64 5078- '\xaba7'# -> unI64 5079- '\xaba8'# -> unI64 5080- '\xaba9'# -> unI64 5081- '\xabaa'# -> unI64 5082- '\xabab'# -> unI64 5083- '\xabac'# -> unI64 5084- '\xabad'# -> unI64 5085- '\xabae'# -> unI64 5086- '\xabaf'# -> unI64 5087- '\xabb0'# -> unI64 5088- '\xabb1'# -> unI64 5089- '\xabb2'# -> unI64 5090- '\xabb3'# -> unI64 5091- '\xabb4'# -> unI64 5092- '\xabb5'# -> unI64 5093- '\xabb6'# -> unI64 5094- '\xabb7'# -> unI64 5095- '\xabb8'# -> unI64 5096- '\xabb9'# -> unI64 5097- '\xabba'# -> unI64 5098- '\xabbb'# -> unI64 5099- '\xabbc'# -> unI64 5100- '\xabbd'# -> unI64 5101- '\xabbe'# -> unI64 5102- '\xabbf'# -> unI64 5103- '\xff41'# -> unI64 65313- '\xff42'# -> unI64 65314- '\xff43'# -> unI64 65315- '\xff44'# -> unI64 65316- '\xff45'# -> unI64 65317- '\xff46'# -> unI64 65318- '\xff47'# -> unI64 65319- '\xff48'# -> unI64 65320- '\xff49'# -> unI64 65321- '\xff4a'# -> unI64 65322- '\xff4b'# -> unI64 65323- '\xff4c'# -> unI64 65324- '\xff4d'# -> unI64 65325- '\xff4e'# -> unI64 65326- '\xff4f'# -> unI64 65327- '\xff50'# -> unI64 65328- '\xff51'# -> unI64 65329- '\xff52'# -> unI64 65330- '\xff53'# -> unI64 65331- '\xff54'# -> unI64 65332- '\xff55'# -> unI64 65333- '\xff56'# -> unI64 65334- '\xff57'# -> unI64 65335- '\xff58'# -> unI64 65336- '\xff59'# -> unI64 65337- '\xff5a'# -> unI64 65338- '\x10428'# -> unI64 66560- '\x10429'# -> unI64 66561- '\x1042a'# -> unI64 66562- '\x1042b'# -> unI64 66563- '\x1042c'# -> unI64 66564- '\x1042d'# -> unI64 66565- '\x1042e'# -> unI64 66566- '\x1042f'# -> unI64 66567- '\x10430'# -> unI64 66568- '\x10431'# -> unI64 66569- '\x10432'# -> unI64 66570- '\x10433'# -> unI64 66571- '\x10434'# -> unI64 66572- '\x10435'# -> unI64 66573- '\x10436'# -> unI64 66574- '\x10437'# -> unI64 66575- '\x10438'# -> unI64 66576- '\x10439'# -> unI64 66577- '\x1043a'# -> unI64 66578- '\x1043b'# -> unI64 66579- '\x1043c'# -> unI64 66580- '\x1043d'# -> unI64 66581- '\x1043e'# -> unI64 66582- '\x1043f'# -> unI64 66583- '\x10440'# -> unI64 66584- '\x10441'# -> unI64 66585- '\x10442'# -> unI64 66586- '\x10443'# -> unI64 66587- '\x10444'# -> unI64 66588- '\x10445'# -> unI64 66589- '\x10446'# -> unI64 66590- '\x10447'# -> unI64 66591- '\x10448'# -> unI64 66592- '\x10449'# -> unI64 66593- '\x1044a'# -> unI64 66594- '\x1044b'# -> unI64 66595- '\x1044c'# -> unI64 66596- '\x1044d'# -> unI64 66597- '\x1044e'# -> unI64 66598- '\x1044f'# -> unI64 66599- '\x104d8'# -> unI64 66736- '\x104d9'# -> unI64 66737- '\x104da'# -> unI64 66738- '\x104db'# -> unI64 66739- '\x104dc'# -> unI64 66740- '\x104dd'# -> unI64 66741- '\x104de'# -> unI64 66742- '\x104df'# -> unI64 66743- '\x104e0'# -> unI64 66744- '\x104e1'# -> unI64 66745- '\x104e2'# -> unI64 66746- '\x104e3'# -> unI64 66747- '\x104e4'# -> unI64 66748- '\x104e5'# -> unI64 66749- '\x104e6'# -> unI64 66750- '\x104e7'# -> unI64 66751- '\x104e8'# -> unI64 66752- '\x104e9'# -> unI64 66753- '\x104ea'# -> unI64 66754- '\x104eb'# -> unI64 66755- '\x104ec'# -> unI64 66756- '\x104ed'# -> unI64 66757- '\x104ee'# -> unI64 66758- '\x104ef'# -> unI64 66759- '\x104f0'# -> unI64 66760- '\x104f1'# -> unI64 66761- '\x104f2'# -> unI64 66762- '\x104f3'# -> unI64 66763- '\x104f4'# -> unI64 66764- '\x104f5'# -> unI64 66765- '\x104f6'# -> unI64 66766- '\x104f7'# -> unI64 66767- '\x104f8'# -> unI64 66768- '\x104f9'# -> unI64 66769- '\x104fa'# -> unI64 66770- '\x104fb'# -> unI64 66771- '\x10597'# -> unI64 66928- '\x10598'# -> unI64 66929- '\x10599'# -> unI64 66930- '\x1059a'# -> unI64 66931- '\x1059b'# -> unI64 66932- '\x1059c'# -> unI64 66933- '\x1059d'# -> unI64 66934- '\x1059e'# -> unI64 66935- '\x1059f'# -> unI64 66936- '\x105a0'# -> unI64 66937- '\x105a1'# -> unI64 66938- '\x105a3'# -> unI64 66940- '\x105a4'# -> unI64 66941- '\x105a5'# -> unI64 66942- '\x105a6'# -> unI64 66943- '\x105a7'# -> unI64 66944- '\x105a8'# -> unI64 66945- '\x105a9'# -> unI64 66946- '\x105aa'# -> unI64 66947- '\x105ab'# -> unI64 66948- '\x105ac'# -> unI64 66949- '\x105ad'# -> unI64 66950- '\x105ae'# -> unI64 66951- '\x105af'# -> unI64 66952- '\x105b0'# -> unI64 66953- '\x105b1'# -> unI64 66954- '\x105b3'# -> unI64 66956- '\x105b4'# -> unI64 66957- '\x105b5'# -> unI64 66958- '\x105b6'# -> unI64 66959- '\x105b7'# -> unI64 66960- '\x105b8'# -> unI64 66961- '\x105b9'# -> unI64 66962- '\x105bb'# -> unI64 66964- '\x105bc'# -> unI64 66965- '\x10cc0'# -> unI64 68736- '\x10cc1'# -> unI64 68737- '\x10cc2'# -> unI64 68738- '\x10cc3'# -> unI64 68739- '\x10cc4'# -> unI64 68740- '\x10cc5'# -> unI64 68741- '\x10cc6'# -> unI64 68742- '\x10cc7'# -> unI64 68743- '\x10cc8'# -> unI64 68744- '\x10cc9'# -> unI64 68745- '\x10cca'# -> unI64 68746- '\x10ccb'# -> unI64 68747- '\x10ccc'# -> unI64 68748- '\x10ccd'# -> unI64 68749- '\x10cce'# -> unI64 68750- '\x10ccf'# -> unI64 68751- '\x10cd0'# -> unI64 68752- '\x10cd1'# -> unI64 68753- '\x10cd2'# -> unI64 68754- '\x10cd3'# -> unI64 68755- '\x10cd4'# -> unI64 68756- '\x10cd5'# -> unI64 68757- '\x10cd6'# -> unI64 68758- '\x10cd7'# -> unI64 68759- '\x10cd8'# -> unI64 68760- '\x10cd9'# -> unI64 68761- '\x10cda'# -> unI64 68762- '\x10cdb'# -> unI64 68763- '\x10cdc'# -> unI64 68764- '\x10cdd'# -> unI64 68765- '\x10cde'# -> unI64 68766- '\x10cdf'# -> unI64 68767- '\x10ce0'# -> unI64 68768- '\x10ce1'# -> unI64 68769- '\x10ce2'# -> unI64 68770- '\x10ce3'# -> unI64 68771- '\x10ce4'# -> unI64 68772- '\x10ce5'# -> unI64 68773- '\x10ce6'# -> unI64 68774- '\x10ce7'# -> unI64 68775- '\x10ce8'# -> unI64 68776- '\x10ce9'# -> unI64 68777- '\x10cea'# -> unI64 68778- '\x10ceb'# -> unI64 68779- '\x10cec'# -> unI64 68780- '\x10ced'# -> unI64 68781- '\x10cee'# -> unI64 68782- '\x10cef'# -> unI64 68783- '\x10cf0'# -> unI64 68784- '\x10cf1'# -> unI64 68785- '\x10cf2'# -> unI64 68786- '\x118c0'# -> unI64 71840- '\x118c1'# -> unI64 71841- '\x118c2'# -> unI64 71842- '\x118c3'# -> unI64 71843- '\x118c4'# -> unI64 71844- '\x118c5'# -> unI64 71845- '\x118c6'# -> unI64 71846- '\x118c7'# -> unI64 71847- '\x118c8'# -> unI64 71848- '\x118c9'# -> unI64 71849- '\x118ca'# -> unI64 71850- '\x118cb'# -> unI64 71851- '\x118cc'# -> unI64 71852- '\x118cd'# -> unI64 71853- '\x118ce'# -> unI64 71854- '\x118cf'# -> unI64 71855- '\x118d0'# -> unI64 71856- '\x118d1'# -> unI64 71857- '\x118d2'# -> unI64 71858- '\x118d3'# -> unI64 71859- '\x118d4'# -> unI64 71860- '\x118d5'# -> unI64 71861- '\x118d6'# -> unI64 71862- '\x118d7'# -> unI64 71863- '\x118d8'# -> unI64 71864- '\x118d9'# -> unI64 71865- '\x118da'# -> unI64 71866- '\x118db'# -> unI64 71867- '\x118dc'# -> unI64 71868- '\x118dd'# -> unI64 71869- '\x118de'# -> unI64 71870- '\x118df'# -> unI64 71871- '\x16e60'# -> unI64 93760- '\x16e61'# -> unI64 93761- '\x16e62'# -> unI64 93762- '\x16e63'# -> unI64 93763- '\x16e64'# -> unI64 93764- '\x16e65'# -> unI64 93765- '\x16e66'# -> unI64 93766- '\x16e67'# -> unI64 93767- '\x16e68'# -> unI64 93768- '\x16e69'# -> unI64 93769- '\x16e6a'# -> unI64 93770- '\x16e6b'# -> unI64 93771- '\x16e6c'# -> unI64 93772- '\x16e6d'# -> unI64 93773- '\x16e6e'# -> unI64 93774- '\x16e6f'# -> unI64 93775- '\x16e70'# -> unI64 93776- '\x16e71'# -> unI64 93777- '\x16e72'# -> unI64 93778- '\x16e73'# -> unI64 93779- '\x16e74'# -> unI64 93780- '\x16e75'# -> unI64 93781- '\x16e76'# -> unI64 93782- '\x16e77'# -> unI64 93783- '\x16e78'# -> unI64 93784- '\x16e79'# -> unI64 93785- '\x16e7a'# -> unI64 93786- '\x16e7b'# -> unI64 93787- '\x16e7c'# -> unI64 93788- '\x16e7d'# -> unI64 93789- '\x16e7e'# -> unI64 93790- '\x16e7f'# -> unI64 93791- '\x1e922'# -> unI64 125184- '\x1e923'# -> unI64 125185- '\x1e924'# -> unI64 125186- '\x1e925'# -> unI64 125187- '\x1e926'# -> unI64 125188- '\x1e927'# -> unI64 125189- '\x1e928'# -> unI64 125190- '\x1e929'# -> unI64 125191- '\x1e92a'# -> unI64 125192- '\x1e92b'# -> unI64 125193- '\x1e92c'# -> unI64 125194- '\x1e92d'# -> unI64 125195- '\x1e92e'# -> unI64 125196- '\x1e92f'# -> unI64 125197- '\x1e930'# -> unI64 125198- '\x1e931'# -> unI64 125199- '\x1e932'# -> unI64 125200- '\x1e933'# -> unI64 125201- '\x1e934'# -> unI64 125202- '\x1e935'# -> unI64 125203- '\x1e936'# -> unI64 125204- '\x1e937'# -> unI64 125205- '\x1e938'# -> unI64 125206- '\x1e939'# -> unI64 125207- '\x1e93a'# -> unI64 125208- '\x1e93b'# -> unI64 125209- '\x1e93c'# -> unI64 125210- '\x1e93d'# -> unI64 125211- '\x1e93e'# -> unI64 125212- '\x1e93f'# -> unI64 125213- '\x1e940'# -> unI64 125214- '\x1e941'# -> unI64 125215- '\x1e942'# -> unI64 125216- '\x1e943'# -> unI64 125217- _ -> unI64 0-foldMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE foldMapping #-}-foldMapping = \case- -- MICRO SIGN- '\x00b5'# -> unI64 956- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 241172595- -- LATIN CAPITAL LETTER I WITH DOT ABOVE- '\x0130'# -> unI64 1625292905- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 230687420- -- LATIN SMALL LETTER LONG S- '\x017f'# -> unI64 115- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778666- -- COMBINING GREEK YPOGEGRAMMENI- '\x0345'# -> unI64 953- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429881- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429893- -- GREEK SMALL LETTER FINAL SIGMA- '\x03c2'# -> unI64 963- -- GREEK BETA SYMBOL- '\x03d0'# -> unI64 946- -- GREEK THETA SYMBOL- '\x03d1'# -> unI64 952- -- GREEK PHI SYMBOL- '\x03d5'# -> unI64 966- -- GREEK PI SYMBOL- '\x03d6'# -> unI64 960- -- GREEK KAPPA SYMBOL- '\x03f0'# -> unI64 954- -- GREEK RHO SYMBOL- '\x03f1'# -> unI64 961- -- GREEK LUNATE EPSILON SYMBOL- '\x03f5'# -> unI64 949- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2956985701- -- CHEROKEE SMALL LETTER YE- '\x13f8'# -> unI64 5104- -- CHEROKEE SMALL LETTER YI- '\x13f9'# -> unI64 5105- -- CHEROKEE SMALL LETTER YO- '\x13fa'# -> unI64 5106- -- CHEROKEE SMALL LETTER YU- '\x13fb'# -> unI64 5107- -- CHEROKEE SMALL LETTER YV- '\x13fc'# -> unI64 5108- -- CHEROKEE SMALL LETTER MV- '\x13fd'# -> unI64 5109- -- CYRILLIC SMALL LETTER ROUNDED VE- '\x1c80'# -> unI64 1074- -- CYRILLIC SMALL LETTER LONG-LEGGED DE- '\x1c81'# -> unI64 1076- -- CYRILLIC SMALL LETTER NARROW O- '\x1c82'# -> unI64 1086- -- CYRILLIC SMALL LETTER WIDE ES- '\x1c83'# -> unI64 1089- -- CYRILLIC SMALL LETTER TALL TE- '\x1c84'# -> unI64 1090- -- CYRILLIC SMALL LETTER THREE-LEGGED TE- '\x1c85'# -> unI64 1090- -- CYRILLIC SMALL LETTER TALL HARD SIGN- '\x1c86'# -> unI64 1098- -- CYRILLIC SMALL LETTER TALL YAT- '\x1c87'# -> unI64 1123- -- CYRILLIC SMALL LETTER UNBLENDED UK- '\x1c88'# -> unI64 42571- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373288- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390068- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584375- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584377- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200801- -- LATIN SMALL LETTER LONG S WITH DOT ABOVE- '\x1e9b'# -> unI64 7777- -- LATIN CAPITAL LETTER SHARP S- '\x1e9e'# -> unI64 241172595- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459589- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987461- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498565- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720325- -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI- '\x1f80'# -> unI64 1998593792- -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI- '\x1f81'# -> unI64 1998593793- -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f82'# -> unI64 1998593794- -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f83'# -> unI64 1998593795- -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f84'# -> unI64 1998593796- -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f85'# -> unI64 1998593797- -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f86'# -> unI64 1998593798- -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f87'# -> unI64 1998593799- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI- '\x1f88'# -> unI64 1998593792- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI- '\x1f89'# -> unI64 1998593793- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f8a'# -> unI64 1998593794- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f8b'# -> unI64 1998593795- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f8c'# -> unI64 1998593796- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f8d'# -> unI64 1998593797- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8e'# -> unI64 1998593798- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8f'# -> unI64 1998593799- -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI- '\x1f90'# -> unI64 1998593824- -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI- '\x1f91'# -> unI64 1998593825- -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f92'# -> unI64 1998593826- -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f93'# -> unI64 1998593827- -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f94'# -> unI64 1998593828- -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f95'# -> unI64 1998593829- -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f96'# -> unI64 1998593830- -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f97'# -> unI64 1998593831- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI- '\x1f98'# -> unI64 1998593824- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI- '\x1f99'# -> unI64 1998593825- -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f9a'# -> unI64 1998593826- -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f9b'# -> unI64 1998593827- -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f9c'# -> unI64 1998593828- -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f9d'# -> unI64 1998593829- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9e'# -> unI64 1998593830- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9f'# -> unI64 1998593831- -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI- '\x1fa0'# -> unI64 1998593888- -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI- '\x1fa1'# -> unI64 1998593889- -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1fa2'# -> unI64 1998593890- -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1fa3'# -> unI64 1998593891- -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1fa4'# -> unI64 1998593892- -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1fa5'# -> unI64 1998593893- -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa6'# -> unI64 1998593894- -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa7'# -> unI64 1998593895- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI- '\x1fa8'# -> unI64 1998593888- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI- '\x1fa9'# -> unI64 1998593889- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1faa'# -> unI64 1998593890- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1fab'# -> unI64 1998593891- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1fac'# -> unI64 1998593892- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1fad'# -> unI64 1998593893- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1fae'# -> unI64 1998593894- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1faf'# -> unI64 1998593895- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1998593904- -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI- '\x1fb3'# -> unI64 1998586801- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1998586796- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025713- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 4191340074107825- -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI- '\x1fbc'# -> unI64 1998586801- -- GREEK PROSGEGRAMMENI- '\x1fbe'# -> unI64 953- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1998593908- -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI- '\x1fc3'# -> unI64 1998586807- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1998586798- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025719- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 4191340074107831- -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI- '\x1fcc'# -> unI64 1998586807- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918777- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429881- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025721- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651641- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918789- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429893- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459585- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025733- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651653- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1998593916- -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI- '\x1ff3'# -> unI64 1998586825- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1998586830- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025737- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 4191340074107849- -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI- '\x1ffc'# -> unI64 1998586825- -- CHEROKEE SMALL LETTER A- '\xab70'# -> unI64 5024- -- CHEROKEE SMALL LETTER E- '\xab71'# -> unI64 5025- -- CHEROKEE SMALL LETTER I- '\xab72'# -> unI64 5026- -- CHEROKEE SMALL LETTER O- '\xab73'# -> unI64 5027- -- CHEROKEE SMALL LETTER U- '\xab74'# -> unI64 5028- -- CHEROKEE SMALL LETTER V- '\xab75'# -> unI64 5029- -- CHEROKEE SMALL LETTER GA- '\xab76'# -> unI64 5030- -- CHEROKEE SMALL LETTER KA- '\xab77'# -> unI64 5031- -- CHEROKEE SMALL LETTER GE- '\xab78'# -> unI64 5032- -- CHEROKEE SMALL LETTER GI- '\xab79'# -> unI64 5033- -- CHEROKEE SMALL LETTER GO- '\xab7a'# -> unI64 5034- -- CHEROKEE SMALL LETTER GU- '\xab7b'# -> unI64 5035- -- CHEROKEE SMALL LETTER GV- '\xab7c'# -> unI64 5036- -- CHEROKEE SMALL LETTER HA- '\xab7d'# -> unI64 5037- -- CHEROKEE SMALL LETTER HE- '\xab7e'# -> unI64 5038- -- CHEROKEE SMALL LETTER HI- '\xab7f'# -> unI64 5039- -- CHEROKEE SMALL LETTER HO- '\xab80'# -> unI64 5040- -- CHEROKEE SMALL LETTER HU- '\xab81'# -> unI64 5041- -- CHEROKEE SMALL LETTER HV- '\xab82'# -> unI64 5042- -- CHEROKEE SMALL LETTER LA- '\xab83'# -> unI64 5043- -- CHEROKEE SMALL LETTER LE- '\xab84'# -> unI64 5044- -- CHEROKEE SMALL LETTER LI- '\xab85'# -> unI64 5045- -- CHEROKEE SMALL LETTER LO- '\xab86'# -> unI64 5046- -- CHEROKEE SMALL LETTER LU- '\xab87'# -> unI64 5047- -- CHEROKEE SMALL LETTER LV- '\xab88'# -> unI64 5048- -- CHEROKEE SMALL LETTER MA- '\xab89'# -> unI64 5049- -- CHEROKEE SMALL LETTER ME- '\xab8a'# -> unI64 5050- -- CHEROKEE SMALL LETTER MI- '\xab8b'# -> unI64 5051- -- CHEROKEE SMALL LETTER MO- '\xab8c'# -> unI64 5052- -- CHEROKEE SMALL LETTER MU- '\xab8d'# -> unI64 5053- -- CHEROKEE SMALL LETTER NA- '\xab8e'# -> unI64 5054- -- CHEROKEE SMALL LETTER HNA- '\xab8f'# -> unI64 5055- -- CHEROKEE SMALL LETTER NAH- '\xab90'# -> unI64 5056- -- CHEROKEE SMALL LETTER NE- '\xab91'# -> unI64 5057- -- CHEROKEE SMALL LETTER NI- '\xab92'# -> unI64 5058- -- CHEROKEE SMALL LETTER NO- '\xab93'# -> unI64 5059- -- CHEROKEE SMALL LETTER NU- '\xab94'# -> unI64 5060- -- CHEROKEE SMALL LETTER NV- '\xab95'# -> unI64 5061- -- CHEROKEE SMALL LETTER QUA- '\xab96'# -> unI64 5062- -- CHEROKEE SMALL LETTER QUE- '\xab97'# -> unI64 5063- -- CHEROKEE SMALL LETTER QUI- '\xab98'# -> unI64 5064- -- CHEROKEE SMALL LETTER QUO- '\xab99'# -> unI64 5065- -- CHEROKEE SMALL LETTER QUU- '\xab9a'# -> unI64 5066- -- CHEROKEE SMALL LETTER QUV- '\xab9b'# -> unI64 5067- -- CHEROKEE SMALL LETTER SA- '\xab9c'# -> unI64 5068- -- CHEROKEE SMALL LETTER S- '\xab9d'# -> unI64 5069- -- CHEROKEE SMALL LETTER SE- '\xab9e'# -> unI64 5070- -- CHEROKEE SMALL LETTER SI- '\xab9f'# -> unI64 5071- -- CHEROKEE SMALL LETTER SO- '\xaba0'# -> unI64 5072- -- CHEROKEE SMALL LETTER SU- '\xaba1'# -> unI64 5073- -- CHEROKEE SMALL LETTER SV- '\xaba2'# -> unI64 5074- -- CHEROKEE SMALL LETTER DA- '\xaba3'# -> unI64 5075- -- CHEROKEE SMALL LETTER TA- '\xaba4'# -> unI64 5076- -- CHEROKEE SMALL LETTER DE- '\xaba5'# -> unI64 5077- -- CHEROKEE SMALL LETTER TE- '\xaba6'# -> unI64 5078- -- CHEROKEE SMALL LETTER DI- '\xaba7'# -> unI64 5079- -- CHEROKEE SMALL LETTER TI- '\xaba8'# -> unI64 5080- -- CHEROKEE SMALL LETTER DO- '\xaba9'# -> unI64 5081- -- CHEROKEE SMALL LETTER DU- '\xabaa'# -> unI64 5082- -- CHEROKEE SMALL LETTER DV- '\xabab'# -> unI64 5083- -- CHEROKEE SMALL LETTER DLA- '\xabac'# -> unI64 5084- -- CHEROKEE SMALL LETTER TLA- '\xabad'# -> unI64 5085- -- CHEROKEE SMALL LETTER TLE- '\xabae'# -> unI64 5086- -- CHEROKEE SMALL LETTER TLI- '\xabaf'# -> unI64 5087- -- CHEROKEE SMALL LETTER TLO- '\xabb0'# -> unI64 5088- -- CHEROKEE SMALL LETTER TLU- '\xabb1'# -> unI64 5089- -- CHEROKEE SMALL LETTER TLV- '\xabb2'# -> unI64 5090- -- CHEROKEE SMALL LETTER TSA- '\xabb3'# -> unI64 5091- -- CHEROKEE SMALL LETTER TSE- '\xabb4'# -> unI64 5092- -- CHEROKEE SMALL LETTER TSI- '\xabb5'# -> unI64 5093- -- CHEROKEE SMALL LETTER TSO- '\xabb6'# -> unI64 5094- -- CHEROKEE SMALL LETTER TSU- '\xabb7'# -> unI64 5095- -- CHEROKEE SMALL LETTER TSV- '\xabb8'# -> unI64 5096- -- CHEROKEE SMALL LETTER WA- '\xabb9'# -> unI64 5097- -- CHEROKEE SMALL LETTER WE- '\xabba'# -> unI64 5098- -- CHEROKEE SMALL LETTER WI- '\xabbb'# -> unI64 5099- -- CHEROKEE SMALL LETTER WO- '\xabbc'# -> unI64 5100- -- CHEROKEE SMALL LETTER WU- '\xabbd'# -> unI64 5101- -- CHEROKEE SMALL LETTER WV- '\xabbe'# -> unI64 5102- -- CHEROKEE SMALL LETTER YA- '\xabbf'# -> unI64 5103- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 213909606- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 220201062- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 226492518- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 461795097575526- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 474989237108838- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 243269747- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 243269747- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2931819892- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2896168308- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2908751220- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2931819902- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2912945524- '\x0041'# -> unI64 97- '\x0042'# -> unI64 98- '\x0043'# -> unI64 99- '\x0044'# -> unI64 100- '\x0045'# -> unI64 101- '\x0046'# -> unI64 102- '\x0047'# -> unI64 103- '\x0048'# -> unI64 104- '\x0049'# -> unI64 105- '\x004a'# -> unI64 106- '\x004b'# -> unI64 107- '\x004c'# -> unI64 108- '\x004d'# -> unI64 109- '\x004e'# -> unI64 110- '\x004f'# -> unI64 111- '\x0050'# -> unI64 112- '\x0051'# -> unI64 113- '\x0052'# -> unI64 114- '\x0053'# -> unI64 115- '\x0054'# -> unI64 116- '\x0055'# -> unI64 117- '\x0056'# -> unI64 118- '\x0057'# -> unI64 119- '\x0058'# -> unI64 120- '\x0059'# -> unI64 121- '\x005a'# -> unI64 122- '\x00c0'# -> unI64 224- '\x00c1'# -> unI64 225- '\x00c2'# -> unI64 226- '\x00c3'# -> unI64 227- '\x00c4'# -> unI64 228- '\x00c5'# -> unI64 229- '\x00c6'# -> unI64 230- '\x00c7'# -> unI64 231- '\x00c8'# -> unI64 232- '\x00c9'# -> unI64 233- '\x00ca'# -> unI64 234- '\x00cb'# -> unI64 235- '\x00cc'# -> unI64 236- '\x00cd'# -> unI64 237- '\x00ce'# -> unI64 238- '\x00cf'# -> unI64 239- '\x00d0'# -> unI64 240- '\x00d1'# -> unI64 241- '\x00d2'# -> unI64 242- '\x00d3'# -> unI64 243- '\x00d4'# -> unI64 244- '\x00d5'# -> unI64 245- '\x00d6'# -> unI64 246- '\x00d8'# -> unI64 248- '\x00d9'# -> unI64 249- '\x00da'# -> unI64 250- '\x00db'# -> unI64 251- '\x00dc'# -> unI64 252- '\x00dd'# -> unI64 253- '\x00de'# -> unI64 254- '\x0100'# -> unI64 257- '\x0102'# -> unI64 259- '\x0104'# -> unI64 261- '\x0106'# -> unI64 263- '\x0108'# -> unI64 265- '\x010a'# -> unI64 267- '\x010c'# -> unI64 269- '\x010e'# -> unI64 271- '\x0110'# -> unI64 273- '\x0112'# -> unI64 275- '\x0114'# -> unI64 277- '\x0116'# -> unI64 279- '\x0118'# -> unI64 281- '\x011a'# -> unI64 283- '\x011c'# -> unI64 285- '\x011e'# -> unI64 287- '\x0120'# -> unI64 289- '\x0122'# -> unI64 291- '\x0124'# -> unI64 293- '\x0126'# -> unI64 295- '\x0128'# -> unI64 297- '\x012a'# -> unI64 299- '\x012c'# -> unI64 301- '\x012e'# -> unI64 303- '\x0132'# -> unI64 307- '\x0134'# -> unI64 309- '\x0136'# -> unI64 311- '\x0139'# -> unI64 314- '\x013b'# -> unI64 316- '\x013d'# -> unI64 318- '\x013f'# -> unI64 320- '\x0141'# -> unI64 322- '\x0143'# -> unI64 324- '\x0145'# -> unI64 326- '\x0147'# -> unI64 328- '\x014a'# -> unI64 331- '\x014c'# -> unI64 333- '\x014e'# -> unI64 335- '\x0150'# -> unI64 337- '\x0152'# -> unI64 339- '\x0154'# -> unI64 341- '\x0156'# -> unI64 343- '\x0158'# -> unI64 345- '\x015a'# -> unI64 347- '\x015c'# -> unI64 349- '\x015e'# -> unI64 351- '\x0160'# -> unI64 353- '\x0162'# -> unI64 355- '\x0164'# -> unI64 357- '\x0166'# -> unI64 359- '\x0168'# -> unI64 361- '\x016a'# -> unI64 363- '\x016c'# -> unI64 365- '\x016e'# -> unI64 367- '\x0170'# -> unI64 369- '\x0172'# -> unI64 371- '\x0174'# -> unI64 373- '\x0176'# -> unI64 375- '\x0178'# -> unI64 255- '\x0179'# -> unI64 378- '\x017b'# -> unI64 380- '\x017d'# -> unI64 382- '\x0181'# -> unI64 595- '\x0182'# -> unI64 387- '\x0184'# -> unI64 389- '\x0186'# -> unI64 596- '\x0187'# -> unI64 392- '\x0189'# -> unI64 598- '\x018a'# -> unI64 599- '\x018b'# -> unI64 396- '\x018e'# -> unI64 477- '\x018f'# -> unI64 601- '\x0190'# -> unI64 603- '\x0191'# -> unI64 402- '\x0193'# -> unI64 608- '\x0194'# -> unI64 611- '\x0196'# -> unI64 617- '\x0197'# -> unI64 616- '\x0198'# -> unI64 409- '\x019c'# -> unI64 623- '\x019d'# -> unI64 626- '\x019f'# -> unI64 629- '\x01a0'# -> unI64 417- '\x01a2'# -> unI64 419- '\x01a4'# -> unI64 421- '\x01a6'# -> unI64 640- '\x01a7'# -> unI64 424- '\x01a9'# -> unI64 643- '\x01ac'# -> unI64 429- '\x01ae'# -> unI64 648- '\x01af'# -> unI64 432- '\x01b1'# -> unI64 650- '\x01b2'# -> unI64 651- '\x01b3'# -> unI64 436- '\x01b5'# -> unI64 438- '\x01b7'# -> unI64 658- '\x01b8'# -> unI64 441- '\x01bc'# -> unI64 445- '\x01c4'# -> unI64 454- '\x01c5'# -> unI64 454- '\x01c7'# -> unI64 457- '\x01c8'# -> unI64 457- '\x01ca'# -> unI64 460- '\x01cb'# -> unI64 460- '\x01cd'# -> unI64 462- '\x01cf'# -> unI64 464- '\x01d1'# -> unI64 466- '\x01d3'# -> unI64 468- '\x01d5'# -> unI64 470- '\x01d7'# -> unI64 472- '\x01d9'# -> unI64 474- '\x01db'# -> unI64 476- '\x01de'# -> unI64 479- '\x01e0'# -> unI64 481- '\x01e2'# -> unI64 483- '\x01e4'# -> unI64 485- '\x01e6'# -> unI64 487- '\x01e8'# -> unI64 489- '\x01ea'# -> unI64 491- '\x01ec'# -> unI64 493- '\x01ee'# -> unI64 495- '\x01f1'# -> unI64 499- '\x01f2'# -> unI64 499- '\x01f4'# -> unI64 501- '\x01f6'# -> unI64 405- '\x01f7'# -> unI64 447- '\x01f8'# -> unI64 505- '\x01fa'# -> unI64 507- '\x01fc'# -> unI64 509- '\x01fe'# -> unI64 511- '\x0200'# -> unI64 513- '\x0202'# -> unI64 515- '\x0204'# -> unI64 517- '\x0206'# -> unI64 519- '\x0208'# -> unI64 521- '\x020a'# -> unI64 523- '\x020c'# -> unI64 525- '\x020e'# -> unI64 527- '\x0210'# -> unI64 529- '\x0212'# -> unI64 531- '\x0214'# -> unI64 533- '\x0216'# -> unI64 535- '\x0218'# -> unI64 537- '\x021a'# -> unI64 539- '\x021c'# -> unI64 541- '\x021e'# -> unI64 543- '\x0220'# -> unI64 414- '\x0222'# -> unI64 547- '\x0224'# -> unI64 549- '\x0226'# -> unI64 551- '\x0228'# -> unI64 553- '\x022a'# -> unI64 555- '\x022c'# -> unI64 557- '\x022e'# -> unI64 559- '\x0230'# -> unI64 561- '\x0232'# -> unI64 563- '\x023a'# -> unI64 11365- '\x023b'# -> unI64 572- '\x023d'# -> unI64 410- '\x023e'# -> unI64 11366- '\x0241'# -> unI64 578- '\x0243'# -> unI64 384- '\x0244'# -> unI64 649- '\x0245'# -> unI64 652- '\x0246'# -> unI64 583- '\x0248'# -> unI64 585- '\x024a'# -> unI64 587- '\x024c'# -> unI64 589- '\x024e'# -> unI64 591- '\x0370'# -> unI64 881- '\x0372'# -> unI64 883- '\x0376'# -> unI64 887- '\x037f'# -> unI64 1011- '\x0386'# -> unI64 940- '\x0388'# -> unI64 941- '\x0389'# -> unI64 942- '\x038a'# -> unI64 943- '\x038c'# -> unI64 972- '\x038e'# -> unI64 973- '\x038f'# -> unI64 974- '\x0391'# -> unI64 945- '\x0392'# -> unI64 946- '\x0393'# -> unI64 947- '\x0394'# -> unI64 948- '\x0395'# -> unI64 949- '\x0396'# -> unI64 950- '\x0397'# -> unI64 951- '\x0398'# -> unI64 952- '\x0399'# -> unI64 953- '\x039a'# -> unI64 954- '\x039b'# -> unI64 955- '\x039c'# -> unI64 956- '\x039d'# -> unI64 957- '\x039e'# -> unI64 958- '\x039f'# -> unI64 959- '\x03a0'# -> unI64 960- '\x03a1'# -> unI64 961- '\x03a3'# -> unI64 963- '\x03a4'# -> unI64 964- '\x03a5'# -> unI64 965- '\x03a6'# -> unI64 966- '\x03a7'# -> unI64 967- '\x03a8'# -> unI64 968- '\x03a9'# -> unI64 969- '\x03aa'# -> unI64 970- '\x03ab'# -> unI64 971- '\x03cf'# -> unI64 983- '\x03d8'# -> unI64 985- '\x03da'# -> unI64 987- '\x03dc'# -> unI64 989- '\x03de'# -> unI64 991- '\x03e0'# -> unI64 993- '\x03e2'# -> unI64 995- '\x03e4'# -> unI64 997- '\x03e6'# -> unI64 999- '\x03e8'# -> unI64 1001- '\x03ea'# -> unI64 1003- '\x03ec'# -> unI64 1005- '\x03ee'# -> unI64 1007- '\x03f4'# -> unI64 952- '\x03f7'# -> unI64 1016- '\x03f9'# -> unI64 1010- '\x03fa'# -> unI64 1019- '\x03fd'# -> unI64 891- '\x03fe'# -> unI64 892- '\x03ff'# -> unI64 893- '\x0400'# -> unI64 1104- '\x0401'# -> unI64 1105- '\x0402'# -> unI64 1106- '\x0403'# -> unI64 1107- '\x0404'# -> unI64 1108- '\x0405'# -> unI64 1109- '\x0406'# -> unI64 1110- '\x0407'# -> unI64 1111- '\x0408'# -> unI64 1112- '\x0409'# -> unI64 1113- '\x040a'# -> unI64 1114- '\x040b'# -> unI64 1115- '\x040c'# -> unI64 1116- '\x040d'# -> unI64 1117- '\x040e'# -> unI64 1118- '\x040f'# -> unI64 1119- '\x0410'# -> unI64 1072- '\x0411'# -> unI64 1073- '\x0412'# -> unI64 1074- '\x0413'# -> unI64 1075- '\x0414'# -> unI64 1076- '\x0415'# -> unI64 1077- '\x0416'# -> unI64 1078- '\x0417'# -> unI64 1079- '\x0418'# -> unI64 1080- '\x0419'# -> unI64 1081- '\x041a'# -> unI64 1082- '\x041b'# -> unI64 1083- '\x041c'# -> unI64 1084- '\x041d'# -> unI64 1085- '\x041e'# -> unI64 1086- '\x041f'# -> unI64 1087- '\x0420'# -> unI64 1088- '\x0421'# -> unI64 1089- '\x0422'# -> unI64 1090- '\x0423'# -> unI64 1091- '\x0424'# -> unI64 1092- '\x0425'# -> unI64 1093- '\x0426'# -> unI64 1094- '\x0427'# -> unI64 1095- '\x0428'# -> unI64 1096- '\x0429'# -> unI64 1097- '\x042a'# -> unI64 1098- '\x042b'# -> unI64 1099- '\x042c'# -> unI64 1100- '\x042d'# -> unI64 1101- '\x042e'# -> unI64 1102- '\x042f'# -> unI64 1103- '\x0460'# -> unI64 1121- '\x0462'# -> unI64 1123- '\x0464'# -> unI64 1125- '\x0466'# -> unI64 1127- '\x0468'# -> unI64 1129- '\x046a'# -> unI64 1131- '\x046c'# -> unI64 1133- '\x046e'# -> unI64 1135- '\x0470'# -> unI64 1137- '\x0472'# -> unI64 1139- '\x0474'# -> unI64 1141- '\x0476'# -> unI64 1143- '\x0478'# -> unI64 1145- '\x047a'# -> unI64 1147- '\x047c'# -> unI64 1149- '\x047e'# -> unI64 1151- '\x0480'# -> unI64 1153- '\x048a'# -> unI64 1163- '\x048c'# -> unI64 1165- '\x048e'# -> unI64 1167- '\x0490'# -> unI64 1169- '\x0492'# -> unI64 1171- '\x0494'# -> unI64 1173- '\x0496'# -> unI64 1175- '\x0498'# -> unI64 1177- '\x049a'# -> unI64 1179- '\x049c'# -> unI64 1181- '\x049e'# -> unI64 1183- '\x04a0'# -> unI64 1185- '\x04a2'# -> unI64 1187- '\x04a4'# -> unI64 1189- '\x04a6'# -> unI64 1191- '\x04a8'# -> unI64 1193- '\x04aa'# -> unI64 1195- '\x04ac'# -> unI64 1197- '\x04ae'# -> unI64 1199- '\x04b0'# -> unI64 1201- '\x04b2'# -> unI64 1203- '\x04b4'# -> unI64 1205- '\x04b6'# -> unI64 1207- '\x04b8'# -> unI64 1209- '\x04ba'# -> unI64 1211- '\x04bc'# -> unI64 1213- '\x04be'# -> unI64 1215- '\x04c0'# -> unI64 1231- '\x04c1'# -> unI64 1218- '\x04c3'# -> unI64 1220- '\x04c5'# -> unI64 1222- '\x04c7'# -> unI64 1224- '\x04c9'# -> unI64 1226- '\x04cb'# -> unI64 1228- '\x04cd'# -> unI64 1230- '\x04d0'# -> unI64 1233- '\x04d2'# -> unI64 1235- '\x04d4'# -> unI64 1237- '\x04d6'# -> unI64 1239- '\x04d8'# -> unI64 1241- '\x04da'# -> unI64 1243- '\x04dc'# -> unI64 1245- '\x04de'# -> unI64 1247- '\x04e0'# -> unI64 1249- '\x04e2'# -> unI64 1251- '\x04e4'# -> unI64 1253- '\x04e6'# -> unI64 1255- '\x04e8'# -> unI64 1257- '\x04ea'# -> unI64 1259- '\x04ec'# -> unI64 1261- '\x04ee'# -> unI64 1263- '\x04f0'# -> unI64 1265- '\x04f2'# -> unI64 1267- '\x04f4'# -> unI64 1269- '\x04f6'# -> unI64 1271- '\x04f8'# -> unI64 1273- '\x04fa'# -> unI64 1275- '\x04fc'# -> unI64 1277- '\x04fe'# -> unI64 1279- '\x0500'# -> unI64 1281- '\x0502'# -> unI64 1283- '\x0504'# -> unI64 1285- '\x0506'# -> unI64 1287- '\x0508'# -> unI64 1289- '\x050a'# -> unI64 1291- '\x050c'# -> unI64 1293- '\x050e'# -> unI64 1295- '\x0510'# -> unI64 1297- '\x0512'# -> unI64 1299- '\x0514'# -> unI64 1301- '\x0516'# -> unI64 1303- '\x0518'# -> unI64 1305- '\x051a'# -> unI64 1307- '\x051c'# -> unI64 1309- '\x051e'# -> unI64 1311- '\x0520'# -> unI64 1313- '\x0522'# -> unI64 1315- '\x0524'# -> unI64 1317- '\x0526'# -> unI64 1319- '\x0528'# -> unI64 1321- '\x052a'# -> unI64 1323- '\x052c'# -> unI64 1325- '\x052e'# -> unI64 1327- '\x0531'# -> unI64 1377- '\x0532'# -> unI64 1378- '\x0533'# -> unI64 1379- '\x0534'# -> unI64 1380- '\x0535'# -> unI64 1381- '\x0536'# -> unI64 1382- '\x0537'# -> unI64 1383- '\x0538'# -> unI64 1384- '\x0539'# -> unI64 1385- '\x053a'# -> unI64 1386- '\x053b'# -> unI64 1387- '\x053c'# -> unI64 1388- '\x053d'# -> unI64 1389- '\x053e'# -> unI64 1390- '\x053f'# -> unI64 1391- '\x0540'# -> unI64 1392- '\x0541'# -> unI64 1393- '\x0542'# -> unI64 1394- '\x0543'# -> unI64 1395- '\x0544'# -> unI64 1396- '\x0545'# -> unI64 1397- '\x0546'# -> unI64 1398- '\x0547'# -> unI64 1399- '\x0548'# -> unI64 1400- '\x0549'# -> unI64 1401- '\x054a'# -> unI64 1402- '\x054b'# -> unI64 1403- '\x054c'# -> unI64 1404- '\x054d'# -> unI64 1405- '\x054e'# -> unI64 1406- '\x054f'# -> unI64 1407- '\x0550'# -> unI64 1408- '\x0551'# -> unI64 1409- '\x0552'# -> unI64 1410- '\x0553'# -> unI64 1411- '\x0554'# -> unI64 1412- '\x0555'# -> unI64 1413- '\x0556'# -> unI64 1414- '\x10a0'# -> unI64 11520- '\x10a1'# -> unI64 11521- '\x10a2'# -> unI64 11522- '\x10a3'# -> unI64 11523- '\x10a4'# -> unI64 11524- '\x10a5'# -> unI64 11525- '\x10a6'# -> unI64 11526- '\x10a7'# -> unI64 11527- '\x10a8'# -> unI64 11528- '\x10a9'# -> unI64 11529- '\x10aa'# -> unI64 11530- '\x10ab'# -> unI64 11531- '\x10ac'# -> unI64 11532- '\x10ad'# -> unI64 11533- '\x10ae'# -> unI64 11534- '\x10af'# -> unI64 11535- '\x10b0'# -> unI64 11536- '\x10b1'# -> unI64 11537- '\x10b2'# -> unI64 11538- '\x10b3'# -> unI64 11539- '\x10b4'# -> unI64 11540- '\x10b5'# -> unI64 11541- '\x10b6'# -> unI64 11542- '\x10b7'# -> unI64 11543- '\x10b8'# -> unI64 11544- '\x10b9'# -> unI64 11545- '\x10ba'# -> unI64 11546- '\x10bb'# -> unI64 11547- '\x10bc'# -> unI64 11548- '\x10bd'# -> unI64 11549- '\x10be'# -> unI64 11550- '\x10bf'# -> unI64 11551- '\x10c0'# -> unI64 11552- '\x10c1'# -> unI64 11553- '\x10c2'# -> unI64 11554- '\x10c3'# -> unI64 11555- '\x10c4'# -> unI64 11556- '\x10c5'# -> unI64 11557- '\x10c7'# -> unI64 11559- '\x10cd'# -> unI64 11565- '\x13a0'# -> unI64 43888- '\x13a1'# -> unI64 43889- '\x13a2'# -> unI64 43890- '\x13a3'# -> unI64 43891- '\x13a4'# -> unI64 43892- '\x13a5'# -> unI64 43893- '\x13a6'# -> unI64 43894- '\x13a7'# -> unI64 43895- '\x13a8'# -> unI64 43896- '\x13a9'# -> unI64 43897- '\x13aa'# -> unI64 43898- '\x13ab'# -> unI64 43899- '\x13ac'# -> unI64 43900- '\x13ad'# -> unI64 43901- '\x13ae'# -> unI64 43902- '\x13af'# -> unI64 43903- '\x13b0'# -> unI64 43904- '\x13b1'# -> unI64 43905- '\x13b2'# -> unI64 43906- '\x13b3'# -> unI64 43907- '\x13b4'# -> unI64 43908- '\x13b5'# -> unI64 43909- '\x13b6'# -> unI64 43910- '\x13b7'# -> unI64 43911- '\x13b8'# -> unI64 43912- '\x13b9'# -> unI64 43913- '\x13ba'# -> unI64 43914- '\x13bb'# -> unI64 43915- '\x13bc'# -> unI64 43916- '\x13bd'# -> unI64 43917- '\x13be'# -> unI64 43918- '\x13bf'# -> unI64 43919- '\x13c0'# -> unI64 43920- '\x13c1'# -> unI64 43921- '\x13c2'# -> unI64 43922- '\x13c3'# -> unI64 43923- '\x13c4'# -> unI64 43924- '\x13c5'# -> unI64 43925- '\x13c6'# -> unI64 43926- '\x13c7'# -> unI64 43927- '\x13c8'# -> unI64 43928- '\x13c9'# -> unI64 43929- '\x13ca'# -> unI64 43930- '\x13cb'# -> unI64 43931- '\x13cc'# -> unI64 43932- '\x13cd'# -> unI64 43933- '\x13ce'# -> unI64 43934- '\x13cf'# -> unI64 43935- '\x13d0'# -> unI64 43936- '\x13d1'# -> unI64 43937- '\x13d2'# -> unI64 43938- '\x13d3'# -> unI64 43939- '\x13d4'# -> unI64 43940- '\x13d5'# -> unI64 43941- '\x13d6'# -> unI64 43942- '\x13d7'# -> unI64 43943- '\x13d8'# -> unI64 43944- '\x13d9'# -> unI64 43945- '\x13da'# -> unI64 43946- '\x13db'# -> unI64 43947- '\x13dc'# -> unI64 43948- '\x13dd'# -> unI64 43949- '\x13de'# -> unI64 43950- '\x13df'# -> unI64 43951- '\x13e0'# -> unI64 43952- '\x13e1'# -> unI64 43953- '\x13e2'# -> unI64 43954- '\x13e3'# -> unI64 43955- '\x13e4'# -> unI64 43956- '\x13e5'# -> unI64 43957- '\x13e6'# -> unI64 43958- '\x13e7'# -> unI64 43959- '\x13e8'# -> unI64 43960- '\x13e9'# -> unI64 43961- '\x13ea'# -> unI64 43962- '\x13eb'# -> unI64 43963- '\x13ec'# -> unI64 43964- '\x13ed'# -> unI64 43965- '\x13ee'# -> unI64 43966- '\x13ef'# -> unI64 43967- '\x13f0'# -> unI64 5112- '\x13f1'# -> unI64 5113- '\x13f2'# -> unI64 5114- '\x13f3'# -> unI64 5115- '\x13f4'# -> unI64 5116- '\x13f5'# -> unI64 5117- '\x1c90'# -> unI64 4304- '\x1c91'# -> unI64 4305- '\x1c92'# -> unI64 4306- '\x1c93'# -> unI64 4307- '\x1c94'# -> unI64 4308- '\x1c95'# -> unI64 4309- '\x1c96'# -> unI64 4310- '\x1c97'# -> unI64 4311- '\x1c98'# -> unI64 4312- '\x1c99'# -> unI64 4313- '\x1c9a'# -> unI64 4314- '\x1c9b'# -> unI64 4315- '\x1c9c'# -> unI64 4316- '\x1c9d'# -> unI64 4317- '\x1c9e'# -> unI64 4318- '\x1c9f'# -> unI64 4319- '\x1ca0'# -> unI64 4320- '\x1ca1'# -> unI64 4321- '\x1ca2'# -> unI64 4322- '\x1ca3'# -> unI64 4323- '\x1ca4'# -> unI64 4324- '\x1ca5'# -> unI64 4325- '\x1ca6'# -> unI64 4326- '\x1ca7'# -> unI64 4327- '\x1ca8'# -> unI64 4328- '\x1ca9'# -> unI64 4329- '\x1caa'# -> unI64 4330- '\x1cab'# -> unI64 4331- '\x1cac'# -> unI64 4332- '\x1cad'# -> unI64 4333- '\x1cae'# -> unI64 4334- '\x1caf'# -> unI64 4335- '\x1cb0'# -> unI64 4336- '\x1cb1'# -> unI64 4337- '\x1cb2'# -> unI64 4338- '\x1cb3'# -> unI64 4339- '\x1cb4'# -> unI64 4340- '\x1cb5'# -> unI64 4341- '\x1cb6'# -> unI64 4342- '\x1cb7'# -> unI64 4343- '\x1cb8'# -> unI64 4344- '\x1cb9'# -> unI64 4345- '\x1cba'# -> unI64 4346- '\x1cbd'# -> unI64 4349- '\x1cbe'# -> unI64 4350- '\x1cbf'# -> unI64 4351- '\x1e00'# -> unI64 7681- '\x1e02'# -> unI64 7683- '\x1e04'# -> unI64 7685- '\x1e06'# -> unI64 7687- '\x1e08'# -> unI64 7689- '\x1e0a'# -> unI64 7691- '\x1e0c'# -> unI64 7693- '\x1e0e'# -> unI64 7695- '\x1e10'# -> unI64 7697- '\x1e12'# -> unI64 7699- '\x1e14'# -> unI64 7701- '\x1e16'# -> unI64 7703- '\x1e18'# -> unI64 7705- '\x1e1a'# -> unI64 7707- '\x1e1c'# -> unI64 7709- '\x1e1e'# -> unI64 7711- '\x1e20'# -> unI64 7713- '\x1e22'# -> unI64 7715- '\x1e24'# -> unI64 7717- '\x1e26'# -> unI64 7719- '\x1e28'# -> unI64 7721- '\x1e2a'# -> unI64 7723- '\x1e2c'# -> unI64 7725- '\x1e2e'# -> unI64 7727- '\x1e30'# -> unI64 7729- '\x1e32'# -> unI64 7731- '\x1e34'# -> unI64 7733- '\x1e36'# -> unI64 7735- '\x1e38'# -> unI64 7737- '\x1e3a'# -> unI64 7739- '\x1e3c'# -> unI64 7741- '\x1e3e'# -> unI64 7743- '\x1e40'# -> unI64 7745- '\x1e42'# -> unI64 7747- '\x1e44'# -> unI64 7749- '\x1e46'# -> unI64 7751- '\x1e48'# -> unI64 7753- '\x1e4a'# -> unI64 7755- '\x1e4c'# -> unI64 7757- '\x1e4e'# -> unI64 7759- '\x1e50'# -> unI64 7761- '\x1e52'# -> unI64 7763- '\x1e54'# -> unI64 7765- '\x1e56'# -> unI64 7767- '\x1e58'# -> unI64 7769- '\x1e5a'# -> unI64 7771- '\x1e5c'# -> unI64 7773- '\x1e5e'# -> unI64 7775- '\x1e60'# -> unI64 7777- '\x1e62'# -> unI64 7779- '\x1e64'# -> unI64 7781- '\x1e66'# -> unI64 7783- '\x1e68'# -> unI64 7785- '\x1e6a'# -> unI64 7787- '\x1e6c'# -> unI64 7789- '\x1e6e'# -> unI64 7791- '\x1e70'# -> unI64 7793- '\x1e72'# -> unI64 7795- '\x1e74'# -> unI64 7797- '\x1e76'# -> unI64 7799- '\x1e78'# -> unI64 7801- '\x1e7a'# -> unI64 7803- '\x1e7c'# -> unI64 7805- '\x1e7e'# -> unI64 7807- '\x1e80'# -> unI64 7809- '\x1e82'# -> unI64 7811- '\x1e84'# -> unI64 7813- '\x1e86'# -> unI64 7815- '\x1e88'# -> unI64 7817- '\x1e8a'# -> unI64 7819- '\x1e8c'# -> unI64 7821- '\x1e8e'# -> unI64 7823- '\x1e90'# -> unI64 7825- '\x1e92'# -> unI64 7827- '\x1e94'# -> unI64 7829- '\x1ea0'# -> unI64 7841- '\x1ea2'# -> unI64 7843- '\x1ea4'# -> unI64 7845- '\x1ea6'# -> unI64 7847- '\x1ea8'# -> unI64 7849- '\x1eaa'# -> unI64 7851- '\x1eac'# -> unI64 7853- '\x1eae'# -> unI64 7855- '\x1eb0'# -> unI64 7857- '\x1eb2'# -> unI64 7859- '\x1eb4'# -> unI64 7861- '\x1eb6'# -> unI64 7863- '\x1eb8'# -> unI64 7865- '\x1eba'# -> unI64 7867- '\x1ebc'# -> unI64 7869- '\x1ebe'# -> unI64 7871- '\x1ec0'# -> unI64 7873- '\x1ec2'# -> unI64 7875- '\x1ec4'# -> unI64 7877- '\x1ec6'# -> unI64 7879- '\x1ec8'# -> unI64 7881- '\x1eca'# -> unI64 7883- '\x1ecc'# -> unI64 7885- '\x1ece'# -> unI64 7887- '\x1ed0'# -> unI64 7889- '\x1ed2'# -> unI64 7891- '\x1ed4'# -> unI64 7893- '\x1ed6'# -> unI64 7895- '\x1ed8'# -> unI64 7897- '\x1eda'# -> unI64 7899- '\x1edc'# -> unI64 7901- '\x1ede'# -> unI64 7903- '\x1ee0'# -> unI64 7905- '\x1ee2'# -> unI64 7907- '\x1ee4'# -> unI64 7909- '\x1ee6'# -> unI64 7911- '\x1ee8'# -> unI64 7913- '\x1eea'# -> unI64 7915- '\x1eec'# -> unI64 7917- '\x1eee'# -> unI64 7919- '\x1ef0'# -> unI64 7921- '\x1ef2'# -> unI64 7923- '\x1ef4'# -> unI64 7925- '\x1ef6'# -> unI64 7927- '\x1ef8'# -> unI64 7929- '\x1efa'# -> unI64 7931- '\x1efc'# -> unI64 7933- '\x1efe'# -> unI64 7935- '\x1f08'# -> unI64 7936- '\x1f09'# -> unI64 7937- '\x1f0a'# -> unI64 7938- '\x1f0b'# -> unI64 7939- '\x1f0c'# -> unI64 7940- '\x1f0d'# -> unI64 7941- '\x1f0e'# -> unI64 7942- '\x1f0f'# -> unI64 7943- '\x1f18'# -> unI64 7952- '\x1f19'# -> unI64 7953- '\x1f1a'# -> unI64 7954- '\x1f1b'# -> unI64 7955- '\x1f1c'# -> unI64 7956- '\x1f1d'# -> unI64 7957- '\x1f28'# -> unI64 7968- '\x1f29'# -> unI64 7969- '\x1f2a'# -> unI64 7970- '\x1f2b'# -> unI64 7971- '\x1f2c'# -> unI64 7972- '\x1f2d'# -> unI64 7973- '\x1f2e'# -> unI64 7974- '\x1f2f'# -> unI64 7975- '\x1f38'# -> unI64 7984- '\x1f39'# -> unI64 7985- '\x1f3a'# -> unI64 7986- '\x1f3b'# -> unI64 7987- '\x1f3c'# -> unI64 7988- '\x1f3d'# -> unI64 7989- '\x1f3e'# -> unI64 7990- '\x1f3f'# -> unI64 7991- '\x1f48'# -> unI64 8000- '\x1f49'# -> unI64 8001- '\x1f4a'# -> unI64 8002- '\x1f4b'# -> unI64 8003- '\x1f4c'# -> unI64 8004- '\x1f4d'# -> unI64 8005- '\x1f59'# -> unI64 8017- '\x1f5b'# -> unI64 8019- '\x1f5d'# -> unI64 8021- '\x1f5f'# -> unI64 8023- '\x1f68'# -> unI64 8032- '\x1f69'# -> unI64 8033- '\x1f6a'# -> unI64 8034- '\x1f6b'# -> unI64 8035- '\x1f6c'# -> unI64 8036- '\x1f6d'# -> unI64 8037- '\x1f6e'# -> unI64 8038- '\x1f6f'# -> unI64 8039- '\x1fb8'# -> unI64 8112- '\x1fb9'# -> unI64 8113- '\x1fba'# -> unI64 8048- '\x1fbb'# -> unI64 8049- '\x1fc8'# -> unI64 8050- '\x1fc9'# -> unI64 8051- '\x1fca'# -> unI64 8052- '\x1fcb'# -> unI64 8053- '\x1fd8'# -> unI64 8144- '\x1fd9'# -> unI64 8145- '\x1fda'# -> unI64 8054- '\x1fdb'# -> unI64 8055- '\x1fe8'# -> unI64 8160- '\x1fe9'# -> unI64 8161- '\x1fea'# -> unI64 8058- '\x1feb'# -> unI64 8059- '\x1fec'# -> unI64 8165- '\x1ff8'# -> unI64 8056- '\x1ff9'# -> unI64 8057- '\x1ffa'# -> unI64 8060- '\x1ffb'# -> unI64 8061- '\x2126'# -> unI64 969- '\x212a'# -> unI64 107- '\x212b'# -> unI64 229- '\x2132'# -> unI64 8526- '\x2160'# -> unI64 8560- '\x2161'# -> unI64 8561- '\x2162'# -> unI64 8562- '\x2163'# -> unI64 8563- '\x2164'# -> unI64 8564- '\x2165'# -> unI64 8565- '\x2166'# -> unI64 8566- '\x2167'# -> unI64 8567- '\x2168'# -> unI64 8568- '\x2169'# -> unI64 8569- '\x216a'# -> unI64 8570- '\x216b'# -> unI64 8571- '\x216c'# -> unI64 8572- '\x216d'# -> unI64 8573- '\x216e'# -> unI64 8574- '\x216f'# -> unI64 8575- '\x2183'# -> unI64 8580- '\x24b6'# -> unI64 9424- '\x24b7'# -> unI64 9425- '\x24b8'# -> unI64 9426- '\x24b9'# -> unI64 9427- '\x24ba'# -> unI64 9428- '\x24bb'# -> unI64 9429- '\x24bc'# -> unI64 9430- '\x24bd'# -> unI64 9431- '\x24be'# -> unI64 9432- '\x24bf'# -> unI64 9433- '\x24c0'# -> unI64 9434- '\x24c1'# -> unI64 9435- '\x24c2'# -> unI64 9436- '\x24c3'# -> unI64 9437- '\x24c4'# -> unI64 9438- '\x24c5'# -> unI64 9439- '\x24c6'# -> unI64 9440- '\x24c7'# -> unI64 9441- '\x24c8'# -> unI64 9442- '\x24c9'# -> unI64 9443- '\x24ca'# -> unI64 9444- '\x24cb'# -> unI64 9445- '\x24cc'# -> unI64 9446- '\x24cd'# -> unI64 9447- '\x24ce'# -> unI64 9448- '\x24cf'# -> unI64 9449- '\x2c00'# -> unI64 11312- '\x2c01'# -> unI64 11313- '\x2c02'# -> unI64 11314- '\x2c03'# -> unI64 11315- '\x2c04'# -> unI64 11316- '\x2c05'# -> unI64 11317- '\x2c06'# -> unI64 11318- '\x2c07'# -> unI64 11319- '\x2c08'# -> unI64 11320- '\x2c09'# -> unI64 11321- '\x2c0a'# -> unI64 11322- '\x2c0b'# -> unI64 11323- '\x2c0c'# -> unI64 11324- '\x2c0d'# -> unI64 11325- '\x2c0e'# -> unI64 11326- '\x2c0f'# -> unI64 11327- '\x2c10'# -> unI64 11328- '\x2c11'# -> unI64 11329- '\x2c12'# -> unI64 11330- '\x2c13'# -> unI64 11331- '\x2c14'# -> unI64 11332- '\x2c15'# -> unI64 11333- '\x2c16'# -> unI64 11334- '\x2c17'# -> unI64 11335- '\x2c18'# -> unI64 11336- '\x2c19'# -> unI64 11337- '\x2c1a'# -> unI64 11338- '\x2c1b'# -> unI64 11339- '\x2c1c'# -> unI64 11340- '\x2c1d'# -> unI64 11341- '\x2c1e'# -> unI64 11342- '\x2c1f'# -> unI64 11343- '\x2c20'# -> unI64 11344- '\x2c21'# -> unI64 11345- '\x2c22'# -> unI64 11346- '\x2c23'# -> unI64 11347- '\x2c24'# -> unI64 11348- '\x2c25'# -> unI64 11349- '\x2c26'# -> unI64 11350- '\x2c27'# -> unI64 11351- '\x2c28'# -> unI64 11352- '\x2c29'# -> unI64 11353- '\x2c2a'# -> unI64 11354- '\x2c2b'# -> unI64 11355- '\x2c2c'# -> unI64 11356- '\x2c2d'# -> unI64 11357- '\x2c2e'# -> unI64 11358- '\x2c2f'# -> unI64 11359- '\x2c60'# -> unI64 11361- '\x2c62'# -> unI64 619- '\x2c63'# -> unI64 7549- '\x2c64'# -> unI64 637- '\x2c67'# -> unI64 11368- '\x2c69'# -> unI64 11370- '\x2c6b'# -> unI64 11372- '\x2c6d'# -> unI64 593- '\x2c6e'# -> unI64 625- '\x2c6f'# -> unI64 592- '\x2c70'# -> unI64 594- '\x2c72'# -> unI64 11379- '\x2c75'# -> unI64 11382- '\x2c7e'# -> unI64 575- '\x2c7f'# -> unI64 576- '\x2c80'# -> unI64 11393- '\x2c82'# -> unI64 11395- '\x2c84'# -> unI64 11397- '\x2c86'# -> unI64 11399- '\x2c88'# -> unI64 11401- '\x2c8a'# -> unI64 11403- '\x2c8c'# -> unI64 11405- '\x2c8e'# -> unI64 11407- '\x2c90'# -> unI64 11409- '\x2c92'# -> unI64 11411- '\x2c94'# -> unI64 11413- '\x2c96'# -> unI64 11415- '\x2c98'# -> unI64 11417- '\x2c9a'# -> unI64 11419- '\x2c9c'# -> unI64 11421- '\x2c9e'# -> unI64 11423- '\x2ca0'# -> unI64 11425- '\x2ca2'# -> unI64 11427- '\x2ca4'# -> unI64 11429- '\x2ca6'# -> unI64 11431- '\x2ca8'# -> unI64 11433- '\x2caa'# -> unI64 11435- '\x2cac'# -> unI64 11437- '\x2cae'# -> unI64 11439- '\x2cb0'# -> unI64 11441- '\x2cb2'# -> unI64 11443- '\x2cb4'# -> unI64 11445- '\x2cb6'# -> unI64 11447- '\x2cb8'# -> unI64 11449- '\x2cba'# -> unI64 11451- '\x2cbc'# -> unI64 11453- '\x2cbe'# -> unI64 11455- '\x2cc0'# -> unI64 11457- '\x2cc2'# -> unI64 11459- '\x2cc4'# -> unI64 11461- '\x2cc6'# -> unI64 11463- '\x2cc8'# -> unI64 11465- '\x2cca'# -> unI64 11467- '\x2ccc'# -> unI64 11469- '\x2cce'# -> unI64 11471- '\x2cd0'# -> unI64 11473- '\x2cd2'# -> unI64 11475- '\x2cd4'# -> unI64 11477- '\x2cd6'# -> unI64 11479- '\x2cd8'# -> unI64 11481- '\x2cda'# -> unI64 11483- '\x2cdc'# -> unI64 11485- '\x2cde'# -> unI64 11487- '\x2ce0'# -> unI64 11489- '\x2ce2'# -> unI64 11491- '\x2ceb'# -> unI64 11500- '\x2ced'# -> unI64 11502- '\x2cf2'# -> unI64 11507- '\xa640'# -> unI64 42561- '\xa642'# -> unI64 42563- '\xa644'# -> unI64 42565- '\xa646'# -> unI64 42567- '\xa648'# -> unI64 42569- '\xa64a'# -> unI64 42571- '\xa64c'# -> unI64 42573- '\xa64e'# -> unI64 42575- '\xa650'# -> unI64 42577- '\xa652'# -> unI64 42579- '\xa654'# -> unI64 42581- '\xa656'# -> unI64 42583- '\xa658'# -> unI64 42585- '\xa65a'# -> unI64 42587- '\xa65c'# -> unI64 42589- '\xa65e'# -> unI64 42591- '\xa660'# -> unI64 42593- '\xa662'# -> unI64 42595- '\xa664'# -> unI64 42597- '\xa666'# -> unI64 42599- '\xa668'# -> unI64 42601- '\xa66a'# -> unI64 42603- '\xa66c'# -> unI64 42605- '\xa680'# -> unI64 42625- '\xa682'# -> unI64 42627- '\xa684'# -> unI64 42629- '\xa686'# -> unI64 42631- '\xa688'# -> unI64 42633- '\xa68a'# -> unI64 42635- '\xa68c'# -> unI64 42637- '\xa68e'# -> unI64 42639- '\xa690'# -> unI64 42641- '\xa692'# -> unI64 42643- '\xa694'# -> unI64 42645- '\xa696'# -> unI64 42647- '\xa698'# -> unI64 42649- '\xa69a'# -> unI64 42651- '\xa722'# -> unI64 42787- '\xa724'# -> unI64 42789- '\xa726'# -> unI64 42791- '\xa728'# -> unI64 42793- '\xa72a'# -> unI64 42795- '\xa72c'# -> unI64 42797- '\xa72e'# -> unI64 42799- '\xa732'# -> unI64 42803- '\xa734'# -> unI64 42805- '\xa736'# -> unI64 42807- '\xa738'# -> unI64 42809- '\xa73a'# -> unI64 42811- '\xa73c'# -> unI64 42813- '\xa73e'# -> unI64 42815- '\xa740'# -> unI64 42817- '\xa742'# -> unI64 42819- '\xa744'# -> unI64 42821- '\xa746'# -> unI64 42823- '\xa748'# -> unI64 42825- '\xa74a'# -> unI64 42827- '\xa74c'# -> unI64 42829- '\xa74e'# -> unI64 42831- '\xa750'# -> unI64 42833- '\xa752'# -> unI64 42835- '\xa754'# -> unI64 42837- '\xa756'# -> unI64 42839- '\xa758'# -> unI64 42841- '\xa75a'# -> unI64 42843- '\xa75c'# -> unI64 42845- '\xa75e'# -> unI64 42847- '\xa760'# -> unI64 42849- '\xa762'# -> unI64 42851- '\xa764'# -> unI64 42853- '\xa766'# -> unI64 42855- '\xa768'# -> unI64 42857- '\xa76a'# -> unI64 42859- '\xa76c'# -> unI64 42861- '\xa76e'# -> unI64 42863- '\xa779'# -> unI64 42874- '\xa77b'# -> unI64 42876- '\xa77d'# -> unI64 7545- '\xa77e'# -> unI64 42879- '\xa780'# -> unI64 42881- '\xa782'# -> unI64 42883- '\xa784'# -> unI64 42885- '\xa786'# -> unI64 42887- '\xa78b'# -> unI64 42892- '\xa78d'# -> unI64 613- '\xa790'# -> unI64 42897- '\xa792'# -> unI64 42899- '\xa796'# -> unI64 42903- '\xa798'# -> unI64 42905- '\xa79a'# -> unI64 42907- '\xa79c'# -> unI64 42909- '\xa79e'# -> unI64 42911- '\xa7a0'# -> unI64 42913- '\xa7a2'# -> unI64 42915- '\xa7a4'# -> unI64 42917- '\xa7a6'# -> unI64 42919- '\xa7a8'# -> unI64 42921- '\xa7aa'# -> unI64 614- '\xa7ab'# -> unI64 604- '\xa7ac'# -> unI64 609- '\xa7ad'# -> unI64 620- '\xa7ae'# -> unI64 618- '\xa7b0'# -> unI64 670- '\xa7b1'# -> unI64 647- '\xa7b2'# -> unI64 669- '\xa7b3'# -> unI64 43859- '\xa7b4'# -> unI64 42933- '\xa7b6'# -> unI64 42935- '\xa7b8'# -> unI64 42937- '\xa7ba'# -> unI64 42939- '\xa7bc'# -> unI64 42941- '\xa7be'# -> unI64 42943- '\xa7c0'# -> unI64 42945- '\xa7c2'# -> unI64 42947- '\xa7c4'# -> unI64 42900- '\xa7c5'# -> unI64 642- '\xa7c6'# -> unI64 7566- '\xa7c7'# -> unI64 42952- '\xa7c9'# -> unI64 42954- '\xa7d0'# -> unI64 42961- '\xa7d6'# -> unI64 42967- '\xa7d8'# -> unI64 42969- '\xa7f5'# -> unI64 42998- '\xff21'# -> unI64 65345- '\xff22'# -> unI64 65346- '\xff23'# -> unI64 65347- '\xff24'# -> unI64 65348- '\xff25'# -> unI64 65349- '\xff26'# -> unI64 65350- '\xff27'# -> unI64 65351- '\xff28'# -> unI64 65352- '\xff29'# -> unI64 65353- '\xff2a'# -> unI64 65354- '\xff2b'# -> unI64 65355- '\xff2c'# -> unI64 65356- '\xff2d'# -> unI64 65357- '\xff2e'# -> unI64 65358- '\xff2f'# -> unI64 65359- '\xff30'# -> unI64 65360- '\xff31'# -> unI64 65361- '\xff32'# -> unI64 65362- '\xff33'# -> unI64 65363- '\xff34'# -> unI64 65364- '\xff35'# -> unI64 65365- '\xff36'# -> unI64 65366- '\xff37'# -> unI64 65367- '\xff38'# -> unI64 65368- '\xff39'# -> unI64 65369- '\xff3a'# -> unI64 65370- '\x10400'# -> unI64 66600- '\x10401'# -> unI64 66601- '\x10402'# -> unI64 66602- '\x10403'# -> unI64 66603- '\x10404'# -> unI64 66604- '\x10405'# -> unI64 66605- '\x10406'# -> unI64 66606- '\x10407'# -> unI64 66607- '\x10408'# -> unI64 66608- '\x10409'# -> unI64 66609- '\x1040a'# -> unI64 66610- '\x1040b'# -> unI64 66611- '\x1040c'# -> unI64 66612- '\x1040d'# -> unI64 66613- '\x1040e'# -> unI64 66614- '\x1040f'# -> unI64 66615- '\x10410'# -> unI64 66616- '\x10411'# -> unI64 66617- '\x10412'# -> unI64 66618- '\x10413'# -> unI64 66619- '\x10414'# -> unI64 66620- '\x10415'# -> unI64 66621- '\x10416'# -> unI64 66622- '\x10417'# -> unI64 66623- '\x10418'# -> unI64 66624- '\x10419'# -> unI64 66625- '\x1041a'# -> unI64 66626- '\x1041b'# -> unI64 66627- '\x1041c'# -> unI64 66628- '\x1041d'# -> unI64 66629- '\x1041e'# -> unI64 66630- '\x1041f'# -> unI64 66631- '\x10420'# -> unI64 66632- '\x10421'# -> unI64 66633- '\x10422'# -> unI64 66634- '\x10423'# -> unI64 66635- '\x10424'# -> unI64 66636- '\x10425'# -> unI64 66637- '\x10426'# -> unI64 66638- '\x10427'# -> unI64 66639- '\x104b0'# -> unI64 66776- '\x104b1'# -> unI64 66777- '\x104b2'# -> unI64 66778- '\x104b3'# -> unI64 66779- '\x104b4'# -> unI64 66780- '\x104b5'# -> unI64 66781- '\x104b6'# -> unI64 66782- '\x104b7'# -> unI64 66783- '\x104b8'# -> unI64 66784- '\x104b9'# -> unI64 66785- '\x104ba'# -> unI64 66786- '\x104bb'# -> unI64 66787- '\x104bc'# -> unI64 66788- '\x104bd'# -> unI64 66789- '\x104be'# -> unI64 66790- '\x104bf'# -> unI64 66791- '\x104c0'# -> unI64 66792- '\x104c1'# -> unI64 66793- '\x104c2'# -> unI64 66794- '\x104c3'# -> unI64 66795- '\x104c4'# -> unI64 66796- '\x104c5'# -> unI64 66797- '\x104c6'# -> unI64 66798- '\x104c7'# -> unI64 66799- '\x104c8'# -> unI64 66800- '\x104c9'# -> unI64 66801- '\x104ca'# -> unI64 66802- '\x104cb'# -> unI64 66803- '\x104cc'# -> unI64 66804- '\x104cd'# -> unI64 66805- '\x104ce'# -> unI64 66806- '\x104cf'# -> unI64 66807- '\x104d0'# -> unI64 66808- '\x104d1'# -> unI64 66809- '\x104d2'# -> unI64 66810- '\x104d3'# -> unI64 66811- '\x10570'# -> unI64 66967- '\x10571'# -> unI64 66968- '\x10572'# -> unI64 66969- '\x10573'# -> unI64 66970- '\x10574'# -> unI64 66971- '\x10575'# -> unI64 66972- '\x10576'# -> unI64 66973- '\x10577'# -> unI64 66974- '\x10578'# -> unI64 66975- '\x10579'# -> unI64 66976- '\x1057a'# -> unI64 66977- '\x1057c'# -> unI64 66979- '\x1057d'# -> unI64 66980- '\x1057e'# -> unI64 66981- '\x1057f'# -> unI64 66982- '\x10580'# -> unI64 66983- '\x10581'# -> unI64 66984- '\x10582'# -> unI64 66985- '\x10583'# -> unI64 66986- '\x10584'# -> unI64 66987- '\x10585'# -> unI64 66988- '\x10586'# -> unI64 66989- '\x10587'# -> unI64 66990- '\x10588'# -> unI64 66991- '\x10589'# -> unI64 66992- '\x1058a'# -> unI64 66993- '\x1058c'# -> unI64 66995- '\x1058d'# -> unI64 66996- '\x1058e'# -> unI64 66997- '\x1058f'# -> unI64 66998- '\x10590'# -> unI64 66999- '\x10591'# -> unI64 67000- '\x10592'# -> unI64 67001- '\x10594'# -> unI64 67003- '\x10595'# -> unI64 67004- '\x10c80'# -> unI64 68800- '\x10c81'# -> unI64 68801- '\x10c82'# -> unI64 68802- '\x10c83'# -> unI64 68803- '\x10c84'# -> unI64 68804- '\x10c85'# -> unI64 68805- '\x10c86'# -> unI64 68806- '\x10c87'# -> unI64 68807- '\x10c88'# -> unI64 68808- '\x10c89'# -> unI64 68809- '\x10c8a'# -> unI64 68810- '\x10c8b'# -> unI64 68811- '\x10c8c'# -> unI64 68812- '\x10c8d'# -> unI64 68813- '\x10c8e'# -> unI64 68814- '\x10c8f'# -> unI64 68815- '\x10c90'# -> unI64 68816- '\x10c91'# -> unI64 68817- '\x10c92'# -> unI64 68818- '\x10c93'# -> unI64 68819- '\x10c94'# -> unI64 68820- '\x10c95'# -> unI64 68821- '\x10c96'# -> unI64 68822- '\x10c97'# -> unI64 68823- '\x10c98'# -> unI64 68824- '\x10c99'# -> unI64 68825- '\x10c9a'# -> unI64 68826- '\x10c9b'# -> unI64 68827- '\x10c9c'# -> unI64 68828- '\x10c9d'# -> unI64 68829- '\x10c9e'# -> unI64 68830- '\x10c9f'# -> unI64 68831- '\x10ca0'# -> unI64 68832- '\x10ca1'# -> unI64 68833- '\x10ca2'# -> unI64 68834- '\x10ca3'# -> unI64 68835- '\x10ca4'# -> unI64 68836- '\x10ca5'# -> unI64 68837- '\x10ca6'# -> unI64 68838- '\x10ca7'# -> unI64 68839- '\x10ca8'# -> unI64 68840- '\x10ca9'# -> unI64 68841- '\x10caa'# -> unI64 68842- '\x10cab'# -> unI64 68843- '\x10cac'# -> unI64 68844- '\x10cad'# -> unI64 68845- '\x10cae'# -> unI64 68846- '\x10caf'# -> unI64 68847- '\x10cb0'# -> unI64 68848- '\x10cb1'# -> unI64 68849- '\x10cb2'# -> unI64 68850- '\x118a0'# -> unI64 71872- '\x118a1'# -> unI64 71873- '\x118a2'# -> unI64 71874- '\x118a3'# -> unI64 71875- '\x118a4'# -> unI64 71876- '\x118a5'# -> unI64 71877- '\x118a6'# -> unI64 71878- '\x118a7'# -> unI64 71879- '\x118a8'# -> unI64 71880- '\x118a9'# -> unI64 71881- '\x118aa'# -> unI64 71882- '\x118ab'# -> unI64 71883- '\x118ac'# -> unI64 71884- '\x118ad'# -> unI64 71885- '\x118ae'# -> unI64 71886- '\x118af'# -> unI64 71887- '\x118b0'# -> unI64 71888- '\x118b1'# -> unI64 71889- '\x118b2'# -> unI64 71890- '\x118b3'# -> unI64 71891- '\x118b4'# -> unI64 71892- '\x118b5'# -> unI64 71893- '\x118b6'# -> unI64 71894- '\x118b7'# -> unI64 71895- '\x118b8'# -> unI64 71896- '\x118b9'# -> unI64 71897- '\x118ba'# -> unI64 71898- '\x118bb'# -> unI64 71899- '\x118bc'# -> unI64 71900- '\x118bd'# -> unI64 71901- '\x118be'# -> unI64 71902- '\x118bf'# -> unI64 71903- '\x16e40'# -> unI64 93792- '\x16e41'# -> unI64 93793- '\x16e42'# -> unI64 93794- '\x16e43'# -> unI64 93795- '\x16e44'# -> unI64 93796- '\x16e45'# -> unI64 93797- '\x16e46'# -> unI64 93798- '\x16e47'# -> unI64 93799- '\x16e48'# -> unI64 93800- '\x16e49'# -> unI64 93801- '\x16e4a'# -> unI64 93802- '\x16e4b'# -> unI64 93803- '\x16e4c'# -> unI64 93804- '\x16e4d'# -> unI64 93805- '\x16e4e'# -> unI64 93806- '\x16e4f'# -> unI64 93807- '\x16e50'# -> unI64 93808- '\x16e51'# -> unI64 93809- '\x16e52'# -> unI64 93810- '\x16e53'# -> unI64 93811- '\x16e54'# -> unI64 93812- '\x16e55'# -> unI64 93813- '\x16e56'# -> unI64 93814- '\x16e57'# -> unI64 93815- '\x16e58'# -> unI64 93816- '\x16e59'# -> unI64 93817- '\x16e5a'# -> unI64 93818- '\x16e5b'# -> unI64 93819- '\x16e5c'# -> unI64 93820- '\x16e5d'# -> unI64 93821- '\x16e5e'# -> unI64 93822- '\x16e5f'# -> unI64 93823- '\x1e900'# -> unI64 125218- '\x1e901'# -> unI64 125219- '\x1e902'# -> unI64 125220- '\x1e903'# -> unI64 125221- '\x1e904'# -> unI64 125222- '\x1e905'# -> unI64 125223- '\x1e906'# -> unI64 125224- '\x1e907'# -> unI64 125225- '\x1e908'# -> unI64 125226- '\x1e909'# -> unI64 125227- '\x1e90a'# -> unI64 125228- '\x1e90b'# -> unI64 125229- '\x1e90c'# -> unI64 125230- '\x1e90d'# -> unI64 125231- '\x1e90e'# -> unI64 125232- '\x1e90f'# -> unI64 125233- '\x1e910'# -> unI64 125234- '\x1e911'# -> unI64 125235- '\x1e912'# -> unI64 125236- '\x1e913'# -> unI64 125237- '\x1e914'# -> unI64 125238- '\x1e915'# -> unI64 125239- '\x1e916'# -> unI64 125240- '\x1e917'# -> unI64 125241- '\x1e918'# -> unI64 125242- '\x1e919'# -> unI64 125243- '\x1e91a'# -> unI64 125244- '\x1e91b'# -> unI64 125245- '\x1e91c'# -> unI64 125246- '\x1e91d'# -> unI64 125247- '\x1e91e'# -> unI64 125248- '\x1e91f'# -> unI64 125249- '\x1e920'# -> unI64 125250- '\x1e921'# -> unI64 125251- _ -> unI64 0+-- AUTOMATICALLY GENERATED - DO NOT EDIT +-- Generated by scripts/CaseMapping.hs +-- CaseFolding-17.0.0.txt +-- Date: 2025-07-30, 23:54:36 GMT +-- SpecialCasing-17.0.0.txt +-- Date: 2025-07-31, 22:11:55 GMT + +{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +module Data.Text.Internal.Fusion.CaseMapping where +import GHC.Int +import GHC.Exts +import Data.Version (Version, makeVersion) +unicodeVersion :: Version +unicodeVersion = makeVersion [17, 0, 0] +unI64 :: Int64 -> _ {- unboxed Int64 -} +unI64 (I64# n) = n + +upperMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE upperMapping #-} +upperMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 174063699 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 146800710 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 153092166 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 159383622 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 321057542111302 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 334251681644614 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 176160851 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 176160851 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2856322357 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2831156548 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2795504964 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2808087876 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2831156558 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2812282180 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1931484936 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1931484937 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1931484938 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1931484939 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1931484940 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1931484941 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1931484942 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1931484943 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1931484936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1931484937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1931484938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1931484939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1931484940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1931484941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1931484942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1931484943 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1931484968 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1931484969 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1931484970 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1931484971 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1931484972 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1931484973 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1931484974 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1931484975 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1931484968 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1931484969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1931484970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1931484971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1931484972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1931484973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1931484974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1931484975 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1931485032 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1931485033 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1931485034 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1931485035 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1931485036 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1931485037 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1931485038 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1931485039 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1931485032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1931485033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1931485034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1931485035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1931485036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1931485037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1931485038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1931485039 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1931477905 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1931477905 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1931477911 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1931477911 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1931477929 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1931477929 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1931485114 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1931477894 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1931485130 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1931477897 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1931485178 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1931477903 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4050602585752465 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4050602585752471 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4050602585752489 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c5'# -> unI64 452 + '\x01c6'# -> unI64 452 + '\x01c8'# -> unI64 455 + '\x01c9'# -> unI64 455 + '\x01cb'# -> unI64 458 + '\x01cc'# -> unI64 458 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f2'# -> unI64 497 + '\x01f3'# -> unI64 497 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x10d0'# -> unI64 7312 + '\x10d1'# -> unI64 7313 + '\x10d2'# -> unI64 7314 + '\x10d3'# -> unI64 7315 + '\x10d4'# -> unI64 7316 + '\x10d5'# -> unI64 7317 + '\x10d6'# -> unI64 7318 + '\x10d7'# -> unI64 7319 + '\x10d8'# -> unI64 7320 + '\x10d9'# -> unI64 7321 + '\x10da'# -> unI64 7322 + '\x10db'# -> unI64 7323 + '\x10dc'# -> unI64 7324 + '\x10dd'# -> unI64 7325 + '\x10de'# -> unI64 7326 + '\x10df'# -> unI64 7327 + '\x10e0'# -> unI64 7328 + '\x10e1'# -> unI64 7329 + '\x10e2'# -> unI64 7330 + '\x10e3'# -> unI64 7331 + '\x10e4'# -> unI64 7332 + '\x10e5'# -> unI64 7333 + '\x10e6'# -> unI64 7334 + '\x10e7'# -> unI64 7335 + '\x10e8'# -> unI64 7336 + '\x10e9'# -> unI64 7337 + '\x10ea'# -> unI64 7338 + '\x10eb'# -> unI64 7339 + '\x10ec'# -> unI64 7340 + '\x10ed'# -> unI64 7341 + '\x10ee'# -> unI64 7342 + '\x10ef'# -> unI64 7343 + '\x10f0'# -> unI64 7344 + '\x10f1'# -> unI64 7345 + '\x10f2'# -> unI64 7346 + '\x10f3'# -> unI64 7347 + '\x10f4'# -> unI64 7348 + '\x10f5'# -> unI64 7349 + '\x10f6'# -> unI64 7350 + '\x10f7'# -> unI64 7351 + '\x10f8'# -> unI64 7352 + '\x10f9'# -> unI64 7353 + '\x10fa'# -> unI64 7354 + '\x10fd'# -> unI64 7357 + '\x10fe'# -> unI64 7358 + '\x10ff'# -> unI64 7359 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +lowerMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE lowerMapping #-} +lowerMapping = \case + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 8064 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 8065 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 8066 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 8067 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 8068 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 8069 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 8070 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 8071 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 8080 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 8081 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 8082 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 8083 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 8084 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 8085 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 8086 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 8087 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 8096 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 8097 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 8098 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 8099 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 8100 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 8101 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 8102 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 8103 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 8115 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 8131 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 8179 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c89'# -> unI64 7306 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1e9e'# -> unI64 223 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7cb'# -> unI64 612 + '\xa7cc'# -> unI64 42957 + '\xa7ce'# -> unI64 42959 + '\xa7d0'# -> unI64 42961 + '\xa7d2'# -> unI64 42963 + '\xa7d4'# -> unI64 42965 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7da'# -> unI64 42971 + '\xa7dc'# -> unI64 411 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x10d50'# -> unI64 68976 + '\x10d51'# -> unI64 68977 + '\x10d52'# -> unI64 68978 + '\x10d53'# -> unI64 68979 + '\x10d54'# -> unI64 68980 + '\x10d55'# -> unI64 68981 + '\x10d56'# -> unI64 68982 + '\x10d57'# -> unI64 68983 + '\x10d58'# -> unI64 68984 + '\x10d59'# -> unI64 68985 + '\x10d5a'# -> unI64 68986 + '\x10d5b'# -> unI64 68987 + '\x10d5c'# -> unI64 68988 + '\x10d5d'# -> unI64 68989 + '\x10d5e'# -> unI64 68990 + '\x10d5f'# -> unI64 68991 + '\x10d60'# -> unI64 68992 + '\x10d61'# -> unI64 68993 + '\x10d62'# -> unI64 68994 + '\x10d63'# -> unI64 68995 + '\x10d64'# -> unI64 68996 + '\x10d65'# -> unI64 68997 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x16ea0'# -> unI64 93883 + '\x16ea1'# -> unI64 93884 + '\x16ea2'# -> unI64 93885 + '\x16ea3'# -> unI64 93886 + '\x16ea4'# -> unI64 93887 + '\x16ea5'# -> unI64 93888 + '\x16ea6'# -> unI64 93889 + '\x16ea7'# -> unI64 93890 + '\x16ea8'# -> unI64 93891 + '\x16ea9'# -> unI64 93892 + '\x16eaa'# -> unI64 93893 + '\x16eab'# -> unI64 93894 + '\x16eac'# -> unI64 93895 + '\x16ead'# -> unI64 93896 + '\x16eae'# -> unI64 93897 + '\x16eaf'# -> unI64 93898 + '\x16eb0'# -> unI64 93899 + '\x16eb1'# -> unI64 93900 + '\x16eb2'# -> unI64 93901 + '\x16eb3'# -> unI64 93902 + '\x16eb4'# -> unI64 93903 + '\x16eb5'# -> unI64 93904 + '\x16eb6'# -> unI64 93905 + '\x16eb7'# -> unI64 93906 + '\x16eb8'# -> unI64 93907 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 +titleMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE titleMapping #-} +titleMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172563 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909574 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201030 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492486 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575494 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108806 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269715 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269715 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985653 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819844 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168260 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751172 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819854 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945476 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 8072 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 8073 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 8074 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 8075 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 8076 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 8077 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 8078 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 8079 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 8088 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 8089 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 8090 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 8091 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 8092 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 8093 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 8094 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 8095 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 8104 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 8105 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 8106 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 8107 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 8108 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 8109 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 8110 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 8111 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 8124 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 8140 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 8188 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1755324346 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1755317126 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1755324362 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1755317129 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1755324410 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1755317135 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 3681166678819729 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 3681166678819735 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 3681166678819753 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c4'# -> unI64 453 + '\x01c6'# -> unI64 453 + '\x01c7'# -> unI64 456 + '\x01c9'# -> unI64 456 + '\x01ca'# -> unI64 459 + '\x01cc'# -> unI64 459 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f1'# -> unI64 498 + '\x01f3'# -> unI64 498 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +foldMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE foldMapping #-} +foldMapping = \case + -- LATIN CAPITAL LETTER A + '\x0041'# -> unI64 97 + -- LATIN CAPITAL LETTER B + '\x0042'# -> unI64 98 + -- LATIN CAPITAL LETTER C + '\x0043'# -> unI64 99 + -- LATIN CAPITAL LETTER D + '\x0044'# -> unI64 100 + -- LATIN CAPITAL LETTER E + '\x0045'# -> unI64 101 + -- LATIN CAPITAL LETTER F + '\x0046'# -> unI64 102 + -- LATIN CAPITAL LETTER G + '\x0047'# -> unI64 103 + -- LATIN CAPITAL LETTER H + '\x0048'# -> unI64 104 + -- LATIN CAPITAL LETTER I + '\x0049'# -> unI64 105 + -- LATIN CAPITAL LETTER J + '\x004a'# -> unI64 106 + -- LATIN CAPITAL LETTER K + '\x004b'# -> unI64 107 + -- LATIN CAPITAL LETTER L + '\x004c'# -> unI64 108 + -- LATIN CAPITAL LETTER M + '\x004d'# -> unI64 109 + -- LATIN CAPITAL LETTER N + '\x004e'# -> unI64 110 + -- LATIN CAPITAL LETTER O + '\x004f'# -> unI64 111 + -- LATIN CAPITAL LETTER P + '\x0050'# -> unI64 112 + -- LATIN CAPITAL LETTER Q + '\x0051'# -> unI64 113 + -- LATIN CAPITAL LETTER R + '\x0052'# -> unI64 114 + -- LATIN CAPITAL LETTER S + '\x0053'# -> unI64 115 + -- LATIN CAPITAL LETTER T + '\x0054'# -> unI64 116 + -- LATIN CAPITAL LETTER U + '\x0055'# -> unI64 117 + -- LATIN CAPITAL LETTER V + '\x0056'# -> unI64 118 + -- LATIN CAPITAL LETTER W + '\x0057'# -> unI64 119 + -- LATIN CAPITAL LETTER X + '\x0058'# -> unI64 120 + -- LATIN CAPITAL LETTER Y + '\x0059'# -> unI64 121 + -- LATIN CAPITAL LETTER Z + '\x005a'# -> unI64 122 + -- MICRO SIGN + '\x00b5'# -> unI64 956 + -- LATIN CAPITAL LETTER A WITH GRAVE + '\x00c0'# -> unI64 224 + -- LATIN CAPITAL LETTER A WITH ACUTE + '\x00c1'# -> unI64 225 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX + '\x00c2'# -> unI64 226 + -- LATIN CAPITAL LETTER A WITH TILDE + '\x00c3'# -> unI64 227 + -- LATIN CAPITAL LETTER A WITH DIAERESIS + '\x00c4'# -> unI64 228 + -- LATIN CAPITAL LETTER A WITH RING ABOVE + '\x00c5'# -> unI64 229 + -- LATIN CAPITAL LETTER AE + '\x00c6'# -> unI64 230 + -- LATIN CAPITAL LETTER C WITH CEDILLA + '\x00c7'# -> unI64 231 + -- LATIN CAPITAL LETTER E WITH GRAVE + '\x00c8'# -> unI64 232 + -- LATIN CAPITAL LETTER E WITH ACUTE + '\x00c9'# -> unI64 233 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX + '\x00ca'# -> unI64 234 + -- LATIN CAPITAL LETTER E WITH DIAERESIS + '\x00cb'# -> unI64 235 + -- LATIN CAPITAL LETTER I WITH GRAVE + '\x00cc'# -> unI64 236 + -- LATIN CAPITAL LETTER I WITH ACUTE + '\x00cd'# -> unI64 237 + -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX + '\x00ce'# -> unI64 238 + -- LATIN CAPITAL LETTER I WITH DIAERESIS + '\x00cf'# -> unI64 239 + -- LATIN CAPITAL LETTER ETH + '\x00d0'# -> unI64 240 + -- LATIN CAPITAL LETTER N WITH TILDE + '\x00d1'# -> unI64 241 + -- LATIN CAPITAL LETTER O WITH GRAVE + '\x00d2'# -> unI64 242 + -- LATIN CAPITAL LETTER O WITH ACUTE + '\x00d3'# -> unI64 243 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX + '\x00d4'# -> unI64 244 + -- LATIN CAPITAL LETTER O WITH TILDE + '\x00d5'# -> unI64 245 + -- LATIN CAPITAL LETTER O WITH DIAERESIS + '\x00d6'# -> unI64 246 + -- LATIN CAPITAL LETTER O WITH STROKE + '\x00d8'# -> unI64 248 + -- LATIN CAPITAL LETTER U WITH GRAVE + '\x00d9'# -> unI64 249 + -- LATIN CAPITAL LETTER U WITH ACUTE + '\x00da'# -> unI64 250 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX + '\x00db'# -> unI64 251 + -- LATIN CAPITAL LETTER U WITH DIAERESIS + '\x00dc'# -> unI64 252 + -- LATIN CAPITAL LETTER Y WITH ACUTE + '\x00dd'# -> unI64 253 + -- LATIN CAPITAL LETTER THORN + '\x00de'# -> unI64 254 + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH MACRON + '\x0100'# -> unI64 257 + -- LATIN CAPITAL LETTER A WITH BREVE + '\x0102'# -> unI64 259 + -- LATIN CAPITAL LETTER A WITH OGONEK + '\x0104'# -> unI64 261 + -- LATIN CAPITAL LETTER C WITH ACUTE + '\x0106'# -> unI64 263 + -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX + '\x0108'# -> unI64 265 + -- LATIN CAPITAL LETTER C WITH DOT ABOVE + '\x010a'# -> unI64 267 + -- LATIN CAPITAL LETTER C WITH CARON + '\x010c'# -> unI64 269 + -- LATIN CAPITAL LETTER D WITH CARON + '\x010e'# -> unI64 271 + -- LATIN CAPITAL LETTER D WITH STROKE + '\x0110'# -> unI64 273 + -- LATIN CAPITAL LETTER E WITH MACRON + '\x0112'# -> unI64 275 + -- LATIN CAPITAL LETTER E WITH BREVE + '\x0114'# -> unI64 277 + -- LATIN CAPITAL LETTER E WITH DOT ABOVE + '\x0116'# -> unI64 279 + -- LATIN CAPITAL LETTER E WITH OGONEK + '\x0118'# -> unI64 281 + -- LATIN CAPITAL LETTER E WITH CARON + '\x011a'# -> unI64 283 + -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX + '\x011c'# -> unI64 285 + -- LATIN CAPITAL LETTER G WITH BREVE + '\x011e'# -> unI64 287 + -- LATIN CAPITAL LETTER G WITH DOT ABOVE + '\x0120'# -> unI64 289 + -- LATIN CAPITAL LETTER G WITH CEDILLA + '\x0122'# -> unI64 291 + -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX + '\x0124'# -> unI64 293 + -- LATIN CAPITAL LETTER H WITH STROKE + '\x0126'# -> unI64 295 + -- LATIN CAPITAL LETTER I WITH TILDE + '\x0128'# -> unI64 297 + -- LATIN CAPITAL LETTER I WITH MACRON + '\x012a'# -> unI64 299 + -- LATIN CAPITAL LETTER I WITH BREVE + '\x012c'# -> unI64 301 + -- LATIN CAPITAL LETTER I WITH OGONEK + '\x012e'# -> unI64 303 + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- LATIN CAPITAL LIGATURE IJ + '\x0132'# -> unI64 307 + -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX + '\x0134'# -> unI64 309 + -- LATIN CAPITAL LETTER K WITH CEDILLA + '\x0136'# -> unI64 311 + -- LATIN CAPITAL LETTER L WITH ACUTE + '\x0139'# -> unI64 314 + -- LATIN CAPITAL LETTER L WITH CEDILLA + '\x013b'# -> unI64 316 + -- LATIN CAPITAL LETTER L WITH CARON + '\x013d'# -> unI64 318 + -- LATIN CAPITAL LETTER L WITH MIDDLE DOT + '\x013f'# -> unI64 320 + -- LATIN CAPITAL LETTER L WITH STROKE + '\x0141'# -> unI64 322 + -- LATIN CAPITAL LETTER N WITH ACUTE + '\x0143'# -> unI64 324 + -- LATIN CAPITAL LETTER N WITH CEDILLA + '\x0145'# -> unI64 326 + -- LATIN CAPITAL LETTER N WITH CARON + '\x0147'# -> unI64 328 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 230687420 + -- LATIN CAPITAL LETTER ENG + '\x014a'# -> unI64 331 + -- LATIN CAPITAL LETTER O WITH MACRON + '\x014c'# -> unI64 333 + -- LATIN CAPITAL LETTER O WITH BREVE + '\x014e'# -> unI64 335 + -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + '\x0150'# -> unI64 337 + -- LATIN CAPITAL LIGATURE OE + '\x0152'# -> unI64 339 + -- LATIN CAPITAL LETTER R WITH ACUTE + '\x0154'# -> unI64 341 + -- LATIN CAPITAL LETTER R WITH CEDILLA + '\x0156'# -> unI64 343 + -- LATIN CAPITAL LETTER R WITH CARON + '\x0158'# -> unI64 345 + -- LATIN CAPITAL LETTER S WITH ACUTE + '\x015a'# -> unI64 347 + -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX + '\x015c'# -> unI64 349 + -- LATIN CAPITAL LETTER S WITH CEDILLA + '\x015e'# -> unI64 351 + -- LATIN CAPITAL LETTER S WITH CARON + '\x0160'# -> unI64 353 + -- LATIN CAPITAL LETTER T WITH CEDILLA + '\x0162'# -> unI64 355 + -- LATIN CAPITAL LETTER T WITH CARON + '\x0164'# -> unI64 357 + -- LATIN CAPITAL LETTER T WITH STROKE + '\x0166'# -> unI64 359 + -- LATIN CAPITAL LETTER U WITH TILDE + '\x0168'# -> unI64 361 + -- LATIN CAPITAL LETTER U WITH MACRON + '\x016a'# -> unI64 363 + -- LATIN CAPITAL LETTER U WITH BREVE + '\x016c'# -> unI64 365 + -- LATIN CAPITAL LETTER U WITH RING ABOVE + '\x016e'# -> unI64 367 + -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + '\x0170'# -> unI64 369 + -- LATIN CAPITAL LETTER U WITH OGONEK + '\x0172'# -> unI64 371 + -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX + '\x0174'# -> unI64 373 + -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + '\x0176'# -> unI64 375 + -- LATIN CAPITAL LETTER Y WITH DIAERESIS + '\x0178'# -> unI64 255 + -- LATIN CAPITAL LETTER Z WITH ACUTE + '\x0179'# -> unI64 378 + -- LATIN CAPITAL LETTER Z WITH DOT ABOVE + '\x017b'# -> unI64 380 + -- LATIN CAPITAL LETTER Z WITH CARON + '\x017d'# -> unI64 382 + -- LATIN SMALL LETTER LONG S + '\x017f'# -> unI64 115 + -- LATIN CAPITAL LETTER B WITH HOOK + '\x0181'# -> unI64 595 + -- LATIN CAPITAL LETTER B WITH TOPBAR + '\x0182'# -> unI64 387 + -- LATIN CAPITAL LETTER TONE SIX + '\x0184'# -> unI64 389 + -- LATIN CAPITAL LETTER OPEN O + '\x0186'# -> unI64 596 + -- LATIN CAPITAL LETTER C WITH HOOK + '\x0187'# -> unI64 392 + -- LATIN CAPITAL LETTER AFRICAN D + '\x0189'# -> unI64 598 + -- LATIN CAPITAL LETTER D WITH HOOK + '\x018a'# -> unI64 599 + -- LATIN CAPITAL LETTER D WITH TOPBAR + '\x018b'# -> unI64 396 + -- LATIN CAPITAL LETTER REVERSED E + '\x018e'# -> unI64 477 + -- LATIN CAPITAL LETTER SCHWA + '\x018f'# -> unI64 601 + -- LATIN CAPITAL LETTER OPEN E + '\x0190'# -> unI64 603 + -- LATIN CAPITAL LETTER F WITH HOOK + '\x0191'# -> unI64 402 + -- LATIN CAPITAL LETTER G WITH HOOK + '\x0193'# -> unI64 608 + -- LATIN CAPITAL LETTER GAMMA + '\x0194'# -> unI64 611 + -- LATIN CAPITAL LETTER IOTA + '\x0196'# -> unI64 617 + -- LATIN CAPITAL LETTER I WITH STROKE + '\x0197'# -> unI64 616 + -- LATIN CAPITAL LETTER K WITH HOOK + '\x0198'# -> unI64 409 + -- LATIN CAPITAL LETTER TURNED M + '\x019c'# -> unI64 623 + -- LATIN CAPITAL LETTER N WITH LEFT HOOK + '\x019d'# -> unI64 626 + -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE + '\x019f'# -> unI64 629 + -- LATIN CAPITAL LETTER O WITH HORN + '\x01a0'# -> unI64 417 + -- LATIN CAPITAL LETTER OI + '\x01a2'# -> unI64 419 + -- LATIN CAPITAL LETTER P WITH HOOK + '\x01a4'# -> unI64 421 + -- LATIN LETTER YR + '\x01a6'# -> unI64 640 + -- LATIN CAPITAL LETTER TONE TWO + '\x01a7'# -> unI64 424 + -- LATIN CAPITAL LETTER ESH + '\x01a9'# -> unI64 643 + -- LATIN CAPITAL LETTER T WITH HOOK + '\x01ac'# -> unI64 429 + -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + '\x01ae'# -> unI64 648 + -- LATIN CAPITAL LETTER U WITH HORN + '\x01af'# -> unI64 432 + -- LATIN CAPITAL LETTER UPSILON + '\x01b1'# -> unI64 650 + -- LATIN CAPITAL LETTER V WITH HOOK + '\x01b2'# -> unI64 651 + -- LATIN CAPITAL LETTER Y WITH HOOK + '\x01b3'# -> unI64 436 + -- LATIN CAPITAL LETTER Z WITH STROKE + '\x01b5'# -> unI64 438 + -- LATIN CAPITAL LETTER EZH + '\x01b7'# -> unI64 658 + -- LATIN CAPITAL LETTER EZH REVERSED + '\x01b8'# -> unI64 441 + -- LATIN CAPITAL LETTER TONE FIVE + '\x01bc'# -> unI64 445 + -- LATIN CAPITAL LETTER DZ WITH CARON + '\x01c4'# -> unI64 454 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + '\x01c5'# -> unI64 454 + -- LATIN CAPITAL LETTER LJ + '\x01c7'# -> unI64 457 + -- LATIN CAPITAL LETTER L WITH SMALL LETTER J + '\x01c8'# -> unI64 457 + -- LATIN CAPITAL LETTER NJ + '\x01ca'# -> unI64 460 + -- LATIN CAPITAL LETTER N WITH SMALL LETTER J + '\x01cb'# -> unI64 460 + -- LATIN CAPITAL LETTER A WITH CARON + '\x01cd'# -> unI64 462 + -- LATIN CAPITAL LETTER I WITH CARON + '\x01cf'# -> unI64 464 + -- LATIN CAPITAL LETTER O WITH CARON + '\x01d1'# -> unI64 466 + -- LATIN CAPITAL LETTER U WITH CARON + '\x01d3'# -> unI64 468 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + '\x01d5'# -> unI64 470 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + '\x01d7'# -> unI64 472 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + '\x01d9'# -> unI64 474 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + '\x01db'# -> unI64 476 + -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + '\x01de'# -> unI64 479 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + '\x01e0'# -> unI64 481 + -- LATIN CAPITAL LETTER AE WITH MACRON + '\x01e2'# -> unI64 483 + -- LATIN CAPITAL LETTER G WITH STROKE + '\x01e4'# -> unI64 485 + -- LATIN CAPITAL LETTER G WITH CARON + '\x01e6'# -> unI64 487 + -- LATIN CAPITAL LETTER K WITH CARON + '\x01e8'# -> unI64 489 + -- LATIN CAPITAL LETTER O WITH OGONEK + '\x01ea'# -> unI64 491 + -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + '\x01ec'# -> unI64 493 + -- LATIN CAPITAL LETTER EZH WITH CARON + '\x01ee'# -> unI64 495 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778666 + -- LATIN CAPITAL LETTER DZ + '\x01f1'# -> unI64 499 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z + '\x01f2'# -> unI64 499 + -- LATIN CAPITAL LETTER G WITH ACUTE + '\x01f4'# -> unI64 501 + -- LATIN CAPITAL LETTER HWAIR + '\x01f6'# -> unI64 405 + -- LATIN CAPITAL LETTER WYNN + '\x01f7'# -> unI64 447 + -- LATIN CAPITAL LETTER N WITH GRAVE + '\x01f8'# -> unI64 505 + -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + '\x01fa'# -> unI64 507 + -- LATIN CAPITAL LETTER AE WITH ACUTE + '\x01fc'# -> unI64 509 + -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + '\x01fe'# -> unI64 511 + -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + '\x0200'# -> unI64 513 + -- LATIN CAPITAL LETTER A WITH INVERTED BREVE + '\x0202'# -> unI64 515 + -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + '\x0204'# -> unI64 517 + -- LATIN CAPITAL LETTER E WITH INVERTED BREVE + '\x0206'# -> unI64 519 + -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + '\x0208'# -> unI64 521 + -- LATIN CAPITAL LETTER I WITH INVERTED BREVE + '\x020a'# -> unI64 523 + -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + '\x020c'# -> unI64 525 + -- LATIN CAPITAL LETTER O WITH INVERTED BREVE + '\x020e'# -> unI64 527 + -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + '\x0210'# -> unI64 529 + -- LATIN CAPITAL LETTER R WITH INVERTED BREVE + '\x0212'# -> unI64 531 + -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + '\x0214'# -> unI64 533 + -- LATIN CAPITAL LETTER U WITH INVERTED BREVE + '\x0216'# -> unI64 535 + -- LATIN CAPITAL LETTER S WITH COMMA BELOW + '\x0218'# -> unI64 537 + -- LATIN CAPITAL LETTER T WITH COMMA BELOW + '\x021a'# -> unI64 539 + -- LATIN CAPITAL LETTER YOGH + '\x021c'# -> unI64 541 + -- LATIN CAPITAL LETTER H WITH CARON + '\x021e'# -> unI64 543 + -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + '\x0220'# -> unI64 414 + -- LATIN CAPITAL LETTER OU + '\x0222'# -> unI64 547 + -- LATIN CAPITAL LETTER Z WITH HOOK + '\x0224'# -> unI64 549 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE + '\x0226'# -> unI64 551 + -- LATIN CAPITAL LETTER E WITH CEDILLA + '\x0228'# -> unI64 553 + -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + '\x022a'# -> unI64 555 + -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON + '\x022c'# -> unI64 557 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE + '\x022e'# -> unI64 559 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + '\x0230'# -> unI64 561 + -- LATIN CAPITAL LETTER Y WITH MACRON + '\x0232'# -> unI64 563 + -- LATIN CAPITAL LETTER A WITH STROKE + '\x023a'# -> unI64 11365 + -- LATIN CAPITAL LETTER C WITH STROKE + '\x023b'# -> unI64 572 + -- LATIN CAPITAL LETTER L WITH BAR + '\x023d'# -> unI64 410 + -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + '\x023e'# -> unI64 11366 + -- LATIN CAPITAL LETTER GLOTTAL STOP + '\x0241'# -> unI64 578 + -- LATIN CAPITAL LETTER B WITH STROKE + '\x0243'# -> unI64 384 + -- LATIN CAPITAL LETTER U BAR + '\x0244'# -> unI64 649 + -- LATIN CAPITAL LETTER TURNED V + '\x0245'# -> unI64 652 + -- LATIN CAPITAL LETTER E WITH STROKE + '\x0246'# -> unI64 583 + -- LATIN CAPITAL LETTER J WITH STROKE + '\x0248'# -> unI64 585 + -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL + '\x024a'# -> unI64 587 + -- LATIN CAPITAL LETTER R WITH STROKE + '\x024c'# -> unI64 589 + -- LATIN CAPITAL LETTER Y WITH STROKE + '\x024e'# -> unI64 591 + -- COMBINING GREEK YPOGEGRAMMENI + '\x0345'# -> unI64 953 + -- GREEK CAPITAL LETTER HETA + '\x0370'# -> unI64 881 + -- GREEK CAPITAL LETTER ARCHAIC SAMPI + '\x0372'# -> unI64 883 + -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA + '\x0376'# -> unI64 887 + -- GREEK CAPITAL LETTER YOT + '\x037f'# -> unI64 1011 + -- GREEK CAPITAL LETTER ALPHA WITH TONOS + '\x0386'# -> unI64 940 + -- GREEK CAPITAL LETTER EPSILON WITH TONOS + '\x0388'# -> unI64 941 + -- GREEK CAPITAL LETTER ETA WITH TONOS + '\x0389'# -> unI64 942 + -- GREEK CAPITAL LETTER IOTA WITH TONOS + '\x038a'# -> unI64 943 + -- GREEK CAPITAL LETTER OMICRON WITH TONOS + '\x038c'# -> unI64 972 + -- GREEK CAPITAL LETTER UPSILON WITH TONOS + '\x038e'# -> unI64 973 + -- GREEK CAPITAL LETTER OMEGA WITH TONOS + '\x038f'# -> unI64 974 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429881 + -- GREEK CAPITAL LETTER ALPHA + '\x0391'# -> unI64 945 + -- GREEK CAPITAL LETTER BETA + '\x0392'# -> unI64 946 + -- GREEK CAPITAL LETTER GAMMA + '\x0393'# -> unI64 947 + -- GREEK CAPITAL LETTER DELTA + '\x0394'# -> unI64 948 + -- GREEK CAPITAL LETTER EPSILON + '\x0395'# -> unI64 949 + -- GREEK CAPITAL LETTER ZETA + '\x0396'# -> unI64 950 + -- GREEK CAPITAL LETTER ETA + '\x0397'# -> unI64 951 + -- GREEK CAPITAL LETTER THETA + '\x0398'# -> unI64 952 + -- GREEK CAPITAL LETTER IOTA + '\x0399'# -> unI64 953 + -- GREEK CAPITAL LETTER KAPPA + '\x039a'# -> unI64 954 + -- GREEK CAPITAL LETTER LAMDA + '\x039b'# -> unI64 955 + -- GREEK CAPITAL LETTER MU + '\x039c'# -> unI64 956 + -- GREEK CAPITAL LETTER NU + '\x039d'# -> unI64 957 + -- GREEK CAPITAL LETTER XI + '\x039e'# -> unI64 958 + -- GREEK CAPITAL LETTER OMICRON + '\x039f'# -> unI64 959 + -- GREEK CAPITAL LETTER PI + '\x03a0'# -> unI64 960 + -- GREEK CAPITAL LETTER RHO + '\x03a1'# -> unI64 961 + -- GREEK CAPITAL LETTER SIGMA + '\x03a3'# -> unI64 963 + -- GREEK CAPITAL LETTER TAU + '\x03a4'# -> unI64 964 + -- GREEK CAPITAL LETTER UPSILON + '\x03a5'# -> unI64 965 + -- GREEK CAPITAL LETTER PHI + '\x03a6'# -> unI64 966 + -- GREEK CAPITAL LETTER CHI + '\x03a7'# -> unI64 967 + -- GREEK CAPITAL LETTER PSI + '\x03a8'# -> unI64 968 + -- GREEK CAPITAL LETTER OMEGA + '\x03a9'# -> unI64 969 + -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + '\x03aa'# -> unI64 970 + -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + '\x03ab'# -> unI64 971 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER FINAL SIGMA + '\x03c2'# -> unI64 963 + -- GREEK CAPITAL KAI SYMBOL + '\x03cf'# -> unI64 983 + -- GREEK BETA SYMBOL + '\x03d0'# -> unI64 946 + -- GREEK THETA SYMBOL + '\x03d1'# -> unI64 952 + -- GREEK PHI SYMBOL + '\x03d5'# -> unI64 966 + -- GREEK PI SYMBOL + '\x03d6'# -> unI64 960 + -- GREEK LETTER ARCHAIC KOPPA + '\x03d8'# -> unI64 985 + -- GREEK LETTER STIGMA + '\x03da'# -> unI64 987 + -- GREEK LETTER DIGAMMA + '\x03dc'# -> unI64 989 + -- GREEK LETTER KOPPA + '\x03de'# -> unI64 991 + -- GREEK LETTER SAMPI + '\x03e0'# -> unI64 993 + -- COPTIC CAPITAL LETTER SHEI + '\x03e2'# -> unI64 995 + -- COPTIC CAPITAL LETTER FEI + '\x03e4'# -> unI64 997 + -- COPTIC CAPITAL LETTER KHEI + '\x03e6'# -> unI64 999 + -- COPTIC CAPITAL LETTER HORI + '\x03e8'# -> unI64 1001 + -- COPTIC CAPITAL LETTER GANGIA + '\x03ea'# -> unI64 1003 + -- COPTIC CAPITAL LETTER SHIMA + '\x03ec'# -> unI64 1005 + -- COPTIC CAPITAL LETTER DEI + '\x03ee'# -> unI64 1007 + -- GREEK KAPPA SYMBOL + '\x03f0'# -> unI64 954 + -- GREEK RHO SYMBOL + '\x03f1'# -> unI64 961 + -- GREEK CAPITAL THETA SYMBOL + '\x03f4'# -> unI64 952 + -- GREEK LUNATE EPSILON SYMBOL + '\x03f5'# -> unI64 949 + -- GREEK CAPITAL LETTER SHO + '\x03f7'# -> unI64 1016 + -- GREEK CAPITAL LUNATE SIGMA SYMBOL + '\x03f9'# -> unI64 1010 + -- GREEK CAPITAL LETTER SAN + '\x03fa'# -> unI64 1019 + -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL + '\x03fd'# -> unI64 891 + -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL + '\x03fe'# -> unI64 892 + -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + '\x03ff'# -> unI64 893 + -- CYRILLIC CAPITAL LETTER IE WITH GRAVE + '\x0400'# -> unI64 1104 + -- CYRILLIC CAPITAL LETTER IO + '\x0401'# -> unI64 1105 + -- CYRILLIC CAPITAL LETTER DJE + '\x0402'# -> unI64 1106 + -- CYRILLIC CAPITAL LETTER GJE + '\x0403'# -> unI64 1107 + -- CYRILLIC CAPITAL LETTER UKRAINIAN IE + '\x0404'# -> unI64 1108 + -- CYRILLIC CAPITAL LETTER DZE + '\x0405'# -> unI64 1109 + -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + '\x0406'# -> unI64 1110 + -- CYRILLIC CAPITAL LETTER YI + '\x0407'# -> unI64 1111 + -- CYRILLIC CAPITAL LETTER JE + '\x0408'# -> unI64 1112 + -- CYRILLIC CAPITAL LETTER LJE + '\x0409'# -> unI64 1113 + -- CYRILLIC CAPITAL LETTER NJE + '\x040a'# -> unI64 1114 + -- CYRILLIC CAPITAL LETTER TSHE + '\x040b'# -> unI64 1115 + -- CYRILLIC CAPITAL LETTER KJE + '\x040c'# -> unI64 1116 + -- CYRILLIC CAPITAL LETTER I WITH GRAVE + '\x040d'# -> unI64 1117 + -- CYRILLIC CAPITAL LETTER SHORT U + '\x040e'# -> unI64 1118 + -- CYRILLIC CAPITAL LETTER DZHE + '\x040f'# -> unI64 1119 + -- CYRILLIC CAPITAL LETTER A + '\x0410'# -> unI64 1072 + -- CYRILLIC CAPITAL LETTER BE + '\x0411'# -> unI64 1073 + -- CYRILLIC CAPITAL LETTER VE + '\x0412'# -> unI64 1074 + -- CYRILLIC CAPITAL LETTER GHE + '\x0413'# -> unI64 1075 + -- CYRILLIC CAPITAL LETTER DE + '\x0414'# -> unI64 1076 + -- CYRILLIC CAPITAL LETTER IE + '\x0415'# -> unI64 1077 + -- CYRILLIC CAPITAL LETTER ZHE + '\x0416'# -> unI64 1078 + -- CYRILLIC CAPITAL LETTER ZE + '\x0417'# -> unI64 1079 + -- CYRILLIC CAPITAL LETTER I + '\x0418'# -> unI64 1080 + -- CYRILLIC CAPITAL LETTER SHORT I + '\x0419'# -> unI64 1081 + -- CYRILLIC CAPITAL LETTER KA + '\x041a'# -> unI64 1082 + -- CYRILLIC CAPITAL LETTER EL + '\x041b'# -> unI64 1083 + -- CYRILLIC CAPITAL LETTER EM + '\x041c'# -> unI64 1084 + -- CYRILLIC CAPITAL LETTER EN + '\x041d'# -> unI64 1085 + -- CYRILLIC CAPITAL LETTER O + '\x041e'# -> unI64 1086 + -- CYRILLIC CAPITAL LETTER PE + '\x041f'# -> unI64 1087 + -- CYRILLIC CAPITAL LETTER ER + '\x0420'# -> unI64 1088 + -- CYRILLIC CAPITAL LETTER ES + '\x0421'# -> unI64 1089 + -- CYRILLIC CAPITAL LETTER TE + '\x0422'# -> unI64 1090 + -- CYRILLIC CAPITAL LETTER U + '\x0423'# -> unI64 1091 + -- CYRILLIC CAPITAL LETTER EF + '\x0424'# -> unI64 1092 + -- CYRILLIC CAPITAL LETTER HA + '\x0425'# -> unI64 1093 + -- CYRILLIC CAPITAL LETTER TSE + '\x0426'# -> unI64 1094 + -- CYRILLIC CAPITAL LETTER CHE + '\x0427'# -> unI64 1095 + -- CYRILLIC CAPITAL LETTER SHA + '\x0428'# -> unI64 1096 + -- CYRILLIC CAPITAL LETTER SHCHA + '\x0429'# -> unI64 1097 + -- CYRILLIC CAPITAL LETTER HARD SIGN + '\x042a'# -> unI64 1098 + -- CYRILLIC CAPITAL LETTER YERU + '\x042b'# -> unI64 1099 + -- CYRILLIC CAPITAL LETTER SOFT SIGN + '\x042c'# -> unI64 1100 + -- CYRILLIC CAPITAL LETTER E + '\x042d'# -> unI64 1101 + -- CYRILLIC CAPITAL LETTER YU + '\x042e'# -> unI64 1102 + -- CYRILLIC CAPITAL LETTER YA + '\x042f'# -> unI64 1103 + -- CYRILLIC CAPITAL LETTER OMEGA + '\x0460'# -> unI64 1121 + -- CYRILLIC CAPITAL LETTER YAT + '\x0462'# -> unI64 1123 + -- CYRILLIC CAPITAL LETTER IOTIFIED E + '\x0464'# -> unI64 1125 + -- CYRILLIC CAPITAL LETTER LITTLE YUS + '\x0466'# -> unI64 1127 + -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS + '\x0468'# -> unI64 1129 + -- CYRILLIC CAPITAL LETTER BIG YUS + '\x046a'# -> unI64 1131 + -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS + '\x046c'# -> unI64 1133 + -- CYRILLIC CAPITAL LETTER KSI + '\x046e'# -> unI64 1135 + -- CYRILLIC CAPITAL LETTER PSI + '\x0470'# -> unI64 1137 + -- CYRILLIC CAPITAL LETTER FITA + '\x0472'# -> unI64 1139 + -- CYRILLIC CAPITAL LETTER IZHITSA + '\x0474'# -> unI64 1141 + -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + '\x0476'# -> unI64 1143 + -- CYRILLIC CAPITAL LETTER UK + '\x0478'# -> unI64 1145 + -- CYRILLIC CAPITAL LETTER ROUND OMEGA + '\x047a'# -> unI64 1147 + -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO + '\x047c'# -> unI64 1149 + -- CYRILLIC CAPITAL LETTER OT + '\x047e'# -> unI64 1151 + -- CYRILLIC CAPITAL LETTER KOPPA + '\x0480'# -> unI64 1153 + -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL + '\x048a'# -> unI64 1163 + -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN + '\x048c'# -> unI64 1165 + -- CYRILLIC CAPITAL LETTER ER WITH TICK + '\x048e'# -> unI64 1167 + -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN + '\x0490'# -> unI64 1169 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE + '\x0492'# -> unI64 1171 + -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK + '\x0494'# -> unI64 1173 + -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + '\x0496'# -> unI64 1175 + -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + '\x0498'# -> unI64 1177 + -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER + '\x049a'# -> unI64 1179 + -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + '\x049c'# -> unI64 1181 + -- CYRILLIC CAPITAL LETTER KA WITH STROKE + '\x049e'# -> unI64 1183 + -- CYRILLIC CAPITAL LETTER BASHKIR KA + '\x04a0'# -> unI64 1185 + -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER + '\x04a2'# -> unI64 1187 + -- CYRILLIC CAPITAL LIGATURE EN GHE + '\x04a4'# -> unI64 1189 + -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK + '\x04a6'# -> unI64 1191 + -- CYRILLIC CAPITAL LETTER ABKHASIAN HA + '\x04a8'# -> unI64 1193 + -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER + '\x04aa'# -> unI64 1195 + -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER + '\x04ac'# -> unI64 1197 + -- CYRILLIC CAPITAL LETTER STRAIGHT U + '\x04ae'# -> unI64 1199 + -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + '\x04b0'# -> unI64 1201 + -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER + '\x04b2'# -> unI64 1203 + -- CYRILLIC CAPITAL LIGATURE TE TSE + '\x04b4'# -> unI64 1205 + -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + '\x04b6'# -> unI64 1207 + -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + '\x04b8'# -> unI64 1209 + -- CYRILLIC CAPITAL LETTER SHHA + '\x04ba'# -> unI64 1211 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE + '\x04bc'# -> unI64 1213 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER + '\x04be'# -> unI64 1215 + -- CYRILLIC LETTER PALOCHKA + '\x04c0'# -> unI64 1231 + -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE + '\x04c1'# -> unI64 1218 + -- CYRILLIC CAPITAL LETTER KA WITH HOOK + '\x04c3'# -> unI64 1220 + -- CYRILLIC CAPITAL LETTER EL WITH TAIL + '\x04c5'# -> unI64 1222 + -- CYRILLIC CAPITAL LETTER EN WITH HOOK + '\x04c7'# -> unI64 1224 + -- CYRILLIC CAPITAL LETTER EN WITH TAIL + '\x04c9'# -> unI64 1226 + -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE + '\x04cb'# -> unI64 1228 + -- CYRILLIC CAPITAL LETTER EM WITH TAIL + '\x04cd'# -> unI64 1230 + -- CYRILLIC CAPITAL LETTER A WITH BREVE + '\x04d0'# -> unI64 1233 + -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS + '\x04d2'# -> unI64 1235 + -- CYRILLIC CAPITAL LIGATURE A IE + '\x04d4'# -> unI64 1237 + -- CYRILLIC CAPITAL LETTER IE WITH BREVE + '\x04d6'# -> unI64 1239 + -- CYRILLIC CAPITAL LETTER SCHWA + '\x04d8'# -> unI64 1241 + -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + '\x04da'# -> unI64 1243 + -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + '\x04dc'# -> unI64 1245 + -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + '\x04de'# -> unI64 1247 + -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE + '\x04e0'# -> unI64 1249 + -- CYRILLIC CAPITAL LETTER I WITH MACRON + '\x04e2'# -> unI64 1251 + -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS + '\x04e4'# -> unI64 1253 + -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS + '\x04e6'# -> unI64 1255 + -- CYRILLIC CAPITAL LETTER BARRED O + '\x04e8'# -> unI64 1257 + -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + '\x04ea'# -> unI64 1259 + -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS + '\x04ec'# -> unI64 1261 + -- CYRILLIC CAPITAL LETTER U WITH MACRON + '\x04ee'# -> unI64 1263 + -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS + '\x04f0'# -> unI64 1265 + -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + '\x04f2'# -> unI64 1267 + -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + '\x04f4'# -> unI64 1269 + -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER + '\x04f6'# -> unI64 1271 + -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + '\x04f8'# -> unI64 1273 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK + '\x04fa'# -> unI64 1275 + -- CYRILLIC CAPITAL LETTER HA WITH HOOK + '\x04fc'# -> unI64 1277 + -- CYRILLIC CAPITAL LETTER HA WITH STROKE + '\x04fe'# -> unI64 1279 + -- CYRILLIC CAPITAL LETTER KOMI DE + '\x0500'# -> unI64 1281 + -- CYRILLIC CAPITAL LETTER KOMI DJE + '\x0502'# -> unI64 1283 + -- CYRILLIC CAPITAL LETTER KOMI ZJE + '\x0504'# -> unI64 1285 + -- CYRILLIC CAPITAL LETTER KOMI DZJE + '\x0506'# -> unI64 1287 + -- CYRILLIC CAPITAL LETTER KOMI LJE + '\x0508'# -> unI64 1289 + -- CYRILLIC CAPITAL LETTER KOMI NJE + '\x050a'# -> unI64 1291 + -- CYRILLIC CAPITAL LETTER KOMI SJE + '\x050c'# -> unI64 1293 + -- CYRILLIC CAPITAL LETTER KOMI TJE + '\x050e'# -> unI64 1295 + -- CYRILLIC CAPITAL LETTER REVERSED ZE + '\x0510'# -> unI64 1297 + -- CYRILLIC CAPITAL LETTER EL WITH HOOK + '\x0512'# -> unI64 1299 + -- CYRILLIC CAPITAL LETTER LHA + '\x0514'# -> unI64 1301 + -- CYRILLIC CAPITAL LETTER RHA + '\x0516'# -> unI64 1303 + -- CYRILLIC CAPITAL LETTER YAE + '\x0518'# -> unI64 1305 + -- CYRILLIC CAPITAL LETTER QA + '\x051a'# -> unI64 1307 + -- CYRILLIC CAPITAL LETTER WE + '\x051c'# -> unI64 1309 + -- CYRILLIC CAPITAL LETTER ALEUT KA + '\x051e'# -> unI64 1311 + -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK + '\x0520'# -> unI64 1313 + -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK + '\x0522'# -> unI64 1315 + -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER + '\x0524'# -> unI64 1317 + -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER + '\x0526'# -> unI64 1319 + -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK + '\x0528'# -> unI64 1321 + -- CYRILLIC CAPITAL LETTER DZZHE + '\x052a'# -> unI64 1323 + -- CYRILLIC CAPITAL LETTER DCHE + '\x052c'# -> unI64 1325 + -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER + '\x052e'# -> unI64 1327 + -- ARMENIAN CAPITAL LETTER AYB + '\x0531'# -> unI64 1377 + -- ARMENIAN CAPITAL LETTER BEN + '\x0532'# -> unI64 1378 + -- ARMENIAN CAPITAL LETTER GIM + '\x0533'# -> unI64 1379 + -- ARMENIAN CAPITAL LETTER DA + '\x0534'# -> unI64 1380 + -- ARMENIAN CAPITAL LETTER ECH + '\x0535'# -> unI64 1381 + -- ARMENIAN CAPITAL LETTER ZA + '\x0536'# -> unI64 1382 + -- ARMENIAN CAPITAL LETTER EH + '\x0537'# -> unI64 1383 + -- ARMENIAN CAPITAL LETTER ET + '\x0538'# -> unI64 1384 + -- ARMENIAN CAPITAL LETTER TO + '\x0539'# -> unI64 1385 + -- ARMENIAN CAPITAL LETTER ZHE + '\x053a'# -> unI64 1386 + -- ARMENIAN CAPITAL LETTER INI + '\x053b'# -> unI64 1387 + -- ARMENIAN CAPITAL LETTER LIWN + '\x053c'# -> unI64 1388 + -- ARMENIAN CAPITAL LETTER XEH + '\x053d'# -> unI64 1389 + -- ARMENIAN CAPITAL LETTER CA + '\x053e'# -> unI64 1390 + -- ARMENIAN CAPITAL LETTER KEN + '\x053f'# -> unI64 1391 + -- ARMENIAN CAPITAL LETTER HO + '\x0540'# -> unI64 1392 + -- ARMENIAN CAPITAL LETTER JA + '\x0541'# -> unI64 1393 + -- ARMENIAN CAPITAL LETTER GHAD + '\x0542'# -> unI64 1394 + -- ARMENIAN CAPITAL LETTER CHEH + '\x0543'# -> unI64 1395 + -- ARMENIAN CAPITAL LETTER MEN + '\x0544'# -> unI64 1396 + -- ARMENIAN CAPITAL LETTER YI + '\x0545'# -> unI64 1397 + -- ARMENIAN CAPITAL LETTER NOW + '\x0546'# -> unI64 1398 + -- ARMENIAN CAPITAL LETTER SHA + '\x0547'# -> unI64 1399 + -- ARMENIAN CAPITAL LETTER VO + '\x0548'# -> unI64 1400 + -- ARMENIAN CAPITAL LETTER CHA + '\x0549'# -> unI64 1401 + -- ARMENIAN CAPITAL LETTER PEH + '\x054a'# -> unI64 1402 + -- ARMENIAN CAPITAL LETTER JHEH + '\x054b'# -> unI64 1403 + -- ARMENIAN CAPITAL LETTER RA + '\x054c'# -> unI64 1404 + -- ARMENIAN CAPITAL LETTER SEH + '\x054d'# -> unI64 1405 + -- ARMENIAN CAPITAL LETTER VEW + '\x054e'# -> unI64 1406 + -- ARMENIAN CAPITAL LETTER TIWN + '\x054f'# -> unI64 1407 + -- ARMENIAN CAPITAL LETTER REH + '\x0550'# -> unI64 1408 + -- ARMENIAN CAPITAL LETTER CO + '\x0551'# -> unI64 1409 + -- ARMENIAN CAPITAL LETTER YIWN + '\x0552'# -> unI64 1410 + -- ARMENIAN CAPITAL LETTER PIWR + '\x0553'# -> unI64 1411 + -- ARMENIAN CAPITAL LETTER KEH + '\x0554'# -> unI64 1412 + -- ARMENIAN CAPITAL LETTER OH + '\x0555'# -> unI64 1413 + -- ARMENIAN CAPITAL LETTER FEH + '\x0556'# -> unI64 1414 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985701 + -- GEORGIAN CAPITAL LETTER AN + '\x10a0'# -> unI64 11520 + -- GEORGIAN CAPITAL LETTER BAN + '\x10a1'# -> unI64 11521 + -- GEORGIAN CAPITAL LETTER GAN + '\x10a2'# -> unI64 11522 + -- GEORGIAN CAPITAL LETTER DON + '\x10a3'# -> unI64 11523 + -- GEORGIAN CAPITAL LETTER EN + '\x10a4'# -> unI64 11524 + -- GEORGIAN CAPITAL LETTER VIN + '\x10a5'# -> unI64 11525 + -- GEORGIAN CAPITAL LETTER ZEN + '\x10a6'# -> unI64 11526 + -- GEORGIAN CAPITAL LETTER TAN + '\x10a7'# -> unI64 11527 + -- GEORGIAN CAPITAL LETTER IN + '\x10a8'# -> unI64 11528 + -- GEORGIAN CAPITAL LETTER KAN + '\x10a9'# -> unI64 11529 + -- GEORGIAN CAPITAL LETTER LAS + '\x10aa'# -> unI64 11530 + -- GEORGIAN CAPITAL LETTER MAN + '\x10ab'# -> unI64 11531 + -- GEORGIAN CAPITAL LETTER NAR + '\x10ac'# -> unI64 11532 + -- GEORGIAN CAPITAL LETTER ON + '\x10ad'# -> unI64 11533 + -- GEORGIAN CAPITAL LETTER PAR + '\x10ae'# -> unI64 11534 + -- GEORGIAN CAPITAL LETTER ZHAR + '\x10af'# -> unI64 11535 + -- GEORGIAN CAPITAL LETTER RAE + '\x10b0'# -> unI64 11536 + -- GEORGIAN CAPITAL LETTER SAN + '\x10b1'# -> unI64 11537 + -- GEORGIAN CAPITAL LETTER TAR + '\x10b2'# -> unI64 11538 + -- GEORGIAN CAPITAL LETTER UN + '\x10b3'# -> unI64 11539 + -- GEORGIAN CAPITAL LETTER PHAR + '\x10b4'# -> unI64 11540 + -- GEORGIAN CAPITAL LETTER KHAR + '\x10b5'# -> unI64 11541 + -- GEORGIAN CAPITAL LETTER GHAN + '\x10b6'# -> unI64 11542 + -- GEORGIAN CAPITAL LETTER QAR + '\x10b7'# -> unI64 11543 + -- GEORGIAN CAPITAL LETTER SHIN + '\x10b8'# -> unI64 11544 + -- GEORGIAN CAPITAL LETTER CHIN + '\x10b9'# -> unI64 11545 + -- GEORGIAN CAPITAL LETTER CAN + '\x10ba'# -> unI64 11546 + -- GEORGIAN CAPITAL LETTER JIL + '\x10bb'# -> unI64 11547 + -- GEORGIAN CAPITAL LETTER CIL + '\x10bc'# -> unI64 11548 + -- GEORGIAN CAPITAL LETTER CHAR + '\x10bd'# -> unI64 11549 + -- GEORGIAN CAPITAL LETTER XAN + '\x10be'# -> unI64 11550 + -- GEORGIAN CAPITAL LETTER JHAN + '\x10bf'# -> unI64 11551 + -- GEORGIAN CAPITAL LETTER HAE + '\x10c0'# -> unI64 11552 + -- GEORGIAN CAPITAL LETTER HE + '\x10c1'# -> unI64 11553 + -- GEORGIAN CAPITAL LETTER HIE + '\x10c2'# -> unI64 11554 + -- GEORGIAN CAPITAL LETTER WE + '\x10c3'# -> unI64 11555 + -- GEORGIAN CAPITAL LETTER HAR + '\x10c4'# -> unI64 11556 + -- GEORGIAN CAPITAL LETTER HOE + '\x10c5'# -> unI64 11557 + -- GEORGIAN CAPITAL LETTER YN + '\x10c7'# -> unI64 11559 + -- GEORGIAN CAPITAL LETTER AEN + '\x10cd'# -> unI64 11565 + -- CHEROKEE SMALL LETTER YE + '\x13f8'# -> unI64 5104 + -- CHEROKEE SMALL LETTER YI + '\x13f9'# -> unI64 5105 + -- CHEROKEE SMALL LETTER YO + '\x13fa'# -> unI64 5106 + -- CHEROKEE SMALL LETTER YU + '\x13fb'# -> unI64 5107 + -- CHEROKEE SMALL LETTER YV + '\x13fc'# -> unI64 5108 + -- CHEROKEE SMALL LETTER MV + '\x13fd'# -> unI64 5109 + -- CYRILLIC SMALL LETTER ROUNDED VE + '\x1c80'# -> unI64 1074 + -- CYRILLIC SMALL LETTER LONG-LEGGED DE + '\x1c81'# -> unI64 1076 + -- CYRILLIC SMALL LETTER NARROW O + '\x1c82'# -> unI64 1086 + -- CYRILLIC SMALL LETTER WIDE ES + '\x1c83'# -> unI64 1089 + -- CYRILLIC SMALL LETTER TALL TE + '\x1c84'# -> unI64 1090 + -- CYRILLIC SMALL LETTER THREE-LEGGED TE + '\x1c85'# -> unI64 1090 + -- CYRILLIC SMALL LETTER TALL HARD SIGN + '\x1c86'# -> unI64 1098 + -- CYRILLIC SMALL LETTER TALL YAT + '\x1c87'# -> unI64 1123 + -- CYRILLIC SMALL LETTER UNBLENDED UK + '\x1c88'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER TJE + '\x1c89'# -> unI64 7306 + -- GEORGIAN MTAVRULI CAPITAL LETTER AN + '\x1c90'# -> unI64 4304 + -- GEORGIAN MTAVRULI CAPITAL LETTER BAN + '\x1c91'# -> unI64 4305 + -- GEORGIAN MTAVRULI CAPITAL LETTER GAN + '\x1c92'# -> unI64 4306 + -- GEORGIAN MTAVRULI CAPITAL LETTER DON + '\x1c93'# -> unI64 4307 + -- GEORGIAN MTAVRULI CAPITAL LETTER EN + '\x1c94'# -> unI64 4308 + -- GEORGIAN MTAVRULI CAPITAL LETTER VIN + '\x1c95'# -> unI64 4309 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN + '\x1c96'# -> unI64 4310 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAN + '\x1c97'# -> unI64 4311 + -- GEORGIAN MTAVRULI CAPITAL LETTER IN + '\x1c98'# -> unI64 4312 + -- GEORGIAN MTAVRULI CAPITAL LETTER KAN + '\x1c99'# -> unI64 4313 + -- GEORGIAN MTAVRULI CAPITAL LETTER LAS + '\x1c9a'# -> unI64 4314 + -- GEORGIAN MTAVRULI CAPITAL LETTER MAN + '\x1c9b'# -> unI64 4315 + -- GEORGIAN MTAVRULI CAPITAL LETTER NAR + '\x1c9c'# -> unI64 4316 + -- GEORGIAN MTAVRULI CAPITAL LETTER ON + '\x1c9d'# -> unI64 4317 + -- GEORGIAN MTAVRULI CAPITAL LETTER PAR + '\x1c9e'# -> unI64 4318 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR + '\x1c9f'# -> unI64 4319 + -- GEORGIAN MTAVRULI CAPITAL LETTER RAE + '\x1ca0'# -> unI64 4320 + -- GEORGIAN MTAVRULI CAPITAL LETTER SAN + '\x1ca1'# -> unI64 4321 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAR + '\x1ca2'# -> unI64 4322 + -- GEORGIAN MTAVRULI CAPITAL LETTER UN + '\x1ca3'# -> unI64 4323 + -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR + '\x1ca4'# -> unI64 4324 + -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR + '\x1ca5'# -> unI64 4325 + -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN + '\x1ca6'# -> unI64 4326 + -- GEORGIAN MTAVRULI CAPITAL LETTER QAR + '\x1ca7'# -> unI64 4327 + -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN + '\x1ca8'# -> unI64 4328 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN + '\x1ca9'# -> unI64 4329 + -- GEORGIAN MTAVRULI CAPITAL LETTER CAN + '\x1caa'# -> unI64 4330 + -- GEORGIAN MTAVRULI CAPITAL LETTER JIL + '\x1cab'# -> unI64 4331 + -- GEORGIAN MTAVRULI CAPITAL LETTER CIL + '\x1cac'# -> unI64 4332 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR + '\x1cad'# -> unI64 4333 + -- GEORGIAN MTAVRULI CAPITAL LETTER XAN + '\x1cae'# -> unI64 4334 + -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN + '\x1caf'# -> unI64 4335 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAE + '\x1cb0'# -> unI64 4336 + -- GEORGIAN MTAVRULI CAPITAL LETTER HE + '\x1cb1'# -> unI64 4337 + -- GEORGIAN MTAVRULI CAPITAL LETTER HIE + '\x1cb2'# -> unI64 4338 + -- GEORGIAN MTAVRULI CAPITAL LETTER WE + '\x1cb3'# -> unI64 4339 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAR + '\x1cb4'# -> unI64 4340 + -- GEORGIAN MTAVRULI CAPITAL LETTER HOE + '\x1cb5'# -> unI64 4341 + -- GEORGIAN MTAVRULI CAPITAL LETTER FI + '\x1cb6'# -> unI64 4342 + -- GEORGIAN MTAVRULI CAPITAL LETTER YN + '\x1cb7'# -> unI64 4343 + -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI + '\x1cb8'# -> unI64 4344 + -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN + '\x1cb9'# -> unI64 4345 + -- GEORGIAN MTAVRULI CAPITAL LETTER AIN + '\x1cba'# -> unI64 4346 + -- GEORGIAN MTAVRULI CAPITAL LETTER AEN + '\x1cbd'# -> unI64 4349 + -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN + '\x1cbe'# -> unI64 4350 + -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + '\x1cbf'# -> unI64 4351 + -- LATIN CAPITAL LETTER A WITH RING BELOW + '\x1e00'# -> unI64 7681 + -- LATIN CAPITAL LETTER B WITH DOT ABOVE + '\x1e02'# -> unI64 7683 + -- LATIN CAPITAL LETTER B WITH DOT BELOW + '\x1e04'# -> unI64 7685 + -- LATIN CAPITAL LETTER B WITH LINE BELOW + '\x1e06'# -> unI64 7687 + -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + '\x1e08'# -> unI64 7689 + -- LATIN CAPITAL LETTER D WITH DOT ABOVE + '\x1e0a'# -> unI64 7691 + -- LATIN CAPITAL LETTER D WITH DOT BELOW + '\x1e0c'# -> unI64 7693 + -- LATIN CAPITAL LETTER D WITH LINE BELOW + '\x1e0e'# -> unI64 7695 + -- LATIN CAPITAL LETTER D WITH CEDILLA + '\x1e10'# -> unI64 7697 + -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + '\x1e12'# -> unI64 7699 + -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + '\x1e14'# -> unI64 7701 + -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + '\x1e16'# -> unI64 7703 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + '\x1e18'# -> unI64 7705 + -- LATIN CAPITAL LETTER E WITH TILDE BELOW + '\x1e1a'# -> unI64 7707 + -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + '\x1e1c'# -> unI64 7709 + -- LATIN CAPITAL LETTER F WITH DOT ABOVE + '\x1e1e'# -> unI64 7711 + -- LATIN CAPITAL LETTER G WITH MACRON + '\x1e20'# -> unI64 7713 + -- LATIN CAPITAL LETTER H WITH DOT ABOVE + '\x1e22'# -> unI64 7715 + -- LATIN CAPITAL LETTER H WITH DOT BELOW + '\x1e24'# -> unI64 7717 + -- LATIN CAPITAL LETTER H WITH DIAERESIS + '\x1e26'# -> unI64 7719 + -- LATIN CAPITAL LETTER H WITH CEDILLA + '\x1e28'# -> unI64 7721 + -- LATIN CAPITAL LETTER H WITH BREVE BELOW + '\x1e2a'# -> unI64 7723 + -- LATIN CAPITAL LETTER I WITH TILDE BELOW + '\x1e2c'# -> unI64 7725 + -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + '\x1e2e'# -> unI64 7727 + -- LATIN CAPITAL LETTER K WITH ACUTE + '\x1e30'# -> unI64 7729 + -- LATIN CAPITAL LETTER K WITH DOT BELOW + '\x1e32'# -> unI64 7731 + -- LATIN CAPITAL LETTER K WITH LINE BELOW + '\x1e34'# -> unI64 7733 + -- LATIN CAPITAL LETTER L WITH DOT BELOW + '\x1e36'# -> unI64 7735 + -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + '\x1e38'# -> unI64 7737 + -- LATIN CAPITAL LETTER L WITH LINE BELOW + '\x1e3a'# -> unI64 7739 + -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + '\x1e3c'# -> unI64 7741 + -- LATIN CAPITAL LETTER M WITH ACUTE + '\x1e3e'# -> unI64 7743 + -- LATIN CAPITAL LETTER M WITH DOT ABOVE + '\x1e40'# -> unI64 7745 + -- LATIN CAPITAL LETTER M WITH DOT BELOW + '\x1e42'# -> unI64 7747 + -- LATIN CAPITAL LETTER N WITH DOT ABOVE + '\x1e44'# -> unI64 7749 + -- LATIN CAPITAL LETTER N WITH DOT BELOW + '\x1e46'# -> unI64 7751 + -- LATIN CAPITAL LETTER N WITH LINE BELOW + '\x1e48'# -> unI64 7753 + -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + '\x1e4a'# -> unI64 7755 + -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + '\x1e4c'# -> unI64 7757 + -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + '\x1e4e'# -> unI64 7759 + -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + '\x1e50'# -> unI64 7761 + -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + '\x1e52'# -> unI64 7763 + -- LATIN CAPITAL LETTER P WITH ACUTE + '\x1e54'# -> unI64 7765 + -- LATIN CAPITAL LETTER P WITH DOT ABOVE + '\x1e56'# -> unI64 7767 + -- LATIN CAPITAL LETTER R WITH DOT ABOVE + '\x1e58'# -> unI64 7769 + -- LATIN CAPITAL LETTER R WITH DOT BELOW + '\x1e5a'# -> unI64 7771 + -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + '\x1e5c'# -> unI64 7773 + -- LATIN CAPITAL LETTER R WITH LINE BELOW + '\x1e5e'# -> unI64 7775 + -- LATIN CAPITAL LETTER S WITH DOT ABOVE + '\x1e60'# -> unI64 7777 + -- LATIN CAPITAL LETTER S WITH DOT BELOW + '\x1e62'# -> unI64 7779 + -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + '\x1e64'# -> unI64 7781 + -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + '\x1e66'# -> unI64 7783 + -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + '\x1e68'# -> unI64 7785 + -- LATIN CAPITAL LETTER T WITH DOT ABOVE + '\x1e6a'# -> unI64 7787 + -- LATIN CAPITAL LETTER T WITH DOT BELOW + '\x1e6c'# -> unI64 7789 + -- LATIN CAPITAL LETTER T WITH LINE BELOW + '\x1e6e'# -> unI64 7791 + -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + '\x1e70'# -> unI64 7793 + -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + '\x1e72'# -> unI64 7795 + -- LATIN CAPITAL LETTER U WITH TILDE BELOW + '\x1e74'# -> unI64 7797 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + '\x1e76'# -> unI64 7799 + -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + '\x1e78'# -> unI64 7801 + -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + '\x1e7a'# -> unI64 7803 + -- LATIN CAPITAL LETTER V WITH TILDE + '\x1e7c'# -> unI64 7805 + -- LATIN CAPITAL LETTER V WITH DOT BELOW + '\x1e7e'# -> unI64 7807 + -- LATIN CAPITAL LETTER W WITH GRAVE + '\x1e80'# -> unI64 7809 + -- LATIN CAPITAL LETTER W WITH ACUTE + '\x1e82'# -> unI64 7811 + -- LATIN CAPITAL LETTER W WITH DIAERESIS + '\x1e84'# -> unI64 7813 + -- LATIN CAPITAL LETTER W WITH DOT ABOVE + '\x1e86'# -> unI64 7815 + -- LATIN CAPITAL LETTER W WITH DOT BELOW + '\x1e88'# -> unI64 7817 + -- LATIN CAPITAL LETTER X WITH DOT ABOVE + '\x1e8a'# -> unI64 7819 + -- LATIN CAPITAL LETTER X WITH DIAERESIS + '\x1e8c'# -> unI64 7821 + -- LATIN CAPITAL LETTER Y WITH DOT ABOVE + '\x1e8e'# -> unI64 7823 + -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + '\x1e90'# -> unI64 7825 + -- LATIN CAPITAL LETTER Z WITH DOT BELOW + '\x1e92'# -> unI64 7827 + -- LATIN CAPITAL LETTER Z WITH LINE BELOW + '\x1e94'# -> unI64 7829 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373288 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390068 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584375 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584377 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200801 + -- LATIN SMALL LETTER LONG S WITH DOT ABOVE + '\x1e9b'# -> unI64 7777 + -- LATIN CAPITAL LETTER SHARP S + '\x1e9e'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH DOT BELOW + '\x1ea0'# -> unI64 7841 + -- LATIN CAPITAL LETTER A WITH HOOK ABOVE + '\x1ea2'# -> unI64 7843 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + '\x1ea4'# -> unI64 7845 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + '\x1ea6'# -> unI64 7847 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ea8'# -> unI64 7849 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + '\x1eaa'# -> unI64 7851 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + '\x1eac'# -> unI64 7853 + -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + '\x1eae'# -> unI64 7855 + -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + '\x1eb0'# -> unI64 7857 + -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + '\x1eb2'# -> unI64 7859 + -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE + '\x1eb4'# -> unI64 7861 + -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + '\x1eb6'# -> unI64 7863 + -- LATIN CAPITAL LETTER E WITH DOT BELOW + '\x1eb8'# -> unI64 7865 + -- LATIN CAPITAL LETTER E WITH HOOK ABOVE + '\x1eba'# -> unI64 7867 + -- LATIN CAPITAL LETTER E WITH TILDE + '\x1ebc'# -> unI64 7869 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + '\x1ebe'# -> unI64 7871 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + '\x1ec0'# -> unI64 7873 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ec2'# -> unI64 7875 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + '\x1ec4'# -> unI64 7877 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + '\x1ec6'# -> unI64 7879 + -- LATIN CAPITAL LETTER I WITH HOOK ABOVE + '\x1ec8'# -> unI64 7881 + -- LATIN CAPITAL LETTER I WITH DOT BELOW + '\x1eca'# -> unI64 7883 + -- LATIN CAPITAL LETTER O WITH DOT BELOW + '\x1ecc'# -> unI64 7885 + -- LATIN CAPITAL LETTER O WITH HOOK ABOVE + '\x1ece'# -> unI64 7887 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + '\x1ed0'# -> unI64 7889 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + '\x1ed2'# -> unI64 7891 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ed4'# -> unI64 7893 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + '\x1ed6'# -> unI64 7895 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + '\x1ed8'# -> unI64 7897 + -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE + '\x1eda'# -> unI64 7899 + -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE + '\x1edc'# -> unI64 7901 + -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + '\x1ede'# -> unI64 7903 + -- LATIN CAPITAL LETTER O WITH HORN AND TILDE + '\x1ee0'# -> unI64 7905 + -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + '\x1ee2'# -> unI64 7907 + -- LATIN CAPITAL LETTER U WITH DOT BELOW + '\x1ee4'# -> unI64 7909 + -- LATIN CAPITAL LETTER U WITH HOOK ABOVE + '\x1ee6'# -> unI64 7911 + -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE + '\x1ee8'# -> unI64 7913 + -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE + '\x1eea'# -> unI64 7915 + -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + '\x1eec'# -> unI64 7917 + -- LATIN CAPITAL LETTER U WITH HORN AND TILDE + '\x1eee'# -> unI64 7919 + -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + '\x1ef0'# -> unI64 7921 + -- LATIN CAPITAL LETTER Y WITH GRAVE + '\x1ef2'# -> unI64 7923 + -- LATIN CAPITAL LETTER Y WITH DOT BELOW + '\x1ef4'# -> unI64 7925 + -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE + '\x1ef6'# -> unI64 7927 + -- LATIN CAPITAL LETTER Y WITH TILDE + '\x1ef8'# -> unI64 7929 + -- LATIN CAPITAL LETTER MIDDLE-WELSH LL + '\x1efa'# -> unI64 7931 + -- LATIN CAPITAL LETTER MIDDLE-WELSH V + '\x1efc'# -> unI64 7933 + -- LATIN CAPITAL LETTER Y WITH LOOP + '\x1efe'# -> unI64 7935 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI + '\x1f08'# -> unI64 7936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA + '\x1f09'# -> unI64 7937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA + '\x1f0a'# -> unI64 7938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA + '\x1f0b'# -> unI64 7939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA + '\x1f0c'# -> unI64 7940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA + '\x1f0d'# -> unI64 7941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI + '\x1f0e'# -> unI64 7942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + '\x1f0f'# -> unI64 7943 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI + '\x1f18'# -> unI64 7952 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA + '\x1f19'# -> unI64 7953 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA + '\x1f1a'# -> unI64 7954 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA + '\x1f1b'# -> unI64 7955 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA + '\x1f1c'# -> unI64 7956 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + '\x1f1d'# -> unI64 7957 + -- GREEK CAPITAL LETTER ETA WITH PSILI + '\x1f28'# -> unI64 7968 + -- GREEK CAPITAL LETTER ETA WITH DASIA + '\x1f29'# -> unI64 7969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA + '\x1f2a'# -> unI64 7970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA + '\x1f2b'# -> unI64 7971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA + '\x1f2c'# -> unI64 7972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA + '\x1f2d'# -> unI64 7973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI + '\x1f2e'# -> unI64 7974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + '\x1f2f'# -> unI64 7975 + -- GREEK CAPITAL LETTER IOTA WITH PSILI + '\x1f38'# -> unI64 7984 + -- GREEK CAPITAL LETTER IOTA WITH DASIA + '\x1f39'# -> unI64 7985 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA + '\x1f3a'# -> unI64 7986 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA + '\x1f3b'# -> unI64 7987 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA + '\x1f3c'# -> unI64 7988 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA + '\x1f3d'# -> unI64 7989 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI + '\x1f3e'# -> unI64 7990 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + '\x1f3f'# -> unI64 7991 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI + '\x1f48'# -> unI64 8000 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA + '\x1f49'# -> unI64 8001 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA + '\x1f4a'# -> unI64 8002 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA + '\x1f4b'# -> unI64 8003 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA + '\x1f4c'# -> unI64 8004 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + '\x1f4d'# -> unI64 8005 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459589 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987461 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498565 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720325 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA + '\x1f59'# -> unI64 8017 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + '\x1f5b'# -> unI64 8019 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + '\x1f5d'# -> unI64 8021 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + '\x1f5f'# -> unI64 8023 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI + '\x1f68'# -> unI64 8032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA + '\x1f69'# -> unI64 8033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA + '\x1f6a'# -> unI64 8034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA + '\x1f6b'# -> unI64 8035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA + '\x1f6c'# -> unI64 8036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA + '\x1f6d'# -> unI64 8037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI + '\x1f6e'# -> unI64 8038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + '\x1f6f'# -> unI64 8039 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1998593792 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1998593793 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1998593794 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1998593795 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1998593796 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1998593797 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1998593798 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1998593799 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1998593792 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1998593793 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1998593794 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1998593795 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1998593796 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1998593797 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1998593798 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1998593799 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1998593824 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1998593825 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1998593826 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1998593827 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1998593828 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1998593829 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1998593830 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1998593831 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1998593824 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1998593825 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1998593826 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1998593827 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1998593828 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1998593829 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1998593830 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1998593831 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1998593888 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1998593889 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1998593890 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1998593891 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1998593892 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1998593893 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1998593894 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1998593895 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1998593888 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1998593889 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1998593890 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1998593891 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1998593892 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1998593893 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1998593894 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1998593895 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1998593904 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1998586801 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1998586796 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025713 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4191340074107825 + -- GREEK CAPITAL LETTER ALPHA WITH VRACHY + '\x1fb8'# -> unI64 8112 + -- GREEK CAPITAL LETTER ALPHA WITH MACRON + '\x1fb9'# -> unI64 8113 + -- GREEK CAPITAL LETTER ALPHA WITH VARIA + '\x1fba'# -> unI64 8048 + -- GREEK CAPITAL LETTER ALPHA WITH OXIA + '\x1fbb'# -> unI64 8049 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1998586801 + -- GREEK PROSGEGRAMMENI + '\x1fbe'# -> unI64 953 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1998593908 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1998586807 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1998586798 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025719 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4191340074107831 + -- GREEK CAPITAL LETTER EPSILON WITH VARIA + '\x1fc8'# -> unI64 8050 + -- GREEK CAPITAL LETTER EPSILON WITH OXIA + '\x1fc9'# -> unI64 8051 + -- GREEK CAPITAL LETTER ETA WITH VARIA + '\x1fca'# -> unI64 8052 + -- GREEK CAPITAL LETTER ETA WITH OXIA + '\x1fcb'# -> unI64 8053 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1998586807 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918777 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025721 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651641 + -- GREEK CAPITAL LETTER IOTA WITH VRACHY + '\x1fd8'# -> unI64 8144 + -- GREEK CAPITAL LETTER IOTA WITH MACRON + '\x1fd9'# -> unI64 8145 + -- GREEK CAPITAL LETTER IOTA WITH VARIA + '\x1fda'# -> unI64 8054 + -- GREEK CAPITAL LETTER IOTA WITH OXIA + '\x1fdb'# -> unI64 8055 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918789 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459585 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025733 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651653 + -- GREEK CAPITAL LETTER UPSILON WITH VRACHY + '\x1fe8'# -> unI64 8160 + -- GREEK CAPITAL LETTER UPSILON WITH MACRON + '\x1fe9'# -> unI64 8161 + -- GREEK CAPITAL LETTER UPSILON WITH VARIA + '\x1fea'# -> unI64 8058 + -- GREEK CAPITAL LETTER UPSILON WITH OXIA + '\x1feb'# -> unI64 8059 + -- GREEK CAPITAL LETTER RHO WITH DASIA + '\x1fec'# -> unI64 8165 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1998593916 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1998586825 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1998586830 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025737 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4191340074107849 + -- GREEK CAPITAL LETTER OMICRON WITH VARIA + '\x1ff8'# -> unI64 8056 + -- GREEK CAPITAL LETTER OMICRON WITH OXIA + '\x1ff9'# -> unI64 8057 + -- GREEK CAPITAL LETTER OMEGA WITH VARIA + '\x1ffa'# -> unI64 8060 + -- GREEK CAPITAL LETTER OMEGA WITH OXIA + '\x1ffb'# -> unI64 8061 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1998586825 + -- OHM SIGN + '\x2126'# -> unI64 969 + -- KELVIN SIGN + '\x212a'# -> unI64 107 + -- ANGSTROM SIGN + '\x212b'# -> unI64 229 + -- TURNED CAPITAL F + '\x2132'# -> unI64 8526 + -- ROMAN NUMERAL ONE + '\x2160'# -> unI64 8560 + -- ROMAN NUMERAL TWO + '\x2161'# -> unI64 8561 + -- ROMAN NUMERAL THREE + '\x2162'# -> unI64 8562 + -- ROMAN NUMERAL FOUR + '\x2163'# -> unI64 8563 + -- ROMAN NUMERAL FIVE + '\x2164'# -> unI64 8564 + -- ROMAN NUMERAL SIX + '\x2165'# -> unI64 8565 + -- ROMAN NUMERAL SEVEN + '\x2166'# -> unI64 8566 + -- ROMAN NUMERAL EIGHT + '\x2167'# -> unI64 8567 + -- ROMAN NUMERAL NINE + '\x2168'# -> unI64 8568 + -- ROMAN NUMERAL TEN + '\x2169'# -> unI64 8569 + -- ROMAN NUMERAL ELEVEN + '\x216a'# -> unI64 8570 + -- ROMAN NUMERAL TWELVE + '\x216b'# -> unI64 8571 + -- ROMAN NUMERAL FIFTY + '\x216c'# -> unI64 8572 + -- ROMAN NUMERAL ONE HUNDRED + '\x216d'# -> unI64 8573 + -- ROMAN NUMERAL FIVE HUNDRED + '\x216e'# -> unI64 8574 + -- ROMAN NUMERAL ONE THOUSAND + '\x216f'# -> unI64 8575 + -- ROMAN NUMERAL REVERSED ONE HUNDRED + '\x2183'# -> unI64 8580 + -- CIRCLED LATIN CAPITAL LETTER A + '\x24b6'# -> unI64 9424 + -- CIRCLED LATIN CAPITAL LETTER B + '\x24b7'# -> unI64 9425 + -- CIRCLED LATIN CAPITAL LETTER C + '\x24b8'# -> unI64 9426 + -- CIRCLED LATIN CAPITAL LETTER D + '\x24b9'# -> unI64 9427 + -- CIRCLED LATIN CAPITAL LETTER E + '\x24ba'# -> unI64 9428 + -- CIRCLED LATIN CAPITAL LETTER F + '\x24bb'# -> unI64 9429 + -- CIRCLED LATIN CAPITAL LETTER G + '\x24bc'# -> unI64 9430 + -- CIRCLED LATIN CAPITAL LETTER H + '\x24bd'# -> unI64 9431 + -- CIRCLED LATIN CAPITAL LETTER I + '\x24be'# -> unI64 9432 + -- CIRCLED LATIN CAPITAL LETTER J + '\x24bf'# -> unI64 9433 + -- CIRCLED LATIN CAPITAL LETTER K + '\x24c0'# -> unI64 9434 + -- CIRCLED LATIN CAPITAL LETTER L + '\x24c1'# -> unI64 9435 + -- CIRCLED LATIN CAPITAL LETTER M + '\x24c2'# -> unI64 9436 + -- CIRCLED LATIN CAPITAL LETTER N + '\x24c3'# -> unI64 9437 + -- CIRCLED LATIN CAPITAL LETTER O + '\x24c4'# -> unI64 9438 + -- CIRCLED LATIN CAPITAL LETTER P + '\x24c5'# -> unI64 9439 + -- CIRCLED LATIN CAPITAL LETTER Q + '\x24c6'# -> unI64 9440 + -- CIRCLED LATIN CAPITAL LETTER R + '\x24c7'# -> unI64 9441 + -- CIRCLED LATIN CAPITAL LETTER S + '\x24c8'# -> unI64 9442 + -- CIRCLED LATIN CAPITAL LETTER T + '\x24c9'# -> unI64 9443 + -- CIRCLED LATIN CAPITAL LETTER U + '\x24ca'# -> unI64 9444 + -- CIRCLED LATIN CAPITAL LETTER V + '\x24cb'# -> unI64 9445 + -- CIRCLED LATIN CAPITAL LETTER W + '\x24cc'# -> unI64 9446 + -- CIRCLED LATIN CAPITAL LETTER X + '\x24cd'# -> unI64 9447 + -- CIRCLED LATIN CAPITAL LETTER Y + '\x24ce'# -> unI64 9448 + -- CIRCLED LATIN CAPITAL LETTER Z + '\x24cf'# -> unI64 9449 + -- GLAGOLITIC CAPITAL LETTER AZU + '\x2c00'# -> unI64 11312 + -- GLAGOLITIC CAPITAL LETTER BUKY + '\x2c01'# -> unI64 11313 + -- GLAGOLITIC CAPITAL LETTER VEDE + '\x2c02'# -> unI64 11314 + -- GLAGOLITIC CAPITAL LETTER GLAGOLI + '\x2c03'# -> unI64 11315 + -- GLAGOLITIC CAPITAL LETTER DOBRO + '\x2c04'# -> unI64 11316 + -- GLAGOLITIC CAPITAL LETTER YESTU + '\x2c05'# -> unI64 11317 + -- GLAGOLITIC CAPITAL LETTER ZHIVETE + '\x2c06'# -> unI64 11318 + -- GLAGOLITIC CAPITAL LETTER DZELO + '\x2c07'# -> unI64 11319 + -- GLAGOLITIC CAPITAL LETTER ZEMLJA + '\x2c08'# -> unI64 11320 + -- GLAGOLITIC CAPITAL LETTER IZHE + '\x2c09'# -> unI64 11321 + -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE + '\x2c0a'# -> unI64 11322 + -- GLAGOLITIC CAPITAL LETTER I + '\x2c0b'# -> unI64 11323 + -- GLAGOLITIC CAPITAL LETTER DJERVI + '\x2c0c'# -> unI64 11324 + -- GLAGOLITIC CAPITAL LETTER KAKO + '\x2c0d'# -> unI64 11325 + -- GLAGOLITIC CAPITAL LETTER LJUDIJE + '\x2c0e'# -> unI64 11326 + -- GLAGOLITIC CAPITAL LETTER MYSLITE + '\x2c0f'# -> unI64 11327 + -- GLAGOLITIC CAPITAL LETTER NASHI + '\x2c10'# -> unI64 11328 + -- GLAGOLITIC CAPITAL LETTER ONU + '\x2c11'# -> unI64 11329 + -- GLAGOLITIC CAPITAL LETTER POKOJI + '\x2c12'# -> unI64 11330 + -- GLAGOLITIC CAPITAL LETTER RITSI + '\x2c13'# -> unI64 11331 + -- GLAGOLITIC CAPITAL LETTER SLOVO + '\x2c14'# -> unI64 11332 + -- GLAGOLITIC CAPITAL LETTER TVRIDO + '\x2c15'# -> unI64 11333 + -- GLAGOLITIC CAPITAL LETTER UKU + '\x2c16'# -> unI64 11334 + -- GLAGOLITIC CAPITAL LETTER FRITU + '\x2c17'# -> unI64 11335 + -- GLAGOLITIC CAPITAL LETTER HERU + '\x2c18'# -> unI64 11336 + -- GLAGOLITIC CAPITAL LETTER OTU + '\x2c19'# -> unI64 11337 + -- GLAGOLITIC CAPITAL LETTER PE + '\x2c1a'# -> unI64 11338 + -- GLAGOLITIC CAPITAL LETTER SHTA + '\x2c1b'# -> unI64 11339 + -- GLAGOLITIC CAPITAL LETTER TSI + '\x2c1c'# -> unI64 11340 + -- GLAGOLITIC CAPITAL LETTER CHRIVI + '\x2c1d'# -> unI64 11341 + -- GLAGOLITIC CAPITAL LETTER SHA + '\x2c1e'# -> unI64 11342 + -- GLAGOLITIC CAPITAL LETTER YERU + '\x2c1f'# -> unI64 11343 + -- GLAGOLITIC CAPITAL LETTER YERI + '\x2c20'# -> unI64 11344 + -- GLAGOLITIC CAPITAL LETTER YATI + '\x2c21'# -> unI64 11345 + -- GLAGOLITIC CAPITAL LETTER SPIDERY HA + '\x2c22'# -> unI64 11346 + -- GLAGOLITIC CAPITAL LETTER YU + '\x2c23'# -> unI64 11347 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS + '\x2c24'# -> unI64 11348 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL + '\x2c25'# -> unI64 11349 + -- GLAGOLITIC CAPITAL LETTER YO + '\x2c26'# -> unI64 11350 + -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS + '\x2c27'# -> unI64 11351 + -- GLAGOLITIC CAPITAL LETTER BIG YUS + '\x2c28'# -> unI64 11352 + -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS + '\x2c29'# -> unI64 11353 + -- GLAGOLITIC CAPITAL LETTER FITA + '\x2c2a'# -> unI64 11354 + -- GLAGOLITIC CAPITAL LETTER IZHITSA + '\x2c2b'# -> unI64 11355 + -- GLAGOLITIC CAPITAL LETTER SHTAPIC + '\x2c2c'# -> unI64 11356 + -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A + '\x2c2d'# -> unI64 11357 + -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE + '\x2c2e'# -> unI64 11358 + -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI + '\x2c2f'# -> unI64 11359 + -- LATIN CAPITAL LETTER L WITH DOUBLE BAR + '\x2c60'# -> unI64 11361 + -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE + '\x2c62'# -> unI64 619 + -- LATIN CAPITAL LETTER P WITH STROKE + '\x2c63'# -> unI64 7549 + -- LATIN CAPITAL LETTER R WITH TAIL + '\x2c64'# -> unI64 637 + -- LATIN CAPITAL LETTER H WITH DESCENDER + '\x2c67'# -> unI64 11368 + -- LATIN CAPITAL LETTER K WITH DESCENDER + '\x2c69'# -> unI64 11370 + -- LATIN CAPITAL LETTER Z WITH DESCENDER + '\x2c6b'# -> unI64 11372 + -- LATIN CAPITAL LETTER ALPHA + '\x2c6d'# -> unI64 593 + -- LATIN CAPITAL LETTER M WITH HOOK + '\x2c6e'# -> unI64 625 + -- LATIN CAPITAL LETTER TURNED A + '\x2c6f'# -> unI64 592 + -- LATIN CAPITAL LETTER TURNED ALPHA + '\x2c70'# -> unI64 594 + -- LATIN CAPITAL LETTER W WITH HOOK + '\x2c72'# -> unI64 11379 + -- LATIN CAPITAL LETTER HALF H + '\x2c75'# -> unI64 11382 + -- LATIN CAPITAL LETTER S WITH SWASH TAIL + '\x2c7e'# -> unI64 575 + -- LATIN CAPITAL LETTER Z WITH SWASH TAIL + '\x2c7f'# -> unI64 576 + -- COPTIC CAPITAL LETTER ALFA + '\x2c80'# -> unI64 11393 + -- COPTIC CAPITAL LETTER VIDA + '\x2c82'# -> unI64 11395 + -- COPTIC CAPITAL LETTER GAMMA + '\x2c84'# -> unI64 11397 + -- COPTIC CAPITAL LETTER DALDA + '\x2c86'# -> unI64 11399 + -- COPTIC CAPITAL LETTER EIE + '\x2c88'# -> unI64 11401 + -- COPTIC CAPITAL LETTER SOU + '\x2c8a'# -> unI64 11403 + -- COPTIC CAPITAL LETTER ZATA + '\x2c8c'# -> unI64 11405 + -- COPTIC CAPITAL LETTER HATE + '\x2c8e'# -> unI64 11407 + -- COPTIC CAPITAL LETTER THETHE + '\x2c90'# -> unI64 11409 + -- COPTIC CAPITAL LETTER IAUDA + '\x2c92'# -> unI64 11411 + -- COPTIC CAPITAL LETTER KAPA + '\x2c94'# -> unI64 11413 + -- COPTIC CAPITAL LETTER LAULA + '\x2c96'# -> unI64 11415 + -- COPTIC CAPITAL LETTER MI + '\x2c98'# -> unI64 11417 + -- COPTIC CAPITAL LETTER NI + '\x2c9a'# -> unI64 11419 + -- COPTIC CAPITAL LETTER KSI + '\x2c9c'# -> unI64 11421 + -- COPTIC CAPITAL LETTER O + '\x2c9e'# -> unI64 11423 + -- COPTIC CAPITAL LETTER PI + '\x2ca0'# -> unI64 11425 + -- COPTIC CAPITAL LETTER RO + '\x2ca2'# -> unI64 11427 + -- COPTIC CAPITAL LETTER SIMA + '\x2ca4'# -> unI64 11429 + -- COPTIC CAPITAL LETTER TAU + '\x2ca6'# -> unI64 11431 + -- COPTIC CAPITAL LETTER UA + '\x2ca8'# -> unI64 11433 + -- COPTIC CAPITAL LETTER FI + '\x2caa'# -> unI64 11435 + -- COPTIC CAPITAL LETTER KHI + '\x2cac'# -> unI64 11437 + -- COPTIC CAPITAL LETTER PSI + '\x2cae'# -> unI64 11439 + -- COPTIC CAPITAL LETTER OOU + '\x2cb0'# -> unI64 11441 + -- COPTIC CAPITAL LETTER DIALECT-P ALEF + '\x2cb2'# -> unI64 11443 + -- COPTIC CAPITAL LETTER OLD COPTIC AIN + '\x2cb4'# -> unI64 11445 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE + '\x2cb6'# -> unI64 11447 + -- COPTIC CAPITAL LETTER DIALECT-P KAPA + '\x2cb8'# -> unI64 11449 + -- COPTIC CAPITAL LETTER DIALECT-P NI + '\x2cba'# -> unI64 11451 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI + '\x2cbc'# -> unI64 11453 + -- COPTIC CAPITAL LETTER OLD COPTIC OOU + '\x2cbe'# -> unI64 11455 + -- COPTIC CAPITAL LETTER SAMPI + '\x2cc0'# -> unI64 11457 + -- COPTIC CAPITAL LETTER CROSSED SHEI + '\x2cc2'# -> unI64 11459 + -- COPTIC CAPITAL LETTER OLD COPTIC SHEI + '\x2cc4'# -> unI64 11461 + -- COPTIC CAPITAL LETTER OLD COPTIC ESH + '\x2cc6'# -> unI64 11463 + -- COPTIC CAPITAL LETTER AKHMIMIC KHEI + '\x2cc8'# -> unI64 11465 + -- COPTIC CAPITAL LETTER DIALECT-P HORI + '\x2cca'# -> unI64 11467 + -- COPTIC CAPITAL LETTER OLD COPTIC HORI + '\x2ccc'# -> unI64 11469 + -- COPTIC CAPITAL LETTER OLD COPTIC HA + '\x2cce'# -> unI64 11471 + -- COPTIC CAPITAL LETTER L-SHAPED HA + '\x2cd0'# -> unI64 11473 + -- COPTIC CAPITAL LETTER OLD COPTIC HEI + '\x2cd2'# -> unI64 11475 + -- COPTIC CAPITAL LETTER OLD COPTIC HAT + '\x2cd4'# -> unI64 11477 + -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA + '\x2cd6'# -> unI64 11479 + -- COPTIC CAPITAL LETTER OLD COPTIC DJA + '\x2cd8'# -> unI64 11481 + -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA + '\x2cda'# -> unI64 11483 + -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA + '\x2cdc'# -> unI64 11485 + -- COPTIC CAPITAL LETTER OLD NUBIAN NGI + '\x2cde'# -> unI64 11487 + -- COPTIC CAPITAL LETTER OLD NUBIAN NYI + '\x2ce0'# -> unI64 11489 + -- COPTIC CAPITAL LETTER OLD NUBIAN WAU + '\x2ce2'# -> unI64 11491 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI + '\x2ceb'# -> unI64 11500 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA + '\x2ced'# -> unI64 11502 + -- COPTIC CAPITAL LETTER BOHAIRIC KHEI + '\x2cf2'# -> unI64 11507 + -- CYRILLIC CAPITAL LETTER ZEMLYA + '\xa640'# -> unI64 42561 + -- CYRILLIC CAPITAL LETTER DZELO + '\xa642'# -> unI64 42563 + -- CYRILLIC CAPITAL LETTER REVERSED DZE + '\xa644'# -> unI64 42565 + -- CYRILLIC CAPITAL LETTER IOTA + '\xa646'# -> unI64 42567 + -- CYRILLIC CAPITAL LETTER DJERV + '\xa648'# -> unI64 42569 + -- CYRILLIC CAPITAL LETTER MONOGRAPH UK + '\xa64a'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER BROAD OMEGA + '\xa64c'# -> unI64 42573 + -- CYRILLIC CAPITAL LETTER NEUTRAL YER + '\xa64e'# -> unI64 42575 + -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER + '\xa650'# -> unI64 42577 + -- CYRILLIC CAPITAL LETTER IOTIFIED YAT + '\xa652'# -> unI64 42579 + -- CYRILLIC CAPITAL LETTER REVERSED YU + '\xa654'# -> unI64 42581 + -- CYRILLIC CAPITAL LETTER IOTIFIED A + '\xa656'# -> unI64 42583 + -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS + '\xa658'# -> unI64 42585 + -- CYRILLIC CAPITAL LETTER BLENDED YUS + '\xa65a'# -> unI64 42587 + -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS + '\xa65c'# -> unI64 42589 + -- CYRILLIC CAPITAL LETTER YN + '\xa65e'# -> unI64 42591 + -- CYRILLIC CAPITAL LETTER REVERSED TSE + '\xa660'# -> unI64 42593 + -- CYRILLIC CAPITAL LETTER SOFT DE + '\xa662'# -> unI64 42595 + -- CYRILLIC CAPITAL LETTER SOFT EL + '\xa664'# -> unI64 42597 + -- CYRILLIC CAPITAL LETTER SOFT EM + '\xa666'# -> unI64 42599 + -- CYRILLIC CAPITAL LETTER MONOCULAR O + '\xa668'# -> unI64 42601 + -- CYRILLIC CAPITAL LETTER BINOCULAR O + '\xa66a'# -> unI64 42603 + -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O + '\xa66c'# -> unI64 42605 + -- CYRILLIC CAPITAL LETTER DWE + '\xa680'# -> unI64 42625 + -- CYRILLIC CAPITAL LETTER DZWE + '\xa682'# -> unI64 42627 + -- CYRILLIC CAPITAL LETTER ZHWE + '\xa684'# -> unI64 42629 + -- CYRILLIC CAPITAL LETTER CCHE + '\xa686'# -> unI64 42631 + -- CYRILLIC CAPITAL LETTER DZZE + '\xa688'# -> unI64 42633 + -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK + '\xa68a'# -> unI64 42635 + -- CYRILLIC CAPITAL LETTER TWE + '\xa68c'# -> unI64 42637 + -- CYRILLIC CAPITAL LETTER TSWE + '\xa68e'# -> unI64 42639 + -- CYRILLIC CAPITAL LETTER TSSE + '\xa690'# -> unI64 42641 + -- CYRILLIC CAPITAL LETTER TCHE + '\xa692'# -> unI64 42643 + -- CYRILLIC CAPITAL LETTER HWE + '\xa694'# -> unI64 42645 + -- CYRILLIC CAPITAL LETTER SHWE + '\xa696'# -> unI64 42647 + -- CYRILLIC CAPITAL LETTER DOUBLE O + '\xa698'# -> unI64 42649 + -- CYRILLIC CAPITAL LETTER CROSSED O + '\xa69a'# -> unI64 42651 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF + '\xa722'# -> unI64 42787 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN + '\xa724'# -> unI64 42789 + -- LATIN CAPITAL LETTER HENG + '\xa726'# -> unI64 42791 + -- LATIN CAPITAL LETTER TZ + '\xa728'# -> unI64 42793 + -- LATIN CAPITAL LETTER TRESILLO + '\xa72a'# -> unI64 42795 + -- LATIN CAPITAL LETTER CUATRILLO + '\xa72c'# -> unI64 42797 + -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA + '\xa72e'# -> unI64 42799 + -- LATIN CAPITAL LETTER AA + '\xa732'# -> unI64 42803 + -- LATIN CAPITAL LETTER AO + '\xa734'# -> unI64 42805 + -- LATIN CAPITAL LETTER AU + '\xa736'# -> unI64 42807 + -- LATIN CAPITAL LETTER AV + '\xa738'# -> unI64 42809 + -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR + '\xa73a'# -> unI64 42811 + -- LATIN CAPITAL LETTER AY + '\xa73c'# -> unI64 42813 + -- LATIN CAPITAL LETTER REVERSED C WITH DOT + '\xa73e'# -> unI64 42815 + -- LATIN CAPITAL LETTER K WITH STROKE + '\xa740'# -> unI64 42817 + -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + '\xa742'# -> unI64 42819 + -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + '\xa744'# -> unI64 42821 + -- LATIN CAPITAL LETTER BROKEN L + '\xa746'# -> unI64 42823 + -- LATIN CAPITAL LETTER L WITH HIGH STROKE + '\xa748'# -> unI64 42825 + -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + '\xa74a'# -> unI64 42827 + -- LATIN CAPITAL LETTER O WITH LOOP + '\xa74c'# -> unI64 42829 + -- LATIN CAPITAL LETTER OO + '\xa74e'# -> unI64 42831 + -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + '\xa750'# -> unI64 42833 + -- LATIN CAPITAL LETTER P WITH FLOURISH + '\xa752'# -> unI64 42835 + -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + '\xa754'# -> unI64 42837 + -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + '\xa756'# -> unI64 42839 + -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + '\xa758'# -> unI64 42841 + -- LATIN CAPITAL LETTER R ROTUNDA + '\xa75a'# -> unI64 42843 + -- LATIN CAPITAL LETTER RUM ROTUNDA + '\xa75c'# -> unI64 42845 + -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + '\xa75e'# -> unI64 42847 + -- LATIN CAPITAL LETTER VY + '\xa760'# -> unI64 42849 + -- LATIN CAPITAL LETTER VISIGOTHIC Z + '\xa762'# -> unI64 42851 + -- LATIN CAPITAL LETTER THORN WITH STROKE + '\xa764'# -> unI64 42853 + -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER + '\xa766'# -> unI64 42855 + -- LATIN CAPITAL LETTER VEND + '\xa768'# -> unI64 42857 + -- LATIN CAPITAL LETTER ET + '\xa76a'# -> unI64 42859 + -- LATIN CAPITAL LETTER IS + '\xa76c'# -> unI64 42861 + -- LATIN CAPITAL LETTER CON + '\xa76e'# -> unI64 42863 + -- LATIN CAPITAL LETTER INSULAR D + '\xa779'# -> unI64 42874 + -- LATIN CAPITAL LETTER INSULAR F + '\xa77b'# -> unI64 42876 + -- LATIN CAPITAL LETTER INSULAR G + '\xa77d'# -> unI64 7545 + -- LATIN CAPITAL LETTER TURNED INSULAR G + '\xa77e'# -> unI64 42879 + -- LATIN CAPITAL LETTER TURNED L + '\xa780'# -> unI64 42881 + -- LATIN CAPITAL LETTER INSULAR R + '\xa782'# -> unI64 42883 + -- LATIN CAPITAL LETTER INSULAR S + '\xa784'# -> unI64 42885 + -- LATIN CAPITAL LETTER INSULAR T + '\xa786'# -> unI64 42887 + -- LATIN CAPITAL LETTER SALTILLO + '\xa78b'# -> unI64 42892 + -- LATIN CAPITAL LETTER TURNED H + '\xa78d'# -> unI64 613 + -- LATIN CAPITAL LETTER N WITH DESCENDER + '\xa790'# -> unI64 42897 + -- LATIN CAPITAL LETTER C WITH BAR + '\xa792'# -> unI64 42899 + -- LATIN CAPITAL LETTER B WITH FLOURISH + '\xa796'# -> unI64 42903 + -- LATIN CAPITAL LETTER F WITH STROKE + '\xa798'# -> unI64 42905 + -- LATIN CAPITAL LETTER VOLAPUK AE + '\xa79a'# -> unI64 42907 + -- LATIN CAPITAL LETTER VOLAPUK OE + '\xa79c'# -> unI64 42909 + -- LATIN CAPITAL LETTER VOLAPUK UE + '\xa79e'# -> unI64 42911 + -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + '\xa7a0'# -> unI64 42913 + -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + '\xa7a2'# -> unI64 42915 + -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + '\xa7a4'# -> unI64 42917 + -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + '\xa7a6'# -> unI64 42919 + -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + '\xa7a8'# -> unI64 42921 + -- LATIN CAPITAL LETTER H WITH HOOK + '\xa7aa'# -> unI64 614 + -- LATIN CAPITAL LETTER REVERSED OPEN E + '\xa7ab'# -> unI64 604 + -- LATIN CAPITAL LETTER SCRIPT G + '\xa7ac'# -> unI64 609 + -- LATIN CAPITAL LETTER L WITH BELT + '\xa7ad'# -> unI64 620 + -- LATIN CAPITAL LETTER SMALL CAPITAL I + '\xa7ae'# -> unI64 618 + -- LATIN CAPITAL LETTER TURNED K + '\xa7b0'# -> unI64 670 + -- LATIN CAPITAL LETTER TURNED T + '\xa7b1'# -> unI64 647 + -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL + '\xa7b2'# -> unI64 669 + -- LATIN CAPITAL LETTER CHI + '\xa7b3'# -> unI64 43859 + -- LATIN CAPITAL LETTER BETA + '\xa7b4'# -> unI64 42933 + -- LATIN CAPITAL LETTER OMEGA + '\xa7b6'# -> unI64 42935 + -- LATIN CAPITAL LETTER U WITH STROKE + '\xa7b8'# -> unI64 42937 + -- LATIN CAPITAL LETTER GLOTTAL A + '\xa7ba'# -> unI64 42939 + -- LATIN CAPITAL LETTER GLOTTAL I + '\xa7bc'# -> unI64 42941 + -- LATIN CAPITAL LETTER GLOTTAL U + '\xa7be'# -> unI64 42943 + -- LATIN CAPITAL LETTER OLD POLISH O + '\xa7c0'# -> unI64 42945 + -- LATIN CAPITAL LETTER ANGLICANA W + '\xa7c2'# -> unI64 42947 + -- LATIN CAPITAL LETTER C WITH PALATAL HOOK + '\xa7c4'# -> unI64 42900 + -- LATIN CAPITAL LETTER S WITH HOOK + '\xa7c5'# -> unI64 642 + -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK + '\xa7c6'# -> unI64 7566 + -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + '\xa7c7'# -> unI64 42952 + -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + '\xa7c9'# -> unI64 42954 + -- LATIN CAPITAL LETTER RAMS HORN + '\xa7cb'# -> unI64 612 + -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE + '\xa7cc'# -> unI64 42957 + -- LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE + '\xa7ce'# -> unI64 42959 + -- LATIN CAPITAL LETTER CLOSED INSULAR G + '\xa7d0'# -> unI64 42961 + -- LATIN CAPITAL LETTER DOUBLE THORN + '\xa7d2'# -> unI64 42963 + -- LATIN CAPITAL LETTER DOUBLE WYNN + '\xa7d4'# -> unI64 42965 + -- LATIN CAPITAL LETTER MIDDLE SCOTS S + '\xa7d6'# -> unI64 42967 + -- LATIN CAPITAL LETTER SIGMOID S + '\xa7d8'# -> unI64 42969 + -- LATIN CAPITAL LETTER LAMBDA + '\xa7da'# -> unI64 42971 + -- LATIN CAPITAL LETTER LAMBDA WITH STROKE + '\xa7dc'# -> unI64 411 + -- LATIN CAPITAL LETTER REVERSED HALF H + '\xa7f5'# -> unI64 42998 + -- CHEROKEE SMALL LETTER A + '\xab70'# -> unI64 5024 + -- CHEROKEE SMALL LETTER E + '\xab71'# -> unI64 5025 + -- CHEROKEE SMALL LETTER I + '\xab72'# -> unI64 5026 + -- CHEROKEE SMALL LETTER O + '\xab73'# -> unI64 5027 + -- CHEROKEE SMALL LETTER U + '\xab74'# -> unI64 5028 + -- CHEROKEE SMALL LETTER V + '\xab75'# -> unI64 5029 + -- CHEROKEE SMALL LETTER GA + '\xab76'# -> unI64 5030 + -- CHEROKEE SMALL LETTER KA + '\xab77'# -> unI64 5031 + -- CHEROKEE SMALL LETTER GE + '\xab78'# -> unI64 5032 + -- CHEROKEE SMALL LETTER GI + '\xab79'# -> unI64 5033 + -- CHEROKEE SMALL LETTER GO + '\xab7a'# -> unI64 5034 + -- CHEROKEE SMALL LETTER GU + '\xab7b'# -> unI64 5035 + -- CHEROKEE SMALL LETTER GV + '\xab7c'# -> unI64 5036 + -- CHEROKEE SMALL LETTER HA + '\xab7d'# -> unI64 5037 + -- CHEROKEE SMALL LETTER HE + '\xab7e'# -> unI64 5038 + -- CHEROKEE SMALL LETTER HI + '\xab7f'# -> unI64 5039 + -- CHEROKEE SMALL LETTER HO + '\xab80'# -> unI64 5040 + -- CHEROKEE SMALL LETTER HU + '\xab81'# -> unI64 5041 + -- CHEROKEE SMALL LETTER HV + '\xab82'# -> unI64 5042 + -- CHEROKEE SMALL LETTER LA + '\xab83'# -> unI64 5043 + -- CHEROKEE SMALL LETTER LE + '\xab84'# -> unI64 5044 + -- CHEROKEE SMALL LETTER LI + '\xab85'# -> unI64 5045 + -- CHEROKEE SMALL LETTER LO + '\xab86'# -> unI64 5046 + -- CHEROKEE SMALL LETTER LU + '\xab87'# -> unI64 5047 + -- CHEROKEE SMALL LETTER LV + '\xab88'# -> unI64 5048 + -- CHEROKEE SMALL LETTER MA + '\xab89'# -> unI64 5049 + -- CHEROKEE SMALL LETTER ME + '\xab8a'# -> unI64 5050 + -- CHEROKEE SMALL LETTER MI + '\xab8b'# -> unI64 5051 + -- CHEROKEE SMALL LETTER MO + '\xab8c'# -> unI64 5052 + -- CHEROKEE SMALL LETTER MU + '\xab8d'# -> unI64 5053 + -- CHEROKEE SMALL LETTER NA + '\xab8e'# -> unI64 5054 + -- CHEROKEE SMALL LETTER HNA + '\xab8f'# -> unI64 5055 + -- CHEROKEE SMALL LETTER NAH + '\xab90'# -> unI64 5056 + -- CHEROKEE SMALL LETTER NE + '\xab91'# -> unI64 5057 + -- CHEROKEE SMALL LETTER NI + '\xab92'# -> unI64 5058 + -- CHEROKEE SMALL LETTER NO + '\xab93'# -> unI64 5059 + -- CHEROKEE SMALL LETTER NU + '\xab94'# -> unI64 5060 + -- CHEROKEE SMALL LETTER NV + '\xab95'# -> unI64 5061 + -- CHEROKEE SMALL LETTER QUA + '\xab96'# -> unI64 5062 + -- CHEROKEE SMALL LETTER QUE + '\xab97'# -> unI64 5063 + -- CHEROKEE SMALL LETTER QUI + '\xab98'# -> unI64 5064 + -- CHEROKEE SMALL LETTER QUO + '\xab99'# -> unI64 5065 + -- CHEROKEE SMALL LETTER QUU + '\xab9a'# -> unI64 5066 + -- CHEROKEE SMALL LETTER QUV + '\xab9b'# -> unI64 5067 + -- CHEROKEE SMALL LETTER SA + '\xab9c'# -> unI64 5068 + -- CHEROKEE SMALL LETTER S + '\xab9d'# -> unI64 5069 + -- CHEROKEE SMALL LETTER SE + '\xab9e'# -> unI64 5070 + -- CHEROKEE SMALL LETTER SI + '\xab9f'# -> unI64 5071 + -- CHEROKEE SMALL LETTER SO + '\xaba0'# -> unI64 5072 + -- CHEROKEE SMALL LETTER SU + '\xaba1'# -> unI64 5073 + -- CHEROKEE SMALL LETTER SV + '\xaba2'# -> unI64 5074 + -- CHEROKEE SMALL LETTER DA + '\xaba3'# -> unI64 5075 + -- CHEROKEE SMALL LETTER TA + '\xaba4'# -> unI64 5076 + -- CHEROKEE SMALL LETTER DE + '\xaba5'# -> unI64 5077 + -- CHEROKEE SMALL LETTER TE + '\xaba6'# -> unI64 5078 + -- CHEROKEE SMALL LETTER DI + '\xaba7'# -> unI64 5079 + -- CHEROKEE SMALL LETTER TI + '\xaba8'# -> unI64 5080 + -- CHEROKEE SMALL LETTER DO + '\xaba9'# -> unI64 5081 + -- CHEROKEE SMALL LETTER DU + '\xabaa'# -> unI64 5082 + -- CHEROKEE SMALL LETTER DV + '\xabab'# -> unI64 5083 + -- CHEROKEE SMALL LETTER DLA + '\xabac'# -> unI64 5084 + -- CHEROKEE SMALL LETTER TLA + '\xabad'# -> unI64 5085 + -- CHEROKEE SMALL LETTER TLE + '\xabae'# -> unI64 5086 + -- CHEROKEE SMALL LETTER TLI + '\xabaf'# -> unI64 5087 + -- CHEROKEE SMALL LETTER TLO + '\xabb0'# -> unI64 5088 + -- CHEROKEE SMALL LETTER TLU + '\xabb1'# -> unI64 5089 + -- CHEROKEE SMALL LETTER TLV + '\xabb2'# -> unI64 5090 + -- CHEROKEE SMALL LETTER TSA + '\xabb3'# -> unI64 5091 + -- CHEROKEE SMALL LETTER TSE + '\xabb4'# -> unI64 5092 + -- CHEROKEE SMALL LETTER TSI + '\xabb5'# -> unI64 5093 + -- CHEROKEE SMALL LETTER TSO + '\xabb6'# -> unI64 5094 + -- CHEROKEE SMALL LETTER TSU + '\xabb7'# -> unI64 5095 + -- CHEROKEE SMALL LETTER TSV + '\xabb8'# -> unI64 5096 + -- CHEROKEE SMALL LETTER WA + '\xabb9'# -> unI64 5097 + -- CHEROKEE SMALL LETTER WE + '\xabba'# -> unI64 5098 + -- CHEROKEE SMALL LETTER WI + '\xabbb'# -> unI64 5099 + -- CHEROKEE SMALL LETTER WO + '\xabbc'# -> unI64 5100 + -- CHEROKEE SMALL LETTER WU + '\xabbd'# -> unI64 5101 + -- CHEROKEE SMALL LETTER WV + '\xabbe'# -> unI64 5102 + -- CHEROKEE SMALL LETTER YA + '\xabbf'# -> unI64 5103 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909606 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201062 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492518 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575526 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108838 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269747 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269747 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819892 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168308 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751220 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819902 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945524 + -- FULLWIDTH LATIN CAPITAL LETTER A + '\xff21'# -> unI64 65345 + -- FULLWIDTH LATIN CAPITAL LETTER B + '\xff22'# -> unI64 65346 + -- FULLWIDTH LATIN CAPITAL LETTER C + '\xff23'# -> unI64 65347 + -- FULLWIDTH LATIN CAPITAL LETTER D + '\xff24'# -> unI64 65348 + -- FULLWIDTH LATIN CAPITAL LETTER E + '\xff25'# -> unI64 65349 + -- FULLWIDTH LATIN CAPITAL LETTER F + '\xff26'# -> unI64 65350 + -- FULLWIDTH LATIN CAPITAL LETTER G + '\xff27'# -> unI64 65351 + -- FULLWIDTH LATIN CAPITAL LETTER H + '\xff28'# -> unI64 65352 + -- FULLWIDTH LATIN CAPITAL LETTER I + '\xff29'# -> unI64 65353 + -- FULLWIDTH LATIN CAPITAL LETTER J + '\xff2a'# -> unI64 65354 + -- FULLWIDTH LATIN CAPITAL LETTER K + '\xff2b'# -> unI64 65355 + -- FULLWIDTH LATIN CAPITAL LETTER L + '\xff2c'# -> unI64 65356 + -- FULLWIDTH LATIN CAPITAL LETTER M + '\xff2d'# -> unI64 65357 + -- FULLWIDTH LATIN CAPITAL LETTER N + '\xff2e'# -> unI64 65358 + -- FULLWIDTH LATIN CAPITAL LETTER O + '\xff2f'# -> unI64 65359 + -- FULLWIDTH LATIN CAPITAL LETTER P + '\xff30'# -> unI64 65360 + -- FULLWIDTH LATIN CAPITAL LETTER Q + '\xff31'# -> unI64 65361 + -- FULLWIDTH LATIN CAPITAL LETTER R + '\xff32'# -> unI64 65362 + -- FULLWIDTH LATIN CAPITAL LETTER S + '\xff33'# -> unI64 65363 + -- FULLWIDTH LATIN CAPITAL LETTER T + '\xff34'# -> unI64 65364 + -- FULLWIDTH LATIN CAPITAL LETTER U + '\xff35'# -> unI64 65365 + -- FULLWIDTH LATIN CAPITAL LETTER V + '\xff36'# -> unI64 65366 + -- FULLWIDTH LATIN CAPITAL LETTER W + '\xff37'# -> unI64 65367 + -- FULLWIDTH LATIN CAPITAL LETTER X + '\xff38'# -> unI64 65368 + -- FULLWIDTH LATIN CAPITAL LETTER Y + '\xff39'# -> unI64 65369 + -- FULLWIDTH LATIN CAPITAL LETTER Z + '\xff3a'# -> unI64 65370 + -- DESERET CAPITAL LETTER LONG I + '\x10400'# -> unI64 66600 + -- DESERET CAPITAL LETTER LONG E + '\x10401'# -> unI64 66601 + -- DESERET CAPITAL LETTER LONG A + '\x10402'# -> unI64 66602 + -- DESERET CAPITAL LETTER LONG AH + '\x10403'# -> unI64 66603 + -- DESERET CAPITAL LETTER LONG O + '\x10404'# -> unI64 66604 + -- DESERET CAPITAL LETTER LONG OO + '\x10405'# -> unI64 66605 + -- DESERET CAPITAL LETTER SHORT I + '\x10406'# -> unI64 66606 + -- DESERET CAPITAL LETTER SHORT E + '\x10407'# -> unI64 66607 + -- DESERET CAPITAL LETTER SHORT A + '\x10408'# -> unI64 66608 + -- DESERET CAPITAL LETTER SHORT AH + '\x10409'# -> unI64 66609 + -- DESERET CAPITAL LETTER SHORT O + '\x1040a'# -> unI64 66610 + -- DESERET CAPITAL LETTER SHORT OO + '\x1040b'# -> unI64 66611 + -- DESERET CAPITAL LETTER AY + '\x1040c'# -> unI64 66612 + -- DESERET CAPITAL LETTER OW + '\x1040d'# -> unI64 66613 + -- DESERET CAPITAL LETTER WU + '\x1040e'# -> unI64 66614 + -- DESERET CAPITAL LETTER YEE + '\x1040f'# -> unI64 66615 + -- DESERET CAPITAL LETTER H + '\x10410'# -> unI64 66616 + -- DESERET CAPITAL LETTER PEE + '\x10411'# -> unI64 66617 + -- DESERET CAPITAL LETTER BEE + '\x10412'# -> unI64 66618 + -- DESERET CAPITAL LETTER TEE + '\x10413'# -> unI64 66619 + -- DESERET CAPITAL LETTER DEE + '\x10414'# -> unI64 66620 + -- DESERET CAPITAL LETTER CHEE + '\x10415'# -> unI64 66621 + -- DESERET CAPITAL LETTER JEE + '\x10416'# -> unI64 66622 + -- DESERET CAPITAL LETTER KAY + '\x10417'# -> unI64 66623 + -- DESERET CAPITAL LETTER GAY + '\x10418'# -> unI64 66624 + -- DESERET CAPITAL LETTER EF + '\x10419'# -> unI64 66625 + -- DESERET CAPITAL LETTER VEE + '\x1041a'# -> unI64 66626 + -- DESERET CAPITAL LETTER ETH + '\x1041b'# -> unI64 66627 + -- DESERET CAPITAL LETTER THEE + '\x1041c'# -> unI64 66628 + -- DESERET CAPITAL LETTER ES + '\x1041d'# -> unI64 66629 + -- DESERET CAPITAL LETTER ZEE + '\x1041e'# -> unI64 66630 + -- DESERET CAPITAL LETTER ESH + '\x1041f'# -> unI64 66631 + -- DESERET CAPITAL LETTER ZHEE + '\x10420'# -> unI64 66632 + -- DESERET CAPITAL LETTER ER + '\x10421'# -> unI64 66633 + -- DESERET CAPITAL LETTER EL + '\x10422'# -> unI64 66634 + -- DESERET CAPITAL LETTER EM + '\x10423'# -> unI64 66635 + -- DESERET CAPITAL LETTER EN + '\x10424'# -> unI64 66636 + -- DESERET CAPITAL LETTER ENG + '\x10425'# -> unI64 66637 + -- DESERET CAPITAL LETTER OI + '\x10426'# -> unI64 66638 + -- DESERET CAPITAL LETTER EW + '\x10427'# -> unI64 66639 + -- OSAGE CAPITAL LETTER A + '\x104b0'# -> unI64 66776 + -- OSAGE CAPITAL LETTER AI + '\x104b1'# -> unI64 66777 + -- OSAGE CAPITAL LETTER AIN + '\x104b2'# -> unI64 66778 + -- OSAGE CAPITAL LETTER AH + '\x104b3'# -> unI64 66779 + -- OSAGE CAPITAL LETTER BRA + '\x104b4'# -> unI64 66780 + -- OSAGE CAPITAL LETTER CHA + '\x104b5'# -> unI64 66781 + -- OSAGE CAPITAL LETTER EHCHA + '\x104b6'# -> unI64 66782 + -- OSAGE CAPITAL LETTER E + '\x104b7'# -> unI64 66783 + -- OSAGE CAPITAL LETTER EIN + '\x104b8'# -> unI64 66784 + -- OSAGE CAPITAL LETTER HA + '\x104b9'# -> unI64 66785 + -- OSAGE CAPITAL LETTER HYA + '\x104ba'# -> unI64 66786 + -- OSAGE CAPITAL LETTER I + '\x104bb'# -> unI64 66787 + -- OSAGE CAPITAL LETTER KA + '\x104bc'# -> unI64 66788 + -- OSAGE CAPITAL LETTER EHKA + '\x104bd'# -> unI64 66789 + -- OSAGE CAPITAL LETTER KYA + '\x104be'# -> unI64 66790 + -- OSAGE CAPITAL LETTER LA + '\x104bf'# -> unI64 66791 + -- OSAGE CAPITAL LETTER MA + '\x104c0'# -> unI64 66792 + -- OSAGE CAPITAL LETTER NA + '\x104c1'# -> unI64 66793 + -- OSAGE CAPITAL LETTER O + '\x104c2'# -> unI64 66794 + -- OSAGE CAPITAL LETTER OIN + '\x104c3'# -> unI64 66795 + -- OSAGE CAPITAL LETTER PA + '\x104c4'# -> unI64 66796 + -- OSAGE CAPITAL LETTER EHPA + '\x104c5'# -> unI64 66797 + -- OSAGE CAPITAL LETTER SA + '\x104c6'# -> unI64 66798 + -- OSAGE CAPITAL LETTER SHA + '\x104c7'# -> unI64 66799 + -- OSAGE CAPITAL LETTER TA + '\x104c8'# -> unI64 66800 + -- OSAGE CAPITAL LETTER EHTA + '\x104c9'# -> unI64 66801 + -- OSAGE CAPITAL LETTER TSA + '\x104ca'# -> unI64 66802 + -- OSAGE CAPITAL LETTER EHTSA + '\x104cb'# -> unI64 66803 + -- OSAGE CAPITAL LETTER TSHA + '\x104cc'# -> unI64 66804 + -- OSAGE CAPITAL LETTER DHA + '\x104cd'# -> unI64 66805 + -- OSAGE CAPITAL LETTER U + '\x104ce'# -> unI64 66806 + -- OSAGE CAPITAL LETTER WA + '\x104cf'# -> unI64 66807 + -- OSAGE CAPITAL LETTER KHA + '\x104d0'# -> unI64 66808 + -- OSAGE CAPITAL LETTER GHA + '\x104d1'# -> unI64 66809 + -- OSAGE CAPITAL LETTER ZA + '\x104d2'# -> unI64 66810 + -- OSAGE CAPITAL LETTER ZHA + '\x104d3'# -> unI64 66811 + -- VITHKUQI CAPITAL LETTER A + '\x10570'# -> unI64 66967 + -- VITHKUQI CAPITAL LETTER BBE + '\x10571'# -> unI64 66968 + -- VITHKUQI CAPITAL LETTER BE + '\x10572'# -> unI64 66969 + -- VITHKUQI CAPITAL LETTER CE + '\x10573'# -> unI64 66970 + -- VITHKUQI CAPITAL LETTER CHE + '\x10574'# -> unI64 66971 + -- VITHKUQI CAPITAL LETTER DE + '\x10575'# -> unI64 66972 + -- VITHKUQI CAPITAL LETTER DHE + '\x10576'# -> unI64 66973 + -- VITHKUQI CAPITAL LETTER EI + '\x10577'# -> unI64 66974 + -- VITHKUQI CAPITAL LETTER E + '\x10578'# -> unI64 66975 + -- VITHKUQI CAPITAL LETTER FE + '\x10579'# -> unI64 66976 + -- VITHKUQI CAPITAL LETTER GA + '\x1057a'# -> unI64 66977 + -- VITHKUQI CAPITAL LETTER HA + '\x1057c'# -> unI64 66979 + -- VITHKUQI CAPITAL LETTER HHA + '\x1057d'# -> unI64 66980 + -- VITHKUQI CAPITAL LETTER I + '\x1057e'# -> unI64 66981 + -- VITHKUQI CAPITAL LETTER IJE + '\x1057f'# -> unI64 66982 + -- VITHKUQI CAPITAL LETTER JE + '\x10580'# -> unI64 66983 + -- VITHKUQI CAPITAL LETTER KA + '\x10581'# -> unI64 66984 + -- VITHKUQI CAPITAL LETTER LA + '\x10582'# -> unI64 66985 + -- VITHKUQI CAPITAL LETTER LLA + '\x10583'# -> unI64 66986 + -- VITHKUQI CAPITAL LETTER ME + '\x10584'# -> unI64 66987 + -- VITHKUQI CAPITAL LETTER NE + '\x10585'# -> unI64 66988 + -- VITHKUQI CAPITAL LETTER NJE + '\x10586'# -> unI64 66989 + -- VITHKUQI CAPITAL LETTER O + '\x10587'# -> unI64 66990 + -- VITHKUQI CAPITAL LETTER PE + '\x10588'# -> unI64 66991 + -- VITHKUQI CAPITAL LETTER QA + '\x10589'# -> unI64 66992 + -- VITHKUQI CAPITAL LETTER RE + '\x1058a'# -> unI64 66993 + -- VITHKUQI CAPITAL LETTER SE + '\x1058c'# -> unI64 66995 + -- VITHKUQI CAPITAL LETTER SHE + '\x1058d'# -> unI64 66996 + -- VITHKUQI CAPITAL LETTER TE + '\x1058e'# -> unI64 66997 + -- VITHKUQI CAPITAL LETTER THE + '\x1058f'# -> unI64 66998 + -- VITHKUQI CAPITAL LETTER U + '\x10590'# -> unI64 66999 + -- VITHKUQI CAPITAL LETTER VE + '\x10591'# -> unI64 67000 + -- VITHKUQI CAPITAL LETTER XE + '\x10592'# -> unI64 67001 + -- VITHKUQI CAPITAL LETTER Y + '\x10594'# -> unI64 67003 + -- VITHKUQI CAPITAL LETTER ZE + '\x10595'# -> unI64 67004 + -- OLD HUNGARIAN CAPITAL LETTER A + '\x10c80'# -> unI64 68800 + -- OLD HUNGARIAN CAPITAL LETTER AA + '\x10c81'# -> unI64 68801 + -- OLD HUNGARIAN CAPITAL LETTER EB + '\x10c82'# -> unI64 68802 + -- OLD HUNGARIAN CAPITAL LETTER AMB + '\x10c83'# -> unI64 68803 + -- OLD HUNGARIAN CAPITAL LETTER EC + '\x10c84'# -> unI64 68804 + -- OLD HUNGARIAN CAPITAL LETTER ENC + '\x10c85'# -> unI64 68805 + -- OLD HUNGARIAN CAPITAL LETTER ECS + '\x10c86'# -> unI64 68806 + -- OLD HUNGARIAN CAPITAL LETTER ED + '\x10c87'# -> unI64 68807 + -- OLD HUNGARIAN CAPITAL LETTER AND + '\x10c88'# -> unI64 68808 + -- OLD HUNGARIAN CAPITAL LETTER E + '\x10c89'# -> unI64 68809 + -- OLD HUNGARIAN CAPITAL LETTER CLOSE E + '\x10c8a'# -> unI64 68810 + -- OLD HUNGARIAN CAPITAL LETTER EE + '\x10c8b'# -> unI64 68811 + -- OLD HUNGARIAN CAPITAL LETTER EF + '\x10c8c'# -> unI64 68812 + -- OLD HUNGARIAN CAPITAL LETTER EG + '\x10c8d'# -> unI64 68813 + -- OLD HUNGARIAN CAPITAL LETTER EGY + '\x10c8e'# -> unI64 68814 + -- OLD HUNGARIAN CAPITAL LETTER EH + '\x10c8f'# -> unI64 68815 + -- OLD HUNGARIAN CAPITAL LETTER I + '\x10c90'# -> unI64 68816 + -- OLD HUNGARIAN CAPITAL LETTER II + '\x10c91'# -> unI64 68817 + -- OLD HUNGARIAN CAPITAL LETTER EJ + '\x10c92'# -> unI64 68818 + -- OLD HUNGARIAN CAPITAL LETTER EK + '\x10c93'# -> unI64 68819 + -- OLD HUNGARIAN CAPITAL LETTER AK + '\x10c94'# -> unI64 68820 + -- OLD HUNGARIAN CAPITAL LETTER UNK + '\x10c95'# -> unI64 68821 + -- OLD HUNGARIAN CAPITAL LETTER EL + '\x10c96'# -> unI64 68822 + -- OLD HUNGARIAN CAPITAL LETTER ELY + '\x10c97'# -> unI64 68823 + -- OLD HUNGARIAN CAPITAL LETTER EM + '\x10c98'# -> unI64 68824 + -- OLD HUNGARIAN CAPITAL LETTER EN + '\x10c99'# -> unI64 68825 + -- OLD HUNGARIAN CAPITAL LETTER ENY + '\x10c9a'# -> unI64 68826 + -- OLD HUNGARIAN CAPITAL LETTER O + '\x10c9b'# -> unI64 68827 + -- OLD HUNGARIAN CAPITAL LETTER OO + '\x10c9c'# -> unI64 68828 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE + '\x10c9d'# -> unI64 68829 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE + '\x10c9e'# -> unI64 68830 + -- OLD HUNGARIAN CAPITAL LETTER OEE + '\x10c9f'# -> unI64 68831 + -- OLD HUNGARIAN CAPITAL LETTER EP + '\x10ca0'# -> unI64 68832 + -- OLD HUNGARIAN CAPITAL LETTER EMP + '\x10ca1'# -> unI64 68833 + -- OLD HUNGARIAN CAPITAL LETTER ER + '\x10ca2'# -> unI64 68834 + -- OLD HUNGARIAN CAPITAL LETTER SHORT ER + '\x10ca3'# -> unI64 68835 + -- OLD HUNGARIAN CAPITAL LETTER ES + '\x10ca4'# -> unI64 68836 + -- OLD HUNGARIAN CAPITAL LETTER ESZ + '\x10ca5'# -> unI64 68837 + -- OLD HUNGARIAN CAPITAL LETTER ET + '\x10ca6'# -> unI64 68838 + -- OLD HUNGARIAN CAPITAL LETTER ENT + '\x10ca7'# -> unI64 68839 + -- OLD HUNGARIAN CAPITAL LETTER ETY + '\x10ca8'# -> unI64 68840 + -- OLD HUNGARIAN CAPITAL LETTER ECH + '\x10ca9'# -> unI64 68841 + -- OLD HUNGARIAN CAPITAL LETTER U + '\x10caa'# -> unI64 68842 + -- OLD HUNGARIAN CAPITAL LETTER UU + '\x10cab'# -> unI64 68843 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE + '\x10cac'# -> unI64 68844 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE + '\x10cad'# -> unI64 68845 + -- OLD HUNGARIAN CAPITAL LETTER EV + '\x10cae'# -> unI64 68846 + -- OLD HUNGARIAN CAPITAL LETTER EZ + '\x10caf'# -> unI64 68847 + -- OLD HUNGARIAN CAPITAL LETTER EZS + '\x10cb0'# -> unI64 68848 + -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN + '\x10cb1'# -> unI64 68849 + -- OLD HUNGARIAN CAPITAL LETTER US + '\x10cb2'# -> unI64 68850 + -- GARAY CAPITAL LETTER A + '\x10d50'# -> unI64 68976 + -- GARAY CAPITAL LETTER CA + '\x10d51'# -> unI64 68977 + -- GARAY CAPITAL LETTER MA + '\x10d52'# -> unI64 68978 + -- GARAY CAPITAL LETTER KA + '\x10d53'# -> unI64 68979 + -- GARAY CAPITAL LETTER BA + '\x10d54'# -> unI64 68980 + -- GARAY CAPITAL LETTER JA + '\x10d55'# -> unI64 68981 + -- GARAY CAPITAL LETTER SA + '\x10d56'# -> unI64 68982 + -- GARAY CAPITAL LETTER WA + '\x10d57'# -> unI64 68983 + -- GARAY CAPITAL LETTER LA + '\x10d58'# -> unI64 68984 + -- GARAY CAPITAL LETTER GA + '\x10d59'# -> unI64 68985 + -- GARAY CAPITAL LETTER DA + '\x10d5a'# -> unI64 68986 + -- GARAY CAPITAL LETTER XA + '\x10d5b'# -> unI64 68987 + -- GARAY CAPITAL LETTER YA + '\x10d5c'# -> unI64 68988 + -- GARAY CAPITAL LETTER TA + '\x10d5d'# -> unI64 68989 + -- GARAY CAPITAL LETTER RA + '\x10d5e'# -> unI64 68990 + -- GARAY CAPITAL LETTER NYA + '\x10d5f'# -> unI64 68991 + -- GARAY CAPITAL LETTER FA + '\x10d60'# -> unI64 68992 + -- GARAY CAPITAL LETTER NA + '\x10d61'# -> unI64 68993 + -- GARAY CAPITAL LETTER PA + '\x10d62'# -> unI64 68994 + -- GARAY CAPITAL LETTER HA + '\x10d63'# -> unI64 68995 + -- GARAY CAPITAL LETTER OLD KA + '\x10d64'# -> unI64 68996 + -- GARAY CAPITAL LETTER OLD NA + '\x10d65'# -> unI64 68997 + -- WARANG CITI CAPITAL LETTER NGAA + '\x118a0'# -> unI64 71872 + -- WARANG CITI CAPITAL LETTER A + '\x118a1'# -> unI64 71873 + -- WARANG CITI CAPITAL LETTER WI + '\x118a2'# -> unI64 71874 + -- WARANG CITI CAPITAL LETTER YU + '\x118a3'# -> unI64 71875 + -- WARANG CITI CAPITAL LETTER YA + '\x118a4'# -> unI64 71876 + -- WARANG CITI CAPITAL LETTER YO + '\x118a5'# -> unI64 71877 + -- WARANG CITI CAPITAL LETTER II + '\x118a6'# -> unI64 71878 + -- WARANG CITI CAPITAL LETTER UU + '\x118a7'# -> unI64 71879 + -- WARANG CITI CAPITAL LETTER E + '\x118a8'# -> unI64 71880 + -- WARANG CITI CAPITAL LETTER O + '\x118a9'# -> unI64 71881 + -- WARANG CITI CAPITAL LETTER ANG + '\x118aa'# -> unI64 71882 + -- WARANG CITI CAPITAL LETTER GA + '\x118ab'# -> unI64 71883 + -- WARANG CITI CAPITAL LETTER KO + '\x118ac'# -> unI64 71884 + -- WARANG CITI CAPITAL LETTER ENY + '\x118ad'# -> unI64 71885 + -- WARANG CITI CAPITAL LETTER YUJ + '\x118ae'# -> unI64 71886 + -- WARANG CITI CAPITAL LETTER UC + '\x118af'# -> unI64 71887 + -- WARANG CITI CAPITAL LETTER ENN + '\x118b0'# -> unI64 71888 + -- WARANG CITI CAPITAL LETTER ODD + '\x118b1'# -> unI64 71889 + -- WARANG CITI CAPITAL LETTER TTE + '\x118b2'# -> unI64 71890 + -- WARANG CITI CAPITAL LETTER NUNG + '\x118b3'# -> unI64 71891 + -- WARANG CITI CAPITAL LETTER DA + '\x118b4'# -> unI64 71892 + -- WARANG CITI CAPITAL LETTER AT + '\x118b5'# -> unI64 71893 + -- WARANG CITI CAPITAL LETTER AM + '\x118b6'# -> unI64 71894 + -- WARANG CITI CAPITAL LETTER BU + '\x118b7'# -> unI64 71895 + -- WARANG CITI CAPITAL LETTER PU + '\x118b8'# -> unI64 71896 + -- WARANG CITI CAPITAL LETTER HIYO + '\x118b9'# -> unI64 71897 + -- WARANG CITI CAPITAL LETTER HOLO + '\x118ba'# -> unI64 71898 + -- WARANG CITI CAPITAL LETTER HORR + '\x118bb'# -> unI64 71899 + -- WARANG CITI CAPITAL LETTER HAR + '\x118bc'# -> unI64 71900 + -- WARANG CITI CAPITAL LETTER SSUU + '\x118bd'# -> unI64 71901 + -- WARANG CITI CAPITAL LETTER SII + '\x118be'# -> unI64 71902 + -- WARANG CITI CAPITAL LETTER VIYO + '\x118bf'# -> unI64 71903 + -- MEDEFAIDRIN CAPITAL LETTER M + '\x16e40'# -> unI64 93792 + -- MEDEFAIDRIN CAPITAL LETTER S + '\x16e41'# -> unI64 93793 + -- MEDEFAIDRIN CAPITAL LETTER V + '\x16e42'# -> unI64 93794 + -- MEDEFAIDRIN CAPITAL LETTER W + '\x16e43'# -> unI64 93795 + -- MEDEFAIDRIN CAPITAL LETTER ATIU + '\x16e44'# -> unI64 93796 + -- MEDEFAIDRIN CAPITAL LETTER Z + '\x16e45'# -> unI64 93797 + -- MEDEFAIDRIN CAPITAL LETTER KP + '\x16e46'# -> unI64 93798 + -- MEDEFAIDRIN CAPITAL LETTER P + '\x16e47'# -> unI64 93799 + -- MEDEFAIDRIN CAPITAL LETTER T + '\x16e48'# -> unI64 93800 + -- MEDEFAIDRIN CAPITAL LETTER G + '\x16e49'# -> unI64 93801 + -- MEDEFAIDRIN CAPITAL LETTER F + '\x16e4a'# -> unI64 93802 + -- MEDEFAIDRIN CAPITAL LETTER I + '\x16e4b'# -> unI64 93803 + -- MEDEFAIDRIN CAPITAL LETTER K + '\x16e4c'# -> unI64 93804 + -- MEDEFAIDRIN CAPITAL LETTER A + '\x16e4d'# -> unI64 93805 + -- MEDEFAIDRIN CAPITAL LETTER J + '\x16e4e'# -> unI64 93806 + -- MEDEFAIDRIN CAPITAL LETTER E + '\x16e4f'# -> unI64 93807 + -- MEDEFAIDRIN CAPITAL LETTER B + '\x16e50'# -> unI64 93808 + -- MEDEFAIDRIN CAPITAL LETTER C + '\x16e51'# -> unI64 93809 + -- MEDEFAIDRIN CAPITAL LETTER U + '\x16e52'# -> unI64 93810 + -- MEDEFAIDRIN CAPITAL LETTER YU + '\x16e53'# -> unI64 93811 + -- MEDEFAIDRIN CAPITAL LETTER L + '\x16e54'# -> unI64 93812 + -- MEDEFAIDRIN CAPITAL LETTER Q + '\x16e55'# -> unI64 93813 + -- MEDEFAIDRIN CAPITAL LETTER HP + '\x16e56'# -> unI64 93814 + -- MEDEFAIDRIN CAPITAL LETTER NY + '\x16e57'# -> unI64 93815 + -- MEDEFAIDRIN CAPITAL LETTER X + '\x16e58'# -> unI64 93816 + -- MEDEFAIDRIN CAPITAL LETTER D + '\x16e59'# -> unI64 93817 + -- MEDEFAIDRIN CAPITAL LETTER OE + '\x16e5a'# -> unI64 93818 + -- MEDEFAIDRIN CAPITAL LETTER N + '\x16e5b'# -> unI64 93819 + -- MEDEFAIDRIN CAPITAL LETTER R + '\x16e5c'# -> unI64 93820 + -- MEDEFAIDRIN CAPITAL LETTER O + '\x16e5d'# -> unI64 93821 + -- MEDEFAIDRIN CAPITAL LETTER AI + '\x16e5e'# -> unI64 93822 + -- MEDEFAIDRIN CAPITAL LETTER Y + '\x16e5f'# -> unI64 93823 + -- BERIA ERFE CAPITAL LETTER ARKAB + '\x16ea0'# -> unI64 93883 + -- BERIA ERFE CAPITAL LETTER BASIGNA + '\x16ea1'# -> unI64 93884 + -- BERIA ERFE CAPITAL LETTER DARBAI + '\x16ea2'# -> unI64 93885 + -- BERIA ERFE CAPITAL LETTER EH + '\x16ea3'# -> unI64 93886 + -- BERIA ERFE CAPITAL LETTER FITKO + '\x16ea4'# -> unI64 93887 + -- BERIA ERFE CAPITAL LETTER GOWAY + '\x16ea5'# -> unI64 93888 + -- BERIA ERFE CAPITAL LETTER HIRDEABO + '\x16ea6'# -> unI64 93889 + -- BERIA ERFE CAPITAL LETTER I + '\x16ea7'# -> unI64 93890 + -- BERIA ERFE CAPITAL LETTER DJAI + '\x16ea8'# -> unI64 93891 + -- BERIA ERFE CAPITAL LETTER KOBO + '\x16ea9'# -> unI64 93892 + -- BERIA ERFE CAPITAL LETTER LAKKO + '\x16eaa'# -> unI64 93893 + -- BERIA ERFE CAPITAL LETTER MERI + '\x16eab'# -> unI64 93894 + -- BERIA ERFE CAPITAL LETTER NINI + '\x16eac'# -> unI64 93895 + -- BERIA ERFE CAPITAL LETTER GNA + '\x16ead'# -> unI64 93896 + -- BERIA ERFE CAPITAL LETTER NGAY + '\x16eae'# -> unI64 93897 + -- BERIA ERFE CAPITAL LETTER OI + '\x16eaf'# -> unI64 93898 + -- BERIA ERFE CAPITAL LETTER PI + '\x16eb0'# -> unI64 93899 + -- BERIA ERFE CAPITAL LETTER ERIGO + '\x16eb1'# -> unI64 93900 + -- BERIA ERFE CAPITAL LETTER ERIGO TAMURA + '\x16eb2'# -> unI64 93901 + -- BERIA ERFE CAPITAL LETTER SERI + '\x16eb3'# -> unI64 93902 + -- BERIA ERFE CAPITAL LETTER SHEP + '\x16eb4'# -> unI64 93903 + -- BERIA ERFE CAPITAL LETTER TATASOUE + '\x16eb5'# -> unI64 93904 + -- BERIA ERFE CAPITAL LETTER UI + '\x16eb6'# -> unI64 93905 + -- BERIA ERFE CAPITAL LETTER WASSE + '\x16eb7'# -> unI64 93906 + -- BERIA ERFE CAPITAL LETTER AY + '\x16eb8'# -> unI64 93907 + -- ADLAM CAPITAL LETTER ALIF + '\x1e900'# -> unI64 125218 + -- ADLAM CAPITAL LETTER DAALI + '\x1e901'# -> unI64 125219 + -- ADLAM CAPITAL LETTER LAAM + '\x1e902'# -> unI64 125220 + -- ADLAM CAPITAL LETTER MIIM + '\x1e903'# -> unI64 125221 + -- ADLAM CAPITAL LETTER BA + '\x1e904'# -> unI64 125222 + -- ADLAM CAPITAL LETTER SINNYIIYHE + '\x1e905'# -> unI64 125223 + -- ADLAM CAPITAL LETTER PE + '\x1e906'# -> unI64 125224 + -- ADLAM CAPITAL LETTER BHE + '\x1e907'# -> unI64 125225 + -- ADLAM CAPITAL LETTER RA + '\x1e908'# -> unI64 125226 + -- ADLAM CAPITAL LETTER E + '\x1e909'# -> unI64 125227 + -- ADLAM CAPITAL LETTER FA + '\x1e90a'# -> unI64 125228 + -- ADLAM CAPITAL LETTER I + '\x1e90b'# -> unI64 125229 + -- ADLAM CAPITAL LETTER O + '\x1e90c'# -> unI64 125230 + -- ADLAM CAPITAL LETTER DHA + '\x1e90d'# -> unI64 125231 + -- ADLAM CAPITAL LETTER YHE + '\x1e90e'# -> unI64 125232 + -- ADLAM CAPITAL LETTER WAW + '\x1e90f'# -> unI64 125233 + -- ADLAM CAPITAL LETTER NUN + '\x1e910'# -> unI64 125234 + -- ADLAM CAPITAL LETTER KAF + '\x1e911'# -> unI64 125235 + -- ADLAM CAPITAL LETTER YA + '\x1e912'# -> unI64 125236 + -- ADLAM CAPITAL LETTER U + '\x1e913'# -> unI64 125237 + -- ADLAM CAPITAL LETTER JIIM + '\x1e914'# -> unI64 125238 + -- ADLAM CAPITAL LETTER CHI + '\x1e915'# -> unI64 125239 + -- ADLAM CAPITAL LETTER HA + '\x1e916'# -> unI64 125240 + -- ADLAM CAPITAL LETTER QAAF + '\x1e917'# -> unI64 125241 + -- ADLAM CAPITAL LETTER GA + '\x1e918'# -> unI64 125242 + -- ADLAM CAPITAL LETTER NYA + '\x1e919'# -> unI64 125243 + -- ADLAM CAPITAL LETTER TU + '\x1e91a'# -> unI64 125244 + -- ADLAM CAPITAL LETTER NHA + '\x1e91b'# -> unI64 125245 + -- ADLAM CAPITAL LETTER VA + '\x1e91c'# -> unI64 125246 + -- ADLAM CAPITAL LETTER KHA + '\x1e91d'# -> unI64 125247 + -- ADLAM CAPITAL LETTER GBE + '\x1e91e'# -> unI64 125248 + -- ADLAM CAPITAL LETTER ZAL + '\x1e91f'# -> unI64 125249 + -- ADLAM CAPITAL LETTER KPO + '\x1e920'# -> unI64 125250 + -- ADLAM CAPITAL LETTER SHA + '\x1e921'# -> unI64 125251 + _ -> unI64 0
src/Data/Text/Internal/Fusion/Common.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns, MagicHash, Rank2Types, PartialTypeSignatures #-}+{-# LANGUAGE BangPatterns, MagicHash, RankNTypes, PartialTypeSignatures #-} {-# OPTIONS_GHC -Wno-partial-type-signatures #-} -- | -- Module : Data.Text.Internal.Fusion.Common@@ -77,6 +77,7 @@ , foldl1' , foldr , foldr1+ , foldlM' -- ** Special folds , concat@@ -123,7 +124,7 @@ import Prelude (Bool(..), Char, Eq, (==), Int, Integral, Maybe(..), Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++), (&&), fromIntegral, otherwise)-import qualified Data.List as L+import qualified Data.List as L hiding (head, tail) import qualified Prelude as P import Data.Bits (shiftL, shiftR, (.&.)) import Data.Char (isLetter, isSpace)@@ -551,7 +552,8 @@ -- | /O(n)/ Convert a string to title case, using simple case -- conversion. ----- The first letter of the input is converted to title case, as is+-- The first letter (as determined by 'Data.Char.isLetter')+-- of the input is converted to title case, as is -- every subsequent letter that immediately follows a non-letter. -- Every letter that immediately follows another letter is converted -- to lower case.@@ -561,6 +563,14 @@ -- sequence Latin capital letter F (U+0046) followed by Latin small -- letter l (U+006C). --+-- This function is not idempotent.+-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).+-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input+-- is converted to title case, becoming two letters.+-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter+-- and as such is recognised as a letter by 'Data.Char.isLetter',+-- so 'T.toTitle' @"ʼN"@ = @"'n"@.+-- -- /Note/: this function does not take language or culture specific -- rules into account. For instance, in English, different style -- guides disagree on whether the book name \"The Hill of the Red@@ -680,6 +690,20 @@ Yield x s' -> loop_foldl1' (f z x) s' {-# INLINE [0] foldl1' #-} +-- | A monadic version of 'foldl'.+--+-- __Properties__+--+-- @ 'foldlM'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldlM'' f z0 @+foldlM' :: P.Monad m => (b -> Char -> m b) -> b -> Stream Char -> m b+foldlM' f z0 (Stream next s0 _len) = loop_foldlM' z0 s0+ where+ loop_foldlM' !z !s = case next s of+ Done -> P.pure z+ Skip s' -> loop_foldlM' z s'+ Yield x s' -> f z x P.>>= \z' -> loop_foldlM' z' s'+{-# INLINE [0] foldlM' #-}+ -- | 'foldr', applied to a binary operator, a starting value (typically the -- right-identity of the operator), and a stream, reduces the stream using the -- binary operator, from right to left.@@ -716,7 +740,7 @@ Yield x' s' -> f x (loop_foldr1 x' s') {-# INLINE [0] foldr1 #-} --- | intercalate str strs interts the stream str in between the streams strs and+-- | intercalate str strs inserts the stream str in between the streams strs and -- concatenates the result. -- -- __Properties__@@ -834,7 +858,7 @@ -- __Properties__ -- -- @'head' ('scanl' f z xs) = z@--- +-- -- @'last' ('scanl' f z xs) = 'foldl' f z xs@ scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low@@ -1008,7 +1032,7 @@ {-# INLINE [0] dropWhile #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns--- 'True' iff the first is a prefix of the second.+-- 'True' if and only if the first is a prefix of the second. -- -- __Properties__ --
src/Data/Text/Internal/IO.hs view
@@ -1,166 +1,308 @@-{-# LANGUAGE BangPatterns, RecordWildCards #-}--- |--- Module : Data.Text.Internal.IO--- Copyright : (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Simon Marlow--- License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Low-level support for text I\/O.--module Data.Text.Internal.IO- (- hGetLineWith- , readChunk- ) where--import qualified Control.Exception as E-import Data.IORef (readIORef, writeIORef)-import Data.Text (Text)-import Data.Text.Internal.Fusion (unstream)-import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Size (exactSize, maxSize)-import Data.Text.Unsafe (inlinePerformIO)-import Foreign.Storable (peekElemOff)-import GHC.IO.Buffer (Buffer(..), CharBuffer, RawCharBuffer, bufferAdjustL,- bufferElems, charSize, isEmptyBuffer, readCharBuf,- withRawBuffer, writeCharBuf)-import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_)-import GHC.IO.Handle.Types (Handle__(..), Newline(..))-import System.IO (Handle)-import System.IO.Error (isEOFError)-import qualified Data.Text as T---- | Read a single line of input from a handle, constructing a list of--- decoded chunks as we go. When we're done, transform them into the--- destination type.-hGetLineWith :: ([Text] -> t) -> Handle -> IO t-hGetLineWith f h = wantReadableHandle_ "hGetLine" h go- where- go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh []--hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text]-hGetLineLoop hh@Handle__{..} = go where- go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do- let findEOL raw r | r == w = return (False, w)- | otherwise = do- (c,r') <- readCharBuf raw r- if c == '\n'- then return (True, r)- else findEOL raw r'- (eol, off) <- findEOL raw0 r0- (t,r') <- if haInputNL == CRLF- then unpack_nl raw0 r0 off- else do t <- unpack raw0 r0 off- return (t,off)- if eol- then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf)- return $ reverse (t:ts)- else do- let buf1 = bufferAdjustL r' buf- maybe_buf <- maybeFillReadBuffer hh buf1- case maybe_buf of- -- Nothing indicates we caught an EOF, and we may have a- -- partial line to return.- Nothing -> do- -- we reached EOF. There might be a lone \r left- -- in the buffer, so check for that and- -- append it to the line if necessary.- let pre | isEmptyBuffer buf1 = T.empty- | otherwise = T.singleton '\r'- writeIORef haCharBuffer buf1{ bufL=0, bufR=0 }- let str = reverse . filter (not . T.null) $ pre:t:ts- if null str- then ioe_EOF- else return str- Just new_buf -> go (t:ts) new_buf---- This function is lifted almost verbatim from GHC.IO.Handle.Text.-maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer)-maybeFillReadBuffer handle_ buf- = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e ->- if isEOFError e- then return Nothing- else ioError e--unpack :: RawCharBuffer -> Int -> Int -> IO Text-unpack !buf !r !w- | charSize /= 4 = sizeError "unpack"- | r >= w = return T.empty- | otherwise = withRawBuffer buf go- where- go pbuf = return $! unstream (Stream next r (exactSize (w-r)))- where- next !i | i >= w = Done- | otherwise = Yield (ix i) (i+1)- ix i = inlinePerformIO $ peekElemOff pbuf i--unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int)-unpack_nl !buf !r !w- | charSize /= 4 = sizeError "unpack_nl"- | r >= w = return (T.empty, 0)- | otherwise = withRawBuffer buf $ go- where- go pbuf = do- let !t = unstream (Stream next r (maxSize (w-r)))- w' = w - 1- return $ if ix w' == '\r'- then (t,w')- else (t,w)- where- next !i | i >= w = Done- | c == '\r' = let i' = i + 1- in if i' < w- then if ix i' == '\n'- then Yield '\n' (i+2)- else Yield '\n' i'- else Done- | otherwise = Yield c (i+1)- where c = ix i- ix i = inlinePerformIO $ peekElemOff pbuf i---- This function is completely lifted from GHC.IO.Handle.Text.-getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer-getSomeCharacters handle_@Handle__{..} buf@Buffer{..} =- case bufferElems buf of- -- buffer empty: read some more- 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf-- -- if the buffer has a single '\r' in it and we're doing newline- -- translation: read some more- 1 | haInputNL == CRLF -> do- (c,_) <- readCharBuf bufRaw bufL- if c == '\r'- then do -- shuffle the '\r' to the beginning. This is only safe- -- if we're about to call readTextDevice, otherwise it- -- would mess up flushCharBuffer.- -- See [note Buffer Flushing], GHC.IO.Handle.Types- _ <- writeCharBuf bufRaw 0 '\r'- let buf' = buf{ bufL=0, bufR=1 }- readTextDevice handle_ buf'- else do- return buf-- -- buffer has some chars in it already: just return it- _otherwise -> {-# SCC "otherwise" #-} return buf---- | Read a single chunk of strict text from a buffer. Used by both--- the strict and lazy implementations of hGetContents.-readChunk :: Handle__ -> CharBuffer -> IO Text-readChunk hh@Handle__{..} buf = do- buf'@Buffer{..} <- getSomeCharacters hh buf- (t,r) <- if haInputNL == CRLF- then unpack_nl bufRaw bufL bufR- else do t <- unpack bufRaw bufL bufR- return (t,bufR)- writeIORef haCharBuffer (bufferAdjustL r buf')- return t--sizeError :: String -> a-sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"+{-# LANGUAGE BangPatterns, RecordWildCards #-} +{-# LANGUAGE MagicHash #-} +-- | +-- Module : Data.Text.Internal.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Low-level support for text I\/O. + +module Data.Text.Internal.IO + ( + hGetLineWith + , readChunk + , hPutStream + , hPutStr + , hPutStrLn + ) where + +import qualified Control.Exception as E +import qualified Data.ByteString as B +import Data.ByteString.Builder (hPutBuilder, charUtf8) +import Data.IORef (readIORef, writeIORef) +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, streamLn, unstream) +import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size (exactSize, maxSize) +import Data.Text.Unsafe (inlinePerformIO) +import Foreign.Storable (peekElemOff) +import GHC.Exts (reallyUnsafePtrEquality#, isTrue#) +import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer, + bufferAdjustL, bufferElems, charSize, emptyBuffer, + isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer, + writeCharBuf) +import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_, + wantWritableHandle) +import GHC.IO.Handle.Text (commitBuffer') +import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..)) +import System.IO (Handle, hPutChar, utf8) +import System.IO.Error (isEOFError) +import qualified Data.Text as T + +-- | Read a single line of input from a handle, constructing a list of +-- decoded chunks as we go. When we're done, transform them into the +-- destination type. +hGetLineWith :: ([Text] -> t) -> Handle -> IO t +hGetLineWith f h = wantReadableHandle_ "hGetLine" h go + where + go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] + +hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] +hGetLineLoop hh@Handle__{..} = go where + go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do + let findEOL raw r | r == w = return (False, w) + | otherwise = do + (c,r') <- readCharBuf raw r + if c == '\n' + then return (True, r) + else findEOL raw r' + (eol, off) <- findEOL raw0 r0 + (t,r') <- if haInputNL == CRLF + then unpack_nl raw0 r0 off + else do t <- unpack raw0 r0 off + return (t,off) + if eol + then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) + return $ reverse (t:ts) + else do + let buf1 = bufferAdjustL r' buf + maybe_buf <- maybeFillReadBuffer hh buf1 + case maybe_buf of + -- Nothing indicates we caught an EOF, and we may have a + -- partial line to return. + Nothing -> do + -- we reached EOF. There might be a lone \r left + -- in the buffer, so check for that and + -- append it to the line if necessary. + let pre | isEmptyBuffer buf1 = T.empty + | otherwise = T.singleton '\r' + writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } + let str = reverse . filter (not . T.null) $ pre:t:ts + if null str + then ioe_EOF + else return str + Just new_buf -> go (t:ts) new_buf + +-- This function is lifted almost verbatim from GHC.IO.Handle.Text. +maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) +maybeFillReadBuffer handle_ buf + = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> + if isEOFError e + then return Nothing + else ioError e + +unpack :: RawCharBuffer -> Int -> Int -> IO Text +unpack !buf !r !w + | charSize /= 4 = sizeError "unpack" + | r >= w = return T.empty + | otherwise = withRawBuffer buf go + where + go pbuf = return $! unstream (Stream next r (exactSize (w-r))) + where + next !i | i >= w = Done + | otherwise = Yield (ix i) (i+1) + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- Variant of 'unpack' with CRLF decoding. If there is a trailing '\r', leave it in the buffer. +unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) +unpack_nl !buf !r !w + | charSize /= 4 = sizeError "unpack_nl" + | r >= w = return (T.empty, 0) + | otherwise = withRawBuffer buf $ go + where + go pbuf = do + let !t = unstream (Stream next r (maxSize (w-r))) + w' = w - 1 + return $ if ix w' == '\r' + then (t,w') + else (t,w) + where + next !i | i >= w = Done + | c == '\r' = let i' = i + 1 + in if i' < w + then if ix i' == '\n' + then Yield '\n' (i+2) + else Yield '\r' i' + else Done + | otherwise = Yield c (i+1) + where c = ix i + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer +getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = + case bufferElems buf of + -- buffer empty: read some more + 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf + + -- if the buffer has a single '\r' in it and we're doing newline + -- translation: read some more + 1 | haInputNL == CRLF -> do + (c,_) <- readCharBuf bufRaw bufL + if c == '\r' + then do -- shuffle the '\r' to the beginning. This is only safe + -- if we're about to call readTextDevice, otherwise it + -- would mess up flushCharBuffer. + -- See [note Buffer Flushing], GHC.IO.Handle.Types + _ <- writeCharBuf bufRaw 0 '\r' + let buf' = buf{ bufL=0, bufR=1 } + readTextDevice handle_ buf' + else do + return buf + + -- buffer has some chars in it already: just return it + _otherwise -> {-# SCC "otherwise" #-} return buf + +-- | Read a single chunk of strict text from a buffer. Used by both +-- the strict and lazy implementations of hGetContents. +readChunk :: Handle__ -> CharBuffer -> IO Text +readChunk hh@Handle__{..} buf = do + buf'@Buffer{..} <- getSomeCharacters hh buf + (t,r) <- if haInputNL == CRLF + then unpack_nl bufRaw bufL bufR + else do t <- unpack bufRaw bufL bufR + return (t,bufR) + writeIORef haCharBuffer (bufferAdjustL r buf') + return t + +-- | Print a @Stream Char@. +hPutStream :: Handle -> Stream Char -> IO () +hPutStream h str = hPutStreamOrUtf8 h str Nothing + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8) + where + putUtf8 = B.hPutStr h (encodeUtf8 t) + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8) + where + -- Not using B.hPutStrLn because it's not necessarily atomic: + -- https://github.com/haskell/bytestring/issues/200 + putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n') + +-- | 'hPutStream' with an optional special case when the output encoding is +-- UTF-8 and without newline conversion. +hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO () +-- This function is modified from GHC.IO.Handle.Text. +hPutStreamOrUtf8 h str mPutUtf8 = do + (buffer_mode, nl, isUtf8) <- + wantWritableHandle "hPutStr" h $ \h_ -> do + bmode <- getSpareBuffer h_ + return (bmode, haOutputNL h_, eqUTF8 h_) + case buffer_mode of + _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8 + (NoBuffering, _) -> hPutChars h str + (LineBuffering, buf) -> writeLines h nl buf str + (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str + + where + -- If the encoding is UTF-8, it's most likely pointer-equal to + -- 'System.IO.utf8', letting us avoid a String comparison. + -- If it is somehow UTF-8 but not pointer-equal to 'utf8', + -- we will just take a slower branch, but the result is still correct. + eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec +{-# INLINE hPutStreamOrUtf8 #-} + +hPutChars :: Handle -> Stream Char -> IO () +hPutChars h (Stream next0 s0 _len) = loop s0 + where + loop !s = case next0 s of + Done -> return () + Skip s' -> loop s' + Yield x s' -> hPutChar h x >> loop s' + +-- The following functions are largely lifted from GHC.IO.Handle.Text, +-- but adapted to a coinductive stream of data instead of an inductive +-- list. +-- +-- We have several variations of more or less the same code for +-- performance reasons. Splitting the original buffered write +-- function into line- and block-oriented versions gave us a 2.1x +-- performance improvement. Lifting out the raw/cooked newline +-- handling gave a few more percent on top. + +writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO () +writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' -> do + n' <- if nl == CRLF + then do n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' + else writeCharBuf' raw len n x + commit n' True{-needs flush-} False >>= outer s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO () +writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + -- Leave room for two characters for CRLF decoding + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' && isCRLF -> do + n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' >>= inner s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +-- | Only modifies the raw buffer and not the buffer attributes +writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int +writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $ + writeCharBuf bufRaw n c + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) +getSpareBuffer Handle__{haCharBuffer=ref, + haBuffers=spare_ref, + haBufferMode=mode} + = do + case mode of + NoBuffering -> return (mode, error "no buffer!") + _ -> do + bufs <- readIORef spare_ref + buf <- readIORef ref + case bufs of + BufferListCons b rest -> do + writeIORef spare_ref rest + return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) + BufferListNil -> do + new_buf <- newCharBuffer (bufSize buf) WriteBuffer + return (mode, new_buf) + + +-- This function is modified from GHC.Internal.IO.Handle.Text. +commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool + -> IO CharBuffer +commitBuffer hdl !raw !sz !count flush release = + wantWritableHandle "commitAndReleaseBuffer" hdl $ + commitBuffer' raw sz count flush release +{-# INLINE commitBuffer #-} + +sizeError :: String -> a +sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"
+ src/Data/Text/Internal/IsAscii.hs view
@@ -0,0 +1,94 @@+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE UnliftedFFITypes #-}+#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'isAscii', using efficient C routines by default.+--+-- Similarly implements asciiPrefixLength, used internally in Data.Text.Encoding.+module Data.Text.Internal.IsAscii where++#if defined(PURE_HASKELL)+import Prelude hiding (all)+import qualified Data.Char as Char+import qualified Data.ByteString as BS+import Data.Text.Unsafe (iter, Iter(..))+#else+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Data.Word (Word8)+import Foreign.C.Types+import Foreign.Ptr (Ptr, plusPtr)+import GHC.Base (ByteArray#)+import Prelude (Bool(..), Int, (==), ($), IO, (<$>))+import qualified Data.Text.Array as A+#endif+import Data.ByteString (ByteString)+import Data.Text.Internal (Text(..))+import qualified Prelude as P++-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only+-- U+0000 through U+007F).+--+-- This is a more efficient version of @'all' 'Data.Char.isAscii'@.+--+-- >>> isAscii ""+-- True+--+-- >>> isAscii "abc\NUL"+-- True+--+-- >>> isAscii "abcd€"+-- False+--+-- prop> isAscii t == all (< '\x80') t+--+-- @since 2.0.2+isAscii :: Text -> Bool+#if defined(PURE_HASKELL)+isAscii = all Char.isAscii++-- | (Re)implemented to avoid circular dependency on Data.Text.+all :: (Char -> Bool) -> Text -> Bool+all p t@(Text _ _ len) = go 0+ where+ go i | i >= len = True+ | otherwise =+ let !(Iter c j) = iter t i+ in p c && go (i+j)+#else+cSizeToInt :: CSize -> Int+cSizeToInt = P.fromIntegral+{-# INLINE cSizeToInt #-}++intToCSize :: Int -> CSize+intToCSize = P.fromIntegral++isAscii (Text (A.ByteArray arr) off len) =+ cSizeToInt (c_is_ascii_offset arr (intToCSize off) (intToCSize len)) == len+#endif+{-# INLINE isAscii #-}++-- | Length of the longest ASCII prefix.+asciiPrefixLength :: ByteString -> Int+#if defined(PURE_HASKELL)+asciiPrefixLength = BS.length P.. BS.takeWhile (P.< 0x80)+#else+asciiPrefixLength bs = unsafeDupablePerformIO $ withBS bs $ \ fp len ->+ unsafeWithForeignPtr fp $ \src -> do+ P.fromIntegral <$> c_is_ascii src (src `plusPtr` len)+#endif+{-# INLINE asciiPrefixLength #-}++#if !defined(PURE_HASKELL)+foreign import ccall unsafe "_hs_text_is_ascii_offset" c_is_ascii_offset+ :: ByteArray# -> CSize -> CSize -> CSize++foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii+ :: Ptr Word8 -> Ptr Word8 -> IO CSize+#endif
src/Data/Text/Internal/Lazy.hs view
@@ -1,135 +1,151 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module : Data.Text.Internal.Lazy--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.--module Data.Text.Internal.Lazy- (- Text(..)- , chunk- , empty- , foldrChunks- , foldlChunks- -- * Data type invariant and abstraction functions-- -- $invariant- , strictInvariant- , lazyInvariant- , showStructure-- -- * Chunk allocation sizes- , defaultChunkSize- , smallChunkSize- , chunkOverhead-- , equal- ) where--import Data.Bits (shiftL)-import Data.Text ()-import Data.Typeable (Typeable)-import Foreign.Storable (sizeOf)-import qualified Data.Text.Array as A-import qualified Data.Text.Internal as T--data Text = Empty- | Chunk {-# UNPACK #-} !T.Text Text- deriving (Typeable)---- $invariant------ The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or--- consists of non-null 'T.Text's. All functions must preserve this,--- and the QC properties must check this.---- | Check the invariant strictly.-strictInvariant :: Text -> Bool-strictInvariant Empty = True-strictInvariant x@(Chunk (T.Text _ _ len) cs)- | len > 0 = strictInvariant cs- | otherwise = error $ "Data.Text.Lazy: invariant violation: "- ++ showStructure x---- | Check the invariant lazily.-lazyInvariant :: Text -> Text-lazyInvariant Empty = Empty-lazyInvariant x@(Chunk c@(T.Text _ _ len) cs)- | len > 0 = Chunk c (lazyInvariant cs)- | otherwise = error $ "Data.Text.Lazy: invariant violation: "- ++ showStructure x---- | Display the internal structure of a lazy 'Text'.-showStructure :: Text -> String-showStructure Empty = "Empty"-showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"-showStructure (Chunk t ts) =- "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"---- | Smart constructor for 'Chunk'. Guarantees the data type invariant.-chunk :: T.Text -> Text -> Text-{-# INLINE chunk #-}-chunk t@(T.Text _ _ len) ts | len == 0 = ts- | otherwise = Chunk t ts---- | Smart constructor for 'Empty'.-empty :: Text-{-# INLINE [0] empty #-}-empty = Empty---- | Consume the chunks of a lazy 'Text' with a natural right fold.-foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a-foldrChunks f z = go- where go Empty = z- go (Chunk c cs) = f c (go cs)-{-# INLINE foldrChunks #-}---- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,--- accumulating left fold.-foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a-foldlChunks f z = go z- where go !a Empty = a- go !a (Chunk c cs) = go (f a c) cs-{-# INLINE foldlChunks #-}---- | Currently set to 16 KiB, less the memory management overhead.-defaultChunkSize :: Int-defaultChunkSize = 16384 - chunkOverhead-{-# INLINE defaultChunkSize #-}---- | Currently set to 128 bytes, less the memory management overhead.-smallChunkSize :: Int-smallChunkSize = 128 - chunkOverhead-{-# INLINE smallChunkSize #-}---- | The memory management overhead. Currently this is tuned for GHC only.-chunkOverhead :: Int-chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1-{-# INLINE chunkOverhead #-}--equal :: Text -> Text -> Bool-equal Empty Empty = True-equal Empty _ = False-equal _ Empty = False-equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =- case compare lenA lenB of- LT -> A.equal arrA offA arrB offB lenA &&- as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs- EQ -> A.equal arrA offA arrB offB lenA &&- as `equal` bs- GT -> A.equal arrA offA arrB offB lenB &&- Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs+{-# LANGUAGE BangPatterns #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal.Lazy +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. + +module Data.Text.Internal.Lazy + ( + Text(..) + , LazyText + , chunk + , empty + , foldrChunks + , foldlChunks + -- * Data type invariant and abstraction functions + + -- $invariant + , strictInvariant + , lazyInvariant + , showStructure + + -- * Chunk allocation sizes + , defaultChunkSize + , smallChunkSize + , chunkOverhead + + , equal + ) where + +import Data.Bits (shiftL) +import Data.Text () +import Foreign.Storable (sizeOf) +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text as T + +data Text = Empty + -- ^ Empty text. + -- + -- @since 2.1.2 + | Chunk {-# UNPACK #-} !T.Text Text + -- ^ Chunks must be non-empty, this invariant is not checked. + +-- | Type synonym for the lazy flavour of 'Text'. +-- +-- @since 2.1.1 +type LazyText = Text + +-- $invariant +-- +-- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or +-- consists of non-null 'T.Text's. All functions must preserve this, +-- and the QC properties must check this. + +-- | Check the invariant strictly. +strictInvariant :: Text -> Bool +strictInvariant Empty = True +strictInvariant x@(Chunk (T.Text _ _ len) cs) + | len > 0 = strictInvariant cs + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Check the invariant lazily. +lazyInvariant :: Text -> Text +lazyInvariant Empty = Empty +lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) + | len > 0 = Chunk c (lazyInvariant cs) + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Display the internal structure of a lazy 'Text'. +showStructure :: Text -> String +showStructure Empty = "Empty" +showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" +showStructure (Chunk t ts) = + "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" + +-- | Smart constructor for 'Chunk'. Guarantees the data type invariant. +chunk :: T.Text -> Text -> Text +{-# INLINE [0] chunk #-} +chunk t ts | T.null t = ts + | otherwise = Chunk t ts + +{-# RULES +"TEXT chunk/text" forall arr off len. + chunk (T.text arr off len) = chunk (T.Text arr off len) +"TEXT chunk/empty" forall ts. + chunk T.empty ts = ts +#-} + +-- | Smart constructor for 'Empty'. +empty :: Text +{-# INLINE [0] empty #-} +empty = Empty + +-- | Consume the chunks of a lazy 'Text' with a natural right fold. +foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a +foldrChunks f z = go + where go Empty = z + go (Chunk c cs) = f c (go cs) +{-# INLINE foldrChunks #-} + +-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, +-- accumulating left fold. +foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a +foldlChunks f z = go z + where go !a Empty = a + go !a (Chunk c cs) = go (f a c) cs +{-# INLINE foldlChunks #-} + +-- | Currently set to 16 KiB, less the memory management overhead. +defaultChunkSize :: Int +defaultChunkSize = 16384 - chunkOverhead +{-# INLINE defaultChunkSize #-} + +-- | Currently set to 128 bytes, less the memory management overhead. +smallChunkSize :: Int +smallChunkSize = 128 - chunkOverhead +{-# INLINE smallChunkSize #-} + +-- | The memory management overhead. Currently this is tuned for GHC only. +chunkOverhead :: Int +chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 +{-# INLINE chunkOverhead #-} + +equal :: Text -> Text -> Bool +equal Empty Empty = True +equal Empty _ = False +equal _ Empty = False +equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + case compare lenA lenB of + LT -> A.equal arrA offA arrB offB lenA && + as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs + EQ -> A.equal arrA offA arrB offB lenA && + as `equal` bs + GT -> A.equal arrA offA arrB offB lenB && + Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs
src/Data/Text/Internal/Lazy/Encoding/Fusion.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} -- | -- Module : Data.Text.Lazy.Encoding.Fusion@@ -49,8 +49,9 @@ import qualified Data.Text.Internal.Encoding.Utf32 as U32 import Data.Text.Unsafe (unsafeDupablePerformIO) import Foreign.ForeignPtr (ForeignPtr)+import Foreign.Marshal.Utils (copyBytes) import Foreign.Storable (pokeByteOff)-import Data.ByteString.Internal (mallocByteString, memcpy)+import Data.ByteString.Internal (mallocByteString) #if defined(ASSERTS) import Control.Exception (assert) #endif@@ -99,10 +100,10 @@ S2 a b -> next (T bs (S3 a b x) (i+1)) S3 a b c -> next (T bs (S4 a b c x) (i+1)) S4 a b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a)- (T bs (S3 b c d) (i+1))+ (T bs (S4 b c d x) (i+1)) where x = B.unsafeIndex ps i consume (T Empty S0 _) = Done- consume st = decodeError "streamUtf8" "UTF-8" onErr Nothing st+ consume (T Empty _ i) = decodeError "streamUtf8" "UTF-8" onErr Nothing (T Empty S0 i) {-# INLINE [0] streamUtf8 #-} -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little@@ -139,10 +140,10 @@ S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) S4 w1 w2 w3 w4 -> decodeError "streamUtf16LE" "UTF-16LE" onErr (Just w1)- (T bs (S3 w2 w3 w4) (i+1))+ (T bs (S4 w2 w3 w4 x) (i+1)) where x = B.unsafeIndex ps i consume (T Empty S0 _) = Done- consume st = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing st+ consume (T Empty _ i) = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (T Empty S0 i) {-# INLINE [0] streamUtf16LE #-} -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big@@ -179,10 +180,10 @@ S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) S4 w1 w2 w3 w4 -> decodeError "streamUtf16BE" "UTF-16BE" onErr (Just w1)- (T bs (S3 w2 w3 w4) (i+1))+ (T bs (S4 w2 w3 w4 x) (i+1)) where x = B.unsafeIndex ps i consume (T Empty S0 _) = Done- consume st = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing st+ consume (T Empty _ i) = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (T Empty S0 i) {-# INLINE [0] streamUtf16BE #-} -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big@@ -223,10 +224,10 @@ S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) S4 w1 w2 w3 w4 -> decodeError "streamUtf32BE" "UTF-32BE" onErr (Just w1)- (T bs (S3 w2 w3 w4) (i+1))+ (T bs (S4 w2 w3 w4 x) (i+1)) where x = B.unsafeIndex ps i consume (T Empty S0 _) = Done- consume st = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing st+ consume (T Empty _ i) = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (T Empty S0 i) {-# INLINE [0] streamUtf32BE #-} -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little@@ -267,10 +268,10 @@ S2 w1 w2 -> next (T bs (S3 w1 w2 x) (i+1)) S3 w1 w2 w3 -> next (T bs (S4 w1 w2 w3 x) (i+1)) S4 w1 w2 w3 w4 -> decodeError "streamUtf32LE" "UTF-32LE" onErr (Just w1)- (T bs (S3 w2 w3 w4) (i+1))+ (T bs (S4 w2 w3 w4 x) (i+1)) where x = B.unsafeIndex ps i consume (T Empty S0 _) = Done- consume st = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing st+ consume (T Empty _ i) = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (T Empty S0 i) {-# INLINE [0] streamUtf32LE #-} -- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.@@ -308,7 +309,7 @@ dest <- mallocByteString destLen unsafeWithForeignPtr src $ \src' -> unsafeWithForeignPtr dest $ \dest' ->- memcpy dest' src' srcLen+ copyBytes dest' src' srcLen return dest -- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.
src/Data/Text/Internal/Lazy/Fusion.hs view
@@ -17,6 +17,7 @@ module Data.Text.Internal.Lazy.Fusion ( stream+ , streamLn , unstream , unstreamChunks , length@@ -47,14 +48,35 @@ HasCallStack => #endif Text -> Stream Char-stream text = Stream next (text :*: 0) unknownSize+stream = stream' False+{-# INLINE [0] stream #-}++-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@+--+-- @since 2.1.2+streamLn ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ Text -> Stream Char+streamLn = stream' True++-- | Shared implementation of 'stream' and 'streamLn'.+stream' ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ Bool -> Text -> Stream Char+stream' addNl text = Stream next (text :*: 0) unknownSize where- next (Empty :*: _) = Done+ next (Empty :*: i)+ | addNl && i <= 0 = Yield '\n' (Empty :*: 1)+ | otherwise = Done next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i) | i >= len = next (ts :*: 0) | otherwise = Yield c (txt :*: i+d) where Iter c d = iter t i-{-# INLINE [0] stream #-}+{-# INLINE [0] stream' #-} -- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given -- chunk size.
src/Data/Text/Internal/Lazy/Search.hs view
@@ -24,19 +24,15 @@ indices ) where -import Data.Bits (unsafeShiftL)+import Data.Bits (unsafeShiftL, (.|.), (.&.)) import qualified Data.Text.Array as A import Data.Int (Int64) import Data.Word (Word8, Word64) import qualified Data.Text.Internal as T import qualified Data.Text as T (concat, isPrefixOf)+import Data.Text.Internal.ArrayUtils (memchr) import Data.Text.Internal.Fusion.Types (PairS(..)) import Data.Text.Internal.Lazy (Text(..), foldrChunks)-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Data.Bits ((.|.), (.&.))-import Foreign.C.Types-import GHC.Exts (ByteArray#)-import System.Posix.Types (CSsize(..)) -- | /O(n+m)/ Find the offsets of all non-overlapping indices of -- @needle@ within @haystack@.@@ -66,10 +62,10 @@ c = index xxs (i + nlast) delta | nextInPattern = nlen + 1 | c == z = skip + 1- | l >= i + nlen = case unsafeDupablePerformIO $- memchr xarr# (intToCSize (xoff + i + nlen)) (intToCSize (l - i - nlen)) z of+ | l >= i + nlen = case+ memchr xarr# (xoff + i + nlen) (l - i - nlen) z of -1 -> max 1 (l - i - nlen)- s -> cSsizeToInt s + 1+ s -> s + 1 | otherwise = 1 nextInPattern = mask .&. swizzle (index xxs (i + nlen)) == 0 @@ -134,12 +130,3 @@ word8ToInt :: Word8 -> Int word8ToInt = fromIntegral--intToCSize :: Int -> CSize-intToCSize = fromIntegral--cSsizeToInt :: CSsize -> Int-cSsizeToInt = fromIntegral--foreign import ccall unsafe "_hs_text_memchr" memchr- :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize
+ src/Data/Text/Internal/Measure.hs view
@@ -0,0 +1,53 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}++#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MultiWayIf #-}+#endif++#if !defined(PURE_HASKELL)+{-# LANGUAGE UnliftedFFITypes #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'measure_off', using efficient C routines by default.+module Data.Text.Internal.Measure+ ( measure_off+ )+where++import GHC.Exts++#if defined(PURE_HASKELL)+import GHC.Word+import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader)+#endif++import Foreign.C.Types (CSize(..))+import System.Posix.Types (CSsize(..))++#if defined(PURE_HASKELL)++measure_off :: ByteArray# -> CSize -> CSize -> CSize -> CSsize+measure_off ba off len cnt = go 0 0+ where+ go !cc !i+ -- return the number of bytes for the first cnt codepoints,+ | cc == cnt = fromIntegral i+ -- return negated number of codepoints if there are fewer than cnt+ | i >= len = negate (fromIntegral cc)+ | otherwise =+ let !(I# o) = fromIntegral (off+i)+ !b = indexWord8Array# ba o+ in go (cc+1) (i + fromIntegral (utf8LengthByLeader (W8# b)))++#else++-- | The input buffer (arr :: ByteArray#, off :: CSize, len :: CSize)+-- must specify a valid UTF-8 sequence, this condition is not checked.+foreign import ccall unsafe "_hs_text_measure_off" measure_off+ :: ByteArray# -> CSize -> CSize -> CSize -> CSsize++#endif
src/Data/Text/Internal/PrimCompat.hs view
@@ -13,13 +13,12 @@ ) where #if MIN_VERSION_base(4,16,0)--import GHC.Base-+import GHC.Exts (wordToWord8#,word8ToWord#,wordToWord16#,word16ToWord#,wordToWord32#,word32ToWord#) #else- import GHC.Prim (Word#)+#endif +#if !(MIN_VERSION_base(4,16,0)) wordToWord8#, word8ToWord# :: Word# -> Word# wordToWord16#, word16ToWord# :: Word# -> Word# wordToWord32#, word32ToWord# :: Word# -> Word#@@ -33,5 +32,4 @@ {-# INLINE word16ToWord# #-} {-# INLINE wordToWord32# #-} {-# INLINE word32ToWord# #-}- #endif
src/Data/Text/Internal/Private.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types, UnboxedTuples #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes, UnboxedTuples #-} -- | -- Module : Data.Text.Internal.Private
+ src/Data/Text/Internal/Reverse.hs view
@@ -0,0 +1,108 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}+{-# LANGUAGE NoImplicitPrelude #-}+#if defined(PURE_HASKELL)+{-# LANGUAGE BangPatterns #-}+#endif++{-# OPTIONS_HADDOCK not-home #-}++-- | Implements 'reverse', using efficient C routines by default.+module Data.Text.Internal.Reverse (reverse, reverseNonEmpty) where++#if !defined(PURE_HASKELL)+import GHC.Exts as Exts+import Control.Monad.ST.Unsafe (unsafeIOToST)+import Foreign.C.Types (CSize(..))+#else+import Control.Monad.ST (ST)+import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader)+#endif+#if defined(ASSERTS)+import GHC.Stack (HasCallStack)+#endif+import Prelude hiding (reverse)+import Data.Text.Internal (Text(..), empty)+import Control.Monad.ST (runST)+import qualified Data.Text.Array as A++-- | /O(n)/ Reverse the characters of a string.+--+-- Example:+--+-- $setup+-- >>> T.reverse "desrever"+-- "reversed"+reverse ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ Text -> Text+reverse (Text _ _ 0) = empty+reverse t = reverseNonEmpty t+{-# INLINE reverse #-}++-- | /O(n)/ Reverse the characters of a string.+-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.+reverseNonEmpty ::+ Text -> Text+#if defined(PURE_HASKELL)+reverseNonEmpty (Text src off len) = runST $ do+ dest <- A.new len+ _ <- reversePoints src off dest len+ result <- A.unsafeFreeze dest+ pure $ Text result 0 len++-- Step 0:+--+-- Input: R E D R U M+-- ^+-- x+-- Output: _ _ _ _ _ _+-- ^+-- y+--+-- Step 1:+--+-- Input: R E D R U M+-- ^+-- x+--+-- Output: _ _ _ _ _ R+-- ^+-- y+reversePoints+ :: A.Array -- ^ Input array+ -> Int -- ^ Input index+ -> A.MArray s -- ^ Output array+ -> Int -- ^ Output index+ -> ST s ()+reversePoints src xx dest yy = go xx yy where+ go !_ y | y <= 0 = pure ()+ go x y =+ let pLen = utf8LengthByLeader (A.unsafeIndex src x)+ -- The next y is also the start of the current point in the output+ yNext = y - pLen+ in do+ A.copyI pLen dest yNext src x+ go (x + pLen) yNext+#else+reverseNonEmpty (Text (A.ByteArray ba) off len) = runST $ do+ marr@(A.MutableByteArray mba) <- A.new len+ unsafeIOToST $ c_reverse mba ba (fromIntegral off) (fromIntegral len)+ brr <- A.unsafeFreeze marr+ return $ Text brr 0 len+#endif+{-# INLINE reverseNonEmpty #-}++#if !defined(PURE_HASKELL)+-- | The input buffer (src :: ByteArray#, off :: CSize, len :: CSize)+-- must specify a valid UTF-8 sequence, this condition is not checked.+foreign import ccall unsafe "_hs_text_reverse" c_reverse+ :: Exts.MutableByteArray# s -> ByteArray# -> CSize -> CSize -> IO ()+#endif++-- $setup+-- >>> :set -XOverloadedStrings+-- >>> import qualified Data.Text.Internal.Reverse as T
src/Data/Text/Internal/Search.hs view
@@ -25,8 +25,8 @@ -- * D. M. Sunday: A Very Fast Substring Search Algorithm. -- Communications of the ACM, 33, 8, 132-142 (1990) ----- * F. Lundh: The Fast Search Algorithm.--- <http://effbot.org/zone/stringlib.htm> (2006)+-- * F. Lundh:+-- <http://web.archive.org/web/20201107074620/http://effbot.org/zone/stringlib.htm The Fast Search Algorithm>. (2006) module Data.Text.Internal.Search (@@ -37,10 +37,7 @@ import Data.Word (Word64, Word8) import Data.Text.Internal (Text(..)) import Data.Bits ((.|.), (.&.), unsafeShiftL)-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Foreign.C.Types-import GHC.Exts (ByteArray#)-import System.Posix.Types (CSsize(..))+import Data.Text.Internal.ArrayUtils (memchr) data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int @@ -88,9 +85,9 @@ | mask .&. swizzle (A.unsafeIndex harr i) == 0 = loop (i + nlen + 1) | otherwise- = case unsafeDupablePerformIO $ memchr harr# (intToCSize i) (intToCSize (hlen + hoff - i)) z of+ = case memchr harr# i (hlen + hoff - i) z of -1 -> []- x -> loop (i + cSsizeToInt x + 1)+ x -> loop (i + x + 1) {-# INLINE indices' #-} scanOne :: Word8 -> Text -> [Int]@@ -104,12 +101,3 @@ word8ToInt :: Word8 -> Int word8ToInt = fromIntegral--intToCSize :: Int -> CSize-intToCSize = fromIntegral--cSsizeToInt :: CSsize -> Int-cSsizeToInt = fromIntegral--foreign import ccall unsafe "_hs_text_memchr" memchr- :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize
+ src/Data/Text/Internal/StrictBuilder.hs view
@@ -0,0 +1,129 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE TypeSynonymInstances #-}++-- |+-- Module : Data.Text.Internal.Builder+-- License : BSD-style (see LICENSE)+-- Stability : experimental+--+-- /Warning/: this is an internal module, and does not have a stable+-- API or name. Functions in this module may not check or enforce+-- preconditions expected by public modules. Use at your own risk!+--+-- @since 2.0.2++module Data.Text.Internal.StrictBuilder+ ( StrictTextBuilder(..)+ , StrictBuilder+ , toText+ , fromChar+ , fromText++ -- * Unsafe+ -- $unsafe+ , unsafeFromByteString+ , unsafeFromWord8+ ) where++import Control.Monad.ST (ST, runST)+import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)+import Data.Functor (void)+import Data.Word (Word8)+import Data.ByteString (ByteString)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup (Semigroup(..))+#endif+import Data.Text.Internal (Text(..), empty, safe)+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Encoding.Utf8 (utf8Length)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import qualified Data.ByteString as B+import qualified Data.Text.Array as A+import qualified Data.Text.Internal.Unsafe.Char as Char++-- | A delayed representation of strict 'Text'.+--+-- @since 2.1.2+data StrictTextBuilder = StrictTextBuilder+ { sbLength :: {-# UNPACK #-} !Int+ , sbWrite :: forall s. A.MArray s -> Int -> ST s ()+ }++-- | A delayed representation of strict 'Text'.+--+-- @since 2.0.2+{-# DEPRECATED StrictBuilder "Use StrictTextBuilder instead" #-}+type StrictBuilder = StrictTextBuilder++-- | Use 'StrictBuilder' to build 'Text'.+--+-- @since 2.0.2+toText :: StrictTextBuilder -> Text+toText (StrictTextBuilder 0 _) = empty+toText (StrictTextBuilder n write) = runST (do+ dst <- A.new n+ write dst 0+ arr <- A.unsafeFreeze dst+ pure (Text arr 0 n))++-- | Concatenation of 'StrictBuilder' is right-biased:+-- the right builder will be run first. This allows a builder to+-- run tail-recursively when it was accumulated left-to-right.+instance Semigroup StrictTextBuilder where+ (<>) = appendRStrictBuilder++instance Monoid StrictTextBuilder where+ mempty = emptyStrictBuilder+ mappend = (<>)++emptyStrictBuilder :: StrictTextBuilder+emptyStrictBuilder = StrictTextBuilder 0 (\_ _ -> pure ())++appendRStrictBuilder :: StrictTextBuilder -> StrictTextBuilder -> StrictTextBuilder+appendRStrictBuilder (StrictTextBuilder 0 _) b2 = b2+appendRStrictBuilder b1 (StrictTextBuilder 0 _) = b1+appendRStrictBuilder (StrictTextBuilder n1 write1) (StrictTextBuilder n2 write2) =+ StrictTextBuilder (n1 + n2) (\dst ofs -> do+ write2 dst (ofs + n1)+ write1 dst ofs)++copyFromByteString :: A.MArray s -> Int -> ByteString -> ST s ()+copyFromByteString dst ofs src = withBS src $ \ srcFPtr len ->+ unsafeIOToST $ unsafeWithForeignPtr srcFPtr $ \ srcPtr -> do+ unsafeSTToIO $ A.copyFromPointer dst ofs srcPtr len++-- | Copy a 'ByteString'.+--+-- Unsafe: This may not be valid UTF-8 text.+--+-- @since 2.0.2+unsafeFromByteString :: ByteString -> StrictTextBuilder+unsafeFromByteString bs =+ StrictTextBuilder (B.length bs) (\dst ofs -> copyFromByteString dst ofs bs)++-- |+-- @since 2.0.2+{-# INLINE fromChar #-}+fromChar :: Char -> StrictTextBuilder+fromChar c =+ StrictTextBuilder (utf8Length c) (\dst ofs -> void (Char.unsafeWrite dst ofs (safe c)))++-- $unsafe+-- For internal purposes, we abuse 'StrictBuilder' as a delayed 'Array' rather+-- than 'Text': it may not actually be valid 'Text'.++-- | Unsafe: This may not be valid UTF-8 text.+--+-- @since 2.0.2+unsafeFromWord8 :: Word8 -> StrictTextBuilder+unsafeFromWord8 !w =+ StrictTextBuilder 1 (\dst ofs -> A.unsafeWrite dst ofs w)++-- | Copy 'Text' in a 'StrictBuilder'+--+-- @since 2.0.2+fromText :: Text -> StrictTextBuilder+fromText (Text src srcOfs n) = StrictTextBuilder n (\dst dstOfs ->+ A.copyI n dst dstOfs src srcOfs)
+ src/Data/Text/Internal/Transformation.hs view
@@ -0,0 +1,340 @@+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text.Internal.Transformation +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- This module holds functions shared between the strict and lazy implementations of @Text@ transformations. + +module Data.Text.Internal.Transformation + ( mapNonEmpty + , toCaseFoldNonEmpty + , toLowerNonEmpty + , toUpperNonEmpty + , toTitleNonEmpty + , filter_ + ) where + +import Prelude (Char, Bool(..), Int, + Ord(..), + Monad(..), pure, + (+), (-), ($), (&&), (||), (==), + not, return, otherwise) +import Data.Bits ((.&.), shiftR, shiftL) +import Data.Char (isLetter, isSpace) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4) +import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping) +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iterArray) +import Data.Word (Word8) +import qualified GHC.Exts as Exts +import GHC.Int (Int64(..)) + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +mapNonEmpty :: (Char -> Char) -> Text -> Text +mapNonEmpty f = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + | otherwise = do + let !(Iter c d) = iterArray src srcOff + d' <- unsafeWrite dst dstOff (safe (f c)) + inner (srcOff + d) (dstOff + d') +{-# INLINE mapNonEmpty #-} + +caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text +caseConvert ascii remap (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff (ascii m0) + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' + +{-# INLINABLE caseConvert #-} + +writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int +writeMapping !_ 0 !dstOff = pure dstOff +writeMapping dst i dstOff = do + let (ch, j) = chopOffChar i + d <- unsafeWrite dst dstOff ch + writeMapping dst j (dstOff + d) + +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (Exts.I# n) = Exts.C# (Exts.chr# n) + mask = (1 `shiftL` 21) - 1 + a = P.fromIntegral $ ab .&. mask + +-- | /O(n)/ Convert a string to folded case. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toCaseFoldNonEmpty :: Text -> Text +toCaseFoldNonEmpty = \xs -> caseConvert asciiToLower foldMapping xs +{-# INLINE toCaseFoldNonEmpty #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toLowerNonEmpty :: Text -> Text +toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs +{-# INLINE toLowerNonEmpty #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toUpperNonEmpty :: Text -> Text +toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs +{-# INLINE toUpperNonEmpty #-} + +asciiToLower :: Word8 -> Word8 +asciiToLower w = if w - 65 <= 25 then w + 32 else w + +asciiToUpper :: Word8 -> Word8 +asciiToUpper w = if w - 97 <= 25 then w - 32 else w + +isAsciiLetter :: Word8 -> Bool +isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25 + +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) + +-- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toTitleNonEmpty :: Text -> Text +toTitleNonEmpty (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 False + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !mode + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff mode + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + + case d of + 1 -> do + let (mode', m0') = asciiAdvance mode m0 + A.unsafeWrite dst dstOff m0' + inner (srcOff + 1) (dstOff + 1) mode' + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' mode' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + isSpace3 ch + = m0 == 0xE1 && m1 == 0x9A && m2 == 0x80 + || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F) + || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80 + !(# mode', c' #) = advance isSpace3 mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' mode' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + !(# mode', c' #) = advance (\_ -> False) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' mode' + + asciiAdvance :: Bool -> Word8 -> (Bool, Word8) + asciiAdvance False w = (isAsciiLetter w, asciiToUpper w) + asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w) + + advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #) + advance _ False c = (# isLetter (Exts.C# c), titleMapping c #) + advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #) + {-# INLINE advance #-} + +-- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@, +-- calls the continuation with the @Text@ containing only the characters satisfying the predicate. +filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a +filter_ mkText p = go + where + go (Text src o l) = runST $ do + -- It's tempting to allocate l elements at once and avoid resizing. + -- However, this can be unacceptable in scenarios where a huge array + -- is filtered with a rare predicate, resulting in a much shorter buffer. + let !dstLen = min l 64 + dst <- A.new dstLen + outer dst dstLen o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return $ mkText arr 0 dstOff + | dstOff + 4 > dstLen = do + -- Double size of the buffer, unless it becomes longer than + -- source string. Ensure to extend it by least 4 bytes. + let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- In case of success, filter writes exactly the same character + -- it just read (this is not a case for map, for example). + -- We leverage this fact below: no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + let !c = unsafeChr8 m0 + if not (p c) then inner (srcOff + 1) dstOff else do + A.unsafeWrite dst dstOff m0 + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !c = chr2 m0 m1 + if not (p c) then inner (srcOff + 2) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + inner (srcOff + 2) (dstOff + 2) + 3 -> do + let !c = chr3 m0 m1 m2 + if not (p c) then inner (srcOff + 3) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + inner (srcOff + 3) (dstOff + 3) + _ -> do + let !c = chr4 m0 m1 m2 m3 + if not (p c) then inner (srcOff + 4) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + inner (srcOff + 4) (dstOff + 4) +{-# INLINE filter_ #-}
+ src/Data/Text/Internal/Validate.hs view
@@ -0,0 +1,107 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- | Test whether or not a sequence of bytes is a valid UTF-8 byte sequence.+-- In the GHC Haskell ecosystem, there are several representations of byte+-- sequences. The only one that the stable @text@ API concerns itself with is+-- 'ByteString'. Part of bytestring-to-text decoding is 'isValidUtf8ByteString',+-- a high-performance UTF-8 validation routine written in C++ with fallbacks+-- for various platforms. The C++ code backing this routine is nontrivial,+-- so in the interest of reuse, this module additionally exports functions+-- for working with the GC-managed @ByteArray@ type. These @ByteArray@+-- functions are not used anywhere else in @text@. They are for the benefit+-- of library and application authors who do not use 'ByteString' but still+-- need to interoperate with @text@.+module Data.Text.Internal.Validate+ (+ -- * ByteString+ isValidUtf8ByteString+ -- * ByteArray+ --+ -- | Is the slice of a byte array a valid UTF-8 byte sequence? These+ -- functions all accept an offset and a length.+ , isValidUtf8ByteArray+ , isValidUtf8ByteArrayUnpinned+ , isValidUtf8ByteArrayPinned+ ) where++import Data.Array.Byte (ByteArray(ByteArray))+import Data.ByteString (ByteString)+import GHC.Exts (isTrue#,isByteArrayPinned#)++#ifdef SIMDUTF+import Data.Text.Unsafe (unsafeDupablePerformIO)+import Data.Text.Internal.ByteStringCompat (withBS)+import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)+import Data.Text.Internal.Validate.Simd (c_is_valid_utf8_bytearray_safe,c_is_valid_utf8_bytearray_unsafe,c_is_valid_utf8_ptr_unsafe)+#else+import qualified Data.ByteString as B+import qualified Data.Text.Internal.Validate.Native as N+#endif++-- | Is the ByteString a valid UTF-8 byte sequence?+isValidUtf8ByteString :: ByteString -> Bool+#ifdef SIMDUTF+isValidUtf8ByteString bs = withBS bs $ \fp len -> unsafeDupablePerformIO $+ unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> c_is_valid_utf8_ptr_unsafe ptr (fromIntegral len)+#else+-- B.isValidUtf8 is buggy before bytestring-0.11.5.3 / bytestring-0.12.1.0.+-- MIN_VERSION_bytestring does not allow us to differentiate+-- between 0.11.5.2 and 0.11.5.3 so no choice except demanding 0.12.1+.+#if MIN_VERSION_bytestring(0,12,1)+isValidUtf8ByteString = B.isValidUtf8+#else+isValidUtf8ByteString = N.isValidUtf8ByteStringHaskell+#endif+#endif++-- | For pinned byte arrays larger than 128KiB, this switches to the safe FFI+-- so that it does not prevent GC. This threshold (128KiB) was chosen+-- somewhat arbitrarily and may change in the future.+isValidUtf8ByteArray ::+ ByteArray -- ^ Bytes+ -> Int -- ^ Offset+ -> Int -- ^ Length+ -> Bool+isValidUtf8ByteArray b@(ByteArray b#) !off !len+ | len >= 131072 -- 128KiB+ , isTrue# (isByteArrayPinned# b#)+ = isValidUtf8ByteArrayPinned b off len+ | otherwise = isValidUtf8ByteArrayUnpinned b off len++-- | This uses the @unsafe@ FFI. GC waits for all @unsafe@ FFI calls+-- to complete before starting. Consequently, an @unsafe@ FFI call does not+-- run concurrently with GC and is not interrupted by GC. Since relocation+-- cannot happen concurrently with an @unsafe@ FFI call, it is safe+-- to call this function with an unpinned byte array argument.+-- It is also safe to call this with a pinned @ByteArray@ argument.+isValidUtf8ByteArrayUnpinned ::+ ByteArray -- ^ Bytes+ -> Int -- ^ Offset+ -> Int -- ^ Length+ -> Bool+#ifdef SIMDUTF+isValidUtf8ByteArrayUnpinned (ByteArray bs) !off !len =+ unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_unsafe bs (fromIntegral off) (fromIntegral len)+#else+isValidUtf8ByteArrayUnpinned = N.isValidUtf8ByteArrayHaskell+#endif++-- | This uses the @safe@ FFI. GC may run concurrently with @safe@+-- FFI calls. Consequently, unpinned objects may be relocated while a+-- @safe@ FFI call is executing. The byte array argument /must/ be pinned,+-- and the calling context is responsible for enforcing this. If the+-- byte array is not pinned, this function's behavior is undefined.+isValidUtf8ByteArrayPinned ::+ ByteArray -- ^ Bytes+ -> Int -- ^ Offset+ -> Int -- ^ Length+ -> Bool+#ifdef SIMDUTF+isValidUtf8ByteArrayPinned (ByteArray bs) !off !len =+ unsafeDupablePerformIO $ (/= 0) <$> c_is_valid_utf8_bytearray_safe bs (fromIntegral off) (fromIntegral len)+#else+isValidUtf8ByteArrayPinned = N.isValidUtf8ByteArrayHaskell+#endif
+ src/Data/Text/Internal/Validate/Native.hs view
@@ -0,0 +1,60 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MagicHash #-}++-- | Native implementation of 'Data.Text.Internal.Validate'.+module Data.Text.Internal.Validate.Native+ ( isValidUtf8ByteStringHaskell+ , isValidUtf8ByteArrayHaskell+ ) where++import Data.Array.Byte (ByteArray(ByteArray))+import Data.ByteString (ByteString)+import GHC.Exts (ByteArray#,Int(I#),indexWord8Array#)+import GHC.Word (Word8(W8#))+import Data.Text.Internal.Encoding.Utf8 (CodePoint(..),DecoderResult(..),utf8DecodeStart,utf8DecodeContinue)+import qualified Data.ByteString as B+import qualified Data.ByteString.Unsafe as B++-- | Native implementation of 'Data.Text.Internal.Validate.isValidUtf8ByteString'.+isValidUtf8ByteStringHaskell :: ByteString -> Bool+isValidUtf8ByteStringHaskell bs = start 0+ where+ start ix+ | ix >= B.length bs = True+ | otherwise = case utf8DecodeStart (B.unsafeIndex bs ix) of+ Accept{} -> start (ix + 1)+ Reject{} -> False+ Incomplete st _ -> step (ix + 1) st+ step ix st+ | ix >= B.length bs = False+ -- We do not use decoded code point, so passing a dummy value to save an argument.+ | otherwise = case utf8DecodeContinue (B.unsafeIndex bs ix) st (CodePoint 0) of+ Accept{} -> start (ix + 1)+ Reject{} -> False+ Incomplete st' _ -> step (ix + 1) st'++-- | Native implementation of+-- 'Data.Text.Internal.Validate.isValidUtf8ByteArrayUnpinned'+-- and 'Data.Text.Internal.Validate.isValidUtf8ByteArrayPinned'.+isValidUtf8ByteArrayHaskell ::+ ByteArray -- ^ Bytes+ -> Int -- ^ Offset+ -> Int -- ^ Length+ -> Bool+isValidUtf8ByteArrayHaskell (ByteArray b) !off !len = start off+ where+ indexWord8 :: ByteArray# -> Int -> Word8+ indexWord8 !x (I# i) = W8# (indexWord8Array# x i)+ start ix+ | ix >= off + len = True+ | otherwise = case utf8DecodeStart (indexWord8 b ix) of+ Accept{} -> start (ix + 1)+ Reject{} -> False+ Incomplete st _ -> step (ix + 1) st+ step ix st+ | ix >= off + len = False+ -- We do not use decoded code point, so passing a dummy value to save an argument.+ | otherwise = case utf8DecodeContinue (indexWord8 b ix) st (CodePoint 0) of+ Accept{} -> start (ix + 1)+ Reject{} -> False+ Incomplete st' _ -> step (ix + 1) st'
+ src/Data/Text/Internal/Validate/Simd.hs view
@@ -0,0 +1,42 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE UnliftedFFITypes #-}++-- | Validate that a byte sequence is UTF-8-encoded text. All of these+-- functions return zero when the byte sequence is not UTF-8-encoded text,+-- and they return an unspecified non-zero value when the byte sequence+-- is UTF-8-encoded text.+--+-- Variants are provided for both @ByteArray#@ and @Ptr@. Additionally,+-- variants are provided that use both the @safe@ and @unsafe@ FFI.+--+-- If compiling with SIMDUTF turned off, this module exports nothing.+module Data.Text.Internal.Validate.Simd+ ( c_is_valid_utf8_ptr_unsafe+ , c_is_valid_utf8_ptr_safe+ , c_is_valid_utf8_bytearray_unsafe+ , c_is_valid_utf8_bytearray_safe+ ) where++import Data.Word (Word8)+import Foreign.C.Types (CSize(..),CInt(..))+import GHC.Exts (Ptr,ByteArray#)++foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_unsafe+ :: Ptr Word8 -- ^ Bytes+ -> CSize -- ^ Length+ -> IO CInt+foreign import ccall safe "_hs_text_is_valid_utf8" c_is_valid_utf8_ptr_safe+ :: Ptr Word8 -- ^ Bytes+ -> CSize -- ^ Length+ -> IO CInt+foreign import ccall unsafe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_unsafe+ :: ByteArray# -- ^ Bytes+ -> CSize -- ^ Offset into bytes+ -> CSize -- ^ Length+ -> IO CInt+foreign import ccall safe "_hs_text_is_valid_utf8_offset" c_is_valid_utf8_bytearray_safe+ :: ByteArray# -- ^ Bytes+ -> CSize -- ^ Offset into bytes+ -> CSize -- ^ Length+ -> IO CInt
src/Data/Text/Lazy.hs view
@@ -1,1668 +1,1911 @@-{-# OPTIONS_GHC -fno-warn-orphans #-}-{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE TemplateHaskellQuotes #-}-{-# LANGUAGE LambdaCase #-}---- |--- Module : Data.Text.Lazy--- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text using--- lists of packed arrays.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ The representation used by this module is suitable for high--- performance use and for streaming large quantities of data. It--- provides a means to manipulate a large body of text without--- requiring that the entire content be resident in memory.------ Some operations, such as 'concat', 'append', 'reverse' and 'cons',--- have better time complexity than their "Data.Text" equivalents, due--- to the underlying representation being a list of chunks. For other--- operations, lazy 'Text's are usually within a few percent of strict--- ones, but often with better heap usage if used in a streaming--- fashion. For data larger than available memory, or if you have--- tight memory constraints, this module will be the only option.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions. eg.------ > import qualified Data.Text.Lazy as L--module Data.Text.Lazy- (- -- * Fusion- -- $fusion-- -- * Acceptable data- -- $replacement-- -- * Types- Text-- -- * Creation and elimination- , pack- , unpack- , singleton- , empty- , fromChunks- , toChunks- , toStrict- , fromStrict- , foldrChunks- , foldlChunks-- -- * Basic interface- , cons- , snoc- , append- , uncons- , unsnoc- , head- , last- , tail- , init- , null- , length- , compareLength-- -- * Transformations- , map- , intercalate- , intersperse- , transpose- , reverse- , replace-- -- ** Case conversion- -- $case- , toCaseFold- , toLower- , toUpper- , toTitle-- -- ** Justification- , justifyLeft- , justifyRight- , center-- -- * Folds- , foldl- , foldl'- , foldl1- , foldl1'- , foldr- , foldr1-- -- ** Special folds- , concat- , concatMap- , any- , all- , maximum- , minimum-- -- * Construction-- -- ** Scans- , scanl- , scanl1- , scanr- , scanr1-- -- ** Accumulating maps- , mapAccumL- , mapAccumR-- -- ** Generation and unfolding- , repeat- , replicate- , cycle- , iterate- , unfoldr- , unfoldrN-- -- * Substrings-- -- ** Breaking strings- , take- , takeEnd- , drop- , dropEnd- , takeWhile- , takeWhileEnd- , dropWhile- , dropWhileEnd- , dropAround- , strip- , stripStart- , stripEnd- , splitAt- , span- , breakOn- , breakOnEnd- , break- , group- , groupBy- , inits- , tails-- -- ** Breaking into many substrings- -- $split- , splitOn- , split- , chunksOf- -- , breakSubstring-- -- ** Breaking into lines and words- , lines- , words- , unlines- , unwords-- -- * Predicates- , isPrefixOf- , isSuffixOf- , isInfixOf-- -- ** View patterns- , stripPrefix- , stripSuffix- , commonPrefixes-- -- * Searching- , filter- , find- , elem- , breakOnAll- , partition-- -- , findSubstring-- -- * Indexing- , index- , count-- -- * Zipping and unzipping- , zip- , zipWith-- -- -* Ordered text- -- , sort- ) where--import Prelude (Char, Bool(..), Maybe(..), String,- Eq, (==), Ord(..), Ordering(..), Read(..), Show(..),- (&&), (+), (-), (.), ($), (++),- error, flip, fmap, fromIntegral, not, otherwise, quot)-import qualified Prelude as P-import Control.Arrow (first)-import Control.DeepSeq (NFData(..))-import Data.Bits (finiteBitSize)-import Data.Int (Int64)-import qualified Data.List as L-import Data.Char (isSpace)-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,- Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Data.Binary (Binary(get, put))-import Data.List.NonEmpty (NonEmpty(..))-import qualified Data.List.NonEmpty as NE-import Data.Monoid (Monoid(..))-import Data.Semigroup (Semigroup(..))-import Data.String (IsString(..))-import qualified Data.Text as T-import qualified Data.Text.Array as A-import qualified Data.Text.Internal as T-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Unsafe as T-import qualified Data.Text.Internal.Lazy.Fusion as S-import Data.Text.Internal.Fusion.Types (PairS(..))-import Data.Text.Internal.Lazy.Fusion (stream, unstream)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,- foldrChunks, smallChunkSize, defaultChunkSize, equal)-import Data.Text.Internal (firstf, safe, text)-import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8)-import Data.Text.Internal.Lazy.Search (indices)-import qualified GHC.CString as GHC-import qualified GHC.Exts as Exts-import GHC.Prim (Addr#)-import GHC.Stack (HasCallStack)-import qualified Language.Haskell.TH.Lib as TH-import qualified Language.Haskell.TH.Syntax as TH-import Text.Printf (PrintfArg, formatArg, formatString)---- $fusion------ Starting from @text-1.3@ fusion is no longer implicit,--- and pipelines of transormations usually allocate intermediate 'Text' values.--- Users, who observe significant changes to performances,--- are encouraged to use fusion framework explicitly, employing--- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values--- (corresponding to 'Data.Char.Surrogate' category) that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"�\" (U+FFFD). Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\". The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)---- $setup--- >>> import Data.Text--- >>> import qualified Data.Text as T--- >>> :seti -XOverloadedStrings--instance Eq Text where- (==) = equal- {-# INLINE (==) #-}--instance Ord Text where- compare = compareText--compareText :: Text -> Text -> Ordering-compareText Empty Empty = EQ-compareText Empty _ = LT-compareText _ Empty = GT-compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =- A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of- LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs)- EQ -> compareText as bs- GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs--- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),--- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering--- of underlying bytearrays, no decoding is needed.--instance Show Text where- showsPrec p ps r = showsPrec p (unpack ps) r--instance Read Text where- readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]---- | @since 1.2.2.0-instance Semigroup Text where- (<>) = append--instance Monoid Text where- mempty = empty- mappend = (<>)- mconcat = concat--instance IsString Text where- fromString = pack---- | @since 1.2.0.0-instance Exts.IsList Text where- type Item Text = Char- fromList = pack- toList = unpack--instance NFData Text where- rnf Empty = ()- rnf (Chunk _ ts) = rnf ts---- | @since 1.2.1.0-instance Binary Text where- put t = put (encodeUtf8 t)- get = do- bs <- get- case decodeUtf8' bs of- P.Left exn -> P.fail (P.show exn)- P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Text".@'Data.Text.Text'-instance Data Text where- gfoldl f z txt = z pack `f` (unpack txt)- toConstr _ = packConstr- gunfold k z c = case constrIndex c of- 1 -> k (z pack)- _ -> error "Data.Text.Lazy.Text.gunfold"- dataTypeOf _ = textDataType---- | This instance has similar considerations to the 'Data' instance:--- it preserves abstraction at the cost of inefficiency.------ @since 1.2.4.0-instance TH.Lift Text where- lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack-#if MIN_VERSION_template_haskell(2,17,0)- liftTyped = TH.unsafeCodeCoerce . TH.lift-#elif MIN_VERSION_template_haskell(2,16,0)- liftTyped = TH.unsafeTExpCoerce . TH.lift-#endif---- | @since 1.2.2.0-instance PrintfArg Text where- formatArg txt = formatString $ unpack txt--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr]---- | /O(n)/ Convert a 'String' into a 'Text'.------ Performs replacement on invalid scalar values.-pack ::-#if defined(ASSERTS)- HasCallStack =>-#endif- String -> Text-pack = unstream . S.streamList . L.map safe-{-# INLINE [1] pack #-}---- | /O(n)/ Convert a 'Text' into a 'String'.-unpack ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> String-unpack t = S.unstreamList (stream t)-{-# INLINE [1] unpack #-}---- | /O(n)/ Convert a literal string into a Text.-unpackCString# :: Addr# -> Text-unpackCString# addr# = unstream (S.streamCString# addr#)-{-# NOINLINE unpackCString# #-}--{-# RULES "TEXT literal" forall a.- unstream (S.streamList (L.map safe (GHC.unpackCString# a)))- = unpackCString# a #-}--{-# RULES "TEXT literal UTF8" forall a.- unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a)))- = unpackCString# a #-}--{-# RULES "LAZY TEXT empty literal"- unstream (S.streamList (L.map safe []))- = Empty #-}--{-# RULES "LAZY TEXT empty literal" forall a.- unstream (S.streamList (L.map safe [a]))- = Chunk (T.singleton a) Empty #-}---- | /O(1)/ Convert a character into a Text.--- Performs replacement on invalid scalar values.-singleton :: Char -> Text-singleton c = Chunk (T.singleton c) Empty-{-# INLINE [1] singleton #-}---- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'.-fromChunks :: [T.Text] -> Text-fromChunks cs = L.foldr chunk Empty cs---- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's.-toChunks :: Text -> [T.Text]-toChunks cs = foldrChunks (:) [] cs---- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'.-toStrict :: Text -> T.Text-toStrict t = T.concat (toChunks t)-{-# INLINE [1] toStrict #-}---- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'.-fromStrict :: T.Text -> Text-fromStrict t = chunk t Empty-{-# INLINE [1] fromStrict #-}---- -------------------------------------------------------------------------------- * Basic functions---- | /O(1)/ Adds a character to the front of a 'Text'.-cons :: Char -> Text -> Text-cons c t = Chunk (T.singleton c) t-{-# INLINE [1] cons #-}--infixr 5 `cons`---- | /O(n)/ Adds a character to the end of a 'Text'. This copies the--- entire array in the process.-snoc :: Text -> Char -> Text-snoc t c = foldrChunks Chunk (singleton c) t-{-# INLINE [1] snoc #-}---- | /O(n\/c)/ Appends one 'Text' to another.-append :: Text -> Text -> Text-append xs ys = foldrChunks Chunk ys xs-{-# INLINE [1] append #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty.-uncons :: Text -> Maybe (Char, Text)-uncons Empty = Nothing-uncons (Chunk t ts) = Just (T.unsafeHead t, ts')- where ts' | T.compareLength t 1 == EQ = ts- | otherwise = Chunk (T.unsafeTail t) ts-{-# INLINE uncons #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'uncons' instead.-head :: HasCallStack => Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty. This is a partial function, consider using 'uncons' instead.-tail :: HasCallStack => Text -> Text-tail (Chunk t ts) = chunk (T.tail t) ts-tail Empty = emptyError "tail"-{-# INLINE [1] tail #-}---- | /O(n\/c)/ Returns all but the last character of a 'Text', which must--- be non-empty. This is a partial function, consider using 'unsnoc' instead.-init :: HasCallStack => Text -> Text-init (Chunk t0 ts0) = go t0 ts0- where go t (Chunk t' ts) = Chunk t (go t' ts)- go t Empty = chunk (T.init t) Empty-init Empty = emptyError "init"-{-# INLINE [1] init #-}---- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if--- empty.------ * It is no faster than using 'init' and 'last'.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc Empty = Nothing-unsnoc ts@(Chunk _ _) = Just (init ts, last ts)-{-# INLINE unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.-null :: Text -> Bool-null Empty = True-null _ = False-{-# INLINE [1] null #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n\/c)/ Returns the last character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'unsnoc' instead.-last :: HasCallStack => Text -> Char-last Empty = emptyError "last"-last (Chunk t ts) = go t ts- where go _ (Chunk t' ts') = go t' ts'- go t' Empty = T.last t'-{-# INLINE [1] last #-}---- | /O(n)/ Returns the number of characters in a 'Text'.-length :: Text -> Int64-length = foldlChunks go 0- where- go :: Int64 -> T.Text -> Int64- go l t = l + intToInt64 (T.length t)-{-# INLINE [1] length #-}--{-# RULES-"TEXT length/map -> length" forall f t.- length (map f t) = length t-"TEXT length/zipWith -> length" forall f t1 t2.- length (zipWith f t1 t2) = min (length t1) (length t2)-"TEXT length/replicate -> n" forall n t.- length (replicate n t) = max 0 n P.* length t-"TEXT length/cons -> length+1" forall c t.- length (cons c t) = 1 + length t-"TEXT length/intersperse -> 2*length-1" forall c t.- length (intersperse c t) = max 0 (2 P.* length t - 1)-"TEXT length/intercalate -> n*length" forall s ts.- length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)- #-}---- | /O(n)/ Compare the count of characters in a 'Text' to a number.------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int64 -> Ordering-compareLength t n = S.compareLengthI (stream t) n-{-# INLINE [1] compareLength #-}---- We don't apply those otherwise appealing length-to-compareLength--- rewrite rules here, because they can change the strictness--- properties of code.---- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@. Performs replacement on--- invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f = foldrChunks (Chunk . T.map f) Empty-{-# INLINE [1] map #-}--{-# RULES-"TEXT map/map -> map" forall f g t.- map f (map g t) = map (f . safe . g) t-#-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.-intercalate :: Text -> [Text] -> Text-intercalate t = concat . L.intersperse t-{-# INLINE [1] intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'. Performs--- replacement on invalid scalar values.-intersperse :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse (safe c) (stream t))-{-# INLINE [1] intersperse #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Performs--- replacement on invalid scalar values.------ Examples:------ > justifyLeft 7 'x' "foo" == "fooxxxx"--- > justifyLeft 3 'x' "foobar" == "foobar"-justifyLeft :: Int64 -> Char -> Text -> Text-justifyLeft k c t- | len >= k = t- | otherwise = t `append` replicateChunk (k-len) (T.singleton c)- where len = length t-{-# INLINE [1] justifyLeft #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Performs replacement on--- invalid scalar values.------ Examples:------ > justifyRight 7 'x' "bar" == "xxxxbar"--- > justifyRight 3 'x' "foobar" == "foobar"-justifyRight :: Int64 -> Char -> Text -> Text-justifyRight k c t- | len >= k = t- | otherwise = replicateChunk (k-len) (T.singleton c) `append` t- where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side. Performs replacement on invalid--- scalar values.------ Examples:------ > center 8 'x' "HS" = "xxxHSxxx"-center :: Int64 -> Char -> Text -> Text-center k c t- | len >= k = t- | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c)- where len = length t- d = k - len- r = d `quot` 2- l = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument. Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.-transpose :: [Text] -> [Text]-transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty)- (L.transpose (L.map unpack ts))--- TODO: make this fast---- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order.-reverse ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Text-reverse = rev Empty- where rev a Empty = a- rev a (Chunk t ts) = rev (Chunk (T.reverse t) a) ts---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =--- 'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once. So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ > replace "oo" "foo" "oo" == "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ > replace "ofo" "bar" "ofofo" == "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: HasCallStack- => Text- -- ^ @needle@ to search for. If this string is empty, an- -- error will occur.- -> Text- -- ^ @replacement@ to replace @needle@ with.- -> Text- -- ^ @haystack@ in which to search.- -> Text-replace s d = intercalate d . splitOn s-{-# INLINE replace #-}---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ With Unicode text, it is incorrect to use combinators like @map--- toUpper@ to case convert each character of a string individually.--- Instead, use the whole-string case conversion functions from this--- module. For correctness in different writing systems, these--- functions may map one input character to two or three output--- characters.---- | /O(n)/ Convert a string to folded case.------ This function is mainly useful for performing caseless (or case--- insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string. For instance,--- the Armenian small ligature men now (U+FB13) is case folded to the--- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is--- case folded to the Greek small letter letter mu (U+03BC) instead of--- itself.-toCaseFold :: Text -> Text-toCaseFold t = unstream (S.toCaseFold (stream t))-{-# INLINE toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the Latin capital letter I with dot above (U+0130) maps--- to the sequence Latin small letter i (U+0069) followed by combining--- dot above (U+0307).-toLower :: Text -> Text-toLower t = unstream (S.toLower (stream t))-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the German eszett (U+00DF) maps to the two-letter--- sequence SS.-toUpper :: Text -> Text-toUpper t = unstream (S.toUpper (stream t))-{-# INLINE toUpper #-}----- | /O(n)/ Convert a string to title case, using simple case--- conversion.------ The first letter of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature fl (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased—but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle t = unstream (S.toTitle (stream t))-{-# INLINE toTitle #-}---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.----foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.-foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat [] = Empty-concat (Empty : css) = concat css-concat (Chunk c Empty : css) = Chunk c (concat css)-concat (Chunk c cs : css) = Chunk c (concat (cs : css))-{-# INLINE concat #-}---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty.-maximum :: HasCallStack => Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty.-minimum :: HasCallStack => Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ Note that------ > last (scanl f z xs) == foldl f z xs.-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))- where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument. Performs replacement on invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t0 = case uncons t0 of- Nothing -> empty- Just (t,ts) -> scanl f t ts-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f v = reverse . scanl g v . reverse- where g a b = safe (f b a)---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument. Performs replacement on invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t = empty- | otherwise = scanr f (last t) (init t)---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'. Performs--- replacement on invalid scalar values.-mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumL f = go- where- go z (Chunk c cs) = (z'', Chunk c' cs')- where (z', c') = T.mapAccumL f z c- (z'', cs') = go z' cs- go z Empty = (z, Empty)-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'. Performs replacement on invalid scalar values.-mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumR f = go- where- go z (Chunk c cs) = (z'', Chunk c' cs')- where (z'', c') = T.mapAccumR f z' c- (z', cs') = go z cs- go z Empty = (z, Empty)-{-# INLINE mapAccumR #-}---- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every--- element.------ @since 1.2.0.5-repeat :: Char -> Text-repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t- in t---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int64 -> Text -> Text-replicate n- | n <= 0 = P.const Empty- | otherwise = \case- Empty -> Empty- Chunk t Empty -> replicateChunk n t- t -> concat (rep n)- where- rep 0 = []- rep i = t : rep (i - 1)-{-# INLINE [1] replicate #-}--replicateChunk :: Int64 -> T.Text -> Text-replicateChunk !n !t@(T.Text _ _ len)- | n <= 0 = Empty- | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk)- where- perChunk = defaultChunkSize `quot` len- normalChunk = T.replicate perChunk t- (q, r) = n `P.quotRem` intToInt64 perChunk- headChunk = T.replicate (int64ToInt r) t-{-# INLINE replicateChunk #-}---- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or--- equivalently, the infinite repetition of the original 'Text'.------ @since 1.2.0.5-cycle :: HasCallStack => Text -> Text-cycle Empty = emptyError "cycle"-cycle t = let t' = foldrChunks Chunk t' t- in t'---- | @'iterate' f x@ returns an infinite 'Text' of repeated applications--- of @f@ to @x@:------ > iterate f x == [x, f x, f (f x), ...]------ @since 1.2.0.5-iterate :: (Char -> Char) -> Char -> Text-iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))- in t c---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.--- Performs replacement on invalid scalar values.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'.--- Performs replacement on invalid scalar values.-unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text.-take :: Int64 -> Text -> Text-take i _ | i <= 0 = Empty-take i t0 = take' i t0- where- take' :: Int64 -> Text -> Text- take' 0 _ = Empty- take' _ Empty = Empty- take' n (Chunk t@(T.Text arr off _) ts)- | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =- if m >= 0- then fromStrict (T.Text arr off m)- else Chunk t (take' (n + intToInt64 m) ts)-- | n < l = Chunk (T.take (int64ToInt n) t) Empty- | otherwise = Chunk t (take' (n - l) ts)- where l = intToInt64 (T.length t)-{-# INLINE [1] take #-}---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ > takeEnd 3 "foobar" == "bar"------ @since 1.1.1.0-takeEnd :: Int64 -> Text -> Text-takeEnd n t0- | n <= 0 = empty- | otherwise = takeChunk n empty . L.reverse . toChunks $ t0- where- takeChunk :: Int64 -> Text -> [T.Text] -> Text- takeChunk _ acc [] = acc- takeChunk i acc (t:ts)- | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc- | otherwise = takeChunk (i-l) (Chunk t acc) ts- where l = intToInt64 (T.length t)---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'.-drop :: Int64 -> Text -> Text-drop i t0- | i <= 0 = t0- | otherwise = drop' i t0- where- drop' :: Int64 -> Text -> Text- drop' 0 ts = ts- drop' _ Empty = Empty- drop' n (Chunk t@(T.Text arr off len) ts)- | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =- if m >= 0- then chunk (T.Text arr (off + m) (len - m)) ts- else drop' (n + intToInt64 m) ts-- | n < l = Chunk (T.drop (int64ToInt n) t) ts- | otherwise = drop' (n - l) ts- where l = intToInt64 (T.length t)-{-# INLINE [1] drop #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ > dropEnd 3 "foobar" == "foo"------ @since 1.1.1.0-dropEnd :: Int64 -> Text -> Text-dropEnd n t0- | n <= 0 = t0- | otherwise = dropChunk n . L.reverse . toChunks $ t0- where- dropChunk :: Int64 -> [T.Text] -> Text- dropChunk _ [] = empty- dropChunk m (t:ts)- | m >= l = dropChunk (m-l) ts- | otherwise = fromChunks . L.reverse $- T.dropEnd (int64ToInt m) t : ts- where l = intToInt64 (T.length t)---- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8'--- values dropped, or the empty 'Text' if @n@ is greater than the--- number of 'Word8' values present.-dropWords :: Int64 -> Text -> Text-dropWords i t0- | i <= 0 = t0- | otherwise = drop' i t0- where- drop' :: Int64 -> Text -> Text- drop' 0 ts = ts- drop' _ Empty = Empty- drop' n (Chunk (T.Text arr off len) ts)- | n < len' = chunk (text arr (off+n') (len-n')) ts- | otherwise = drop' (n - len') ts- where len' = intToInt64 len- n' = int64ToInt n---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t0 = takeWhile' t0- where takeWhile' Empty = Empty- takeWhile' (Chunk t ts) =- case T.findIndex (not . p) t of- Just n | n > 0 -> Chunk (T.take n t) Empty- | otherwise -> Empty- Nothing -> Chunk t (takeWhile' ts)-{-# INLINE [1] takeWhile #-}---- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.--- Examples:------ > takeWhileEnd (=='o') "foo" == "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p = takeChunk empty . L.reverse . toChunks- where takeChunk acc [] = acc- takeChunk acc (t:ts)- | T.lengthWord8 t' < T.lengthWord8 t- = chunk t' acc- | otherwise = takeChunk (Chunk t' acc) ts- where t' = T.takeWhileEnd p t-{-# INLINE takeWhileEnd #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t0 = dropWhile' t0- where dropWhile' Empty = Empty- dropWhile' (Chunk t ts) =- case T.findIndex (not . p) t of- Just n -> Chunk (T.drop n t) ts- Nothing -> dropWhile' ts-{-# INLINE [1] dropWhile #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.------ Examples:------ > dropWhileEnd (=='.') "foo..." == "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p = go- where go Empty = Empty- go (Chunk t Empty) = if T.null t'- then Empty- else Chunk t' Empty- where t' = T.dropWhileEnd p t- go (Chunk t ts) = case go ts of- Empty -> go (Chunk t Empty)- ts' -> Chunk t ts'-{-# INLINE dropWhileEnd #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string. Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile isSpace-{-# INLINE stripStart #-}---- | /O(n)/ Remove trailing white space from a string. Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int64 -> Text -> (Text, Text)-splitAt = loop- where- loop :: Int64 -> Text -> (Text, Text)- loop _ Empty = (empty, empty)- loop n t | n <= 0 = (empty, t)- loop n (Chunk t ts)- | n < len = let (t',t'') = T.splitAt (int64ToInt n) t- in (Chunk t' Empty, Chunk t'' ts)- | otherwise = let (ts',ts'') = loop (n - len) ts- in (Chunk t ts', ts'')- where len = intToInt64 (T.length t)---- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first--- element is a prefix of @t@ whose chunks contain @n@ 'Word8'--- values, and whose second is the remainder of the string.-splitAtWord :: Int64 -> Text -> PairS Text Text-splitAtWord _ Empty = empty :*: empty-splitAtWord x (Chunk c@(T.Text arr off len) cs)- | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs- in Chunk c h :*: t- | otherwise = chunk (text arr off y) empty :*:- chunk (text arr (off+y) (len-y)) cs- where y = int64ToInt x---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@. The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched. The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ > breakOn "::" "a::b::c" ==> ("a", "::b::c")--- > breakOn "/" "foobar" ==> ("foobar", "")------ Laws:------ > append prefix match == haystack--- > where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: HasCallStack => Text -> Text -> (Text, Text)-breakOn pat src- | null pat = emptyError "breakOn"- | otherwise = case indices pat src of- [] -> (src, empty)- (x:_) -> let h :*: t = splitAtWord x src- in (h, t)---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@. The second is the--- remainder of @haystack@, following the match.------ > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")-breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)-breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)- in (reverse b, reverse a)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@. Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ > breakOnAll "::" ""--- > ==> []--- > breakOnAll "/" "a/b/c/"--- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: HasCallStack- => Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-breakOnAll pat src- | null pat = emptyError "breakOnAll"- | otherwise = go 0 empty src (indices pat src)- where- go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s- h' = append p h- in (h',t) : go x h' t xs- go _ _ _ _ = []---- | /O(n)/ 'break' is like 'span', but the prefix returned is over--- elements that fail the predicate @p@.------ >>> T.break (=='c') "180cm"--- ("180","cm")-break :: (Char -> Bool) -> Text -> (Text, Text)-break p t0 = break' t0- where break' Empty = (empty, empty)- break' c@(Chunk t ts) =- case T.findIndex p t of- Nothing -> let (ts', ts'') = break' ts- in (Chunk t ts', ts'')- Just n | n == 0 -> (Empty, c)- | otherwise -> let (a,b) = T.splitAt n t- in (Chunk a Empty, Chunk b ts)---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the list.------ >>> T.span (=='0') "000AB"--- ("000","AB")-span :: (Char -> Bool) -> Text -> (Text, Text)-span p = break (not . p)-{-# INLINE span #-}---- | The 'group' function takes a 'Text' and returns a list of 'Text's--- such that the concatenation of the result is equal to the argument.--- Moreover, each sublist in the result contains only equal elements.--- For example,------ > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"]------ It is a special case of 'groupBy', which allows the programmer to--- supply their own equality test.-group :: Text -> [Text]-group = groupBy (==)-{-# INLINE group #-}---- | The 'groupBy' function is the non-overloaded version of 'group'.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy _ Empty = []-groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs- where (ys,zs) = span (eq x) xs- x = T.unsafeHead t- xs = chunk (T.unsafeTail t) ts---- | /O(n)/ Return all initial segments of the given 'Text',--- shortest first.-inits :: Text -> [Text]-inits = (Empty :) . inits'- where inits' Empty = []- inits' (Chunk t ts) = L.map (\t' -> Chunk t' Empty) (L.tail (T.inits t))- ++ L.map (Chunk t) (inits' ts)---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails Empty = Empty : []-tails ts@(Chunk t ts')- | T.length t == 1 = ts : tails ts'- | otherwise = ts : tails (Chunk (T.unsafeTail t) ts')---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be an empty string), consuming the--- delimiter. An empty delimiter is invalid, and will cause an error--- to be raised.------ Examples:------ > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]--- > splitOn "x" "x" == ["",""]------ and------ > intercalate s . splitOn s == id--- > splitOn (singleton c) == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: HasCallStack- => Text- -- ^ String to split on. If this string is empty, an error- -- will occur.- -> Text- -- ^ Input text.- -> [Text]-splitOn pat src- | null pat = emptyError "splitOn"- | isSingleton pat = split (== head pat) src- | otherwise = go 0 (indices pat src) src- where- go _ [] cs = [cs]- go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs- in h : go (x+l) xs (dropWords l t)- l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat-{-# INLINE [1] splitOn #-}--{-# RULES-"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.- splitOn (singleton c) t = split (==c) t- #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element. The--- resulting components do not contain the separators. Two adjacent--- separators result in an empty component in the output. eg.------ > split (=='a') "aabbaca" == ["","","bb","c",""]--- > split (=='a') [] == [""]-split :: (Char -> Bool) -> Text -> [Text]-split _ Empty = [Empty]-split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0- where comb acc (s:[]) Empty = revChunks (s:acc) : []- comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts- comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts- comb _ [] _ = impossibleError "split"-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@. The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ > chunksOf 3 "foobarbaz" == ["foo","bar","baz"]--- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]-chunksOf :: Int64 -> Text -> [Text]-chunksOf k = go- where- go t = case splitAt k t of- (a,b) | null a -> []- | otherwise -> a : go b-{-# INLINE chunksOf #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters--- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.------ 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.-lines :: Text -> [Text]-lines Empty = []-lines t = NE.toList $ go t- where- go :: Text -> NonEmpty Text- go Empty = Empty :| []- go (Chunk x xs)- -- x is non-empty, so T.lines x is non-empty as well- | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs- | otherwise = case unsnocList (T.lines x) of- Nothing -> impossibleError "lines"- Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls--prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text-prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial!--unsnocList :: [a] -> Maybe ([a], a)-unsnocList [] = Nothing-unsnocList (x : xs) = Just $ go x xs- where- go y [] = ([], y)- go y (z : zs) = first (y :) (go z zs)--hasNlEnd :: T.Text -> Bool-hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words = L.filter (not . null) . split isSpace-{-# INLINE words #-}---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' iff the first is a prefix of the second.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf Empty _ = True-isPrefixOf _ Empty = False-isPrefixOf (Chunk x xs) (Chunk y ys)- | lx == ly = x == y && isPrefixOf xs ys- | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys)- | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys- where (xh,xt) = T.splitAt ly x- (yh,yt) = T.splitAt lx y- lx = T.length x- ly = T.length y-{-# INLINE [1] isPrefixOf #-}---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' iff the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf x y = reverse x `isPrefixOf` reverse y-{-# INLINE isSuffixOf #-}--- TODO: a better implementation---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' iff the first is contained, wholly and intact, anywhere--- within the second.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf :: Text -> Text -> Bool-isInfixOf needle haystack- | null needle = True- | isSingleton needle = S.elem (head needle) . S.stream $ haystack- | otherwise = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ > stripPrefix "foo" "foobar" == Just "bar"--- > stripPrefix "" "baz" == Just "baz"--- > stripPrefix "foo" "quux" == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _ = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p t- | null p = Just t- | otherwise = case commonPrefixes p t of- Just (_,c,r) | null c -> Just r- _ -> Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")--- > commonPrefixes "veeble" "fetzer" == Nothing--- > commonPrefixes "" "baz" == Nothing-commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)-commonPrefixes Empty _ = Nothing-commonPrefixes _ Empty = Nothing-commonPrefixes a0 b0 = Just (go a0 b0 [])- where- go t0@(Chunk x xs) t1@(Chunk y ys) ps- = case T.commonPrefixes x y of- Just (p,a,b)- | T.null a -> go xs (chunk b ys) (p:ps)- | T.null b -> go (chunk a xs) ys (p:ps)- | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys)- Nothing -> (fromChunks (L.reverse ps),t0,t1)- go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1)---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ > stripSuffix "bar" "foobar" == Just "foo"--- > stripSuffix "" "baz" == Just "baz"--- > stripSuffix "foo" "quux" == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _ = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t)---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p = foldrChunks (chunk . T.filter p) Empty-{-# INLINE [1] filter #-}--{-# RULES-"TEXT filter/filter -> filter" forall p q t.- filter p (filter q t) = filter (\c -> p c && q c) t-#-}---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'elem' function takes a character and a 'Text', and--- returns 'True' if the element is found in the given 'Text', or--- 'False' otherwise.-elem :: Char -> Text -> Bool-elem c t = S.any (== c) (stream t)-{-# INLINE elem #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: HasCallStack => Text -> Int64 -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: HasCallStack => Text -> Text -> Int64-count pat- | null pat = emptyError "count"- | otherwise = go 0 . indices pat- where go !n [] = n- go !n (_:xs) = go (n+1) xs-{-# INLINE [1] count #-}--{-# RULES-"LAZY TEXT count/singleton -> countChar" [~1] forall c t.- count (singleton c) t = countChar c t- #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'.-countChar :: Char -> Text -> Int64-countChar c t = S.countChar c (stream t)---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE [0] zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))- where g a b = safe (f a b)-{-# INLINE [0] zipWith #-}--revChunks :: [T.Text] -> Text-revChunks = L.foldl' (flip chunk) Empty--emptyError :: HasCallStack => String -> a-emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input")--impossibleError :: HasCallStack => String -> a-impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case")--intToInt64 :: Exts.Int -> Int64-intToInt64 = fromIntegral--int64ToInt :: Int64 -> Exts.Int-int64ToInt = fromIntegral+{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy +-- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text using +-- lists of packed arrays. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- The representation used by this module is suitable for high +-- performance use and for streaming large quantities of data. It +-- provides a means to manipulate a large body of text without +-- requiring that the entire content be resident in memory. +-- +-- Some operations, such as 'concat', 'append', 'reverse' and 'cons', +-- have better time complexity than their "Data.Text" equivalents, due +-- to the underlying representation being a list of chunks. For other +-- operations, lazy 'Text's are usually within a few percent of strict +-- ones, but often with better heap usage if used in a streaming +-- fashion. For data larger than available memory, or if you have +-- tight memory constraints, this module will be the only option. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions. eg. +-- +-- > import qualified Data.Text.Lazy as L + +module Data.Text.Lazy + ( + -- * Fusion + -- $fusion + + -- * Acceptable data + -- $replacement + + -- * Types + Text + , LazyText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + , fromChunks + , toChunks + , toStrict + , fromStrict + , foldrChunks + , foldlChunks + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , repeat + , replicate + , cycle + , iterate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , span + , spanM + , spanEndM + , breakOn + , breakOnEnd + , break + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + -- , breakSubstring + + -- ** Breaking into lines and words + , lines + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , find + , elem + , breakOnAll + , partition + + -- , findSubstring + + -- * Indexing + , index + , count + + -- * Zipping and unzipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + ) where + +import Prelude (Char, Bool(..), Maybe(..), String, + Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec), + Monad(..), pure, (<$>), + (&&), (+), (-), (.), ($), (++), + error, flip, fmap, fromIntegral, not, otherwise, quot) +import qualified Prelude as P +import Control.Arrow (first) +import Control.DeepSeq (NFData(..)) +import Data.Bits (finiteBitSize, toIntegralSized) +import Data.Int (Int64) +import qualified Data.List as L hiding (head, tail) +import Data.Char (isSpace) +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Unsafe as T +import qualified Data.Text.Internal.Lazy.Fusion as S +import Data.Text.Internal.Fusion.Types (PairS(..)) +import Data.Text.Internal.Lazy.Fusion (stream, unstream) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, + foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText) +import Data.Text.Internal (firstf, safe, text) +import Data.Text.Internal.Reverse (reverseNonEmpty) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) +import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Lazy.Search (indices) +import qualified GHC.CString as GHC +import qualified GHC.Exts as Exts +import GHC.Prim (Addr#) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Syntax as TH +import qualified Language.Haskell.TH.Lib as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >. +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >) +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.) + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +instance Eq Text where + (==) = equal + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +compareText :: Text -> Text -> Ordering +compareText Empty Empty = EQ +compareText Empty _ = LT +compareText _ Empty = GT +compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of + LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) + EQ -> compareText as bs + GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +instance Show Text where + showsPrec p ps r = showsPrec p (unpack ps) r + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +instance Semigroup Text where + (<>) = append + stimes n _ | n < 0 = P.error "Data.Text.Lazy.stimes: given number is negative!" + stimes n a = + let nInt64 = fromIntegral n :: Int64 + len = if n == fromIntegral nInt64 && nInt64 >= 0 then nInt64 else P.maxBound + -- We clamp the length to maxBound :: Int64. + -- To tell the difference, the caller would have to skip through 2^63 chunks. + in replicate len a + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Data.Text.Lazy.Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Data.Text.Lazy.Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Text".@'Data.Text.Text' +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> error "Data.Text.Lazy.Text.gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 900 + lift x = [| fromStrict $(TH.lift . toStrict $ x) |] +#else + lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- +-- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555") +-- "\65533" +pack :: +#if defined(ASSERTS) + HasCallStack => +#endif + String -> Text +pack = unstream . S.streamList . L.map safe +{-# INLINE [1] pack #-} + +-- | /O(n)/ Convert a 'Text' into a 'String'. +unpack :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> String +unpack t = S.unstreamList (stream t) +{-# NOINLINE unpack #-} + +foldrFB :: (Char -> b -> b) -> b -> Text -> b +foldrFB = foldr +{-# INLINE [0] foldrFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us `foldrFB (:) []` +-- and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrFB` inlines and `foldr` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.Lazy.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrFB lcons lnil t) +"Text.Lazy.unpackBack" [1] foldrFB (:) [] = unpack + #-} + +-- | /O(n)/ Convert a literal string into a Text. +unpackCString# :: Addr# -> Text +unpackCString# addr# = unstream (S.streamCString# addr#) +{-# NOINLINE unpackCString# #-} + +{-# RULES "TEXT literal" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCString# a))) + = unpackCString# a #-} + +{-# RULES "TEXT literal UTF8" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) + = unpackCString# a #-} + +{-# RULES "LAZY TEXT empty literal" + unstream (S.streamList (L.map safe [])) + = Empty #-} + +{-# RULES "LAZY TEXT empty literal" forall a. + unstream (S.streamList (L.map safe [a])) + = Chunk (T.singleton a) Empty #-} + +-- | /O(1)/ Convert a character into a Text. +-- Performs replacement on invalid scalar values. +singleton :: Char -> Text +singleton c = Chunk (T.singleton c) Empty +{-# INLINE [1] singleton #-} + +-- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. +fromChunks :: [T.Text] -> Text +fromChunks cs = L.foldr chunk Empty cs + +-- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. +toChunks :: Text -> [T.Text] +toChunks cs = foldrChunks (:) [] cs + +-- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. +toStrict :: LazyText -> T.StrictText +toStrict t = T.concat (toChunks t) +{-# INLINE [1] toStrict #-} + +-- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. +fromStrict :: T.StrictText -> LazyText +fromStrict t = chunk t Empty +{-# INLINE [1] fromStrict #-} + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(1)/ Adds a character to the front of a 'Text'. +cons :: Char -> Text -> Text +cons c t = Chunk (T.singleton c) t +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +snoc :: Text -> Char -> Text +snoc t c = foldrChunks Chunk (singleton c) t +{-# INLINE [1] snoc #-} + +-- | /O(n\/c)/ Appends one 'Text' to another. +append :: Text -> Text -> Text +append xs ys = foldrChunks Chunk ys xs +{-# INLINE [1] append #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons Empty = Nothing +uncons (Chunk t ts) = Just (T.unsafeHead t, ts') + where ts' | T.compareLength t 1 == EQ = ts + | otherwise = Chunk (T.unsafeTail t) ts +{-# INLINE uncons #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t = S.head (stream t) +{-# INLINE head #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail (Chunk t ts) = chunk (T.tail t) ts +tail Empty = emptyError "tail" +{-# INLINE [1] tail #-} + +-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init (Chunk t0 ts0) = go t0 ts0 + where go t (Chunk t' ts) = Chunk t (go t' ts) + go t Empty = chunk (T.init t) Empty +init Empty = emptyError "init" +{-# INLINE [1] init #-} + +-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if +-- empty. +-- +-- * It is no faster than using 'init' and 'last'. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc Empty = Nothing +unsnoc ts@(Chunk _ _) = Just (init ts, last ts) +{-# INLINE unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null Empty = True +null _ = False +{-# INLINE [1] null #-} + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + +-- | /O(n\/c)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last Empty = emptyError "last" +last (Chunk t ts) = go t ts + where go _ (Chunk t' ts') = go t' ts' + go t' Empty = T.last t' +{-# INLINE [1] last #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: Text -> Int64 +length = foldlChunks go 0 + where + go :: Int64 -> T.Text -> Int64 + go l t = l + intToInt64 (T.length t) +{-# INLINE [1] length #-} + +{-# RULES +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = max 0 n P.* length t +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (2 P.* length t - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int64 -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +-- We don't apply those otherwise appealing length-to-compareLength +-- rewrite rules here, because they can change the strictness +-- properties of code. + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. Performs replacement on +-- invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = foldrChunks (Chunk . mapNonEmpty f) Empty +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. Performs +-- replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t = unstream (S.intersperse (safe c) (stream t)) +{-# INLINE [1] intersperse #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. Performs +-- replacement on invalid scalar values. +-- +-- Examples: +-- +-- > justifyLeft 7 'x' "foo" == "fooxxxx" +-- > justifyLeft 3 'x' "foobar" == "foobar" +justifyLeft :: Int64 -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChunk (k-len) (T.singleton c) + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- > justifyRight 7 'x' "bar" == "xxxxbar" +-- > justifyRight 3 'x' "foobar" == "foobar" +justifyRight :: Int64 -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChunk (k-len) (T.singleton c) `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- > center 8 'x' "HS" = "xxxHSxxx" +center :: Int64 -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +transpose :: [Text] -> [Text] +transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) + (L.transpose (L.map unpack ts)) +-- TODO: make this fast + +-- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text +reverse = rev Empty + where rev a Empty = a + rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- > replace "oo" "foo" "oo" == "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- > replace "ofo" "bar" "ofofo" == "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace s d = intercalate d . splitOn s +{-# INLINE replace #-} + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- With Unicode text, it is incorrect to use combinators like @map +-- toUpper@ to case convert each character of a string individually. +-- Instead, use the whole-string case conversion functions from this +-- module. For correctness in different writing systems, these +-- functions may map one input character to two or three output +-- characters. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (or case +-- insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature men now (U+FB13) is case folded to the +-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is +-- case folded to the Greek small letter letter mu (U+03BC) instead of +-- itself. +toCaseFold :: Text -> Text +toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the Latin capital letter I with dot above (U+0130) maps +-- to the sequence Latin small letter i (U+0069) followed by combining +-- dot above (U+0307). +toLower :: Text -> Text +toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German eszett (U+00DF) maps to the two-letter +-- sequence SS. +toUpper :: Text -> Text +toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty +{-# INLINE toUpper #-} + + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty +{-# INLINE toTitle #-} + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z t = S.foldl f z (stream t) +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +-- +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z t = S.foldl' f z (stream t) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = S.foldl1 f (stream t) +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = S.foldl1' f (stream t) +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z t = S.foldlM' f z (stream t) +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z t = S.foldr f z (stream t) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = S.foldr1 f (stream t) +{-# INLINE foldr1 #-} + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat [] = Empty +concat (Empty : css) = concat css +concat (Chunk c Empty : css) = Chunk c (concat css) +concat (Chunk c cs : css) = Chunk c (concat (cs : css)) +{-# INLINE concat #-} + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p t = S.any p (stream t) +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p t = S.all p (stream t) +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum t = S.maximum (stream t) +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum t = S.minimum (stream t) +{-# INLINE minimum #-} + +-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only +-- U+0000 through U+007F). +-- +-- This is a more efficient version of @'all' 'Data.Char.isAscii'@. +-- +-- >>> isAscii "" +-- True +-- +-- >>> isAscii "abc\NUL" +-- True +-- +-- >>> isAscii "abcd€" +-- False +-- +-- prop> isAscii t == all (< '\x80') t +-- +-- @since 2.0.2 +isAscii :: Text -> Bool +isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True + +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- Note that +-- +-- > last (scanl f z xs) == foldl f z xs. +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f z t = cons z $ P.snd $ + mapAccumL (\acc c -> let c' = f acc c in (c', c')) (safe z) t +-- This is a bit suboptimal: we could have used +-- Data.Text.scanl for the first chunk and mapAccumL +-- for subsequent ones, but but I doubt anyone cares +-- about the performance of 'scanl' much. +{-# INLINE scanl #-} + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t0 = case uncons t0 of + Nothing -> empty + Just (t,ts) -> scanl f t ts +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f z t = (`snoc` z) $ P.snd $ + mapAccumR (\acc c -> let c' = f c acc in (c', c')) (safe z) t +-- See the comment for 'scanl' above. +{-# INLINE scanr #-} + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumL f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z', c') = T.mapAccumL f z c + (z'', cs') = go z' cs + go z Empty = (z, Empty) +{-# INLINE mapAccumL #-} + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. Performs replacement on invalid scalar values. +mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumR f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z'', c') = T.mapAccumR f z' c + (z', cs') = go z cs + go z Empty = (z, Empty) +{-# INLINE mapAccumR #-} + +-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every +-- element. +-- +-- @since 1.2.0.5 +repeat :: Char -> Text +repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t + in t + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int64 -> Text -> Text +replicate n + | n <= 0 = P.const Empty + | otherwise = \case + Empty -> Empty + Chunk t Empty -> replicateChunk n t + t -> concat (rep n) + where + rep 0 = [] + rep i = t : rep (i - 1) +{-# INLINE [1] replicate #-} + +replicateChunk :: Int64 -> T.Text -> Text +replicateChunk !n !t@(T.Text _ _ len) + | n <= 0 = Empty + | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) + where + perChunk = defaultChunkSize `quot` len + normalChunk = T.replicate perChunk t + (q, r) = n `P.quotRem` intToInt64 perChunk + headChunk = T.replicate (int64ToInt r) t +{-# INLINE replicateChunk #-} + +-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or +-- equivalently, the infinite repetition of the original 'Text'. +-- +-- @since 1.2.0.5 +cycle :: HasCallStack => Text -> Text +cycle Empty = emptyError "cycle" +cycle t = let t' = foldrChunks Chunk t' t + in t' + +-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications +-- of @f@ to @x@: +-- +-- > iterate f x == [x, f x, f (f x), ...] +-- +-- @since 1.2.0.5 +iterate :: (Char -> Char) -> Char -> Text +iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) + in t c + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int64 -> Text -> Text +take i _ | i <= 0 = Empty +take i t0 = take' i t0 + where + take' :: Int64 -> Text -> Text + take' 0 _ = Empty + take' _ Empty = Empty + take' n (Chunk t@(T.Text arr off _) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then fromStrict (T.Text arr off m) + else Chunk t (take' (n + intToInt64 m) ts) + + | n < l = Chunk (T.take (int64ToInt n) t) Empty + | otherwise = Chunk t (take' (n - l) ts) + where l = intToInt64 (T.length t) +{-# INLINE [1] take #-} + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > takeEnd 3 "foobar" == "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int64 -> Text -> Text +takeEnd n t0 + | n <= 0 = empty + | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 + where + takeChunk :: Int64 -> Text -> [T.Text] -> Text + takeChunk _ acc [] = acc + takeChunk i acc (t:ts) + | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc + | otherwise = takeChunk (i-l) (Chunk t acc) ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int64 -> Text -> Text +drop i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk t@(T.Text arr off len) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then chunk (T.Text arr (off + m) (len - m)) ts + else drop' (n + intToInt64 m) ts + + | n < l = Chunk (T.drop (int64ToInt n) t) ts + | otherwise = drop' (n - l) ts + where l = intToInt64 (T.length t) +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > dropEnd 3 "foobar" == "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int64 -> Text -> Text +dropEnd n t0 + | n <= 0 = t0 + | otherwise = dropChunk n . L.reverse . toChunks $ t0 + where + dropChunk :: Int64 -> [T.Text] -> Text + dropChunk _ [] = empty + dropChunk m (t:ts) + | m >= l = dropChunk (m-l) ts + | otherwise = fromChunks . L.reverse $ + T.dropEnd (int64ToInt m) t : ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' +-- values dropped, or the empty 'Text' if @n@ is greater than the +-- number of 'Word8' values present. +dropWords :: Int64 -> Text -> Text +dropWords i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk (T.Text arr off len) ts) + | n < len' = chunk (text arr (off+n') (len-n')) ts + | otherwise = drop' (n - len') ts + where len' = intToInt64 len + n' = int64ToInt n + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t0 = takeWhile' t0 + where takeWhile' Empty = Empty + takeWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n | n > 0 -> Chunk (T.take n t) Empty + | otherwise -> Empty + Nothing -> Chunk t (takeWhile' ts) +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- > takeWhileEnd (=='o') "foo" == "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p = takeChunk empty . L.reverse . toChunks + where takeChunk acc [] = acc + takeChunk acc (t:ts) + | T.lengthWord8 t' < T.lengthWord8 t + = chunk t' acc + | otherwise = takeChunk (Chunk t' acc) ts + where t' = T.takeWhileEnd p t +{-# INLINE takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t0 = dropWhile' t0 + where dropWhile' Empty = Empty + dropWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n -> Chunk (T.drop n t) ts + Nothing -> dropWhile' ts +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- > dropWhileEnd (=='.') "foo..." == "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p = go + where go Empty = Empty + go (Chunk t Empty) = if T.null t' + then Empty + else Chunk t' Empty + where t' = T.dropWhileEnd p t + go (Chunk t ts) = case go ts of + Empty -> go (Chunk t Empty) + ts' -> Chunk t ts' +{-# INLINE dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int64 -> Text -> (Text, Text) +splitAt = loop + where + loop :: Int64 -> Text -> (Text, Text) + loop !_ Empty = (empty, empty) + loop n t | n <= 0 = (empty, t) + loop n (Chunk t@(T.Text arr off len) ts) + | n > mx = let (ts', ts'') = loop (n - intToInt64 (T.length t)) ts + in (Chunk t ts', ts'') + | m > 0, m >= len = (Chunk t Empty, ts) + | m > 0 = let t' = T.Text arr off m + t'' = T.Text arr (off+m) (len-m) + in (Chunk t' Empty, Chunk t'' ts) + | otherwise = let (ts', ts'') = loop (n + intToInt64 m) ts + in (Chunk t ts', ts'') + where + mx = intToInt64 P.maxBound + m = T.measureOff (int64ToInt n) t + + +-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first +-- element is a prefix of @t@ whose chunks contain @n@ 'Word8' +-- values, and whose second is the remainder of the string. +splitAtWord :: Int64 -> Text -> PairS Text Text +splitAtWord !_ Empty = empty :*: empty +splitAtWord x (Chunk c@(T.Text arr off len) cs) + | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs + in Chunk c h :*: t + | otherwise = chunk (text arr off y) empty :*: + chunk (text arr (off+y) (len-y)) cs + where y = int64ToInt x + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> let h :*: t = splitAtWord x src + in (h, t) + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) + in (reverse b, reverse a) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- > breakOnAll "::" "" +-- > ==> [] +-- > breakOnAll "/" "a/b/c/" +-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src + | null pat = emptyError "breakOnAll" + | otherwise = go 0 empty src (indices pat src) + where + go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s + h' = append p h + in (h',t) : go x h' t xs + go _ _ _ _ = [] + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is over +-- elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p t0 = break' t0 + where break' Empty = (empty, empty) + break' c@(Chunk t ts) = + case T.findIndex p t of + Nothing -> let (ts', ts'') = break' ts + in (Chunk t ts', ts'') + Just n | n == 0 -> (Empty, c) + | otherwise -> let (a,b) = T.splitAt n t + in (Chunk a Empty, Chunk b ts) + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p = break (not . p) +{-# INLINE span #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t1, t2) <- T.spanM p t + if T.null t2 then first (chunk t) <$> go ts + else pure (chunk t1 empty, Chunk t2 ts) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t3, t4) <- go ts + if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t + else pure (Chunk t t3, t4) +{-# INLINE spanEndM #-} + +-- | The 'group' function takes a 'Text' and returns a list of 'Text's +-- such that the concatenation of the result is equal to the argument. +-- Moreover, each sublist in the result contains only equal elements. +-- For example, +-- +-- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] +-- +-- It is a special case of 'groupBy', which allows the programmer to +-- supply their own equality test. +group :: Text -> [Text] +group = groupBy (==) +{-# INLINE group #-} + +-- | The 'groupBy' function is the non-overloaded version of 'group'. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy _ Empty = [] +groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs + where (ys,zs) = span (eq x) xs + x = T.unsafeHead t + xs = chunk (T.unsafeTail t) ts + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +inits :: Text -> [Text] +inits = (NE.toList P.$!) . initsNE + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmpty Text +initsNE ts0 = Empty NE.:| inits' 0 ts0 + where + inits' :: Int64 -- Number of previous chunks i + -> Text -- The remainder after dropping i chunks from ts0 + -> [Text] -- Prefixes longer than the first i chunks of ts0. + inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t)) + ++ inits' (i + 1) ts + inits' _ Empty = [] + +takeChunks :: Int64 -> Text -> T.Text -> Text +takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk) +takeChunks _ _ lastChunk = Chunk lastChunk Empty + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NE.toList P.$!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmpty Text +tailsNE Empty = Empty :| [] +tailsNE ts@(Chunk t ts') + | T.length t == 1 = ts :| tails ts' + | otherwise = ts :| tails (Chunk (T.unsafeTail t) ts') + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be an empty string), consuming the +-- delimiter. An empty delimiter is invalid, and will cause an error +-- to be raised. +-- +-- Examples: +-- +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat src + | null pat = emptyError "splitOn" + | isSingleton pat = split (== head pat) src + | otherwise = go 0 (indices pat src) src + where + go _ [] cs = [cs] + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in h : go (x+l) xs (dropWords l t) + l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat +{-# INLINE [1] splitOn #-} + +{-# RULES +"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') [] == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ Empty = [Empty] +split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 + where comb acc (s:[]) Empty = revChunks (s:acc) : [] + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts + comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts + comb _ [] _ = impossibleError "split" +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] +-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] +chunksOf :: Int64 -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines Empty = [] +lines t = NE.toList $ go t + where + go :: Text -> NonEmpty Text + go Empty = Empty :| [] + go (Chunk x xs) + -- x is non-empty, so T.lines x is non-empty as well + | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs + | otherwise = case unsnocList (T.lines x) of + Nothing -> impossibleError "lines" + Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls + +prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text +prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! + +unsnocList :: [a] -> Maybe ([a], a) +unsnocList [] = Nothing +unsnocList (x : xs) = Just $ go x xs + where + go y [] = ([], y) + go y (z : zs) = first (y :) (go z zs) + +hasNlEnd :: T.Text -> Bool +hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words = L.filter (not . null) . split isSpace +{-# INLINE words #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf Empty _ = True +isPrefixOf _ Empty = False +isPrefixOf (Chunk x xs) (Chunk y ys) + | lx == ly = x == y && isPrefixOf xs ys + | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) + | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys + where (xh,xt) = T.splitAt ly x + (yh,yt) = T.splitAt lx y + lx = T.length x + ly = T.length y + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf x y = reverse x `isPrefixOf` reverse y +{-# INLINE isSuffixOf #-} +-- TODO: a better implementation + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (head needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripPrefix "foo" "foobar" == Just "bar" +-- > stripPrefix "" "baz" == Just "baz" +-- > stripPrefix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p t + | null p = Just t + | otherwise = case commonPrefixes p t of + Just (_,c,r) | null c -> Just r + _ -> Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") +-- > commonPrefixes "veeble" "fetzer" == Nothing +-- > commonPrefixes "" "baz" == Nothing +commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) +commonPrefixes Empty _ = Nothing +commonPrefixes _ Empty = Nothing +commonPrefixes a0 b0 = Just (go a0 b0 []) + where + go t0@(Chunk x xs) t1@(Chunk y ys) ps + = case T.commonPrefixes x y of + Just (p,a,b) + | T.null a -> go xs (chunk b ys) (p:ps) + | T.null b -> go (chunk a xs) ys (p:ps) + | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) + Nothing -> (fromChunks (L.reverse ps),t0,t1) + go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripSuffix "bar" "foobar" == Just "foo" +-- > stripSuffix "" "baz" == Just "baz" +-- > stripSuffix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = foldrChunks (chunk . filter_ T.Text p) Empty +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' +-- if there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem c t = S.any (== c) (stream t) +{-# INLINE elem #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int64 -> Char +index lazyText ix + | ix < 0 = P.error $ "Data.Text.Lazy.index: negative index " ++ P.show ix + | otherwise = go lazyText ix + where + go :: Text -> Int64 -> Char + go Empty _ = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + go (Chunk t@(T.Text _ _ lenInBytes) ts) n = case toIntegralSized n of + Nothing -> + go ts (n - fromIntegral (T.length t)) + Just n' + | off < 0 -> go ts (n + fromIntegral off) + | off == lenInBytes -> go ts 0 + | otherwise -> ch + where + off = T.measureOff n' t + T.Iter ch _ = T.iter t off +{-# INLINE index #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int64 +count pat + | null pat = emptyError "count" + | otherwise = go 0 . indices pat + where go !n [] = n + go !n (_:xs) = go (n+1) xs +{-# INLINE [1] count #-} + +{-# RULES +"LAZY TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int64 +countChar c t = S.countChar c (stream t) + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE [0] zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [0] zipWith #-} + +-- | Convert a value to lazy 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +revChunks :: [T.Text] -> Text +revChunks = L.foldl' (flip chunk) Empty + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") + +impossibleError :: HasCallStack => String -> a +impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") + +intToInt64 :: Exts.Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Exts.Int +int64ToInt = fromIntegral
src/Data/Text/Lazy/Builder.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} {-# LANGUAGE Trustworthy #-} -----------------------------------------------------------------------------@@ -39,6 +39,7 @@ module Data.Text.Lazy.Builder ( -- * The Builder type Builder+ , LazyTextBuilder , toLazyText , toLazyTextWith
src/Data/Text/Lazy/Builder/RealFloat.hs view
@@ -1,6 +1,9 @@ {-# LANGUAGE CPP, OverloadedStrings #-} {-# LANGUAGE Trustworthy #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}+ -- | -- Module: Data.Text.Lazy.Builder.RealFloat -- Copyright: (c) The University of Glasgow 1994-2002
src/Data/Text/Lazy/Encoding.hs view
@@ -1,236 +1,254 @@-{-# LANGUAGE BangPatterns,CPP #-}-{-# LANGUAGE Trustworthy #-}---- |--- Module : Data.Text.Lazy.Encoding--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Functions for converting lazy 'Text' values to and from lazy--- 'ByteString', using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Lazy.Encoding- (- -- * Decoding ByteStrings to Text- -- $strict- decodeASCII- , decodeLatin1- , decodeUtf8- , decodeUtf16LE- , decodeUtf16BE- , decodeUtf32LE- , decodeUtf32BE-- -- ** Catchable failure- , decodeUtf8'-- -- ** Controllable error handling- , decodeUtf8With- , decodeUtf16LEWith- , decodeUtf16BEWith- , decodeUtf32LEWith- , decodeUtf32BEWith-- -- * Encoding Text to ByteStrings- , encodeUtf8- , encodeUtf16LE- , encodeUtf16BE- , encodeUtf32LE- , encodeUtf32BE-- -- * Encoding Text using ByteString Builders- , encodeUtf8Builder- , encodeUtf8BuilderEscaped- ) where--import Control.Exception (evaluate, try)-import Data.Monoid (Monoid(..))-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)-import Data.Word (Word8)-import qualified Data.ByteString as S-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Lazy as B-import qualified Data.ByteString.Lazy.Internal as B-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE-import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E-import qualified Data.Text.Internal.Lazy.Fusion as F-import Data.Text.Unsafe (unsafeDupablePerformIO)---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- | Decode a 'ByteString' containing 7-bit ASCII--- encoded text.-decodeASCII :: B.ByteString -> Text-decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.-decodeLatin1 :: B.ByteString -> Text-decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks---- | Decode a 'ByteString' containing UTF-8 encoded text.-decodeUtf8With :: OnDecodeError -> B.ByteString -> Text-decodeUtf8With onErr (B.Chunk b0 bs0) =- case TE.streamDecodeUtf8With onErr b0 of- TE.Some t l f -> chunk t (go f l bs0)- where- go f0 _ (B.Chunk b bs) =- case f0 b of- TE.Some t l f -> chunk t (go f l bs)- go _ l _- | S.null l = empty- | otherwise =- let !t = T.pack (skipBytes l)- skipBytes = S.foldr (\x s' ->- case onErr desc (Just x) of- Just c -> c : s'- Nothing -> s') [] in- Chunk t Empty- desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"-decodeUtf8With _ _ = empty---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code. For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.-decodeUtf8 :: B.ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}---- | Decode a 'ByteString' containing UTF-8 encoded text..------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.------ /Note/: this function is /not/ lazy, as it must decode its entire--- input before it can return a result. If you need lazy (streaming)--- decoding, use 'decodeUtf8With' in lenient mode.-decodeUtf8' :: B.ByteString -> Either UnicodeException Text-decodeUtf8' bs = unsafeDupablePerformIO $ do- let t = decodeUtf8 bs- try (evaluate (rnf t `seq` t))- where- rnf Empty = ()- rnf (Chunk _ ts) = rnf ts-{-# INLINE decodeUtf8' #-}---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> B.ByteString-encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder =- foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped prim =- foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: B.ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: B.ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> B.ByteString-encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt)-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> B.ByteString-encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt)-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: B.ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: B.ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> B.ByteString-encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt)-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> B.ByteString-encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt)-{-# INLINE encodeUtf32BE #-}+{-# LANGUAGE BangPatterns,CPP #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy.Encoding +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting lazy 'Text' values to and from lazy +-- 'ByteString', using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- <http://hackage.haskell.org/package/text-icu text-icu package>. + +module Data.Text.Lazy.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeUtf8Lenient + + -- *** Catchable failure + , decodeUtf8' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + ) where + +import Control.Exception (evaluate, try) +import Data.Monoid (Monoid(..)) +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) +import Data.Word (Word8) +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Lazy as B +import qualified Data.ByteString.Lazy.Internal as B +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Internal.Encoding as TE +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as F +import qualified Data.Text.Internal.StrictBuilder as SB +import Data.Text.Unsafe (unsafeDupablePerformIO) + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing 7-bit ASCII +-- encoded text. +decodeASCII :: B.ByteString -> Text +decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +decodeLatin1 :: B.ByteString -> Text +decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +decodeUtf8With :: OnDecodeError -> B.ByteString -> Text +decodeUtf8With onErr = loop TE.startUtf8State + where + chunkb builder t | SB.sbLength builder == 0 = t + | otherwise = Chunk (TE.strictBuilderToText builder) t + loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of + (builder, _, s') -> chunkb builder (loop s' bs) + loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty + msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +decodeUtf8 :: B.ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text.. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +-- +-- /Note/: this function is /not/ lazy, as it must decode its entire +-- input before it can return a result. If you need lazy (streaming) +-- decoding, use 'decodeUtf8With' in lenient mode. +decodeUtf8' :: B.ByteString -> Either UnicodeException Text +decodeUtf8' bs = unsafeDupablePerformIO $ do + let t = decodeUtf8 bs + try (evaluate (rnf t `seq` t)) + where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts +{-# INLINE decodeUtf8' #-} + +-- | Decode a lazy 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.1.4 +decodeUtf8Lenient :: B.ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> B.ByteString +encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped prim = + foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: B.ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: B.ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> B.ByteString +encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> B.ByteString +encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: B.ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: B.ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> B.ByteString +encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> B.ByteString +encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) +{-# INLINE encodeUtf32BE #-}
src/Data/Text/Lazy/IO.hs view
@@ -41,15 +41,15 @@ import Data.Text.Lazy (Text) import Prelude hiding (appendFile, getContents, getLine, interact, putStr, putStrLn, readFile, writeFile)-import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout,+import System.IO (Handle, IOMode(..), openFile, stdin, stdout, withFile)-import qualified Data.Text.IO as T import qualified Data.Text.Lazy as L import qualified Control.Exception as E import Control.Monad (when) import Data.IORef (readIORef)-import Data.Text.Internal.IO (hGetLineWith, readChunk)+import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStream) import Data.Text.Internal.Lazy (chunk, empty)+import Data.Text.Internal.Lazy.Fusion (stream, streamLn) import GHC.IO.Buffer (isEmptyBuffer) import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException) import GHC.IO.Handle.Internals (augmentIOError, hClose_help,@@ -129,11 +129,11 @@ -- | Write a string to a handle. hPutStr :: Handle -> Text -> IO ()-hPutStr h = mapM_ (T.hPutStr h) . L.toChunks+hPutStr h = hPutStream h . stream -- | Write a string to a handle, followed by a newline. hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h t = hPutStr h t >> hPutChar h '\n'+hPutStrLn h = hPutStream h . streamLn -- | The 'interact' function takes a function of type @Text -> Text@ -- as its argument. The entire input from the standard input device is
src/Data/Text/Lazy/Internal.hs view
@@ -1,4 +1,3 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} -- | -- Module : Data.Text.Lazy.Internal -- Copyright : (c) 2013 Bryan O'Sullivan
src/Data/Text/Lazy/Read.hs view
@@ -36,7 +36,7 @@ -- | Read some text. If the read succeeds, return its value and the -- remaining text, otherwise an error message.-type Reader a = IReader Text a+type Reader a = Text -> Either String (a, Text) type Parser = IParser Text -- | Read a decimal integer. The input must begin with at least one@@ -134,8 +134,16 @@ -- >rational "3e" == Right (3.0, "e") rational :: Fractional a => Reader a {-# SPECIALIZE rational :: Reader Double #-}-rational = floaty $ \real frac fracDenom -> fromRational $- real % 1 + frac % fracDenom+rational = floaty $ \real frac fracDenom power ->+ -- We must be careful to prevent DDoS attacks: if the return type is 'Double',+ -- a client rightfully expects 'rational' to operate within bounded memory.+ -- Thus if power is small, we can compute fraction with full precision and divide.+ -- Otherwise divide first, apply fromRational and scale last:+ -- the small loss of precision for Double does not matter much because the result is+ -- likely infinity or zero anyway.+ if abs power < 1000+ then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power))+ else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) -- | Read a rational number. --@@ -150,9 +158,9 @@ -- around the 15th decimal place. For 0.001% of numbers, this -- function will lose precision at the 13th or 14th decimal place. double :: Reader Double-double = floaty $ \real frac fracDenom ->- fromInteger real +- fromInteger frac / fromInteger fracDenom+double = floaty $ \real frac fracDenom power ->+ (fromInteger real ++ fromInteger frac / fromInteger fracDenom) * (10 ^^ power) signa :: Num a => Parser a -> Parser a {-# SPECIALIZE signa :: Parser Int -> Parser Int #-}@@ -174,7 +182,7 @@ then Right (c, if len <= 1 then ts else Chunk (T.Text arr (off + 1) (len - 1)) ts) else Left "character does not match" -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a+floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')@@ -190,9 +198,7 @@ then if power == 0 then fromInteger real else fromInteger real * (10 ^^ power)- else if power == 0- then f real fraction (10 ^ fracDigits)- else f real fraction (10 ^ fracDigits) * (10 ^^ power)+ else f real fraction (10 ^ fracDigits) power return $! if sign == ord8 '+' then n else -n
src/Data/Text/Read.hs view
@@ -33,7 +33,7 @@ -- | Read some text. If the read succeeds, return its value and the -- remaining text, otherwise an error message.-type Reader a = IReader Text a+type Reader a = Text -> Either String (a, Text) type Parser a = IParser Text a -- | Read a decimal integer. The input must begin with at least one@@ -140,8 +140,16 @@ -- >rational "3e" == Right (3.0, "e") rational :: Fractional a => Reader a {-# SPECIALIZE rational :: Reader Double #-}-rational = floaty $ \real frac fracDenom -> fromRational $- real % 1 + frac % fracDenom+rational = floaty $ \real frac fracDenom power ->+ -- We must be careful to prevent DDoS attacks: if the return type is 'Double',+ -- a client rightfully expects 'rational' to operate within bounded memory.+ -- Thus if power is small, we can compute fraction with full precision and divide.+ -- Otherwise divide first, apply fromRational and scale last:+ -- the small loss of precision for Double does not matter much because the result is+ -- likely infinity or zero anyway.+ if abs power < 1000+ then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power))+ else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) -- | Read a rational number. --@@ -156,9 +164,9 @@ -- around the 15th decimal place. For 0.001% of numbers, this -- function will lose precision at the 13th or 14th decimal place. double :: Reader Double-double = floaty $ \real frac fracDenom ->- fromInteger real +- fromInteger frac / fromInteger fracDenom+double = floaty $ \real frac fracDenom power ->+ (fromInteger real ++ fromInteger frac / fromInteger fracDenom) * (10 ^^ power) signa :: Num a => Parser a -> Parser a {-# SPECIALIZE signa :: Parser Int -> Parser Int #-}@@ -177,7 +185,7 @@ then Right (c, Text arr (off + 1) (len - 1)) else Left "character does not match" -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a+floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+')@@ -193,9 +201,7 @@ then if power == 0 then fromInteger real else fromInteger real * (10 ^^ power)- else if power == 0- then f real fraction (10 ^ fracDigits)- else f real fraction (10 ^ fracDigits) * (10 ^^ power)+ else f real fraction (10 ^ fracDigits) power return $! if sign == ord8 '+' then n else -n
src/Data/Text/Show.hs view
@@ -1,6 +1,8 @@-{-# LANGUAGE CPP, MagicHash #-}-{-# LANGUAGE Trustworthy #-}+{-# LANGUAGE BangPatterns #-} {-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE Trustworthy #-} {-# LANGUAGE ViewPatterns #-} {-# OPTIONS_GHC -fno-warn-orphans #-}@@ -16,23 +18,23 @@ module Data.Text.Show (- singleton+ addrLen+ , singleton , unpack , unpackCString# , unpackCStringAscii# ) where import Control.Monad.ST (ST, runST)-import Data.Text.Internal (Text(..), empty_, safe)+import Data.Text.Internal (Text(..), empty, safe, pack) import Data.Text.Internal.Encoding.Utf8 (utf8Length)-import Data.Text.Internal.Fusion (stream, unstream) import Data.Text.Internal.Unsafe.Char (unsafeWrite)+import Data.Text.Unsafe (Iter(..), iterArray) import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#)+import qualified GHC.Exts as Exts import GHC.Word (Word8(..)) import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Fusion.Common as S #if !MIN_VERSION_ghc_prim(0,7,0)-import Data.Text.Internal.Unsafe (inlinePerformIO) import Foreign.C.String (CString) import Foreign.C.Types (CSize(..)) #endif@@ -52,9 +54,33 @@ HasCallStack => #endif Text -> String-unpack = S.unstreamList . stream-{-# INLINE [1] unpack #-}+unpack t = foldrText (:) [] t+{-# NOINLINE unpack #-} +foldrText :: (Char -> b -> b) -> b -> Text -> b+foldrText f z (Text arr off len) = go off+ where+ go !i+ | i >= off + len = z+ | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l))+{-# INLINE foldrText #-}++foldrTextFB :: (Char -> b -> b) -> b -> Text -> b+foldrTextFB = foldrText+{-# INLINE [0] foldrTextFB #-}++-- List fusion rules for `unpack`:+-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build`+-- fuses if `foldr` is applied to it.+-- * If it doesn't fuse: In phase 1, `build` inlines to give us+-- `foldrTextFB (:) []` and we rewrite that back to `unpack`.+-- * If it fuses: In phase 0, `foldrTextFB` inlines and `foldrText` inlines. GHC+-- optimizes the fused code.+{-# RULES+"Text.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrTextFB lcons lnil t)+"Text.unpackBack" [1] foldrTextFB (:) [] = unpack+ #-}+ -- | /O(n)/ Convert a null-terminated -- <https://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 modified UTF-8> -- (but with a standard UTF-8 representation of characters from supplementary planes)@@ -86,8 +112,12 @@ A.shrinkM marr actualLen arr <- A.unsafeFreeze marr return $ Text arr 0 actualLen-{-# INLINE unpackCString# #-} +-- When a module contains many literal strings, 'unpackCString#' can easily+-- bloat generated code to insane size. There is also very little to gain+-- from inlining. Thus explicit NOINLINE is desired.+{-# NOINLINE unpackCString# #-}+ -- | /O(n)/ Convert a null-terminated ASCII string to a 'Text'. -- Counterpart to 'GHC.unpackCString#'. -- No validation is performed, malformed input can lead to memory access violation.@@ -101,32 +131,28 @@ marr <- A.new l A.copyFromPointer marr 0 (Ptr addr#) l A.unsafeFreeze marr-{-# INLINE unpackCStringAscii# #-}+{-# NOINLINE unpackCStringAscii# #-} addrLen :: Addr# -> Int #if MIN_VERSION_ghc_prim(0,7,0) addrLen addr# = I# (GHC.cstringLength# addr#) #else-addrLen addr# = fromIntegral (inlinePerformIO (c_strlen (Ptr addr#)))+addrLen addr# = fromIntegral (c_strlen (Ptr addr#)) -foreign import capi unsafe "string.h strlen" c_strlen :: CString -> IO CSize+foreign import capi unsafe "string.h strlen" c_strlen :: CString -> CSize #endif -{-# RULES "TEXT literal" [1] forall a.- unstream (S.map safe (S.streamList (GHC.unpackCString# a)))- = unpackCStringAscii# a #-}+{-# RULES "TEXT literal" forall a.+ pack (GHC.unpackCString# a) = unpackCStringAscii# a #-} -{-# RULES "TEXT literal UTF8" [1] forall a.- unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a)))- = unpackCString# a #-}+{-# RULES "TEXT literal UTF8" forall a.+ pack (GHC.unpackCStringUtf8# a) = unpackCString# a #-} -{-# RULES "TEXT empty literal" [1]- unstream (S.map safe (S.streamList []))- = empty_ #-}+{-# RULES "TEXT empty literal"+ pack [] = empty #-} -{-# RULES "TEXT singleton literal" [1] forall a.- unstream (S.map safe (S.streamList [a]))- = singleton_ a #-}+{-# RULES "TEXT singleton literal" forall a.+ pack [a] = singleton a #-} -- | /O(1)/ Convert a character into a Text. -- Performs replacement on invalid scalar values.@@ -135,24 +161,11 @@ HasCallStack => #endif Char -> Text-singleton = unstream . S.singleton . safe-{-# INLINE [1] singleton #-}--{-# RULES "TEXT singleton" forall a.- unstream (S.singleton (safe a))- = singleton_ a #-}---- This is intended to reduce inlining bloat.-singleton_ ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> Text-singleton_ c = Text (A.run x) 0 len+singleton c = Text (A.run x) 0 len where x :: ST s (A.MArray s) x = do arr <- A.new len _ <- unsafeWrite arr 0 d return arr len = utf8Length d d = safe c-{-# NOINLINE singleton_ #-}+{-# NOINLINE singleton #-}
src/Data/Text/Unsafe.hs view
@@ -80,7 +80,11 @@ {-# INLINE iter #-} -- | @since 2.0-iterArray :: A.Array -> Int -> Iter+iterArray ::+#if defined(ASSERTS)+ HasCallStack =>+#endif+ A.Array -> Int -> Iter iterArray arr j = Iter chr l where m0 = A.unsafeIndex arr j m1 = A.unsafeIndex arr (j+1)
tests/Tests.hs view
@@ -9,10 +9,14 @@ import qualified Tests.Lift as Lift import qualified Tests.Properties as Properties import qualified Tests.Regressions as Regressions+import qualified Tests.ShareEmpty as ShareEmpty+import qualified Tests.RebindableSyntaxTest as RST main :: IO () main = defaultMain $ testGroup "All" [ Lift.tests , Properties.tests , Regressions.tests+ , ShareEmpty.tests+ , RST.tests ]
tests/Tests/Lift.hs view
@@ -1,28 +1,39 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TemplateHaskell #-}-module Tests.Lift- ( tests- )- where--import qualified Data.Text as S-import qualified Data.Text.Lazy as L-import Language.Haskell.TH.Syntax (lift)-import Test.Tasty.HUnit (testCase, assertEqual)-import Test.Tasty (TestTree, testGroup)--tests :: TestTree-tests = testGroup "TH lifting Text"- [ testCase "strict" $ assertEqual "strict"- $(lift ("foo" :: S.Text))- ("foo" :: S.Text)- , testCase "strict0" $ assertEqual "strict0"- $(lift ("f\0o\1o\2" :: S.Text))- ("f\0o\1o\2" :: S.Text)- , testCase "lazy" $ assertEqual "lazy"- $(lift ("foo" :: L.Text))- ("foo" :: L.Text)- , testCase "lazy0" $ assertEqual "lazy0"- $(lift ("f\0o\1o\2" :: L.Text))- ("f\0o\1o\2" :: L.Text)- ]+{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +module Tests.Lift + ( tests + ) + where + +import qualified Data.Text as S +import qualified Data.Text.Lazy as L +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) + +tests :: TestTree +tests = testGroup "TH lifting Text" + [ testCase "strict" $ assertEqual "strict" + $(lift ("foo" :: S.Text)) + ("foo" :: S.Text) + , testCase "strict0" $ assertEqual "strict0" + $(lift ("f\0o\1o\2" :: S.Text)) + ("f\0o\1o\2" :: S.Text) + , testCase "strict-nihao" $ assertEqual "strict-nihao" + $(lift ("\20320\22909" :: S.Text)) + ("\20320\22909" :: S.Text) + , testCase "lazy" $ assertEqual "lazy" + $(lift ("foo" :: L.Text)) + ("foo" :: L.Text) + , testCase "lazy0" $ assertEqual "lazy0" + $(lift ("f\0o\1o\2" :: L.Text)) + ("f\0o\1o\2" :: L.Text) + , testCase "lazy-nihao" $ assertEqual "lazy-nihao" + $(lift ("\20320\22909" :: L.Text)) + ("\20320\22909" :: L.Text) + ]
tests/Tests/Properties.hs view
@@ -16,6 +16,8 @@ import Tests.Properties.Read (testRead) import Tests.Properties.Text (testText) import Tests.Properties.Transcoding (testTranscoding)+import Tests.Properties.Validate (testValidate)+import Tests.Properties.CornerCases (testCornerCases) tests :: TestTree tests =@@ -28,5 +30,7 @@ testSubstrings, testBuilder, testLowLevel,- testRead+ testRead,+ testCornerCases,+ testValidate ]
tests/Tests/Properties/Basics.hs view
@@ -2,7 +2,11 @@ {-# LANGUAGE ViewPatterns #-} -{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+{-# OPTIONS_GHC -Wno-missing-signatures #-}+{-# OPTIONS_GHC -Wno-warnings-deprecations #-}+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}+ module Tests.Properties.Basics ( testBasics ) where
+ tests/Tests/Properties/CornerCases.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++-- | Check that the definitions that are partial crash in the expected ways or+-- return sensible defaults.+module Tests.Properties.CornerCases (testCornerCases) where++import Control.Exception+import Data.Either+import Data.Semigroup+import Data.Text+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils ()++testCornerCases :: TestTree+testCornerCases =+ testGroup+ "corner cases"+ [ testGroup+ "stimes"+ $ let specimen = stimes :: Integer -> Text -> Text+ in [ testProperty+ "given a negative number, evaluate to error call"+ $ \(Negative number) text ->+ (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+ specimen+ (fromIntegral (number :: Int))+ text+ , testProperty+ "given a number that does not fit into Int, evaluate to error call"+ $ \(NonNegative number) text ->+ (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+ specimen+ (fromIntegral (number :: Int) + fromIntegral (maxBound :: Int) + 1)+ text+ ]+ ]
tests/Tests/Properties/Folds.hs view
@@ -1,263 +1,357 @@--- | Test folds, scans, and unfolds--{-# LANGUAGE ViewPatterns #-}--{-# OPTIONS_GHC -fno-warn-missing-signatures #-}-module Tests.Properties.Folds- ( testFolds- ) where--import Control.Arrow (second)-import Data.Word (Word8, Word16)-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2)-import Tests.QuickCheckUtils-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Lazy as TL---- Folds--sf_foldl (applyFun -> p) (applyFun2 -> f) z =- (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z)- where _types = f :: Char -> Char -> Char-tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z)- where _types = f :: Char -> Char -> Char-sf_foldl' (applyFun -> p) (applyFun2 -> f) z =- (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z- where _types = f :: Char -> Char -> Char-tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z- where _types = f :: Char -> Char -> Char-sf_foldl1 (applyFun -> p) (applyFun2 -> f) =- (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p)-t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f-tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f-sf_foldl1' (applyFun -> p) (applyFun2 -> f) =- (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p)-t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f-tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f-sf_foldr (applyFun -> p) (applyFun2 -> f) z =- (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z- where _types = f :: Char -> Char -> Char-tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z- where _types = f :: Char -> Char -> Char-sf_foldr1 (applyFun -> p) (applyFun2 -> f) =- (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p)-t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f-tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f---- Special folds--s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt)-sf_concat (applyFun -> p)- = (L.concat . map (L.filter p) . unSqrt) `eq`- (unpackS . S.concat . map (S.filter p . packS) . unSqrt)-t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt)-tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt)-sf_concatMap (applyFun -> p) (applyFun -> f) =- (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p)-t_concatMap (applyFun -> f)- = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f))-tl_concatMap (applyFun -> f)- = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f))-sf_any (applyFun -> q) (applyFun -> p)- = (L.any p . L.filter q) `eqP` (S.any p . S.filter q)-t_any (applyFun -> p)- = L.any p `eqP` T.any p-tl_any (applyFun -> p)- = L.any p `eqP` TL.any p-sf_all (applyFun -> q) (applyFun -> p)- = (L.all p . L.filter q) `eqP` (S.all p . S.filter q)-t_all (applyFun -> p)- = L.all p `eqP` T.all p-tl_all (applyFun -> p)- = L.all p `eqP` TL.all p-sf_maximum (applyFun -> p)- = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p)-t_maximum = L.maximum `eqP` T.maximum-tl_maximum = L.maximum `eqP` TL.maximum-sf_minimum (applyFun -> p)- = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p)-t_minimum = L.minimum `eqP` T.minimum-tl_minimum = L.minimum `eqP` TL.minimum---- Scans--sf_scanl (applyFun -> p) (applyFun2 -> f) z =- (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p)-t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z)-tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z)-t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f)-tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f)-t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z)-tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z)-t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f)-tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f)--t_mapAccumL_char c t =- snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)-t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z)- where _types = f :: Int -> Char -> (Int,Char)-tl_mapAccumL_char c t =- snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)-tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z)- where _types = f :: Int -> Char -> (Int,Char)-t_mapAccumR_char c t =- snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)-t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)- where _types = f :: Int -> Char -> (Int,Char)-tl_mapAccumR_char c t =- snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)-tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)- where _types = f :: Int -> Char -> (Int,Char)---- Unfolds--tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat)--s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS)--t_replicate_char (Small n) c =- L.replicate n c === T.unpack (T.replicate n (T.singleton c))-tl_replicate_char (Small n) c =- L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c))-t_length_replicate_char (Small n) c =- L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c))-tl_length_replicate_char (Small n) c =- L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c))--t_replicate (Small n) =- (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n)-tl_replicate (Small n) =- (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n))-t_length_replicate (Small n) =- (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n)-tl_length_replicate (Small n) =- (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n))--tl_cycle n = (L.take m . L.cycle) `eq`- (unpackS . TL.take (fromIntegral m) . TL.cycle . packS)- where m = fromIntegral (n :: Word8)--tl_iterate (applyFun -> f) n- = (L.take m . L.iterate f) `eq`- (unpackS . TL.take (fromIntegral m) . TL.iterate f)- where m = fromIntegral (n :: Word8)--unf :: Int -> Char -> Maybe (Char, Char)-unf n c | fromEnum c * 100 > n = Nothing- | otherwise = Just (c, succ c)--t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m))- where m = fromIntegral (n :: Word16)-tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m))- where m = fromIntegral (n :: Word16)-t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq`- (unpackS . T.unfoldrN i (unf j))- where i = fromIntegral (n :: Word16)- j = fromIntegral (m :: Word16)-tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq`- (unpackS . TL.unfoldrN (fromIntegral i) (unf j))- where i = fromIntegral (n :: Word16)- j = fromIntegral (m :: Word16)--testFolds :: TestTree-testFolds =- testGroup "folds-unfolds" [- testGroup "folds" [- testProperty "sf_foldl" sf_foldl,- testProperty "t_foldl" t_foldl,- testProperty "tl_foldl" tl_foldl,- testProperty "sf_foldl'" sf_foldl',- testProperty "t_foldl'" t_foldl',- testProperty "tl_foldl'" tl_foldl',- testProperty "sf_foldl1" sf_foldl1,- testProperty "t_foldl1" t_foldl1,- testProperty "tl_foldl1" tl_foldl1,- testProperty "t_foldl1'" t_foldl1',- testProperty "sf_foldl1'" sf_foldl1',- testProperty "tl_foldl1'" tl_foldl1',- testProperty "sf_foldr" sf_foldr,- testProperty "t_foldr" t_foldr,- testProperty "tl_foldr" tl_foldr,- testProperty "sf_foldr1" sf_foldr1,- testProperty "t_foldr1" t_foldr1,- testProperty "tl_foldr1" tl_foldr1,-- testGroup "special" [- testProperty "s_concat_s" s_concat_s,- testProperty "sf_concat" sf_concat,- testProperty "t_concat" t_concat,- testProperty "tl_concat" tl_concat,- testProperty "sf_concatMap" sf_concatMap,- testProperty "t_concatMap" t_concatMap,- testProperty "tl_concatMap" tl_concatMap,- testProperty "sf_any" sf_any,- testProperty "t_any" t_any,- testProperty "tl_any" tl_any,- testProperty "sf_all" sf_all,- testProperty "t_all" t_all,- testProperty "tl_all" tl_all,- testProperty "sf_maximum" sf_maximum,- testProperty "t_maximum" t_maximum,- testProperty "tl_maximum" tl_maximum,- testProperty "sf_minimum" sf_minimum,- testProperty "t_minimum" t_minimum,- testProperty "tl_minimum" tl_minimum- ]- ],-- testGroup "scans" [- testProperty "sf_scanl" sf_scanl,- testProperty "t_scanl" t_scanl,- testProperty "tl_scanl" tl_scanl,- testProperty "t_scanl1" t_scanl1,- testProperty "tl_scanl1" tl_scanl1,- testProperty "t_scanr" t_scanr,- testProperty "tl_scanr" tl_scanr,- testProperty "t_scanr1" t_scanr1,- testProperty "tl_scanr1" tl_scanr1- ],-- testGroup "mapAccum" [- testProperty "t_mapAccumL_char" t_mapAccumL_char,- testProperty "t_mapAccumL" t_mapAccumL,- testProperty "tl_mapAccumL_char" tl_mapAccumL_char,- testProperty "tl_mapAccumL" tl_mapAccumL,- testProperty "t_mapAccumR_char" t_mapAccumR_char,- testProperty "t_mapAccumR" t_mapAccumR,- testProperty "tl_mapAccumR_char" tl_mapAccumR_char,- testProperty "tl_mapAccumR" tl_mapAccumR- ],-- testGroup "unfolds" [- testProperty "tl_cycle" tl_cycle,- testProperty "tl_iterate" tl_iterate,- testProperty "t_unfoldr" t_unfoldr,- testProperty "tl_unfoldr" tl_unfoldr,- testProperty "t_unfoldrN" t_unfoldrN,- testProperty "tl_unfoldrN" tl_unfoldrN- ],-- testGroup "replicate" [- testProperty "tl_repeat" tl_repeat,- testProperty "s_replicate" s_replicate,- testProperty "t_replicate_char" t_replicate_char,- testProperty "tl_replicate_char" tl_replicate_char,- testProperty "t_length_replicate_char" t_length_replicate_char,- testProperty "tl_length_replicate_char" tl_length_replicate_char,- testProperty "t_replicate" t_replicate,- testProperty "tl_replicate" tl_replicate,- testProperty "t_length_replicate" t_length_replicate,- testProperty "tl_length_replicate" tl_length_replicate- ]-- ]+-- | Test folds, scans, and unfolds + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.Folds + ( testFolds + ) where + +import Control.Arrow (second) +import Control.Exception (ErrorCall, evaluate, try) +import Data.Functor.Identity (Identity(..)) +import Control.Monad.Trans.State (runState, state) +import Data.Word (Word8, Word16) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertFailure, assertBool) +import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Lazy as TL +import qualified Data.Char as Char + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectTest, (==~)) +import GHC.Exts (inline) +#endif + +-- Folds + +sf_foldl (applyFun -> p) (applyFun2 -> f) z = + (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) + where _types = f :: Char -> Char -> Char +tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) + where _types = f :: Char -> Char -> Char +sf_foldl' (applyFun -> p) (applyFun2 -> f) z = + (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z + where _types = f :: Char -> Char -> Char +tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z + where _types = f :: Char -> Char -> Char +sf_foldl1 (applyFun -> p) (applyFun2 -> f) = + (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' (applyFun -> p) (applyFun2 -> f) = + (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr (applyFun -> p) (applyFun2 -> f) z = + (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z + where _types = f :: Char -> Char -> Char +t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z + where _types = f :: Char -> Char -> Char +tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z + where _types = f :: Char -> Char -> Char +sf_foldr1 (applyFun -> p) (applyFun2 -> f) = + (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) +t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f + +-- Distinguish foldl/foldr from foldl'/foldr' + +fold_apart :: IO () +fold_apart = do + ok (T.foldr f () (T.pack "az")) + ko (T.foldr' f () (T.pack "az")) + ok (T.foldl (flip f) () (T.pack "za")) + ko (T.foldl' (flip f) () (T.pack "za")) + where + f c _ = if c == 'z' then error "catchme" else () + ok = evaluate + ko t = do + x <- try (evaluate t) + case x :: Either ErrorCall () of + Left _ -> pure () + Right _ -> assertFailure "test should have failed but didn't" + +-- Special folds + +s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) +sf_concat (applyFun -> p) + = (L.concat . map (L.filter p) . unSqrt) `eq` + (unpackS . S.concat . map (S.filter p . packS) . unSqrt) +t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) +tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) +sf_concatMap (applyFun -> p) (applyFun -> f) = + (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) +t_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) +tl_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) +sf_any (applyFun -> q) (applyFun -> p) + = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any (applyFun -> p) + = L.any p `eqP` T.any p +tl_any (applyFun -> p) + = L.any p `eqP` TL.any p +sf_all (applyFun -> q) (applyFun -> p) + = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all (applyFun -> p) + = L.all p `eqP` T.all p +tl_all (applyFun -> p) + = L.all p `eqP` TL.all p +sf_maximum (applyFun -> p) + = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +t_maximum = L.maximum `eqP` T.maximum +tl_maximum = L.maximum `eqP` TL.maximum +sf_minimum (applyFun -> p) + = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +t_minimum = L.minimum `eqP` T.minimum +tl_minimum = L.minimum `eqP` TL.minimum +t_isAscii = L.all Char.isAscii `eqP` T.isAscii +tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii + +-- Scans + +sf_scanl (applyFun -> p) (applyFun2 -> f) z = + (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) + +t_scanl_is_safe = let c = '\55296' in + T.scanl undefined c mempty === T.singleton c +tl_scanl_is_safe = let c = '\55296' in + TL.scanl undefined c mempty === TL.singleton c +t_scanr_is_safe = let c = '\55296' in + T.scanr undefined c mempty === T.singleton c +tl_scanr_is_safe = let c = '\55296' in + TL.scanr undefined c mempty === TL.singleton c + +t_mapAccumL_char c t = + snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumL_char c t = + snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +t_mapAccumR_char c t = + snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumR_char c t = + snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) + +-- Unfolds + +tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) + +s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) + +t_replicate_char (Small n) c = + L.replicate n c === T.unpack (T.replicate n (T.singleton c)) +tl_replicate_char (Small n) c = + L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) +t_length_replicate_char (Small n) c = + L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) +tl_length_replicate_char (Small n) c = + L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) + +t_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) +tl_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) +t_length_replicate (Small n) = + (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) +tl_length_replicate (Small n) = + (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) + +tl_cycle n = (L.take m . L.cycle) `eq` + (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) + where m = fromIntegral (n :: Word8) + +tl_iterate (applyFun -> f) n + = (L.take m . L.iterate f) `eq` + (unpackS . TL.take (fromIntegral m) . TL.iterate f) + where m = fromIntegral (n :: Word8) + +unf :: Int -> Char -> Maybe (Char, Char) +unf n c | fromEnum c * 100 > n = Nothing + | otherwise = Just (c, succ c) + +t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . T.unfoldrN i (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) +tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) + +-- Monadic folds + +-- Parametric polymorphism allows us to only test foldlM' specialized to +-- one function in the state monad (called @logger@ in the following tests) +-- that just logs the arguments it was applied to and produces a fresh +-- accumulator. That alone determines the general behavior of foldlM' with an +-- arbitrary function in any monad. +-- Reference: "Testing Polymorphic Properties" by Bernardy et al. +-- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf + +t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order +tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order + +#ifdef MIN_VERSION_tasty_inspection_testing +-- As a sanity check for performance, the simplified Core +-- foldlM' specialized to Identity is the same as foldl'. + +_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x + +_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl' = inline S.foldl' +#endif + +isAscii_border :: IO () +isAscii_border = do + let text = T.drop 2 $ T.pack "XX1234五" + assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text + +testFolds :: TestTree +testFolds = + testGroup "folds-unfolds" [ + testGroup "folds" [ + testProperty "sf_foldl" sf_foldl, + testProperty "t_foldl" t_foldl, + testProperty "tl_foldl" tl_foldl, + testProperty "sf_foldl'" sf_foldl', + testProperty "t_foldl'" t_foldl', + testProperty "tl_foldl'" tl_foldl', + testProperty "sf_foldl1" sf_foldl1, + testProperty "t_foldl1" t_foldl1, + testProperty "tl_foldl1" tl_foldl1, + testProperty "t_foldl1'" t_foldl1', + testProperty "sf_foldl1'" sf_foldl1', + testProperty "tl_foldl1'" tl_foldl1', + testProperty "sf_foldr" sf_foldr, + testProperty "t_foldr" t_foldr, + testProperty "t_foldr'" t_foldr', + testProperty "tl_foldr" tl_foldr, + testProperty "sf_foldr1" sf_foldr1, + testProperty "t_foldr1" t_foldr1, + testProperty "tl_foldr1" tl_foldr1, + testProperty "t_foldlM'" t_foldlM', + testProperty "tl_foldlM'" tl_foldlM', +#ifdef MIN_VERSION_tasty_inspection_testing + let _unused = ['_S_foldl'_from_foldlM', '_S_foldl'] in + $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')), +#endif + testCase "fold_apart" fold_apart, + + testGroup "special" [ + testProperty "s_concat_s" s_concat_s, + testProperty "sf_concat" sf_concat, + testProperty "t_concat" t_concat, + testProperty "tl_concat" tl_concat, + testProperty "sf_concatMap" sf_concatMap, + testProperty "t_concatMap" t_concatMap, + testProperty "tl_concatMap" tl_concatMap, + testProperty "sf_any" sf_any, + testProperty "t_any" t_any, + testProperty "tl_any" tl_any, + testProperty "sf_all" sf_all, + testProperty "t_all" t_all, + testProperty "tl_all" tl_all, + testProperty "sf_maximum" sf_maximum, + testProperty "t_maximum" t_maximum, + testProperty "tl_maximum" tl_maximum, + testProperty "sf_minimum" sf_minimum, + testProperty "t_minimum" t_minimum, + testProperty "tl_minimum" tl_minimum, + testProperty "t_isAscii " t_isAscii, + testProperty "tl_isAscii " tl_isAscii, + testCase "isAscii_border" isAscii_border + ] + ], + + testGroup "scans" [ + testProperty "sf_scanl" sf_scanl, + testProperty "t_scanl" t_scanl, + testProperty "tl_scanl" tl_scanl, + testProperty "t_scanl1" t_scanl1, + testProperty "tl_scanl1" tl_scanl1, + testProperty "t_scanr" t_scanr, + testProperty "tl_scanr" tl_scanr, + testProperty "t_scanr1" t_scanr1, + testProperty "tl_scanr1" tl_scanr1, + + testProperty "t_scanl_is_safe" t_scanl_is_safe, + testProperty "tl_scanl_is_safe" tl_scanl_is_safe, + testProperty "t_scanr_is_safe" t_scanr_is_safe, + testProperty "tl_scanr_is_safe" tl_scanr_is_safe + ], + + testGroup "mapAccum" [ + testProperty "t_mapAccumL_char" t_mapAccumL_char, + testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL_char" tl_mapAccumL_char, + testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR_char" t_mapAccumR_char, + testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR_char" tl_mapAccumR_char, + testProperty "tl_mapAccumR" tl_mapAccumR + ], + + testGroup "unfolds" [ + testProperty "tl_cycle" tl_cycle, + testProperty "tl_iterate" tl_iterate, + testProperty "t_unfoldr" t_unfoldr, + testProperty "tl_unfoldr" tl_unfoldr, + testProperty "t_unfoldrN" t_unfoldrN, + testProperty "tl_unfoldrN" tl_unfoldrN + ], + + testGroup "replicate" [ + testProperty "tl_repeat" tl_repeat, + testProperty "s_replicate" s_replicate, + testProperty "t_replicate_char" t_replicate_char, + testProperty "tl_replicate_char" tl_replicate_char, + testProperty "t_length_replicate_char" t_length_replicate_char, + testProperty "tl_length_replicate_char" tl_length_replicate_char, + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_length_replicate" t_length_replicate, + testProperty "tl_length_replicate" tl_length_replicate + ] + + ]
tests/Tests/Properties/Instances.hs view
@@ -6,6 +6,8 @@ ( testInstances ) where +import Data.Binary (encode, decodeOrFail)+import Data.Semigroup import Data.String (IsString(fromString)) import Test.QuickCheck import Test.Tasty (TestTree, testGroup)@@ -36,6 +38,12 @@ tl_Show = show `eq` (show . TL.pack) t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s))+t_stimes = \ number -> eq+ ((stimes :: Int -> String -> String) number . unSqrt)+ (unpackS . (stimes :: Int -> T.Text -> T.Text) number . T.pack . unSqrt)+tl_stimes = \ number -> eq+ ((stimes :: Int -> String -> String) number . unSqrt)+ (unpackS . (stimes :: Int -> TL.Text -> TL.Text) number . TL.pack . unSqrt) t_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt) tl_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt) t_mempty = mempty === (unpackS (mempty :: T.Text))@@ -43,6 +51,16 @@ t_IsString = fromString `eqP` (T.unpack . fromString) tl_IsString = fromString `eqP` (TL.unpack . fromString) +t_Binary s =+ case decodeOrFail . encode $ (s :: T.Text) of+ Left _ -> counterexample (show (T.unpack s)) (property False)+ Right (_, _, s') -> s === s'++tl_Binary s =+ case decodeOrFail . encode $ (s :: TL.Text) of+ Left _ -> counterexample (show (TL.unpack s)) (property False)+ Right (_, _, s') -> s === s'+ testInstances :: TestTree testInstances = testGroup "instances" [@@ -60,10 +78,14 @@ testProperty "tl_Show" tl_Show, testProperty "t_mappend" t_mappend, testProperty "tl_mappend" tl_mappend,+ testProperty "t_stimes" t_stimes,+ testProperty "tl_stimes" tl_stimes, testProperty "t_mconcat" t_mconcat, testProperty "tl_mconcat" tl_mconcat, testProperty "t_mempty" t_mempty, testProperty "tl_mempty" tl_mempty, testProperty "t_IsString" t_IsString,- testProperty "tl_IsString" tl_IsString+ testProperty "tl_IsString" tl_IsString,+ testProperty "t_Binary" t_Binary,+ testProperty "tl_Binary" tl_Binary ]
tests/Tests/Properties/LowLevel.hs view
@@ -1,146 +1,176 @@--- | Test low-level operations--{-# LANGUAGE CPP #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE ScopedTypeVariables #-}--{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-}--#ifdef MIN_VERSION_tasty_inspection_testing-{-# LANGUAGE TemplateHaskell #-}-{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-}-#endif--module Tests.Properties.LowLevel (testLowLevel) where--import Control.Applicative ((<$>), pure)-import Control.Exception as E (SomeException, catch, evaluate)-import Data.Int (Int32, Int64)-import Data.Text.Foreign-import Data.Text.Internal (Text(..), mul, mul32, mul64, safe)-import Data.Word (Word8, Word16, Word32)-import System.IO.Unsafe (unsafePerformIO)-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.HUnit (testCase, assertEqual)-import Test.Tasty.QuickCheck (testProperty)-import Test.QuickCheck hiding ((.&.))-import Tests.QuickCheckUtils-import Tests.Utils-import qualified Data.Text as T-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.IO as TL-import qualified System.IO as IO--#ifdef MIN_VERSION_tasty_inspection_testing-import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf)-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified GHC.CString as GHC-#endif--mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a-mulRef a b- | ab < bot || ab > top = Nothing- | otherwise = Just (fromIntegral ab)- where ab = fromIntegral a * fromIntegral b- top = fromIntegral (maxBound `asTypeOf` a) :: Integer- bot = fromIntegral (minBound `asTypeOf` a) :: Integer--eval :: (a -> b -> c) -> a -> b -> Maybe c-eval f a b = unsafePerformIO $- (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing)--t_mul32 :: Int32 -> Int32 -> Property-t_mul32 a b = mulRef a b === eval mul32 a b--t_mul64 :: Int64 -> Int64 -> Property-t_mul64 a b = mulRef a b === eval mul64 a b--t_mul :: Int -> Int -> Property-t_mul a b = mulRef a b === eval mul a b---- Misc.--t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t-t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t-t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t-t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr--t_copy t = T.copy t === t--t_literal_length1 = assertEqual xs (length xs) byteLen- where- xs = "\0\1\0\1\0"- Text _ _ byteLen = T.pack xs-t_literal_length2 = assertEqual xs (length xs) byteLen- where- xs = "\1\2\3\4\5"- Text _ _ byteLen = T.pack xs-t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys)- where- ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000"- xs = map safe ys--#ifdef MIN_VERSION_tasty_inspection_testing-t_literal_foo :: Text-t_literal_foo = T.pack "foo"-#endif---- Input and output.---- t_put_get = write_read T.unlines T.filter put get--- where put h = withRedirect h IO.stdout . T.putStr--- get h = withRedirect h IO.stdin T.getContents--- tl_put_get = write_read TL.unlines TL.filter put get--- where put h = withRedirect h IO.stdout . TL.putStr--- get h = withRedirect h IO.stdin TL.getContents-t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents-tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents--t_write_read_line m b t = write_read head T.filter T.hPutStrLn- T.hGetLine m b [t]-tl_write_read_line m b t = write_read head TL.filter TL.hPutStrLn- TL.hGetLine m b [t]---testLowLevel :: TestTree-testLowLevel =- testGroup "lowlevel" [- testGroup "mul" [- testProperty "t_mul" t_mul,- testProperty "t_mul32" t_mul32,- testProperty "t_mul64" t_mul64- ],-- testGroup "misc" [- testProperty "t_dropWord8" t_dropWord8,- testProperty "t_takeWord8" t_takeWord8,- testProperty "t_take_drop_8" t_take_drop_8,- testProperty "t_use_from" t_use_from,- testProperty "t_copy" t_copy,- testCase "t_literal_length1" t_literal_length1,- testCase "t_literal_length2" t_literal_length2,- testCase "t_literal_surrogates" t_literal_surrogates-#ifdef MIN_VERSION_tasty_inspection_testing- , $(inspectObligations- [ (`hasNoTypes` [''Char, ''[]])- , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList])- , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#])- , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#])- ]- 't_literal_foo)-#endif- ],-- testGroup "input-output" [- testProperty "t_write_read" t_write_read,- testProperty "tl_write_read" tl_write_read,- testProperty "t_write_read_line" t_write_read_line,- testProperty "tl_write_read_line" tl_write_read_line- -- These tests are subject to I/O race conditions- -- testProperty "t_put_get" t_put_get,- -- testProperty "tl_put_get" tl_put_get- ]- ]-+-- | Test low-level operations + +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.LowLevel (testLowLevel) where + +import Prelude hiding (head, tail) +import Control.Applicative ((<$>), pure) +import Control.Exception as E (SomeException, catch, evaluate) +import Data.Functor.Identity (Identity(..)) +import Data.Int (Int32, Int64) +import Data.Text.Foreign +import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) +import Data.Word (Word8, Word16, Word32) +import System.IO.Unsafe (unsafePerformIO) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty.QuickCheck (testProperty) +import Test.QuickCheck hiding ((.&.)) +import Tests.QuickCheckUtils +import Tests.Utils +import qualified Data.Text as T +import qualified Data.Text.Foreign as T +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.IO.Utf8 as TU +import qualified System.IO as IO + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified GHC.CString as GHC +#endif + +mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a +mulRef a b + | ab < bot || ab > top = Nothing + | otherwise = Just (fromIntegral ab) + where ab = fromIntegral a * fromIntegral b + top = fromIntegral (maxBound `asTypeOf` a) :: Integer + bot = fromIntegral (minBound `asTypeOf` a) :: Integer + +eval :: (a -> b -> c) -> a -> b -> Maybe c +eval f a b = unsafePerformIO $ + (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) + +t_mul32 :: Int32 -> Int32 -> Property +t_mul32 a b = mulRef a b === eval mul32 a b + +t_mul64 :: Int64 -> Int64 -> Property +t_mul64 a b = mulRef a b === eval mul64 a b + +t_mul :: Int -> Int -> Property +t_mul a b = mulRef a b === eval mul a b + +-- Misc. + +t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t +t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t +t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t +t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr +t_use_from0 t = ioProperty $ do + let t' = t `T.snoc` '\0' + (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0) + +t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do + roundTrip <- T.withCString t T.peekCString + assertEqual "cstring" t roundTrip + +t_peek_cstring_len t = ioProperty $ do + roundTrip <- T.withCStringLen t T.peekCStringLen + assertEqual "cstring_len" t roundTrip + +t_copy t = T.copy t === t + +t_literal_length1 = assertEqual xs (length xs) byteLen + where + xs = "\0\1\0\1\0" + Text _ _ byteLen = T.pack xs +t_literal_length2 = assertEqual xs (length xs) byteLen + where + xs = "\1\2\3\4\5" + Text _ _ byteLen = T.pack xs +t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) + where + ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" + xs = map safe ys + +#ifdef MIN_VERSION_tasty_inspection_testing +t_literal_foo :: Text +t_literal_foo = T.pack "foo" +#endif + +-- Input and output. + +-- t_put_get = write_read T.unlines T.filter put get +-- where put h = withRedirect h IO.stdout . T.putStr +-- get h = withRedirect h IO.stdin T.getContents +-- tl_put_get = write_read TL.unlines TL.filter put get +-- where put h = withRedirect h IO.stdout . TL.putStr +-- get h = withRedirect h IO.stdin TL.getContents + +inputOutput :: TestTree +inputOutput = testGroup "input-output" [ + testProperty "t_write_read" $ write_read arbitrary shrink (T.replace "\n" "\r\n") T.hPutStr T.hGetContents, + testProperty "tl_write_read" $ write_read arbitrary shrink (TL.replace "\n" "\r\n") TL.hPutStr TL.hGetContents, + testProperty "t_write_read_line" $ write_read genTLine shrinkTLine (`T.append` "\r") T.hPutStrLn T.hGetLine, + testProperty "tl_write_read_line" $ write_read genTLLine shrinkTLLine (`TL.append` "\r") TL.hPutStrLn TL.hGetLine, + -- Note: Data.Text.IO.Utf8 does NO newline translation + testProperty "utf8_write_read" $ write_read arbitrary shrink id TU.hPutStr TU.hGetContents, + testProperty "utf8_write_read_line" $ write_read genTLine shrinkTLine id TU.hPutStrLn TU.hGetLine + -- These tests are subject to I/O race conditions + -- testProperty "t_put_get" t_put_get, + -- testProperty "tl_put_get" tl_put_get + ] + +genTLine :: Gen T.Text +genTLine = T.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +genTLLine :: Gen TL.Text +genTLLine = TL.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +shrinkTLine :: T.Text -> [T.Text] +shrinkTLine = filter (T.all (/= '\n')) . shrink + +shrinkTLLine :: TL.Text -> [TL.Text] +shrinkTLLine = filter (TL.all (/= '\n')) . shrink + +testLowLevel :: TestTree +testLowLevel = + testGroup "lowlevel" [ + testGroup "mul" [ + testProperty "t_mul" t_mul, + testProperty "t_mul32" t_mul32, + testProperty "t_mul64" t_mul64 + ], + + testGroup "misc" [ + testProperty "t_dropWord8" t_dropWord8, + testProperty "t_takeWord8" t_takeWord8, + testProperty "t_take_drop_8" t_take_drop_8, + testProperty "t_use_from" t_use_from, + testProperty "t_use_from0" t_use_from0, + testProperty "t_copy" t_copy, + testProperty "t_peek_cstring" t_peek_cstring, + testProperty "t_peek_cstring_len" t_peek_cstring_len, + testCase "t_literal_length1" t_literal_length1, + testCase "t_literal_length2" t_literal_length2, + testCase "t_literal_surrogates" t_literal_surrogates +#ifdef MIN_VERSION_tasty_inspection_testing + -- Hack to force GHC to keep the binding around until this is fixed: https://gitlab.haskell.org/ghc/ghc/-/issues/26436 + , let _unused = 't_literal_foo in + $(inspectObligations + [ (`hasNoTypes` [''Char, ''[]]) + , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) + , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) + , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) + ] + 't_literal_foo) +#endif + ], + + inputOutput + ]
tests/Tests/Properties/Read.hs view
@@ -8,7 +8,7 @@ import Data.Char (isDigit, isHexDigit) import Numeric (showHex)-import Test.Tasty (TestTree, testGroup)+import Test.Tasty (TestTree, testGroup, localOption, mkTimeout) import Test.Tasty.QuickCheck (testProperty) import Test.QuickCheck import Tests.QuickCheckUtils ()@@ -38,23 +38,39 @@ isFloaty c = c `elem` ("+-.0123456789eE" :: String) -t_read_rational p tol (n::Double) s =- case p (T.pack (show n) `T.append` t) of- Left err -> counterexample err $ property False- Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol)- where t = T.dropWhile isFloaty s+t_read_rational :: Double -> T.Text -> Property+t_read_rational n s =+ case T.rational (T.pack (show n) `T.append` t) of+ Left err -> counterexample err $ property False+ Right (n', t') -> t === t' .&&. n' === n''+ where+ t = T.dropWhile isFloaty s+ n'' = read (show n) :: Double -tl_read_rational p tol (n::Double) s =- case p (TL.pack (show n) `TL.append` t) of- Left err -> counterexample err $ property False- Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol)- where t = TL.dropWhile isFloaty s+t_read_double :: Double -> Double -> T.Text -> Property+t_read_double tol n s =+ case T.double (T.pack (show n) `T.append` t) of+ Left err -> counterexample err $ property False+ Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol)+ where+ t = T.dropWhile isFloaty s -t_double = t_read_rational T.double 1e-13-tl_double = tl_read_rational TL.double 1e-13-t_rational = t_read_rational T.rational 1e-16-tl_rational = tl_read_rational TL.rational 1e-16+tl_read_rational :: Double -> TL.Text -> Property+tl_read_rational n s =+ case TL.rational (TL.pack (show n) `TL.append` t) of+ Left err -> counterexample err $ property False+ Right (n', t') -> t === t' .&&. n' === n''+ where+ t = TL.dropWhile isFloaty s+ n'' = read (show n) :: Double +tl_read_double :: Double -> Double -> TL.Text -> Property+tl_read_double tol n s =+ case TL.rational (TL.pack (show n) `TL.append` t) of+ Left err -> counterexample err $ property False+ Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol)+ where+ t = TL.dropWhile isFloaty s testRead :: TestTree testRead =@@ -63,8 +79,33 @@ testProperty "tl_decimal" tl_decimal, testProperty "t_hexadecimal" t_hexadecimal, testProperty "tl_hexadecimal" tl_hexadecimal,- testProperty "t_double" t_double,- testProperty "tl_double" tl_double,- testProperty "t_rational" t_rational,- testProperty "tl_rational" tl_rational- ]++ testProperty "t_double" $ t_read_double 1e-13,+ testProperty "tl_double" $ tl_read_double 1e-13,++ testProperty "t_rational" t_read_rational,+ testProperty "t_rational 1.3e-2" (t_read_rational 1.3e-2),+ testProperty "tl_rational" tl_read_rational,+ testProperty "tl_rational 9e-3" (tl_read_rational 9e-3),++ localOption (mkTimeout 100000) $ testGroup "DDoS attacks" [+ testProperty "t_double large positive exponent" $+ T.double (T.pack "1.1e1000000000") === Right (1 / 0, mempty),+ testProperty "t_double large negative exponent" $+ T.double (T.pack "1.1e-1000000000") === Right (0.0, mempty),+ testProperty "tl_double large positive exponent" $+ TL.double (TL.pack "1.1e1000000000") === Right (1 / 0, mempty),+ testProperty "tl_double large negative exponent" $+ TL.double (TL.pack "1.1e-1000000000") === Right (0.0, mempty),++ testProperty "t_rational large positive exponent" $+ T.rational (T.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty),+ testProperty "t_rational large negative exponent" $+ T.rational (T.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty),+ testProperty "tl_rational large positive exponent" $+ TL.rational (TL.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty),+ testProperty "tl_rational large negative exponent" $+ TL.rational (TL.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty)+ ]++ ]
tests/Tests/Properties/Substrings.hs view
@@ -7,12 +7,15 @@ ( testSubstrings ) where +import Prelude hiding (head, tail)+import Control.Monad.Trans.State (State, state, runState) import Data.Char (isSpace) import Test.QuickCheck import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils import qualified Data.List as L+import qualified Data.List.NonEmpty as NonEmptyList import qualified Data.Text as T import qualified Data.Text.Internal.Fusion as S import qualified Data.Text.Internal.Fusion.Common as S@@ -154,9 +157,44 @@ tl_groupBy (applyFun2 -> p) = L.groupBy p `eqP` (map unpackS . TL.groupBy p) t_inits = L.inits `eqP` (map unpackS . T.inits)+t_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . T.initsNE) tl_inits = L.inits `eqP` (map unpackS . TL.inits)+tl_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . TL.initsNE) t_tails = L.tails `eqP` (map unpackS . T.tails)+t_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . T.tailsNE) tl_tails = L.tails `eqPSqrt` (map unpackS . TL.tails)+tl_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . TL.tailsNE)++spanML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b])+spanML p s = go [] s+ where+ go acc [] = pure (reverse acc, [])+ go acc ccs@(c : cs) = do+ continue <- p c+ if continue then go (c : acc) cs else pure (reverse acc, ccs)++spanEndML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b])+spanEndML p s = (\(s1, s2) -> (reverse s2, reverse s1)) <$> spanML p (reverse s)++eqSP :: Stringy s => (String -> State Int (String, String)) -> (s -> State Int (s, s)) -> Property+eqSP a b = property $ ((`runState` 0) . a) `eqP` ((`runState` 0) . (fmap unpack2 . b))++t_spanM (applyFun2 -> f')+ = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+ spanML f `eqSP` T.spanM f++t_spanEndM (applyFun2 -> f')+ = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+ spanEndML f `eqSP` T.spanEndM f++tl_spanM (applyFun2 -> f')+ = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+ spanML f `eqSP` TL.spanM f++tl_spanEndM (applyFun2 -> f')+ = let f c = state $ \i -> (getSkewed (f' i c), i+1) in+ spanEndML f `eqSP` TL.spanEndM f+ t_findAppendId = \(Sqrt (NotEmpty s)) ts -> let t = T.intercalate s ts in conjoin $ map (=== t) $ map (uncurry T.append) (T.breakOnAll s t)@@ -186,13 +224,13 @@ split :: (a -> Bool) -> [a] -> [[a]] split _ [] = [[]] split p xs = loop xs- where loop s | null s' = [l]- | otherwise = l : loop (tail s')- where (l, s') = break p s+ where+ loop s = case break p s of+ (l, []) -> [l]+ (l, _ : s') -> l : loop s' -t_chunksOf_same_lengths k = conjoin . map ((===k) . T.length) . ini . T.chunksOf k- where ini [] = []- ini xs = init xs+t_chunksOf_same_lengths k =+ conjoin . map ((===k) . T.length) . drop 1 . reverse . T.chunksOf k t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0) where len = L.sum . L.map T.length $ T.chunksOf k t@@ -205,7 +243,9 @@ t_lines_spacy = (L.lines `eqP` (map unpackS . T.lines)) . getSpacyString tl_lines_spacy = (L.lines `eqP` (map unpackS . TL.lines)) . getSpacyString -tl_lines_laziness = TL.head (head (TL.lines (TL.replicate 1000000000000000 (TL.singleton 'a')))) === 'a'+tl_lines_laziness = case TL.lines (TL.replicate 1000000000000000 (TL.singleton 'a')) of+ [] -> property False+ hd : _ -> TL.head hd === 'a' tl_lines_specialCase = TL.lines (TL.Chunk (T.pack "foo") $ TL.Chunk (T.pack "bar\nbaz\n") $ TL.Empty) === [TL.pack "foobar", TL.pack "baz"] @@ -325,9 +365,17 @@ testProperty "t_groupBy" t_groupBy, testProperty "tl_groupBy" tl_groupBy, testProperty "t_inits" t_inits,+ testProperty "t_initsNE" t_initsNE, testProperty "tl_inits" tl_inits,+ testProperty "tl_initsNE" tl_initsNE, testProperty "t_tails" t_tails,- testProperty "tl_tails" tl_tails+ testProperty "t_tailsNE" t_tailsNE,+ testProperty "tl_tails" tl_tails,+ testProperty "tl_tailsNE" tl_tailsNE,+ testProperty "t_spanM" t_spanM,+ testProperty "t_spanEndM" t_spanEndM,+ testProperty "tl_spanM" tl_spanM,+ testProperty "tl_spanEndM" tl_spanEndM ], testGroup "breaking many" [
tests/Tests/Properties/Text.hs view
@@ -1,401 +1,491 @@--- | Tests for operations that don't fit in the other @Test.Properties.*@ modules.--{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE ViewPatterns #-}-{-# OPTIONS_GHC -fno-warn-missing-signatures #-}-module Tests.Properties.Text- ( testText- ) where--import Data.Char (isLower, isLetter, isUpper)-import Data.Maybe (mapMaybe)-import Data.Text.Internal.Fusion.Size-import Data.Word (Word8)-import Test.QuickCheck-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.QuickCheck (testProperty)-import Tests.QuickCheckUtils-import qualified Data.Char as C-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Internal.Lazy.Fusion as SL-import qualified Data.Text.Internal.Lazy.Search as S (indices)-import qualified Data.Text.Internal.Search as T (indices)-import qualified Data.Text.Lazy as TL-import qualified Tests.SlowFunctions as Slow--t_pack_unpack = (T.unpack . T.pack) `eq` id-tl_pack_unpack = (TL.unpack . TL.pack) `eq` id-t_stream_unstream = (S.unstream . S.stream) `eq` id-tl_stream_unstream = (SL.unstream . SL.stream) `eq` id-t_reverse_stream t = (S.reverse . S.reverseStream) t === t-t_singleton c = [c] === (T.unpack . T.singleton) c-tl_singleton c = [c] === (TL.unpack . TL.singleton) c-tl_unstreamChunks x = f 11 x === f 1000 x- where f n = SL.unstreamChunks n . S.streamList-tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id-tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id--s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f)-s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f)-sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p)--t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f)-tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f)-t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g)-tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g)-t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f)-tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f)--s_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . S.intercalate (packS c) . map packS . unSqrt)-t_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . T.intercalate (packS c) . map packS . unSqrt)-tl_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)-t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq`- (T.length . T.intercalate (packS c) . map packS . unSqrt)-tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq`- (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)-s_intersperse c = L.intersperse c `eqP`- (unpackS . S.intersperse c)-s_intersperse_s c = L.intersperse c `eqP`- (unpackS . S.unstream . S.intersperse c)-sf_intersperse (applyFun -> p) c- = (L.intersperse c . L.filter p) `eqP`- (unpackS . S.intersperse c . S.filter p)-t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c)-tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c)-t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c)-tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c)-t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt)-tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt)-t_reverse = L.reverse `eqP` (unpackS . T.reverse)-tl_reverse = L.reverse `eqP` (unpackS . TL.reverse)-t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)--t_replace s d = (L.intercalate d . splitOn s) `eqP`- (unpackS . T.replace (T.pack s) (T.pack d))-tl_replace s d = (L.intercalate d . splitOn s) `eqP`- (unpackS . TL.replace (TL.pack s) (TL.pack d))--splitOn :: (Eq a) => [a] -> [a] -> [[a]]-splitOn pat src0- | l == 0 = error "splitOn: empty"- | otherwise = go src0- where- l = length pat- go src = search 0 src- where- search _ [] = [src]- search !n s@(_:s')- | pat `L.isPrefixOf` s = take n src : go (drop l s)- | otherwise = search (n+1) s'--s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs- where s = S.streamList xs-sf_toCaseFold_length (applyFun -> p) xs =- (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs)- where s = S.streamList xs-t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t-tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t-t_toLower_length t = T.length (T.toLower t) >= T.length t-t_toLower_lower t = p (T.toLower t) >= p t- where p = T.length . T.filter isLower-tl_toLower_lower t = p (TL.toLower t) >= p t- where p = TL.length . TL.filter isLower-t_toUpper_length t = T.length (T.toUpper t) >= T.length t-t_toUpper_upper t = p (T.toUpper t) >= p t- where p = T.length . T.filter isUpper-tl_toUpper_upper t = p (TL.toUpper t) >= p t- where p = TL.length . TL.filter isUpper-t_toTitle_title t = all (<= 1) (caps w)- where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle- -- TIL: there exist uppercase-only letters- w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t-t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian)- where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words- -- Surprise! The Spanish/Portuguese ordinal indicators changed- -- from category Ll (letter, lowercase) to Lo (letter, other)- -- in Unicode 7.0- -- Oh, and there exist lowercase-only letters (see previous test)- stable c = if isLower c- then C.toUpper c /= c- else c /= '\170' && c /= '\186'- -- Georgian text does not have a concept of title case- -- https://en.wikipedia.org/wiki/Georgian_Extended- isGeorgian c = c >= '\4256' && c < '\4352'--ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs))-ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs))-ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs))--ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs))- where- referenceToTitle _ [] = []- referenceToTitle False (y : ys)- | C.isLetter y = C.toUpper y : referenceToTitle True ys- | otherwise = y : referenceToTitle False ys- referenceToTitle True (y : ys)- | C.isLetter y = C.toLower y : referenceToTitle True ys- | otherwise = y : referenceToTitle (not (C.isSpace y)) ys--justifyLeft k c xs = xs ++ L.replicate (k - length xs) c-justifyRight m n xs = L.replicate (m - length xs) n ++ xs-center k c xs- | len >= k = xs- | otherwise = L.replicate l c ++ xs ++ L.replicate r c- where len = length xs- d = k - len- r = d `div` 2- l = d - r--s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c)- where j = fromIntegral (k :: Word8)-s_justifyLeft_s k c = justifyLeft j c `eqP`- (unpackS . S.unstream . S.justifyLeftI j c)- where j = fromIntegral (k :: Word8)-sf_justifyLeft (applyFun -> p) k c- = (justifyLeft j c . L.filter p) `eqP`- (unpackS . S.justifyLeftI j c . S.filter p)- where j = fromIntegral (k :: Word8)-t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c)- where j = fromIntegral (k :: Word8)-tl_justifyLeft k c = justifyLeft j c `eqP`- (unpackS . TL.justifyLeft (fromIntegral j) c)- where j = fromIntegral (k :: Word8)-t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c)- where j = fromIntegral (k :: Word8)-tl_justifyRight k c = justifyRight j c `eqP`- (unpackS . TL.justifyRight (fromIntegral j) c)- where j = fromIntegral (k :: Word8)-t_center k c = center j c `eqP` (unpackS . T.center j c)- where j = fromIntegral (k :: Word8)-tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c)- where j = fromIntegral (k :: Word8)--t_elem c = L.elem c `eqP` T.elem c-tl_elem c = L.elem c `eqP` TL.elem c-sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p)-sf_filter (applyFun -> q) (applyFun -> p)- = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q)--t_filter (applyFun -> p)- = L.filter p `eqP` (unpackS . T.filter p)-tl_filter (applyFun -> p)- = L.filter p `eqP` (unpackS . TL.filter p)-t_filter_filter (applyFun -> p) (applyFun -> q)- = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q)-tl_filter_filter (applyFun -> p) (applyFun -> q)- = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q)-t_length_filter (applyFun -> p)- = (L.length . L.filter p) `eqP` (T.length . T.filter p)-tl_length_filter (applyFun -> p)- = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p)--sf_findBy (applyFun -> q) (applyFun -> p)- = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)-t_find (applyFun -> p) = L.find p `eqP` T.find p-tl_find (applyFun -> p) = L.find p `eqP` TL.find p-t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p)-tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p)--sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l-t_index s i = ((s L.!!) `eq` T.index (packS s)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l--tl_index s i = ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l--t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p-t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t-tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`- TL.count t-t_zip s = L.zip s `eqP` T.zip (packS s)-tl_zip s = L.zip s `eqP` TL.zip (packS s)-sf_zipWith (applyFun -> p) (applyFun2 -> c) s- = (L.zipWith c (L.filter p s) . L.filter p) `eqP`- (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p)-t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))-tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))-t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s))-tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s))--t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s-tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s- where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t)- conc = T.concat . TL.toChunks-t_indices_occurs = \(Sqrt (NotEmpty t)) ts ->- let s = T.intercalate t ts- in Slow.indices t s === T.indices t s--t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8]-tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8]--t_indices_drop n s pref suff = T.indices s t === Slow.indices s t- where- t = T.drop n $ pref `T.append` s `T.append` suff-tl_indices_drop n s pref suff =- map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- t = TL.drop n $ pref `TL.append` s `TL.append` suff--tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0]-tl_indices_drop_chunked n s pref suff =- map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- -- constructing a pathologically chunked haystack- t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff--t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t- where- s = T.singleton c- t = T.drop n $ pref `T.append` s `T.append` suff-tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- s = TL.singleton c- t = TL.drop n $ pref `TL.append` s `TL.append` suff---- Make a stream appear shorter than it really is, to ensure that--- functions that consume inaccurately sized streams behave--- themselves.-shorten :: Int -> S.Stream a -> S.Stream a-shorten n t@(S.Stream arr off len)- | n > 0 = S.Stream arr off (smaller (exactSize n) len)- | otherwise = t--testText :: TestTree-testText =- testGroup "Text" [- testGroup "creation/elimination" [- testProperty "t_pack_unpack" t_pack_unpack,- testProperty "tl_pack_unpack" tl_pack_unpack,- testProperty "t_stream_unstream" t_stream_unstream,- testProperty "tl_stream_unstream" tl_stream_unstream,- testProperty "t_reverse_stream" t_reverse_stream,- testProperty "t_singleton" t_singleton,- testProperty "tl_singleton" tl_singleton,- testProperty "tl_unstreamChunks" tl_unstreamChunks,- testProperty "tl_chunk_unchunk" tl_chunk_unchunk,- testProperty "tl_from_to_strict" tl_from_to_strict- ],-- testGroup "transformations" [- testProperty "s_map" s_map,- testProperty "s_map_s" s_map_s,- testProperty "sf_map" sf_map,-- testProperty "t_map" t_map,- testProperty "tl_map" tl_map,- testProperty "t_map_map" t_map_map,- testProperty "tl_map_map" tl_map_map,- testProperty "t_length_map" t_length_map,- testProperty "tl_length_map" tl_length_map,-- testProperty "s_intercalate" s_intercalate,- testProperty "t_intercalate" t_intercalate,- testProperty "tl_intercalate" tl_intercalate,- testProperty "t_length_intercalate" t_length_intercalate,- testProperty "tl_length_intercalate" tl_length_intercalate,- testProperty "s_intersperse" s_intersperse,- testProperty "s_intersperse_s" s_intersperse_s,- testProperty "sf_intersperse" sf_intersperse,- testProperty "t_intersperse" t_intersperse,- testProperty "tl_intersperse" tl_intersperse,- testProperty "t_length_intersperse" t_length_intersperse,- testProperty "tl_length_intersperse" tl_length_intersperse,- testProperty "t_transpose" t_transpose,- testProperty "tl_transpose" tl_transpose,- testProperty "t_reverse" t_reverse,- testProperty "tl_reverse" tl_reverse,- testProperty "t_reverse_short" t_reverse_short,- testProperty "t_replace" t_replace,- testProperty "tl_replace" tl_replace,-- testGroup "case conversion" [- testProperty "s_toCaseFold_length" s_toCaseFold_length,- testProperty "sf_toCaseFold_length" sf_toCaseFold_length,- testProperty "t_toCaseFold_length" t_toCaseFold_length,- testProperty "tl_toCaseFold_length" tl_toCaseFold_length,- testProperty "t_toLower_length" t_toLower_length,- testProperty "t_toLower_lower" t_toLower_lower,- testProperty "tl_toLower_lower" tl_toLower_lower,- testProperty "t_toUpper_length" t_toUpper_length,- testProperty "t_toUpper_upper" t_toUpper_upper,- testProperty "tl_toUpper_upper" tl_toUpper_upper,- testProperty "t_toTitle_title" t_toTitle_title,- testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower,- testProperty "ascii_toLower" ascii_toLower,- testProperty "ascii_toUpper" ascii_toUpper,- testProperty "ascii_toTitle" ascii_toTitle,- testProperty "ascii_toCaseFold" ascii_toCaseFold- ],-- testGroup "justification" [- testProperty "s_justifyLeft" s_justifyLeft,- testProperty "s_justifyLeft_s" s_justifyLeft_s,- testProperty "sf_justifyLeft" sf_justifyLeft,- testProperty "t_justifyLeft" t_justifyLeft,- testProperty "tl_justifyLeft" tl_justifyLeft,- testProperty "t_justifyRight" t_justifyRight,- testProperty "tl_justifyRight" tl_justifyRight,- testProperty "t_center" t_center,- testProperty "tl_center" tl_center- ]- ],-- testGroup "searching" [- testProperty "t_elem" t_elem,- testProperty "tl_elem" tl_elem,- testProperty "sf_elem" sf_elem,- testProperty "sf_filter" sf_filter,- testProperty "t_filter" t_filter,- testProperty "tl_filter" tl_filter,- testProperty "t_filter_filter" t_filter_filter,- testProperty "tl_filter_filter" tl_filter_filter,- testProperty "t_length_filter" t_length_filter,- testProperty "tl_length_filter" tl_length_filter,- testProperty "sf_findBy" sf_findBy,- testProperty "t_find" t_find,- testProperty "tl_find" tl_find,- testProperty "t_partition" t_partition,- testProperty "tl_partition" tl_partition- ],-- testGroup "indexing" [- testProperty "sf_index" sf_index,- testProperty "t_index" t_index,- testProperty "tl_index" tl_index,- testProperty "t_findIndex" t_findIndex,- testProperty "t_count" t_count,- testProperty "tl_count" tl_count,- testProperty "t_indices" t_indices,- testProperty "tl_indices" tl_indices,- testProperty "t_indices_occurs" t_indices_occurs,-- testProperty "t_indices_drop5" t_indices_drop5,- testProperty "tl_indices_drop5" tl_indices_drop5,- testProperty "t_indices_drop" t_indices_drop,- testProperty "tl_indices_drop" tl_indices_drop,- testProperty "tl_indices_chunked" tl_indices_chunked,- testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked,- testProperty "t_indices_char_drop" t_indices_char_drop,- testProperty "tl_indices_char_drop" tl_indices_char_drop- ],-- testGroup "zips" [- testProperty "t_zip" t_zip,- testProperty "tl_zip" tl_zip,- testProperty "sf_zipWith" sf_zipWith,- testProperty "t_zipWith" t_zipWith,- testProperty "tl_zipWith" tl_zipWith,- testProperty "t_length_zipWith" t_length_zipWith,- testProperty "tl_length_zipWith" tl_length_zipWith- ]- ]+-- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. + +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Avoid restricted function" #-} + +module Tests.Properties.Text + ( testText + ) where + +import Control.Exception (SomeException, evaluate, try) +import Data.Char (isLower, isLetter, isUpper) +import Data.Maybe (mapMaybe) +import Data.Text.Internal.Fusion.Size +import Data.Word (Word8) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.Char as C +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Internal.Lazy.Search as S (indices) +import qualified Data.Text.Internal.Search as T (indices) +import qualified Data.Text.Lazy as TL +import qualified Tests.SlowFunctions as Slow +#if MIN_VERSION_base(4, 15, 0) +import qualified GHC.Unicode as G (unicodeVersion) +import qualified Data.Text.Internal.Fusion.CaseMapping as T (unicodeVersion) +#endif + +t_pack_unpack = (T.unpack . T.pack) `eq` id +tl_pack_unpack = (TL.unpack . TL.pack) `eq` id +t_stream_unstream = (S.unstream . S.stream) `eq` id +tl_stream_unstream = (SL.unstream . SL.stream) `eq` id +t_reverse_stream t = (S.reverse . S.reverseStream) t === t +t_singleton c = [c] === (T.unpack . T.singleton) c +tl_singleton c = [c] === (TL.unpack . TL.singleton) c +tl_unstreamChunks x = f 11 x === f 1000 x + where f n = SL.unstreamChunks n . S.streamList +tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id +tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id + +s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) +s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) + +t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) +tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) +t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) +tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) +t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) +tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) + +s_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . S.intercalate (packS c) . map packS . unSqrt) +t_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . T.intercalate (packS c) . map packS . unSqrt) +tl_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` + (T.length . T.intercalate (packS c) . map packS . unSqrt) +tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` + (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +s_intersperse c = L.intersperse c `eqP` + (unpackS . S.intersperse c) +s_intersperse_s c = L.intersperse c `eqP` + (unpackS . S.unstream . S.intersperse c) +sf_intersperse (applyFun -> p) c + = (L.intersperse c . L.filter p) `eqP` + (unpackS . S.intersperse c . S.filter p) +t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) +t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) +tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) +t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) +tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) +t_reverse = L.reverse `eqP` (unpackS . T.reverse) +tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) +t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) + +t_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . T.replace (T.pack s) (T.pack d)) +tl_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . TL.replace (TL.pack s) (TL.pack d)) + +splitOn :: (Eq a) => [a] -> [a] -> [[a]] +splitOn pat src0 + | l == 0 = error "splitOn: empty" + | otherwise = go src0 + where + l = length pat + go src = search 0 src + where + search _ [] = [src] + search !n s@(_:s') + | pat `L.isPrefixOf` s = take n src : go (drop l s) + | otherwise = search (n+1) s' + +s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs + where s = S.streamList xs +sf_toCaseFold_length (applyFun -> p) xs = + (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) + where s = S.streamList xs +t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t + +tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t +t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==> + T.toCaseFold (T.singleton c) === T.singleton (C.toLower c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389" +t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower)) +t_toCaseFold_cherokeeUpper = conjoin $ + map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $ +-- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c)) +-- [minBound .. maxBound] +toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" +cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase +cherokeeLower = ['\xAB70'..'\xABBF'] + +t_toLower_length t = T.length (T.toLower t) >= T.length t +t_toLower_lower t = p (T.toLower t) >= p t + where p = T.length . T.filter isLower +tl_toLower_lower t = p (TL.toLower t) >= p t + where p = TL.length . TL.filter isLower +t_toLower_char c = c /= '\304' ==> + T.toLower (T.singleton c) === T.singleton (C.toLower c) +t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775" + +t_toUpper_length t = T.length (T.toUpper t) >= T.length t +t_toUpper_upper t = p (T.toUpper t) >= p t + where p = T.length . T.filter isUpper +tl_toUpper_upper t = p (TL.toUpper t) >= p t + where p = TL.length . TL.filter isUpper +t_toUpper_char c = c `notElem` toUpperExceptions ==> + T.toUpper (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toTitle_title t = all (<= 1) (caps w) + where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle + -- TIL: there exist uppercase-only letters + w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t +t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian) + where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words + -- Surprise! The Spanish/Portuguese ordinal indicators changed + -- from category Ll (letter, lowercase) to Lo (letter, other) + -- in Unicode 7.0 + -- Oh, and there exist lowercase-only letters (see previous test) + stable c = if isLower c + then C.toUpper c /= c + else c /= '\170' && c /= '\186' + -- Georgian text does not have a concept of title case + -- https://en.wikipedia.org/wiki/Georgian_Extended + isGeorgian c = c >= '\4256' && c < '\4352' +t_toTitle_char c = c `notElem` toTitleExceptions ==> + T.toTitle (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t +t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t +t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t + +ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) +ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) +ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) + +ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) + where + referenceToTitle _ [] = [] + referenceToTitle False (y : ys) + | C.isLetter y = C.toUpper y : referenceToTitle True ys + | otherwise = y : referenceToTitle False ys + referenceToTitle True (y : ys) + | C.isLetter y = C.toLower y : referenceToTitle True ys + | otherwise = y : referenceToTitle (not (C.isSpace y)) ys + +justifyLeft k c xs = xs ++ L.replicate (k - length xs) c +justifyRight m n xs = L.replicate (m - length xs) n ++ xs +center k c xs + | len >= k = xs + | otherwise = L.replicate l c ++ xs ++ L.replicate r c + where len = length xs + d = k - len + r = d `div` 2 + l = d - r + +s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +s_justifyLeft_s k c = justifyLeft j c `eqP` + (unpackS . S.unstream . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +sf_justifyLeft (applyFun -> p) k c + = (justifyLeft j c . L.filter p) `eqP` + (unpackS . S.justifyLeftI j c . S.filter p) + where j = fromIntegral (k :: Word8) +t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) + where j = fromIntegral (k :: Word8) +tl_justifyLeft k c = justifyLeft j c `eqP` + (unpackS . TL.justifyLeft (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) + where j = fromIntegral (k :: Word8) +tl_justifyRight k c = justifyRight j c `eqP` + (unpackS . TL.justifyRight (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_center k c = center j c `eqP` (unpackS . T.center j c) + where j = fromIntegral (k :: Word8) +tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) + where j = fromIntegral (k :: Word8) + +t_elem c = L.elem c `eqP` T.elem c +tl_elem c = L.elem c `eqP` TL.elem c +sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter (applyFun -> q) (applyFun -> p) + = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) + +t_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . T.filter p) +tl_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . TL.filter p) +t_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) +tl_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) +t_length_filter (applyFun -> p) + = (L.length . L.filter p) `eqP` (T.length . T.filter p) +tl_length_filter (applyFun -> p) + = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) + +sf_findBy (applyFun -> q) (applyFun -> p) + = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_find (applyFun -> p) = L.find p `eqP` T.find p +tl_find (applyFun -> p) = L.find p `eqP` TL.find p +t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) + +sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j + where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l + +t_index :: T.Text -> Int -> Property +t_index xs i = ioProperty $ do + ch <- try (evaluate (T.index xs i)) + pure $ case ch of + Left (_ :: SomeException) -> i < 0 .||. i >= T.length xs + Right c -> i >= 0 .&&. i < T.length xs .&&. c === T.unpack xs L.!! i + +tl_index :: TL.Text -> Int -> Property +tl_index xs i = ioProperty $ do + let i' = fromIntegral i + ch <- try (evaluate (TL.index xs i')) + pure $ case ch of + Left (_ :: SomeException) -> i' < 0 .||. i' >= TL.length xs + Right c -> i >= 0 .&&. i' < TL.length xs .&&. c === TL.unpack xs L.!! i + +t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p +t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t +tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` + TL.count t +t_zip s = L.zip s `eqP` T.zip (packS s) +tl_zip s = L.zip s `eqP` TL.zip (packS s) +sf_zipWith (applyFun -> p) (applyFun2 -> c) s + = (L.zipWith c (L.filter p s) . L.filter p) `eqP` + (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) +t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) +tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) + +t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s +tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s + where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) + conc = T.concat . TL.toChunks +t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> + let s = T.intercalate t ts + in Slow.indices t s === T.indices t s + +t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] +tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] + +t_indices_drop n s pref suff = T.indices s t === Slow.indices s t + where + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_drop n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] +tl_indices_drop_chunked n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + -- constructing a pathologically chunked haystack + t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff + +t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t + where + s = T.singleton c + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + s = TL.singleton c + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +-- Make a stream appear shorter than it really is, to ensure that +-- functions that consume inaccurately sized streams behave +-- themselves. +shorten :: Int -> S.Stream a -> S.Stream a +shorten n t@(S.Stream arr off len) + | n > 0 = S.Stream arr off (smaller (exactSize n) len) + | otherwise = t + +testText :: TestTree +testText = + testGroup "Text" [ + testGroup "creation/elimination" [ + testProperty "t_pack_unpack" t_pack_unpack, + testProperty "tl_pack_unpack" tl_pack_unpack, + testProperty "t_stream_unstream" t_stream_unstream, + testProperty "tl_stream_unstream" tl_stream_unstream, + testProperty "t_reverse_stream" t_reverse_stream, + testProperty "t_singleton" t_singleton, + testProperty "tl_singleton" tl_singleton, + testProperty "tl_unstreamChunks" tl_unstreamChunks, + testProperty "tl_chunk_unchunk" tl_chunk_unchunk, + testProperty "tl_from_to_strict" tl_from_to_strict + ], + + testGroup "transformations" [ + testProperty "s_map" s_map, + testProperty "s_map_s" s_map_s, + testProperty "sf_map" sf_map, + + testProperty "t_map" t_map, + testProperty "tl_map" tl_map, + testProperty "t_map_map" t_map_map, + testProperty "tl_map_map" tl_map_map, + testProperty "t_length_map" t_length_map, + testProperty "tl_length_map" tl_length_map, + + testProperty "s_intercalate" s_intercalate, + testProperty "t_intercalate" t_intercalate, + testProperty "tl_intercalate" tl_intercalate, + testProperty "t_length_intercalate" t_length_intercalate, + testProperty "tl_length_intercalate" tl_length_intercalate, + testProperty "s_intersperse" s_intersperse, + testProperty "s_intersperse_s" s_intersperse_s, + testProperty "sf_intersperse" sf_intersperse, + testProperty "t_intersperse" t_intersperse, + testProperty "tl_intersperse" tl_intersperse, + testProperty "t_length_intersperse" t_length_intersperse, + testProperty "tl_length_intersperse" tl_length_intersperse, + testProperty "t_transpose" t_transpose, + testProperty "tl_transpose" tl_transpose, + testProperty "t_reverse" t_reverse, + testProperty "tl_reverse" tl_reverse, + testProperty "t_reverse_short" t_reverse_short, + testProperty "t_replace" t_replace, + testProperty "tl_replace" tl_replace, + + testGroup "case conversion" [ + testProperty "s_toCaseFold_length" s_toCaseFold_length, + testProperty "sf_toCaseFold_length" sf_toCaseFold_length, + testProperty "t_toCaseFold_length" t_toCaseFold_length, + testProperty "tl_toCaseFold_length" tl_toCaseFold_length, + testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions, + testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower, + testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper, + + testProperty "t_toLower_length" t_toLower_length, + testProperty "t_toLower_lower" t_toLower_lower, + testProperty "tl_toLower_lower" tl_toLower_lower, + testProperty "t_toLower_dotted_i" t_toLower_dotted_i, + + testProperty "t_toUpper_length" t_toUpper_length, + testProperty "t_toUpper_upper" t_toUpper_upper, + testProperty "tl_toUpper_upper" tl_toUpper_upper, + testProperty "t_toUpper_exceptions" t_toUpper_exceptions, + + testProperty "t_toTitle_title" t_toTitle_title, + testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, + testProperty "t_toTitle_exceptions" t_toTitle_exceptions, + + testProperty "t_toUpper_idempotent" t_toUpper_idempotent, + testProperty "t_toLower_idempotent" t_toLower_idempotent, + testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent, + + testProperty "ascii_toLower" ascii_toLower, + testProperty "ascii_toUpper" ascii_toUpper, + testProperty "ascii_toTitle" ascii_toTitle, + testProperty "ascii_toCaseFold" ascii_toCaseFold + ], + +#if MIN_VERSION_base(4, 15, 0) + -- Requires matching version of Unicode in base and text + testGroup "char case conversion" $ if T.unicodeVersion == G.unicodeVersion then [ + testProperty "t_toCaseFold_char" t_toCaseFold_char, + testProperty "t_toLower_char" t_toLower_char, + testProperty "t_toUpper_char" t_toUpper_char, + testProperty "t_toTitle_char" t_toTitle_char + ] else [], +#endif + + testGroup "justification" [ + testProperty "s_justifyLeft" s_justifyLeft, + testProperty "s_justifyLeft_s" s_justifyLeft_s, + testProperty "sf_justifyLeft" sf_justifyLeft, + testProperty "t_justifyLeft" t_justifyLeft, + testProperty "tl_justifyLeft" tl_justifyLeft, + testProperty "t_justifyRight" t_justifyRight, + testProperty "tl_justifyRight" tl_justifyRight, + testProperty "t_center" t_center, + testProperty "tl_center" tl_center + ] + ], + + testGroup "searching" [ + testProperty "t_elem" t_elem, + testProperty "tl_elem" tl_elem, + testProperty "sf_elem" sf_elem, + testProperty "sf_filter" sf_filter, + testProperty "t_filter" t_filter, + testProperty "tl_filter" tl_filter, + testProperty "t_filter_filter" t_filter_filter, + testProperty "tl_filter_filter" tl_filter_filter, + testProperty "t_length_filter" t_length_filter, + testProperty "tl_length_filter" tl_length_filter, + testProperty "sf_findBy" sf_findBy, + testProperty "t_find" t_find, + testProperty "tl_find" tl_find, + testProperty "t_partition" t_partition, + testProperty "tl_partition" tl_partition + ], + + testGroup "indexing" [ + testProperty "sf_index" sf_index, + testProperty "t_index" t_index, + testProperty "tl_index" tl_index, + testProperty "t_findIndex" t_findIndex, + testProperty "t_count" t_count, + testProperty "tl_count" tl_count, + testProperty "t_indices" t_indices, + testProperty "tl_indices" tl_indices, + testProperty "t_indices_occurs" t_indices_occurs, + + testProperty "t_indices_drop5" t_indices_drop5, + testProperty "tl_indices_drop5" tl_indices_drop5, + testProperty "t_indices_drop" t_indices_drop, + testProperty "tl_indices_drop" tl_indices_drop, + testProperty "tl_indices_chunked" tl_indices_chunked, + testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, + testProperty "t_indices_char_drop" t_indices_char_drop, + testProperty "tl_indices_char_drop" tl_indices_char_drop + ], + + testGroup "zips" [ + testProperty "t_zip" t_zip, + testProperty "tl_zip" tl_zip, + testProperty "sf_zipWith" sf_zipWith, + testProperty "t_zipWith" t_zipWith, + testProperty "tl_zipWith" tl_zipWith, + testProperty "t_length_zipWith" t_length_zipWith, + testProperty "tl_length_zipWith" tl_length_zipWith + ] + ]
tests/Tests/Properties/Transcoding.hs view
@@ -1,16 +1,27 @@ -- | Tests for encoding and decoding -{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-}-{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+{-# LANGUAGE CPP, OverloadedStrings, ScopedTypeVariables #-}+{-# OPTIONS_GHC -Wno-missing-signatures #-}+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}+ module Tests.Properties.Transcoding ( testTranscoding ) where +import Prelude hiding (head, tail) import Data.Bits ((.&.), shiftR) import Data.Char (chr, ord)+import Data.Functor (void)+import Data.Maybe (isNothing)+#if !MIN_VERSION_base(4,11,0)+import Data.Semigroup ((<>))+#endif+import Data.Word (Word8) import Test.QuickCheck hiding ((.&.)) import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty)+import Test.Tasty.HUnit ((@?=), assertBool, assertFailure, testCase) import Tests.QuickCheckUtils import qualified Control.Exception as Exception import qualified Data.Bits as Bits (shiftL, shiftR)@@ -24,6 +35,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as E import qualified Data.Text.Encoding.Error as E+import qualified Data.Text.Internal.Encoding as E import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as EL @@ -35,6 +47,167 @@ t_latin1 = E.decodeLatin1 `eq` (T.pack . BC.unpack) tl_latin1 = EL.decodeLatin1 `eq` (TL.pack . BLC.unpack) +t_p_utf8_1 = testValidateUtf8_ [0x63] 1+t_p_utf8_2 = testValidateUtf8_ [0x63, 0x63, 0x63] 3+t_p_utf8_3 = testValidateUtf8_ [0x63, 0x63, 0xc2, 0x80, 0x63] 5+t_p_utf8_4 = testValidateUtf8_ [0x63, 0xe1, 0x80, 0x80, 0x63] 5+t_p_utf8_5 = testValidateUtf8_ [0xF0, 0x90, 0x80, 0x80, 0x63] 5+t_p_utf8_6 = testValidateUtf8_ [0x63, 0x63, 0xF0, 0x90, 0x80] 2+t_p_utf8_7 = testValidateUtf8_ [0x63, 0x63, 0x63, 0xF0, 0x90] 3+t_p_utf8_8 = testValidateUtf8Fail [0xF0, 0x90, 0x80, 0x63, 0x63] 0+t_p_utf8_9 = testValidateUtf8Fail [0x63, 0x63, 0x80, 0x63, 0x63] 2+t_p_utf8_0 = testValidateUtf8Fail [0x63, 0x63, 0xe1, 0x63, 0x63] 2++testValidateUtf8With ::+ (B.ByteString -> (Int, Maybe E.Utf8State)) ->+ (Maybe E.Utf8State -> IO r) ->+ [Word8] -> Int -> IO r+testValidateUtf8With validate k xs expectedLen = case validate (B.pack xs) of+ (len, s) -> do+ len @?= expectedLen+ k s++expectJust :: Maybe a -> IO a+expectJust Nothing = assertFailure "Unexpected Nothing"+expectJust (Just s) = pure s++expectNothing :: Maybe a -> IO ()+expectNothing Nothing = pure ()+expectNothing (Just _) = assertFailure "Unexpected Just"++testValidateUtf8 :: [Word8] -> Int -> IO E.Utf8State+testValidateUtf8 = testValidateUtf8With E.validateUtf8Chunk expectJust++testValidateUtf8_ :: [Word8] -> Int -> IO ()+testValidateUtf8_ = testValidateUtf8With E.validateUtf8Chunk (void . expectJust)++testValidateUtf8Fail :: [Word8] -> Int -> IO ()+testValidateUtf8Fail = testValidateUtf8With E.validateUtf8Chunk expectNothing++testValidateUtf8More :: E.Utf8State -> [Word8] -> Int -> IO E.Utf8State+testValidateUtf8More s = testValidateUtf8With (E.validateUtf8More s) expectJust++testValidateUtf8MoreFail :: E.Utf8State -> [Word8] -> Int -> IO ()+testValidateUtf8MoreFail s = testValidateUtf8With (E.validateUtf8More s) expectNothing++t_pn_utf8_1 = do+ s <- testValidateUtf8 [0xF0, 0x90, 0x80] 0+ _ <- testValidateUtf8More s [0x80] 1+ testValidateUtf8MoreFail s [0x7f] (-3)+t_pn_utf8_2 = do+ s0 <- testValidateUtf8 [0xF0] 0+ testValidateUtf8MoreFail s0 [0x7f] (-1)+ s1 <- testValidateUtf8More s0 [0x90] (-1)+ testValidateUtf8MoreFail s1 [0x7f] (-2)+ s2 <- testValidateUtf8More s1 [0x80] (-2)+ testValidateUtf8MoreFail s2 [0x7f] (-3)+ _ <- testValidateUtf8More s2 [0x80] 1+ pure ()+t_pn_utf8_3 = do+ s1 <- testValidateUtf8 [0xc2] 0+ assertBool "PartialUtf8 must be partial" $ B.length (E.getPartialUtf8 s1) < E.getCompleteLen s1+ testValidateUtf8MoreFail s1 [0x80, 0x80] 1++-- Precondition: (i, ms1) = E.validateUtf8More s chunk+--+-- The index points to the end of the longest valid prefix+-- of prechunk `B.append` chunk+pre_validateUtf8More_validPrefix s chunk i =+ let prechunk = E.getPartialUtf8 s in+ -- Note: i <= 0 implies take i = id+ let (j, ms2) = E.validateUtf8Chunk (B.take (B.length prechunk + i) (prechunk `B.append` chunk)) in+ counterexample (show prechunk) $+ (B.length prechunk + i, ms2) === (j, Just E.startUtf8State)++-- Precondition: (i, Nothing) = E.validateUtf8More s chunk+--+-- Appending to an invalid chunk yields another invalid chunk.+pre_validateUtf8More_maximalPrefix s chunk i more =+ E.validateUtf8More s (chunk `B.append` more) === (i, Nothing)++-- Precondition: (i, Just s1) = E.validateUtf8More s chunk+pre_validateUtf8More_suffix s chunk i s1 =+ if 0 <= i+ then B.drop i chunk === p2b s1 -- The state s1 contains a suffix of the chunk.+ else p2b s `B.append` chunk === p2b s1 -- Or the chunk extends the incomplete code point in s1.++-- Precondition: (i, Just s1) = E.validateUtf8More s chunk1+--+-- Validating two chunks sequentially is equivalent to validating them at once.+pre_validateUtf8More_append s chunk1 s1 chunk2 =+ let (j, ms2) = E.validateUtf8More s1 chunk2 in+ (B.length chunk1 + j, ms2) === E.validateUtf8More s (chunk1 `B.append` chunk2)++-- These wrappers use custom generators to satisfy the above properties.++t_validateUtf8More_validPrefix = property $ do+ cex@(s, chunk, i, _ms1) <- randomMoreChunk+ pure $ counterexample (show cex) $+ pre_validateUtf8More_validPrefix s chunk i++t_validateUtf8More_maximalPrefix = property $ do+ -- We want chunks that fail validation: force their size to be big,..+ cex@(s, chunk, i, ms1) <- scale (* 3) arbitraryMoreChunk+ pure $ counterexample (show cex) $+ -- ... and just use rejection sampling+ isNothing ms1 ==>+ pre_validateUtf8More_maximalPrefix s chunk i++t_validateUtf8More_valid = property $ do+ cex@(s, chunk1, i, s1, chunk2) <- validMoreChunks+ pure $ counterexample (show cex) $+ pre_validateUtf8More_suffix s chunk1 i s1 .&&.+ pre_validateUtf8More_append s chunk1 s1 chunk2++randomMoreChunk, arbitraryMoreChunk, validMoreChunk :: Gen (E.Utf8State, B.ByteString, Int, Maybe E.Utf8State)+randomMoreChunk = oneof [arbitraryMoreChunk, validMoreChunk]++arbitraryMoreChunk = do+ s <- randomUtf8State+ chunk <- arbitrary+ let (i, ms1) = E.validateUtf8More s chunk+ pure (s, chunk, i, ms1)++-- | Generate a random state by parsing a prefix of a Char+randomUtf8State :: Gen E.Utf8State+randomUtf8State = do+ c <- arbitrary+ chunk <- elements (B.inits (E.encodeUtf8 (T.singleton c)))+ case E.validateUtf8Chunk chunk of+ (_, Just s) -> pure s+ (_, Nothing) -> error "should not happen"++-- | Make a valid chunk, i.e., (s, chunk) such that+--+-- validateUtf8More s chunk = (i, Just s1)+--+-- Also returning i and s1 to not repeat work.+validMoreChunk = do+ (s, chunk, i, s1, _chunk2) <- validMoreChunks+ pure (s, chunk, i, Just s1)++-- | Make a valid chunk by slicing a valid UTF8 bs,+-- and also provide a second chunk which is a valid extension+-- with 0.5 probability.+validMoreChunks :: Gen (E.Utf8State, B.ByteString, Int, E.Utf8State, B.ByteString)+validMoreChunks = do+ bs <- E.encodeUtf8 <$> scale (* 3) arbitrary+ -- Take an intermediate state.+ -- No need to go too far since code points are at most 4 bytes long+ i <- choose (0, 3)+ let (bs0, bs1) = B.splitAt i bs+ case E.validateUtf8Chunk bs0 of+ (_, Just s) -> do+ j <- choose (0, B.length bs1)+ let (chunk1, chunk2') = B.splitAt j bs1+ case E.validateUtf8More s chunk1 of+ (n1, Just s1) -> do+ chunk2 <- oneof [pure chunk2', arbitrary]+ pure (s, chunk1, n1, s1, chunk2)+ (_, Nothing) -> error "should not happen"+ (_, Nothing) -> error "should not happen"++t_utf8_c = (E.strictBuilderToText . fst3 . E.decodeUtf8Chunk . E.encodeUtf8) `eq` id t_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id t_utf8' = (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) tl_utf8 = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id@@ -48,6 +221,9 @@ t_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id tl_utf32BE = (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id +fst3 :: (a, b, c) -> a+fst3 (a, _, _) = a+ runBuilder :: B.Builder -> B.ByteString runBuilder = -- Use smallish buffers to exercise bufferFull case as well@@ -82,7 +258,7 @@ t_utf8_undecoded t = let b = E.encodeUtf8 t ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t- leftover = (++ [B.empty]) . init . tail . B.inits+ leftover = (++ [B.empty]) . init . drop 1 . B.inits in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls data InvalidUtf8 = InvalidUtf8@@ -160,7 +336,7 @@ upTo n gen = do k <- choose (0,n) vectorOf k gen- -- Data.Text.Internal.Encoding.Utf8.ord{2,3,4} withous sanity checks+ -- Data.Text.Internal.Encoding.Utf8.ord{2,3,4} without sanity checks ord2_ n = map fromIntegral [(n `shiftR` 6) + 0xC0, (n .&. 0x3F) + 0x80] ord3_ n = map fromIntegral [(n `shiftR` 12) + 0xE0, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80] ord4_ n = map fromIntegral [(n `shiftR` 18) + 0xF0, ((n `shiftR` 12) .&. 0x3F) + 0x80, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80]@@ -190,26 +366,124 @@ t_decode_with_error4 = E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa" +t_decode_with_error1' = do+ E.Some x1 bs1 f1 <- pure $ E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xc2])+ x1 @?= ""+ bs1 @?= B.pack [0xc2]+ E.Some x2 bs2 _ <- pure $ f1 $ B.pack [0x80, 0x80]+ x2 @?= "\x80x"+ bs2 @?= mempty t_decode_with_error2' = case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of- E.Some x _ _ -> x === "xa"+ E.Some x _ _ -> x @?= "xa" t_decode_with_error3' = case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of- E.Some x _ _ -> x === "xaa"+ E.Some x _ _ -> x @?= "xaa" t_decode_with_error4' = case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of- E.Some x _ _ -> x === "xaaa"-t_decode_with_error5' = ioProperty $ do+ E.Some x _ _ -> x @?= "xaaa"+t_decode_with_error5' = do ret <- Exception.try $ Exception.evaluate $ E.streamDecodeUtf8 (B.pack [0x81])- pure $ case ret of- Left (_ :: E.UnicodeException) -> True- Right{} -> False+ case ret of+ Left (_ :: E.UnicodeException) -> pure ()+ Right{} -> assertFailure "Unexpected success" +testDecodeUtf8With :: (Maybe E.Utf8State -> IO r) -> E.Utf8State -> [Word8] -> T.Text -> IO r+testDecodeUtf8With k s xs expected =+ let xs' = B.pack xs in+ case E.decodeUtf8More s xs' of+ (prefix, bs, s') -> do+ let txt = E.strictBuilderToText prefix+ txt @?= expected+ if T.null txt then+ bs @?= xs'+ else+ E.encodeUtf8 txt `B.append` bs @?= E.getPartialUtf8 s `B.append` xs'+ k s'++testDecodeUtf8 :: E.Utf8State -> [Word8] -> T.Text -> IO E.Utf8State+testDecodeUtf8 = testDecodeUtf8With (\ms -> case ms of+ Just s -> pure s+ Nothing -> assertFailure "Unexpected failure")++testDecodeUtf8Fail :: E.Utf8State -> [Word8] -> T.Text -> IO ()+testDecodeUtf8Fail = testDecodeUtf8With (\ms -> case ms of+ Just _ -> assertFailure "Unexpected failure"+ Nothing -> pure ())++t_decode_chunk1 = do+ s1 <- testDecodeUtf8 E.startUtf8State [0xc2] ""+ B.length (E.getPartialUtf8 s1) @?= 1+ testDecodeUtf8Fail s1 [0x80, 0x80] "\128"++t_decode_chunk2 = do+ s1 <- testDecodeUtf8 E.startUtf8State [0xf0] ""+ s2 <- testDecodeUtf8 s1 [0x90, 0x80] ""+ _ <- testDecodeUtf8 s2 [0x80, 0x41] "\65536A"+ pure ()+ t_infix_concat bs1 text bs2 = forAll (Blind <$> genDecodeErr Replace) $ \(Blind onErr) -> text `T.isInfixOf` E.decodeUtf8With onErr (B.concat [bs1, E.encodeUtf8 text, bs2]) +t_textToStrictBuilder =+ (E.strictBuilderToText . E.textToStrictBuilder) `eq` id++-- decodeUtf8Chunk splits the input bytestring+t_decodeUtf8Chunk_split chunk =+ let (pre, suf, _ms) = E.decodeUtf8Chunk chunk+ in s2b pre `B.append` suf === chunk++-- decodeUtf8More mostly splits the input bytestring,+-- also inserting bytes from the partial code point in s.+--+-- This is wrapped by t_decodeUtf8More_split to have more+-- likely valid chunks.+t_decodeUtf8More_split' s chunk =+ let (pre, suf, _ms) = E.decodeUtf8More s chunk+ in if B.length chunk > B.length suf+ then s2b pre `B.append` suf === p2b s `B.append` chunk+ else suf === chunk++-- The output state of decodeUtf8More contains the suffix.+--+-- Precondition (valid chunk): ms = Just s'+pre_decodeUtf8More_suffix s chunk =+ let (_pre, suf, ms) = E.decodeUtf8More s chunk+ in case ms of+ Nothing -> discard+ Just s' -> if B.length chunk > B.length suf+ then p2b s' === suf+ else p2b s' === p2b s `B.append` suf++-- Decoding chunks separately is equivalent to decoding their concatenation.+pre_decodeUtf8More_append s chunk1 chunk2 =+ let (pre1, _, ms1) = E.decodeUtf8More s chunk1 in+ case ms1 of+ Nothing -> discard+ Just s1 ->+ let (pre2, _, ms2) = E.decodeUtf8More s1 chunk2 in+ let (pre3, _, ms3) = E.decodeUtf8More s (chunk1 `B.append` chunk2) in+ (s2b (pre1 <> pre2), ms2) === (s2b pre3, ms3)++-- Properties for any chunk+-- (but do try to generate valid chunks often enough)+t_decodeUtf8More1 = property $ do+ cex@(s, chunk, _, _) <- randomMoreChunk+ pure $ counterexample (show cex) $+ t_decodeUtf8More_split' s chunk++-- Properties that require valid chunks+t_decodeUtf8More2 = property $ do+ cex@(s, chunk, _, _, chunk2) <- validMoreChunks+ pure $ counterexample (show cex) $+ pre_decodeUtf8More_suffix s chunk .&&.+ pre_decodeUtf8More_append s chunk chunk2++s2b = E.encodeUtf8 . E.strictBuilderToText+p2b = E.getPartialUtf8+ testTranscoding :: TestTree testTranscoding = testGroup "transcoding" [@@ -219,8 +493,8 @@ testProperty "tl_latin1" tl_latin1, testProperty "t_utf8" t_utf8, testProperty "t_utf8'" t_utf8',- testProperty "t_utf8_incr" t_utf8_incr, testProperty "t_utf8_undecoded" t_utf8_undecoded,+ testProperty "t_utf8_incr" t_utf8_incr, testProperty "tl_utf8" tl_utf8, testProperty "tl_utf8'" tl_utf8', testProperty "t_utf16LE" t_utf16LE,@@ -247,10 +521,40 @@ testProperty "t_decode_with_error2" t_decode_with_error2, testProperty "t_decode_with_error3" t_decode_with_error3, testProperty "t_decode_with_error4" t_decode_with_error4,- testProperty "t_decode_with_error2'" t_decode_with_error2',- testProperty "t_decode_with_error3'" t_decode_with_error3',- testProperty "t_decode_with_error4'" t_decode_with_error4',- testProperty "t_decode_with_error5'" t_decode_with_error5',+ testCase "t_decode_with_error1'" t_decode_with_error1',+ testCase "t_decode_with_error2'" t_decode_with_error2',+ testCase "t_decode_with_error3'" t_decode_with_error3',+ testCase "t_decode_with_error4'" t_decode_with_error4',+ testCase "t_decode_with_error5'" t_decode_with_error5', testProperty "t_infix_concat" t_infix_concat+ ],+ testGroup "validate" [+ testProperty "t_validateUtf8More_validPrefix" t_validateUtf8More_validPrefix,+ testProperty "t_validateUtf8More_maximalPrefix" t_validateUtf8More_maximalPrefix,+ testProperty "t_validateUtf8More_valid" t_validateUtf8More_valid+ ],+ testGroup "streaming" [+ testProperty "t_utf8_c" t_utf8_c,+ testCase "t_p_utf8_1" t_p_utf8_1,+ testCase "t_p_utf8_2" t_p_utf8_2,+ testCase "t_p_utf8_3" t_p_utf8_3,+ testCase "t_p_utf8_4" t_p_utf8_4,+ testCase "t_p_utf8_5" t_p_utf8_5,+ testCase "t_p_utf8_6" t_p_utf8_6,+ testCase "t_p_utf8_7" t_p_utf8_7,+ testCase "t_p_utf8_8" t_p_utf8_8,+ testCase "t_p_utf8_9" t_p_utf8_9,+ testCase "t_p_utf8_0" t_p_utf8_0,+ testCase "t_pn_utf8_1" t_pn_utf8_1,+ testCase "t_pn_utf8_2" t_pn_utf8_2,+ testCase "t_pn_utf8_3" t_pn_utf8_3,+ testCase "t_decode_chunk1" t_decode_chunk1,+ testCase "t_decode_chunk2" t_decode_chunk2,+ testProperty "t_decodeUtf8Chunk_split" t_decodeUtf8Chunk_split,+ testProperty "t_decodeUtf8More1" t_decodeUtf8More1,+ testProperty "t_decodeUtf8More2" t_decodeUtf8More2+ ],+ testGroup "strictBuilder" [+ testProperty "textToStrictBuilder" t_textToStrictBuilder ] ]
+ tests/Tests/Properties/Validate.hs view
@@ -0,0 +1,50 @@+{-# LANGUAGE CPP #-}+module Tests.Properties.Validate (testValidate) where++import Data.Array.Byte (ByteArray)+import Data.ByteString (ByteString)+import qualified Data.ByteString as B+import Data.ByteString.Short (toShort)+import Data.Either (isRight)+import Data.Text.Encoding (decodeUtf8', encodeUtf8)+import Data.Text.Internal.Validate (isValidUtf8ByteString, isValidUtf8ByteArray)+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck ((===), Gen, Property,+ testProperty, arbitrary, forAllShrink, oneof, shrink)+import Tests.QuickCheckUtils ()+#if MIN_VERSION_bytestring(0,12,0)+import Data.ByteString.Short (unShortByteString)+#else+#if MIN_VERSION_bytestring(0,11,1)+import Data.ByteString.Short (ShortByteString(SBS))+#else+import Data.ByteString.Short.Internal (ShortByteString(SBS))+#endif+import Data.Array.Byte (ByteArray(ByteArray))++unShortByteString :: ShortByteString -> ByteArray+unShortByteString (SBS ba) = ByteArray ba+#endif++testValidate :: TestTree+testValidate = testGroup "validate"+ [ testProperty "bytestring" $ forAllShrink genByteString shrink $ \bs ->+ isValidUtf8ByteString bs === isRight (decodeUtf8' bs)+ , testProperty "bytearray" $ forAllByteArray $ \ba off len bs ->+ isValidUtf8ByteArray ba off len === isRight (decodeUtf8' bs)+ ]++genByteString :: Gen ByteString+genByteString = oneof+ [ arbitrary+ , encodeUtf8 <$> arbitrary+ ]++-- | We want to test 'isValidUtf8ByteArray' with various offsets, so we insert a random+-- prefix and remember its length.+forAllByteArray :: (ByteArray -> Int -> Int -> ByteString -> Property) -> Property+forAllByteArray prop =+ forAllShrink genByteString shrink $ \mainSlice ->+ forAllShrink arbitrary shrink $ \prefix ->+ let bs2ba = unShortByteString . toShort in+ prop (bs2ba (prefix `B.append` mainSlice)) (B.length prefix) (B.length mainSlice) mainSlice
tests/Tests/QuickCheckUtils.hs view
@@ -1,279 +1,326 @@--- | This module provides quickcheck utilities, e.g. arbitrary and show--- instances, and comparison functions, so we can focus on the actual properties--- in the 'Tests.Properties' module.----{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE DeriveFunctor #-}--{-# OPTIONS_GHC -fno-warn-orphans #-}--module Tests.QuickCheckUtils- ( BigInt(..)- , NotEmpty(..)- , Sqrt(..)- , SpacyString(..)-- , Precision(..)- , precision-- , DecodeErr(..)- , genDecodeErr-- , Stringy(..)- , unpack2- , eq- , eqP- , eqPSqrt-- , write_read- ) where--import Control.Arrow ((***))-import Control.DeepSeq (NFData (..), deepseq)-import Control.Exception (bracket)-import Data.Char (isSpace)-import Data.Text.Foreign (I8)-import Data.Text.Lazy.Builder.RealFloat (FPFormat(..))-import Data.Word (Word8, Word16)-import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, discard, counterexample, scale, (===), (.&&.), NonEmptyList(..))-import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized)-import Tests.Utils-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL-import qualified Data.Text as T-import qualified Data.Text.Encoding.Error as T-import qualified Data.Text.Internal.Fusion as TF-import qualified Data.Text.Internal.Fusion.Common as TF-import qualified Data.Text.Internal.Lazy as TL-import qualified Data.Text.Internal.Lazy.Fusion as TLF-import qualified Data.Text.Lazy as TL-import qualified System.IO as IO--genWord8 :: Gen Word8-genWord8 = chooseAny--instance Arbitrary I8 where- arbitrary = arbitrarySizedIntegral- shrink = shrinkIntegral--instance Arbitrary B.ByteString where- arbitrary = B.pack `fmap` listOf genWord8- shrink = map B.pack . shrink . B.unpack--instance Arbitrary BL.ByteString where- arbitrary = oneof- [ BL.fromChunks <$> arbitrary- -- so that a single utf8 code point could appear split over up to 4 chunks- , BL.fromChunks . map B.singleton <$> listOf genWord8- -- so that a code point with 4 byte long utf8 representation- -- could appear split over 3 non-singleton chunks- , (\a b c -> BL.fromChunks [a, b, c])- <$> arbitrary- <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8)- <*> arbitrary- ]- shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs)---- | For tests that have O(n^2) running times or input sizes, resize--- their inputs to the square root of the originals.-newtype Sqrt a = Sqrt { unSqrt :: a }- deriving (Eq, Show)--instance Arbitrary a => Arbitrary (Sqrt a) where- arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary- where- smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs- shrink = map Sqrt . shrink . unSqrt--instance Arbitrary T.Text where- arbitrary = do- t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary- -- Generate chunks that start in the middle of their buffers.- (\i -> T.drop i t) <$> choose (0, T.length t)- shrink = map T.pack . shrink . T.unpack--instance Arbitrary TL.Text where- arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary- shrink = map TL.pack . shrink . TL.unpack--newtype BigInt = Big Integer- deriving (Eq, Show)--instance Arbitrary BigInt where- arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e)- shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l]- where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer--newtype NotEmpty a = NotEmpty { notEmpty :: a }- deriving (Eq, Ord, Show)--instance Arbitrary (NotEmpty T.Text) where- arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary- shrink = fmap (NotEmpty . T.pack . getNonEmpty)- . shrink . NonEmpty . T.unpack . notEmpty--instance Arbitrary (NotEmpty TL.Text) where- arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary- shrink = fmap (NotEmpty . TL.pack . getNonEmpty)- . shrink . NonEmpty . TL.unpack . notEmpty--data DecodeErr = Lenient | Ignore | Strict | Replace- deriving (Show, Eq, Bounded, Enum)--genDecodeErr :: DecodeErr -> Gen T.OnDecodeError-genDecodeErr Lenient = return T.lenientDecode-genDecodeErr Ignore = return T.ignore-genDecodeErr Strict = return T.strictDecode-genDecodeErr Replace = (\c _ _ -> c) <$> frequency- [ (1, return Nothing)- , (50, Just <$> arbitraryUnicodeChar)- ]--instance Arbitrary DecodeErr where- arbitrary = arbitraryBoundedEnum--class Stringy s where- packS :: String -> s- unpackS :: s -> String- splitAtS :: Int -> s -> (s,s)- packSChunkSize :: Int -> String -> s- packSChunkSize _ = packS--instance Stringy String where- packS = id- unpackS = id- splitAtS = splitAt--instance Stringy (TF.Stream Char) where- packS = TF.streamList- unpackS = TF.unstreamList- splitAtS n s = (TF.take n s, TF.drop n s)--instance Stringy T.Text where- packS = T.pack- unpackS = T.unpack- splitAtS = T.splitAt--instance Stringy TL.Text where- packSChunkSize k = TLF.unstreamChunks k . TF.streamList- packS = TL.pack- unpackS = TL.unpack- splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) .- TL.splitAt . fromIntegral--unpack2 :: (Stringy s) => (s,s) -> (String,String)-unpack2 = unpackS *** unpackS---- Do two functions give the same answer?-eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property-eq a b s = a s =^= b s---- What about with the RHS packed?-eqP :: (Eq a, Show a, Stringy s) =>- (String -> a) -> (s -> a) -> String -> Word8 -> Property-eqP f g s w = counterexample "orig" (f s =^= g t) .&&.- counterexample "mini" (f s =^= g mini) .&&.- counterexample "head" (f sa =^= g ta) .&&.- counterexample "tail" (f sb =^= g tb)- where t = packS s- mini = packSChunkSize 10 s- (sa,sb) = splitAt m s- (ta,tb) = splitAtS m t- l = length s- m | l == 0 = n- | otherwise = n `mod` l- n = fromIntegral w--eqPSqrt :: (Eq a, Show a, Stringy s) =>- (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property-eqPSqrt f g s = eqP f g (unSqrt s)--instance Arbitrary FPFormat where- arbitrary = arbitraryBoundedEnum--newtype Precision a = Precision (Maybe Int)- deriving (Eq, Show)--precision :: a -> Precision a -> Maybe Int-precision _ (Precision prec) = prec--arbitraryPrecision :: Int -> Gen (Precision a)-arbitraryPrecision maxDigits = Precision <$> do- n <- choose (-1,maxDigits)- return $ if n == -1- then Nothing- else Just n--instance Arbitrary (Precision Float) where- arbitrary = arbitraryPrecision 11- shrink = map Precision . shrink . precision undefined--instance Arbitrary (Precision Double) where- arbitrary = arbitraryPrecision 22- shrink = map Precision . shrink . precision undefined--instance Arbitrary IO.Newline where- arbitrary = oneof [return IO.LF, return IO.CRLF]--instance Arbitrary IO.NewlineMode where- arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary--instance Arbitrary IO.BufferMode where- arbitrary = oneof [ return IO.NoBuffering,- return IO.LineBuffering,- return (IO.BlockBuffering Nothing),- (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap`- (arbitrary :: Gen Word16) ]---- This test harness is complex! What property are we checking?------ Reading after writing a multi-line file should give the same--- results as were written.------ What do we vary while checking this property?--- * The lines themselves, scrubbed to contain neither CR nor LF. (By--- working with a list of lines, we ensure that the data will--- sometimes contain line endings.)--- * Newline translation mode.--- * Buffering.-write_read :: (NFData a, Eq a, Show a)- => ([b] -> a)- -> ((Char -> Bool) -> a -> b)- -> (IO.Handle -> a -> IO ())- -> (IO.Handle -> IO a)- -> IO.NewlineMode- -> IO.BufferMode- -> [a]- -> Property-write_read _ _ _ _ (IO.NewlineMode IO.LF IO.CRLF) _ _ = discard-write_read unline filt writer reader nl buf ts = ioProperty $- (===t) <$> act- where- t = unline . map (filt (not . (`elem` "\r\n"))) $ ts-- act = withTempFile $ \path h -> do- IO.hSetEncoding h IO.utf8- IO.hSetNewlineMode h nl- IO.hSetBuffering h buf- () <- writer h t- IO.hClose h- bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do- IO.hSetEncoding h' IO.utf8- IO.hSetNewlineMode h' nl- IO.hSetBuffering h' buf- r <- reader h'- r `deepseq` return r---- Generate various Unicode space characters with high probability-arbitrarySpacyChar :: Gen Char-arbitrarySpacyChar = oneof- [ arbitraryUnicodeChar- , elements $ filter isSpace [minBound..maxBound]- ]--newtype SpacyString = SpacyString { getSpacyString :: String }- deriving (Eq, Ord, Show, Read)--instance Arbitrary SpacyString where- arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar- shrink (SpacyString xs) = SpacyString `fmap` shrink xs+-- | This module provides quickcheck utilities, e.g. arbitrary and show +-- instances, and comparison functions, so we can focus on the actual properties +-- in the 'Tests.Properties' module. +-- +{-# LANGUAGE CPP #-} +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Tests.QuickCheckUtils + ( BigInt(..) + , NotEmpty(..) + , Sqrt(..) + , SpacyString(..) + , SkewedBool(..) + + , Precision(..) + , precision + + , DecodeErr(..) + , genDecodeErr + + , Stringy(..) + , unpack2 + , eq + , eqP + , eqPSqrt + + , write_read + ) where + +import Control.Arrow ((***)) +import Control.Monad (when) +import Data.Char (isSpace) +import Data.IORef (writeIORef) +import Data.Text.Foreign (I8) +import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) +import Data.Word (Word8) +#if !MIN_VERSION_QuickCheck(2,17,0) +import Data.Word (Word16) +#endif +import qualified GHC.IO.Buffer as GIO +import qualified GHC.IO.Handle.Internals as GIO +import qualified GHC.IO.Handle.Types as GIO +import GHC.IO.Encoding.Types (TextEncoding(textEncodingName)) +import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, counterexample, scale, (.&&.), NonEmptyList(..), forAllShrink) +import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) +import Tests.Utils +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import qualified Data.Text as T +import qualified Data.Text.Encoding.Error as T +import qualified Data.Text.Internal.Fusion as TF +import qualified Data.Text.Internal.Fusion.Common as TF +import qualified Data.Text.Internal.Lazy as TL +import qualified Data.Text.Internal.Lazy.Fusion as TLF +import qualified Data.Text.Lazy as TL +import qualified System.IO as IO + +genWord8 :: Gen Word8 +genWord8 = chooseAny + +instance Arbitrary I8 where + arbitrary = arbitrarySizedIntegral + shrink = shrinkIntegral + +instance Arbitrary B.ByteString where + arbitrary = B.pack `fmap` listOf genWord8 + shrink = map B.pack . shrink . B.unpack + +instance Arbitrary BL.ByteString where + arbitrary = oneof + [ BL.fromChunks <$> arbitrary + -- so that a single utf8 code point could appear split over up to 4 chunks + , BL.fromChunks . map B.singleton <$> listOf genWord8 + -- so that a code point with 4 byte long utf8 representation + -- could appear split over 3 non-singleton chunks + , (\a b c -> BL.fromChunks [a, b, c]) + <$> arbitrary + <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8) + <*> arbitrary + ] + shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) + +-- | For tests that have O(n^2) running times or input sizes, resize +-- their inputs to the square root of the originals. +newtype Sqrt a = Sqrt { unSqrt :: a } + deriving (Eq, Show) + +instance Arbitrary a => Arbitrary (Sqrt a) where + arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary + where + smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs + shrink = map Sqrt . shrink . unSqrt + +instance Arbitrary T.Text where + arbitrary = do + t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary + -- Generate chunks that start in the middle of their buffers. + (\i -> T.drop i t) <$> choose (0, T.length t) + shrink = map T.pack . shrink . T.unpack + +instance Arbitrary TL.Text where + arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary + shrink = map TL.pack . shrink . TL.unpack + +newtype BigInt = Big Integer + deriving (Eq, Show) + +instance Arbitrary BigInt where + arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) + shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] + where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer + +newtype NotEmpty a = NotEmpty { notEmpty :: a } + deriving (Eq, Ord, Show) + +instance Arbitrary (NotEmpty T.Text) where + arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . T.pack . getNonEmpty) + . shrink . NonEmpty . T.unpack . notEmpty + +instance Arbitrary (NotEmpty TL.Text) where + arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . TL.pack . getNonEmpty) + . shrink . NonEmpty . TL.unpack . notEmpty + +data DecodeErr = Lenient | Ignore | Strict | Replace + deriving (Show, Eq, Bounded, Enum) + +genDecodeErr :: DecodeErr -> Gen T.OnDecodeError +genDecodeErr Lenient = return T.lenientDecode +genDecodeErr Ignore = return T.ignore +genDecodeErr Strict = return T.strictDecode +genDecodeErr Replace = (\c _ _ -> c) <$> frequency + [ (1, return Nothing) + , (50, Just <$> arbitraryUnicodeChar) + ] + +instance Arbitrary DecodeErr where + arbitrary = arbitraryBoundedEnum + +class Stringy s where + packS :: String -> s + unpackS :: s -> String + splitAtS :: Int -> s -> (s,s) + packSChunkSize :: Int -> String -> s + packSChunkSize _ = packS + +instance Stringy String where + packS = id + unpackS = id + splitAtS = splitAt + +instance Stringy (TF.Stream Char) where + packS = TF.streamList + unpackS = TF.unstreamList + splitAtS n s = (TF.take n s, TF.drop n s) + +instance Stringy T.Text where + packS = T.pack + unpackS = T.unpack + splitAtS = T.splitAt + +instance Stringy TL.Text where + packSChunkSize k = TLF.unstreamChunks k . TF.streamList + packS = TL.pack + unpackS = TL.unpack + splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . + TL.splitAt . fromIntegral + +unpack2 :: (Stringy s) => (s,s) -> (String,String) +unpack2 = unpackS *** unpackS + +-- Do two functions give the same answer? +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property +eq a b s = a s =^= b s + +-- What about with the RHS packed? +eqP :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> String -> Word8 -> Property +eqP f g s w = counterexample "orig" (f s =^= g t) .&&. + counterexample "mini" (f s =^= g mini) .&&. + counterexample "head" (f sa =^= g ta) .&&. + counterexample "tail" (f sb =^= g tb) + where t = packS s + mini = packSChunkSize 10 s + (sa,sb) = splitAt m s + (ta,tb) = splitAtS m t + l = length s + m | l == 0 = n + | otherwise = n `mod` l + n = fromIntegral w + +eqPSqrt :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property +eqPSqrt f g s = eqP f g (unSqrt s) + +instance Arbitrary FPFormat where + arbitrary = arbitraryBoundedEnum + +newtype Precision a = Precision (Maybe Int) + deriving (Eq, Show) + +precision :: a -> Precision a -> Maybe Int +precision _ (Precision prec) = prec + +arbitraryPrecision :: Int -> Gen (Precision a) +arbitraryPrecision maxDigits = Precision <$> do + n <- choose (-1,maxDigits) + return $ if n == -1 + then Nothing + else Just n + +instance Arbitrary (Precision Float) where + arbitrary = arbitraryPrecision 11 + shrink = map Precision . shrink . precision undefined + +instance Arbitrary (Precision Double) where + arbitrary = arbitraryPrecision 22 + shrink = map Precision . shrink . precision undefined + +#if !MIN_VERSION_QuickCheck(2,14,3) +instance Arbitrary IO.Newline where + arbitrary = oneof [return IO.LF, return IO.CRLF] + +instance Arbitrary IO.NewlineMode where + arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary +#endif + +#if !MIN_VERSION_QuickCheck(2,17,0) +instance Arbitrary IO.BufferMode where + arbitrary = oneof [ return IO.NoBuffering, + return IO.LineBuffering, + return (IO.BlockBuffering Nothing), + (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` + genWord16 ] + +genWord16 :: Gen Word16 +genWord16 = chooseAny +#endif + +-- This test harness is complex! What property are we checking? +-- +-- Reading after writing a multi-line file should give the same +-- results as were written. +-- +-- What do we vary while checking this property? +-- * The lines themselves, scrubbed to contain neither CR nor LF. (By +-- working with a list of lines, we ensure that the data will +-- sometimes contain line endings.) +-- * Newline translation mode. +-- * Buffering. +write_read :: forall a. + (Eq a, Show a) + => Gen a + -> (a -> [a]) + -> (a -> a) -- ^ replace '\n' with '\r\n' (for multiline tests) or append '\r' (for single-line tests) + -> (IO.Handle -> a -> IO ()) + -> (IO.Handle -> IO a) + -> Property +write_read genTxt shrinkTxt expandNl writer reader + = forAllShrink genEncoding shrinkEncoding propTest + where + propTest :: TextEncoding -> IO.BufferMode -> Property + propTest enc mode = forAllShrink genTxt shrinkTxt $ \txt -> ioProperty $ do + file <- emptyTempFile + let with nl k = IO.withFile file IO.ReadWriteMode $ \h -> do + IO.hSetEncoding h enc + IO.hSetBuffering h mode + IO.hSetNewlineMode h nl + setSmallBuffer h + k h + -- Put a very small buffer in Handle to easily test boundary conditions in `writeBlocks` + setSmallBuffer h = GIO.withHandle_ "setSmallBuffer" h $ \h_ -> do + buf <- GIO.newCharBuffer 9 GIO.WriteBuffer + writeIORef (GIO.haCharBuffer h_) buf + readExpecting h txt' msg = do + out <- reader h + when (txt' /= out) $ error (show txt' ++ " /= " ++ show out ++ msg) + -- 'reader' may be 'hGetContents', which closes the handle + -- So we reopen a new file every time. + + -- Test with CRLF encoding + with (IO.NewlineMode IO.CRLF IO.CRLF) $ \h -> do + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 1)" + + -- Re-read without CRLF decoding to check that we did encode CRLF correctly + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + readExpecting h (expandNl txt) " (at location 2)" + + -- Test without CRLF encoding + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + IO.hSetFileSize h 0 + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 3)" + + genEncoding = elements [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be] + shrinkEncoding enc = if textEncodingName enc == textEncodingName IO.utf8 then [] else [IO.utf8] + +-- Generate various Unicode space characters with high probability +arbitrarySpacyChar :: Gen Char +arbitrarySpacyChar = oneof + [ arbitraryUnicodeChar + , elements $ filter isSpace [minBound..maxBound] + ] + +newtype SpacyString = SpacyString { getSpacyString :: String } + deriving (Eq, Ord, Show, Read) + +instance Arbitrary SpacyString where + arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar + shrink (SpacyString xs) = SpacyString `fmap` shrink xs + +newtype SkewedBool = Skewed { getSkewed :: Bool } + deriving Show + +instance Arbitrary SkewedBool where + arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)]
+ tests/Tests/RebindableSyntaxTest.hs view
@@ -0,0 +1,18 @@+{-# LANGUAGE CPP, RebindableSyntax, TemplateHaskell #-} + +module Tests.RebindableSyntaxTest where + +import qualified Data.Text as Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) +import Prelude (($)) + +tests :: TestTree +tests = testGroup "RebindableSyntax" + [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a") + ]
tests/Tests/Regressions.hs view
@@ -1,162 +1,236 @@--- | Regression tests for specific bugs.----{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE ScopedTypeVariables #-}--module Tests.Regressions- (- tests- ) where--import Control.Exception (SomeException, handle)-import Data.Char (isLetter)-import GHC.Exts (Int(..), sizeofByteArray#)-import System.IO-import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure)-import qualified Data.ByteString as B-import Data.ByteString.Char8 ()-import qualified Data.ByteString.Lazy as LB-import qualified Data.Text as T-import qualified Data.Text.Array as TA-import qualified Data.Text.Encoding as TE-import qualified Data.Text.Encoding.Error as E-import qualified Data.Text.Internal as T-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as LT-import qualified Data.Text.Lazy.Builder as TB-import qualified Data.Text.Lazy.Encoding as LE-import qualified Data.Text.Unsafe as T-import qualified Test.Tasty as F-import qualified Test.Tasty.HUnit as F-import System.Directory (removeFile)--import Tests.Utils (withTempFile)---- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring--- caused either a segfault or attempt to allocate a negative number--- of bytes.-lazy_encode_crash :: IO ()-lazy_encode_crash = withTempFile $ \ _ h ->- LB.hPut h . LE.encodeUtf8 . LT.pack . replicate 100000 $ 'a'---- Reported by Pieter Laeremans: attempting to read an incorrectly--- encoded file can result in a crash in the RTS (i.e. not merely an--- exception).-hGetContents_crash :: IO ()-hGetContents_crash = do- (path, h) <- openTempFile "." "crashy.txt"- B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) >> hClose h- h' <- openFile path ReadMode- hSetEncoding h' utf8- handle (\(_::SomeException) -> return ()) $- T.hGetContents h' >> assertFailure "T.hGetContents should crash"- hClose h'- removeFile path---- Reported by Ian Lynagh: attempting to allocate a sufficiently large--- string (via either Array.new or Text.replicate) could result in an--- integer overflow.-replicate_crash :: IO ()-replicate_crash = handle (\(_::SomeException) -> return ()) $- T.replicate (2^power) "0123456789abcdef" `seq`- assertFailure "T.replicate should crash"- where- power | maxBound == (2147483647::Int) = 28- | otherwise = 60 :: Int---- Reported by John Millikin: a UTF-8 decode error handler could--- return a bogus substitution character, which we would write without--- checking.-utf8_decode_unsafe :: IO ()-utf8_decode_unsafe = do- let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80"- assertBool "broken error recovery shouldn't break us" (t == "\xfffd")---- Reported by Eric Seidel: we mishandled mapping Chars that fit in a--- single Word16 to Chars that require two.-mapAccumL_resize :: IO ()-mapAccumL_resize = do- let f a _ = (a, '\65536')- count = 5- val = T.mapAccumL f (0::Int) (T.replicate count "a")- assertEqual "mapAccumL should correctly fill buffers for four-byte results"- (0, T.replicate count "\65536") val- assertEqual "mapAccumL should correctly size buffers for four-byte results"- (count * 4) (T.lengthWord8 (snd val))---- See GitHub #197-t197 :: IO ()-t197 =- assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")- where- currencyParser x = cond == 1- where- cond = length fltr- fltr = filter (== ',') x--t221 :: IO ()-t221 =- assertEqual "toLower of large input shouldn't crash"- (T.toLower (T.replicate 200000 "0") `seq` ())- ()--t227 :: IO ()-t227 =- assertEqual "take (-3) shouldn't crash with overflow"- (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?")- 0--t280_fromString :: IO ()-t280_fromString =- assertEqual "TB.fromString performs replacement on invalid scalar values"- (TB.toLazyText (TB.fromString "\xD800"))- (LT.pack "\xFFFD")--t280_singleton :: IO ()-t280_singleton =- assertEqual "TB.singleton performs replacement on invalid scalar values"- (TB.toLazyText (TB.singleton '\xD800'))- (LT.pack "\xFFFD")---- See GitHub issue #301--- This tests whether the "TEXT take . drop -> unfused" rule is applied to the--- slice function. When the slice function is fused, a new array will be--- constructed that is shorter than the original array. Without fusion the--- array remains unmodified.-t301 :: IO ()-t301 = do- assertEqual "The length of the array remains the same despite slicing"- (I# (sizeofByteArray# originalArr))- (I# (sizeofByteArray# newArr))-- assertEqual "The new array still contains the original value"- (T.Text (TA.ByteArray newArr) originalOff originalLen)- original- where- !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890"- !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original--t330 :: IO ()-t330 = do- let decodeL = LE.decodeUtf8With E.lenientDecode- assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked"- (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]]))- (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]]))--tests :: F.TestTree-tests = F.testGroup "Regressions"- [ F.testCase "hGetContents_crash" hGetContents_crash- , F.testCase "lazy_encode_crash" lazy_encode_crash- , F.testCase "mapAccumL_resize" mapAccumL_resize- , F.testCase "replicate_crash" replicate_crash- , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe- , F.testCase "t197" t197- , F.testCase "t221" t221- , F.testCase "t227" t227- , F.testCase "t280/fromString" t280_fromString- , F.testCase "t280/singleton" t280_singleton- , F.testCase "t301" t301- , F.testCase "t330" t330- ]+-- | Regression tests for specific bugs. +-- +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Tests.Regressions + ( + tests + ) where + +import Control.Exception (ErrorCall, SomeException, handle, evaluate, displayException, try) +import Data.Char (isLetter, chr) +import GHC.Exts (Int(..), sizeofByteArray#) +import System.IO +import System.IO.Temp (withSystemTempFile) +import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure, (@?=)) +import qualified Data.ByteString as B +import Data.ByteString.Char8 () +import qualified Data.ByteString.Lazy as LB +import Data.Semigroup (stimes) +import qualified Data.Text as T +import qualified Data.Text.Array as TA +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as E +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as LF +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as LT +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Text.Lazy.Encoding as LE +import qualified Data.Text.Unsafe as T +import qualified Test.Tasty as F +import qualified Test.Tasty.HUnit as F +import Tests.Utils (withTempFile) +import System.IO.Error (isFullError) + +-- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring +-- caused either a segfault or attempt to allocate a negative number +-- of bytes. +lazy_encode_crash :: IO () +lazy_encode_crash = withTempFile $ \ _ h -> do + putRes <- try $ LB.hPut h $ LE.encodeUtf8 $ LT.pack $ replicate 100000 'a' + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> pure () + +-- Reported by Pieter Laeremans: attempting to read an incorrectly +-- encoded file can result in a crash in the RTS (i.e. not merely an +-- exception). +hGetContents_crash :: IO () +hGetContents_crash = withSystemTempFile "crashy.txt" $ \path h -> do + putRes <- try $ B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> do + hClose h + h' <- openFile path ReadMode + hSetEncoding h' utf8 + handle (\(_::SomeException) -> pure ()) $ + T.hGetContents h' >> assertFailure "T.hGetContents should crash" + hClose h' + +-- Reported by Ian Lynagh: attempting to allocate a sufficiently large +-- string (via either Array.new or Text.replicate) could result in an +-- integer overflow. +replicate_crash :: IO () +replicate_crash = handle (\(_::SomeException) -> return ()) $ + T.replicate (2^power) "0123456789abcdef" `seq` + assertFailure "T.replicate should crash" + where + power | maxBound == (2147483647::Int) = 28 + | otherwise = 60 :: Int + +-- Reported by John Millikin: a UTF-8 decode error handler could +-- return a bogus substitution character, which we would write without +-- checking. +utf8_decode_unsafe :: IO () +utf8_decode_unsafe = do + let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" + assertBool "broken error recovery shouldn't break us" (t == "\xfffd") + +-- Reported by Eric Seidel: we mishandled mapping Chars that fit in a +-- single Word16 to Chars that require two. +mapAccumL_resize :: IO () +mapAccumL_resize = do + let f a _ = (a, '\65536') + count = 5 + val = T.mapAccumL f (0::Int) (T.replicate count "a") + assertEqual "mapAccumL should correctly fill buffers for four-byte results" + (0, T.replicate count "\65536") val + assertEqual "mapAccumL should correctly size buffers for four-byte results" + (count * 4) (T.lengthWord8 (snd val)) + +-- See GitHub #197 +t197 :: IO () +t197 = + assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00") + where + currencyParser x = cond == 1 + where + cond = length fltr + fltr = filter (== ',') x + +t221 :: IO () +t221 = + assertEqual "toLower of large input shouldn't crash" + (T.toLower (T.replicate 200000 "0") `seq` ()) + () + +t227 :: IO () +t227 = + assertEqual "take (-3) shouldn't crash with overflow" + (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?") + 0 + +t280_fromString :: IO () +t280_fromString = + assertEqual "TB.fromString performs replacement on invalid scalar values" + (TB.toLazyText (TB.fromString "\xD800")) + (LT.pack "\xFFFD") + +t280_singleton :: IO () +t280_singleton = + assertEqual "TB.singleton performs replacement on invalid scalar values" + (TB.toLazyText (TB.singleton '\xD800')) + (LT.pack "\xFFFD") + +-- See GitHub issue #301 +-- This tests whether the "TEXT take . drop -> unfused" rule is applied to the +-- slice function. When the slice function is fused, a new array will be +-- constructed that is shorter than the original array. Without fusion the +-- array remains unmodified. +t301 :: IO () +t301 = do + assertEqual "The length of the array remains the same despite slicing" + (I# (sizeofByteArray# originalArr)) + (I# (sizeofByteArray# newArr)) + + assertEqual "The new array still contains the original value" + (T.Text (TA.ByteArray newArr) originalOff originalLen) + original + where + !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" + !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original + +t330 :: IO () +t330 = do + let decodeL = LE.decodeUtf8With E.lenientDecode + assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked" + (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]])) + (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]])) + +-- Stream decoders should not loop on incomplete code points +t525 :: IO () +t525 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0" @?= "\65533" + LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533" + +-- Stream decoders skip one invalid byte at a time +t528 :: IO () +t528 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536" + LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536" + LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536" + LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0" + LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280" + +t529 :: IO () +t529 = do + let decode = TE.decodeUtf8With E.lenientDecode + -- https://github.com/haskell/bytestring/issues/575 + assertEqual "Data.ByteString.isValidUtf8 should work correctly" + (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0])) + (decode (B.pack (33 : replicate 31 0 ++ [128, 0]))) + +-- See Github #559 +-- filter/filter fusion rules should apply predicates in the right order. +t559 :: IO () +t559 = do + T.filter undefined (T.filter (const False) "a") @?= "" + LT.filter undefined (LT.filter (const False) "a") @?= "" + +-- Github #633 +-- stimes checked for an `a` to `Int` to `a` roundtrip, but the `a` and `Int` values could represent different integers. +t633 :: IO () +t633 = + handle (\(_ :: ErrorCall) -> return ()) $ do + _ <- evaluate (stimes (maxBound :: Word) "a" :: T.Text) + assertFailure "should fail" + +t648 :: IO () +t648 = withTempFile $ \_ h -> do + hSetEncoding h utf8 + hSetNewlineMode h (NewlineMode LF CRLF) + hSetBuffering h (BlockBuffering $ Just 4) + let line = T.replicate 2047 "_" + T.hPutStrLn h line + hSeek h AbsoluteSeek 0 + line' <- T.hGetLine h + T.append line "\r" @?= line' + +tests :: F.TestTree +tests = F.testGroup "Regressions" + [ F.testCase "hGetContents_crash" hGetContents_crash + , F.testCase "lazy_encode_crash" lazy_encode_crash + , F.testCase "mapAccumL_resize" mapAccumL_resize + , F.testCase "replicate_crash" replicate_crash + , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe + , F.testCase "t197" t197 + , F.testCase "t221" t221 + , F.testCase "t227" t227 + , F.testCase "t280/fromString" t280_fromString + , F.testCase "t280/singleton" t280_singleton + , F.testCase "t301" t301 + , F.testCase "t330" t330 + , F.testCase "t525" t525 + , F.testCase "t528" t528 + , F.testCase "t529" t529 + , F.testCase "t559" t559 + , F.testCase "t633" t633 + , F.testCase "t648" t648 + ]
@@ -0,0 +1,138 @@+{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE BangPatterns #-} + +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.ShareEmpty + ( tests + ) where + +import Control.Exception (evaluate) +import Data.Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertFailure, assertEqual) +import Test.Tasty (TestTree, testGroup) +import GHC.Exts +import GHC.Stack +import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList +import qualified Data.Text as T + + +-- | assert that a text value is represented by the same pointer +-- as the 'empty' value. +assertPtrEqEmpty :: HasCallStack => Text -> IO () +assertPtrEqEmpty t = do + t' <- evaluate t + empty' <- evaluate empty + assertEqual "" empty' t' + case reallyUnsafePtrEquality# empty' t' of + 1# -> pure () + _ -> assertFailure "Pointers are not equal" +{-# NOINLINE assertPtrEqEmpty #-} + +tests :: TestTree +tests = testGroup "empty Text values are shared" + [ testCase "empty = empty" $ assertPtrEqEmpty T.empty + , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack "" + , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString "" + , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack "")) + , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a" + , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b" + , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty + , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " [] + , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' "" + , testCase "reverse empty = empty" $ assertPtrEqEmpty $ + T.reverse empty + , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $ + T.replace "needle" "replacement" empty + , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold "" + , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower "" + , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper "" + , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle "" + , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyLeft 0 ' ' empty + , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyRight 0 ' ' empty + , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $ + T.center 0 ' ' empty + , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $ + T.transpose [empty] + , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat [] + , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty] + , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x" + , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty + , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $ + T.unfoldr (const Nothing) () + , testCase "take 0 _ = empty" $ assertPtrEqEmpty $ + T.take 0 "xyz" + , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $ + T.takeEnd 0 "xyz" + , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhile (const False) "xyz" + , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhileEnd (const False) "xyz" + , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $ + T.drop 5 "xyz" + , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $ + T.dropEnd 5 "xyz" + , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $ + T.dropWhile (const True) "xyz" + , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $ + dropWhileEnd (const True) "xyz" + , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $ + dropAround (const True) empty + , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty + , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty + , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty + , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123" + , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $ + snd $ T.splitAt 5 "123" + , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (spanM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123" + , testCase "snd (spanM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123" + , testCase "fst (spanEndM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123" + , testCase "snd (spanEndM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123" + , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty + , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty + , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty + , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123" + , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123" + , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty + , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty + , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" + , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123" + , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') "" + , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234" + , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" "" + , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines [] + , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords [] + , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripPrefix empty empty + , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripSuffix empty empty + , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $ + mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123" + , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $ + mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz" + , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy "" + ]
tests/Tests/Utils.hs view
@@ -1,51 +1,50 @@--- | Miscellaneous testing utilities----{-# LANGUAGE ScopedTypeVariables #-}-module Tests.Utils- (- (=^=)- , withRedirect- , withTempFile- ) where--import Control.Exception (SomeException, bracket, bracket_, evaluate, try)-import Control.Monad (when)-import GHC.IO.Handle.Internals (withHandle)-import System.Directory (removeFile)-import System.IO (Handle, hClose, hFlush, hIsOpen, hIsWritable, openTempFile)-import Test.QuickCheck (Property, ioProperty, property, (===), counterexample)---- Ensure that two potentially bottom values (in the sense of crashing--- for some inputs, not looping infinitely) either both crash, or both--- give comparable results for some input.-(=^=) :: (Eq a, Show a) => a -> a -> Property-i =^= j = ioProperty $ do- x <- try (evaluate i)- y <- try (evaluate j)- return $ case (x, y) of- (Left (_ :: SomeException), Left (_ :: SomeException))- -> property True- (Right a, Right b) -> a === b- e -> counterexample ("Divergence: " ++ show e) $ property False-infix 4 =^=-{-# NOINLINE (=^=) #-}--withTempFile :: (FilePath -> Handle -> IO a) -> IO a-withTempFile = bracket (openTempFile "." "crashy.txt") cleanupTemp . uncurry- where- cleanupTemp (path,h) = do- open <- hIsOpen h- when open (hClose h)- removeFile path--withRedirect :: Handle -> Handle -> IO a -> IO a-withRedirect tmp h = bracket_ swap swap- where- whenM p a = p >>= (`when` a)- swap = do- whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp- whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h- withHandle "spam" tmp $ \tmph -> do- hh <- withHandle "spam" h $ \hh ->- return (tmph,hh)- return (hh,())+-- | Miscellaneous testing utilities +-- +{-# LANGUAGE ScopedTypeVariables #-} +module Tests.Utils + ( + (=^=) + , withRedirect + , withTempFile + , emptyTempFile + ) where + +import Control.Exception (SomeException, bracket_, evaluate, try) +import Control.Monad (when) +import System.IO.Temp (withSystemTempFile, emptySystemTempFile) +import GHC.IO.Handle.Internals (withHandle) +import System.IO (Handle, hFlush, hIsOpen, hIsWritable) +import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) + +-- Ensure that two potentially bottom values (in the sense of crashing +-- for some inputs, not looping infinitely) either both crash, or both +-- give comparable results for some input. +(=^=) :: (Eq a, Show a) => a -> a -> Property +i =^= j = ioProperty $ do + x <- try (evaluate i) + y <- try (evaluate j) + return $ case (x, y) of + (Left (_ :: SomeException), Left (_ :: SomeException)) + -> property True + (Right a, Right b) -> a === b + e -> counterexample ("Divergence: " ++ show e) $ property False +infix 4 =^= +{-# NOINLINE (=^=) #-} + +withTempFile :: (FilePath -> Handle -> IO a) -> IO a +withTempFile = withSystemTempFile "crashy.txt" + +emptyTempFile :: IO FilePath +emptyTempFile = emptySystemTempFile "crashy.txt" + +withRedirect :: Handle -> Handle -> IO a -> IO a +withRedirect tmp h = bracket_ swap swap + where + whenM p a = p >>= (`when` a) + swap = do + whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp + whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h + withHandle "spam" tmp $ \tmph -> do + hh <- withHandle "spam" h $ \hh -> + return (tmph,hh) + return (hh,())
text.cabal view
@@ -1,287 +1,385 @@-cabal-version: 2.2-name: text-version: 2.0--homepage: https://github.com/haskell/text-bug-reports: https://github.com/haskell/text/issues-synopsis: An efficient packed Unicode text type.-description:- .- An efficient packed, immutable Unicode text type (both strict and- lazy).- .- The 'Text' type represents Unicode character strings, in a time and- space-efficient manner. This package provides text processing- capabilities that are optimized for performance critical use, both- in terms of large data quantities and high speed.- .- The 'Text' type provides character-encoding, type-safe case- conversion via whole-string case conversion functions (see "Data.Text").- It also provides a range of functions for converting 'Text' values to- and from 'ByteStrings', using several standard encodings- (see "Data.Text.Encoding").- .- Efficient locale-sensitive support for text IO is also supported- (see "Data.Text.IO").- .- These modules are intended to be imported qualified, to avoid name- clashes with Prelude functions, e.g.- .- > import qualified Data.Text as T- .- == ICU Support- .- To use an extended and very rich family of functions for working- with Unicode text (including normalization, regular expressions,- non-standard encodings, text breaking, and locales), see- the [text-icu package](https://hackage.haskell.org/package/text-icu)- based on the well-respected and liberally- licensed [ICU library](http://site.icu-project.org/).--license: BSD-2-Clause-license-file: LICENSE-author: Bryan O'Sullivan <bos@serpentine.com>-maintainer: Haskell Text Team <andrew.lelechenko@gmail.com>, Core Libraries Committee-copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko-category: Data, Text-build-type: Simple-tested-with:- GHC == 8.0.2- GHC == 8.2.2- GHC == 8.4.4- GHC == 8.6.5- GHC == 8.8.4- GHC == 8.10.7- GHC == 9.0.1- GHC == 9.2.1--extra-source-files:- -- scripts/CaseFolding.txt- -- scripts/SpecialCasing.txt- README.markdown- changelog.md- scripts/*.hs- simdutf/LICENSE-APACHE- simdutf/LICENSE-MIT- simdutf/simdutf.h- tests/literal-rule-test.sh- tests/LiteralRuleTest.hs--flag developer- description: operate in developer mode- default: False- manual: True--flag simdutf- description: use simdutf library- default: True- manual: True--library- c-sources: cbits/is_ascii.c- cbits/measure_off.c- cbits/reverse.c- cbits/utils.c- hs-source-dirs: src-- if flag(simdutf)- include-dirs: simdutf- cxx-sources: simdutf/simdutf.cpp- cbits/validate_utf8.cpp- cxx-options: -std=c++17- cpp-options: -DSIMDUTF- if os(windows)- if arch(x86_64)- extra-libraries: stdc++-6 gcc_s_seh-1- else- extra-libraries: stdc++-6 gcc_s_dw2-1- else- if os(darwin)- extra-libraries: c++- else- extra-libraries: stdc++-- exposed-modules:- Data.Text- Data.Text.Array- Data.Text.Encoding- Data.Text.Encoding.Error- Data.Text.Foreign- Data.Text.IO- Data.Text.Internal- Data.Text.Internal.Builder- Data.Text.Internal.Builder.Functions- Data.Text.Internal.Builder.Int.Digits- Data.Text.Internal.Builder.RealFloat.Functions- Data.Text.Internal.ByteStringCompat- Data.Text.Internal.PrimCompat- Data.Text.Internal.Encoding.Fusion- Data.Text.Internal.Encoding.Fusion.Common- Data.Text.Internal.Encoding.Utf16- Data.Text.Internal.Encoding.Utf32- Data.Text.Internal.Encoding.Utf8- Data.Text.Internal.Fusion- Data.Text.Internal.Fusion.CaseMapping- Data.Text.Internal.Fusion.Common- Data.Text.Internal.Fusion.Size- Data.Text.Internal.Fusion.Types- Data.Text.Internal.IO- Data.Text.Internal.Lazy- Data.Text.Internal.Lazy.Encoding.Fusion- Data.Text.Internal.Lazy.Fusion- Data.Text.Internal.Lazy.Search- Data.Text.Internal.Private- Data.Text.Internal.Read- Data.Text.Internal.Search- Data.Text.Internal.Unsafe- Data.Text.Internal.Unsafe.Char- Data.Text.Lazy- Data.Text.Lazy.Builder- Data.Text.Lazy.Builder.Int- Data.Text.Lazy.Builder.RealFloat- Data.Text.Lazy.Encoding- Data.Text.Lazy.IO- Data.Text.Lazy.Internal- Data.Text.Lazy.Read- Data.Text.Read- Data.Text.Unsafe-- other-modules:- Data.Text.Show-- build-depends:- array >= 0.3 && < 0.6,- base >= 4.9 && < 5,- binary >= 0.5 && < 0.9,- bytestring >= 0.10.4 && < 0.12,- deepseq >= 1.1 && < 1.5,- ghc-prim >= 0.2 && < 0.9,- template-haskell >= 2.5 && < 2.19-- ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2- if flag(developer)- ghc-options: -fno-ignore-asserts- cpp-options: -DASSERTS-- -- https://gitlab.haskell.org/ghc/ghc/-/issues/19900- if os(windows)- extra-libraries: gcc_s-- default-language: Haskell2010- default-extensions:- NondecreasingIndentation- other-extensions:- BangPatterns- CPP- DeriveDataTypeable- ExistentialQuantification- ForeignFunctionInterface- GeneralizedNewtypeDeriving- MagicHash- OverloadedStrings- Rank2Types- RankNTypes- RecordWildCards- Safe- ScopedTypeVariables- TemplateHaskellQuotes- Trustworthy- TypeFamilies- UnboxedTuples- UnliftedFFITypes--source-repository head- type: git- location: https://github.com/haskell/text--test-suite tests- type: exitcode-stdio-1.0- ghc-options:- -Wall -threaded -rtsopts-- hs-source-dirs: tests- main-is: Tests.hs- other-modules:- Tests.Lift- Tests.Properties- Tests.Properties.Basics- Tests.Properties.Builder- Tests.Properties.Folds- Tests.Properties.Instances- Tests.Properties.LowLevel- Tests.Properties.Read- Tests.Properties.Substrings- Tests.Properties.Text- Tests.Properties.Transcoding- Tests.QuickCheckUtils- Tests.Regressions- Tests.SlowFunctions- Tests.Utils-- build-depends:- QuickCheck >= 2.12.6 && < 2.15,- base <5,- bytestring,- deepseq,- directory,- ghc-prim,- tasty,- tasty-hunit,- tasty-quickcheck,- template-haskell,- text-- -- Starting from 9.2 ghc library depends on parsec,- -- which causes circular dependency.- if impl(ghc < 9.2)- build-depends: tasty-inspection-testing-- default-language: Haskell2010- default-extensions: NondecreasingIndentation--benchmark text-benchmarks- type: exitcode-stdio-1.0-- ghc-options: -Wall -O2 -rtsopts- if impl(ghc >= 8.10)- ghc-options: "-with-rtsopts=-A32m --nonmoving-gc"- else- ghc-options: "-with-rtsopts=-A32m"-- build-depends: base,- bytestring >= 0.10.4,- containers,- deepseq,- directory,- filepath,- tasty-bench >= 0.2,- text,- transformers-- hs-source-dirs: benchmarks/haskell- main-is: Benchmarks.hs- other-modules:- Benchmarks.Builder- Benchmarks.Concat- Benchmarks.DecodeUtf8- Benchmarks.EncodeUtf8- Benchmarks.Equality- Benchmarks.FileRead- Benchmarks.FoldLines- Benchmarks.Multilang- Benchmarks.Programs.BigTable- Benchmarks.Programs.Cut- Benchmarks.Programs.Fold- Benchmarks.Programs.Sort- Benchmarks.Programs.StripTags- Benchmarks.Programs.Throughput- Benchmarks.Pure- Benchmarks.ReadNumbers- Benchmarks.Replace- Benchmarks.Search- Benchmarks.Stream- Benchmarks.WordFrequencies-- default-language: Haskell2010- default-extensions: NondecreasingIndentation- other-extensions: DeriveGeneric+cabal-version: 2.2 +name: text +version: 2.1.4 + +homepage: https://github.com/haskell/text +bug-reports: https://github.com/haskell/text/issues +synopsis: An efficient packed Unicode text type. +description: + . + An efficient packed, immutable Unicode text type (both strict and + lazy). + . + The 'Text' type represents Unicode character strings, in a time and + space-efficient manner. This package provides text processing + capabilities that are optimized for performance critical use, both + in terms of large data quantities and high speed. + . + The 'Text' type provides character-encoding, type-safe case + conversion via whole-string case conversion functions (see "Data.Text"). + It also provides a range of functions for converting 'Text' values to + and from 'ByteStrings', using several standard encodings + (see "Data.Text.Encoding"). + . + Efficient locale-sensitive support for text IO is also supported + (see "Data.Text.IO"). + . + These modules are intended to be imported qualified, to avoid name + clashes with Prelude functions, e.g. + . + > import qualified Data.Text as T + . + == ICU Support + . + To use an extended and very rich family of functions for working + with Unicode text (including normalization, regular expressions, + non-standard encodings, text breaking, and locales), see + the [text-icu package](https://hackage.haskell.org/package/text-icu) + based on the well-respected and liberally + licensed [ICU library](http://site.icu-project.org/). + +license: BSD-2-Clause +license-file: LICENSE +author: Bryan O'Sullivan <bos@serpentine.com> +maintainer: Haskell Text Team <andrew.lelechenko@gmail.com>, Core Libraries Committee +copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko +category: Data, Text +build-type: Simple +tested-with: + GHC == 8.4.4 + GHC == 8.6.5 + GHC == 8.8.4 + GHC == 8.10.7 + GHC == 9.0.2 + GHC == 9.2.8 + GHC == 9.4.8 + GHC == 9.6.7 + GHC == 9.8.4 + GHC == 9.10.1 + GHC == 9.12.2 + +extra-source-files: + -- scripts/CaseFolding.txt + -- scripts/SpecialCasing.txt + scripts/*.hs + simdutf/LICENSE-APACHE + simdutf/LICENSE-MIT + simdutf/simdutf_c.h + simdutf/simdutf.h + tests/literal-rule-test.sh + tests/LiteralRuleTest.hs +extra-doc-files: + README.md + changelog.md + +flag developer + description: operate in developer mode + default: False + manual: True + +flag simdutf + description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed + default: True + manual: True + +flag pure-haskell + description: Don't use text's standard C routines + NB: This feature is not fully implemented. Several C routines are still in + use. + + When this flag is true, text will use pure Haskell variants of the + routines. This is not recommended except for use with GHC's JavaScript + backend. + + This flag also disables simdutf. + + default: False + manual: True + +flag ExtendedBenchmarks + description: Runs extra benchmarks which can be very slow. + default: False + manual: True + +library + if arch(javascript) || flag(pure-haskell) + cpp-options: -DPURE_HASKELL + else + c-sources: cbits/is_ascii.c + cbits/reverse.c + cbits/utils.c + if (arch(aarch64)) + c-sources: cbits/aarch64/measure_off.c + else + c-sources: cbits/measure_off.c + + hs-source-dirs: src + + if flag(simdutf) && !(arch(javascript) || flag(pure-haskell)) + exposed-modules: Data.Text.Internal.Validate.Simd + include-dirs: simdutf + c-sources: simdutf/hs_simdutf.c + cxx-sources: simdutf/simdutf.cpp + cxx-options: -std=c++17 + cpp-options: -DSIMDUTF + if impl(ghc >= 9.4) + build-depends: system-cxx-std-lib == 1.0 + elif os(darwin) || os(freebsd) + extra-libraries: c++ + elif os(openbsd) + extra-libraries: c++ c++abi pthread + elif os(windows) + -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later + if impl(ghc < 9.3) + extra-libraries: stdc++ + else + extra-libraries: c++ c++abi + elif arch(wasm32) + cpp-options: -DSIMDUTF_NO_THREADS + cxx-options: -fno-exceptions + extra-libraries: c++ c++abi + else + extra-libraries: stdc++ + + -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417 + if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1) + build-depends: base < 0 + + -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker. + if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1) + build-depends: base < 0 + + -- GHC 8.10 has linking issues (probably TH-related) on ARM. + if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*) + build-depends: base < 0 + + -- Subword primitives in GHC 9.2.1 are broken on ARM platforms. + if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1) + build-depends: base < 0 + + -- NetBSD + GHC 9.2.1 + TH + C++ does not work together. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577 + if flag(simdutf) && os(netbsd) && impl(ghc < 9.4) + build-depends: base < 0 + + exposed-modules: + Data.Text + Data.Text.Array + Data.Text.Encoding + Data.Text.Encoding.Error + Data.Text.Foreign + Data.Text.IO + Data.Text.IO.Utf8 + Data.Text.Internal + Data.Text.Internal.ArrayUtils + Data.Text.Internal.Builder + Data.Text.Internal.Builder.Functions + Data.Text.Internal.Builder.Int.Digits + Data.Text.Internal.Builder.RealFloat.Functions + Data.Text.Internal.ByteStringCompat + Data.Text.Internal.PrimCompat + Data.Text.Internal.Encoding + Data.Text.Internal.Encoding.Fusion + Data.Text.Internal.Encoding.Fusion.Common + Data.Text.Internal.Encoding.Utf16 + Data.Text.Internal.Encoding.Utf32 + Data.Text.Internal.Encoding.Utf8 + Data.Text.Internal.Fusion + Data.Text.Internal.Fusion.CaseMapping + Data.Text.Internal.Fusion.Common + Data.Text.Internal.Fusion.Size + Data.Text.Internal.Fusion.Types + Data.Text.Internal.IO + Data.Text.Internal.Lazy + Data.Text.Internal.Lazy.Encoding.Fusion + Data.Text.Internal.Lazy.Fusion + Data.Text.Internal.Lazy.Search + Data.Text.Internal.Private + Data.Text.Internal.Read + Data.Text.Internal.Search + Data.Text.Internal.StrictBuilder + Data.Text.Internal.Unsafe + Data.Text.Internal.Unsafe.Char + Data.Text.Internal.Validate + Data.Text.Internal.Validate.Native + Data.Text.Lazy + Data.Text.Lazy.Builder + Data.Text.Lazy.Builder.Int + Data.Text.Lazy.Builder.RealFloat + Data.Text.Lazy.Encoding + Data.Text.Lazy.IO + Data.Text.Lazy.Internal + Data.Text.Lazy.Read + Data.Text.Read + Data.Text.Unsafe + + other-modules: + Data.Text.Show + Data.Text.Internal.Measure + Data.Text.Internal.Reverse + Data.Text.Internal.Transformation + Data.Text.Internal.IsAscii + + build-depends: + array >= 0.3 && < 0.6, + base >= 4.11 && < 5, + binary >= 0.8.3 && < 0.9, + bytestring >= 0.10.4 && < 0.13, + deepseq >= 1.1 && < 1.6, + ghc-prim >= 0.2 && < 0.15, + + -- template-haskell-lift was added as a boot library in GHC-9.14 + -- once we no longer wish to backport releases to older major releases of GHC, + -- this conditional can be dropped + if impl(ghc < 9.14) + build-depends: template-haskell >= 2.5 && < 3 + else + build-depends: template-haskell-lift >= 0.1 && <0.2 + + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + + ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 + if flag(developer) + ghc-options: -fno-ignore-asserts + cpp-options: -DASSERTS + if impl(ghc >= 9.2.2) + ghc-options: -fcheck-prim-bounds + + default-language: Haskell2010 + default-extensions: + NondecreasingIndentation + other-extensions: + BangPatterns + CPP + DeriveDataTypeable + ExistentialQuantification + ForeignFunctionInterface + GeneralizedNewtypeDeriving + MagicHash + OverloadedStrings + Rank2Types + RankNTypes + RecordWildCards + Safe + ScopedTypeVariables + TemplateHaskellQuotes + Trustworthy + TypeFamilies + UnboxedTuples + UnliftedFFITypes + +source-repository head + type: git + location: https://github.com/haskell/text + +test-suite tests + type: exitcode-stdio-1.0 + ghc-options: + -Wall -threaded -rtsopts -with-rtsopts=-N + + hs-source-dirs: tests + main-is: Tests.hs + other-modules: + Tests.Lift + Tests.Properties + Tests.Properties.Basics + Tests.Properties.Builder + Tests.Properties.Folds + Tests.Properties.Instances + Tests.Properties.LowLevel + Tests.Properties.Read + Tests.Properties.Substrings + Tests.Properties.Text + Tests.Properties.Transcoding + Tests.Properties.CornerCases + Tests.Properties.Validate + Tests.QuickCheckUtils + Tests.RebindableSyntaxTest + Tests.Regressions + Tests.SlowFunctions + Tests.ShareEmpty + Tests.Utils + + build-depends: + QuickCheck >= 2.12.6 && < 2.18, + base <5, + binary, + bytestring, + deepseq, + ghc-prim, + tasty, + tasty-hunit, + tasty-quickcheck, + temporary, + transformers, + text + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + if impl(ghc < 9.14) + build-depends: template-haskell + else + build-depends: template-haskell-lift + + -- Plugin infrastructure does not work properly in 8.6.1, and + -- ghc-9.2.1 library depends on parsec, which causes a circular dependency. + if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2) + build-depends: tasty-inspection-testing + + -- https://github.com/haskellari/splitmix/issues/101 + if os(openbsd) + build-depends: splitmix < 0.1.3 || > 0.1.3.1 + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + +benchmark text-benchmarks + type: exitcode-stdio-1.0 + + ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m" + if impl(ghc >= 8.6) + ghc-options: -fproc-alignment=64 + if flag(ExtendedBenchmarks) + cpp-options: -DExtendedBenchmarks + + build-depends: base, + bytestring >= 0.10.4, + containers, + deepseq, + directory, + filepath, + tasty-bench >= 0.2, + temporary, + text, + transformers + + hs-source-dirs: benchmarks/haskell + main-is: Benchmarks.hs + other-modules: + Benchmarks.Builder + Benchmarks.Concat + Benchmarks.DecodeUtf8 + Benchmarks.EncodeUtf8 + Benchmarks.Equality + Benchmarks.FileRead + Benchmarks.FileWrite + Benchmarks.FoldLines + Benchmarks.Micro + Benchmarks.Multilang + Benchmarks.Programs.BigTable + Benchmarks.Programs.Cut + Benchmarks.Programs.Fold + Benchmarks.Programs.Sort + Benchmarks.Programs.StripTags + Benchmarks.Programs.Throughput + Benchmarks.Pure + Benchmarks.ReadNumbers + Benchmarks.Replace + Benchmarks.Search + Benchmarks.Stream + Benchmarks.WordFrequencies + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + other-extensions: DeriveGeneric